diff --git a/Makefile b/Makefile index 52e16ecb528..5ceb601fa0f 100644 --- a/Makefile +++ b/Makefile @@ -129,11 +129,19 @@ test-compile-es-scripts: .PHONY: index-cleaner-integration-test index-cleaner-integration-test: docker-images-elastic - # Expire tests results for storage integration tests since the environment might change + # Expire test results for storage integration tests since the environment might change # even though the code remains the same. go clean -testcache bash -c "set -e; set -o pipefail; $(GOTEST) -tags index_cleaner $(STORAGE_PKGS) | $(COLORIZE)" +.PHONY: index-rollover-integration-test +index-rollover-integration-test: docker-images-elastic + # Expire test results for storage integration tests since the environment might change + # even though the code remains the same. + go clean -testcache + bash -c "set -e; set -o pipefail; $(GOTEST) -tags index_rollover $(STORAGE_PKGS) | $(COLORIZE)" + + .PHONY: token-propagation-integration-test token-propagation-integration-test: go clean -testcache @@ -217,6 +225,14 @@ build-tracegen: build-anonymizer: $(GOBUILD) -o ./cmd/anonymizer/anonymizer-$(GOOS)-$(GOARCH) ./cmd/anonymizer/main.go +.PHONY: build-templatizer +build-templatizer: + $(GOBUILD) -o ./plugin/storage/es/templatizer-$(GOOS)-$(GOARCH) ./cmd/templatizer/main.go + +.PHONY: build-templatizer-linux +build-templatizer-linux: + GOOS=linux GOARCH=amd64 $(GOBUILD) -o ./plugin/storage/es/templatizer ./cmd/templatizer/main.go + .PHONY: docker-hotrod docker-hotrod: GOOS=linux $(MAKE) build-examples @@ -336,7 +352,7 @@ docker-images-cassandra: @echo "Finished building jaeger-cassandra-schema ==============" .PHONY: docker-images-elastic -docker-images-elastic: +docker-images-elastic: build-templatizer-linux docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} plugin/storage/es docker build -t $(DOCKER_NAMESPACE)/jaeger-es-rollover:${DOCKER_TAG} plugin/storage/es -f plugin/storage/es/Dockerfile.rollover @echo "Finished building jaeger-es-indices-clean ==============" diff --git a/cmd/opentelemetry/app/exporter/elasticsearchexporter/exporter.go b/cmd/opentelemetry/app/exporter/elasticsearchexporter/exporter.go index d9118c984b5..90462a2f2b5 100644 --- a/cmd/opentelemetry/app/exporter/elasticsearchexporter/exporter.go +++ b/cmd/opentelemetry/app/exporter/elasticsearchexporter/exporter.go @@ -20,6 +20,7 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/exporter/exporterhelper" + esTemplate "github.com/jaegertracing/jaeger/pkg/es" "github.com/jaegertracing/jaeger/plugin/storage/es" ) @@ -31,7 +32,10 @@ func newExporter(ctx context.Context, config *Config, params component.ExporterC return nil, err } if config.Primary.IsCreateIndexTemplates() { - spanMapping, serviceMapping := es.GetSpanServiceMappings(esCfg.GetNumShards(), esCfg.GetNumReplicas(), uint(w.esClientVersion())) + spanMapping, serviceMapping, err := es.GetSpanServiceMappings(esTemplate.TextTemplateBuilder{}, esCfg.GetNumShards(), esCfg.GetNumReplicas(), uint(w.esClientVersion()), esCfg.GetIndexPrefix(), esCfg.GetUseILM()) + if err != nil { + return nil, err + } if err = w.CreateTemplates(ctx, spanMapping, serviceMapping); err != nil { return nil, err } diff --git a/cmd/opentelemetry/app/exporter/elasticsearchexporter/integration_test.go b/cmd/opentelemetry/app/exporter/elasticsearchexporter/integration_test.go index 2d28d20efe6..aaed59e33db 100644 --- a/cmd/opentelemetry/app/exporter/elasticsearchexporter/integration_test.go +++ b/cmd/opentelemetry/app/exporter/elasticsearchexporter/integration_test.go @@ -32,6 +32,7 @@ import ( "github.com/jaegertracing/jaeger/cmd/opentelemetry/app/internal/esclient" "github.com/jaegertracing/jaeger/cmd/opentelemetry/app/internal/reader/es/esdependencyreader" "github.com/jaegertracing/jaeger/cmd/opentelemetry/app/internal/reader/es/esspanreader" + esTemplate "github.com/jaegertracing/jaeger/pkg/es" "github.com/jaegertracing/jaeger/pkg/es/config" "github.com/jaegertracing/jaeger/pkg/testutils" "github.com/jaegertracing/jaeger/plugin/storage/es" @@ -104,7 +105,10 @@ func (s *IntegrationTest) initSpanstore(allTagsAsFields bool) error { return err } esVersion := uint(w.esClientVersion()) - spanMapping, serviceMapping := es.GetSpanServiceMappings(numShards, numReplicas, esVersion) + spanMapping, serviceMapping, err := es.GetSpanServiceMappings(esTemplate.TextTemplateBuilder{}, numShards, numReplicas, esVersion, "", false) + if err != nil { + return err + } err = w.CreateTemplates(context.Background(), spanMapping, serviceMapping) if err != nil { return err @@ -127,7 +131,10 @@ func (s *IntegrationTest) initSpanstore(allTagsAsFields bool) error { }) s.SpanReader = reader - depMapping := es.GetDependenciesMappings(numShards, numReplicas, esVersion) + depMapping, err := es.GetDependenciesMappings(esTemplate.TextTemplateBuilder{}, numShards, numReplicas, esVersion) + if err != nil { + return err + } depStore := esdependencyreader.NewDependencyStore(elasticsearchClient, s.logger, indexPrefix, indexDateLayout, defaultMaxDocCount) if err := depStore.CreateTemplates(depMapping); err != nil { return nil diff --git a/cmd/opentelemetry/go.sum b/cmd/opentelemetry/go.sum index cf5faef06f8..33d45f7fc5b 100644 --- a/cmd/opentelemetry/go.sum +++ b/cmd/opentelemetry/go.sum @@ -1024,8 +1024,10 @@ github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5/go.mod h1:ppEjwdhyy7Y31EnHRDm1JkChoC7LXIJ7Ex0VYLWtZtQ= github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= diff --git a/cmd/templatizer/app/flags.go b/cmd/templatizer/app/flags.go new file mode 100644 index 00000000000..3f3efe39d8d --- /dev/null +++ b/cmd/templatizer/app/flags.go @@ -0,0 +1,81 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +import ( + "github.com/spf13/cobra" +) + +// Options represent configurable parameters for jaeger-templatizer +type Options struct { + Mapping string + EsVersion int64 + Shards int64 + Replicas int64 + EsPrefix string + UseILM string // using string as util is being used in python and using bool leads to type issues. +} + +const ( + mappingFlag = "mapping" + esVersionFlag = "esVersion" + shardsFlag = "shards" + replicasFlag = "replicas" + esPrefixFlag = "esPrefix" + useILMFlag = "useILM" +) + +// AddFlags adds flags for templatizer main program +func (o *Options) AddFlags(command *cobra.Command) { + command.Flags().StringVarP( + &o.Mapping, + mappingFlag, + "m", + "", + "Pass either jaeger-span or jaeger-service") + command.Flags().Int64VarP( + &o.EsVersion, + esVersionFlag, + "v", + 7, + "the major Elasticsearch version") + command.Flags().Int64VarP( + &o.Shards, + shardsFlag, + "s", + 5, + "the number of shards per index in Elasticsearch") + command.Flags().Int64VarP( + &o.Replicas, + replicasFlag, + "r", + 1, + "the number of replicas per index in Elasticsearch") + command.Flags().StringVarP( + &o.EsPrefix, + esPrefixFlag, + "e", + "", + "specifies index prefix") + command.Flags().StringVarP( + &o.UseILM, + useILMFlag, + "u", + "false", + "set to true to use ILM for managing lifecycle of jaeger indices") + + // mark mapping flag as mandatory + command.MarkFlagRequired(mappingFlag) +} diff --git a/cmd/templatizer/app/flags_test.go b/cmd/templatizer/app/flags_test.go new file mode 100644 index 00000000000..2d3149a3c6b --- /dev/null +++ b/cmd/templatizer/app/flags_test.go @@ -0,0 +1,57 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +import ( + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" +) + +func TestOptionsWithDefaultFlags(t *testing.T) { + o := Options{} + c := cobra.Command{} + o.AddFlags(&c) + + assert.Equal(t, "", o.Mapping) + assert.Equal(t, int64(7), o.EsVersion) + assert.Equal(t, int64(5), o.Shards) + assert.Equal(t, int64(1), o.Replicas) + assert.Equal(t, "", o.EsPrefix) + assert.Equal(t, "false", o.UseILM) +} + +func TestOptionsWithFlags(t *testing.T) { + o := Options{} + c := cobra.Command{} + + o.AddFlags(&c) + c.ParseFlags([]string{ + "--mapping=jaeger-span", + "--esVersion=6", + "--shards=5", + "--replicas=1", + "--esPrefix=test", + "--useILM=true", + }) + + assert.Equal(t, "jaeger-span", o.Mapping) + assert.Equal(t, int64(6), o.EsVersion) + assert.Equal(t, int64(5), o.Shards) + assert.Equal(t, int64(1), o.Replicas) + assert.Equal(t, "test", o.EsPrefix) + assert.Equal(t, "true", o.UseILM) +} diff --git a/cmd/templatizer/app/renderer/render.go b/cmd/templatizer/app/renderer/render.go new file mode 100644 index 00000000000..3bbedf7d126 --- /dev/null +++ b/cmd/templatizer/app/renderer/render.go @@ -0,0 +1,61 @@ +// Copyright (c) 2021 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package renderer + +import ( + "strconv" + + "github.com/jaegertracing/jaeger/cmd/templatizer/app" + esTemplate "github.com/jaegertracing/jaeger/pkg/es" + "github.com/jaegertracing/jaeger/plugin/storage/es" +) + +var fixMappingFunc = es.FixMapping +var loadMappingFunc = es.LoadMapping +var mapping string +var err error + +// GetMappingAsString returns rendered index templates as string +func GetMappingAsString(opt *app.Options) (string, error) { + + if opt.EsVersion == 7 { + enableILM, err := strconv.ParseBool(opt.UseILM) + if err != nil { + return "", err + } + mapping, err = fixMappingFunc(esTemplate.TextTemplateBuilder{}, loadMappingFunc("/"+opt.Mapping+"-7.json"), opt.Shards, opt.Replicas, opt.EsPrefix, enableILM) + if err != nil { + return "", err + } + } else { + mapping, err = fixMappingFunc(esTemplate.TextTemplateBuilder{}, loadMappingFunc("/"+opt.Mapping+".json"), opt.Shards, opt.Replicas, "", false) + if err != nil { + return "", err + } + } + return mapping, nil + +} + +// IsValidOption checks if passed option is a valid index template. +func IsValidOption(val string) bool { + allowedValues := []string{"jaeger-span", "jaeger-service"} + for _, value := range allowedValues { + if val == value { + return true + } + } + return false +} diff --git a/cmd/templatizer/app/renderer/render_test.go b/cmd/templatizer/app/renderer/render_test.go new file mode 100644 index 00000000000..272717c005d --- /dev/null +++ b/cmd/templatizer/app/renderer/render_test.go @@ -0,0 +1,102 @@ +// Copyright (c) 2021 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package renderer + +import ( + "errors" + "testing" + + "github.com/jaegertracing/jaeger/cmd/templatizer/app" + esTemplate "github.com/jaegertracing/jaeger/pkg/es" +) + +func TestIsValidOption(t *testing.T) { + tests := []struct { + name string + arg string + expectedValue bool + }{{name: "span mapping", arg: "jaeger-span", expectedValue: true}, + {name: "service mapping", arg: "jaeger-service", expectedValue: true}, + {name: "Invalid mapping", arg: "dependency-service", expectedValue: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := IsValidOption(test.arg); got != test.expectedValue { + t.Errorf("isValidOption() = %v, want %v", got, test.expectedValue) + } + }) + } +} + +func Test_getMappingAsString(t *testing.T) { + + tests := []struct { + name string + args app.Options + fixMappingFunc func(esTemplate.TemplateBuilder, string, int64, int64, string, bool) (string, error) + want string + wantErr bool + }{ + {name: "ES version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "true"}, + fixMappingFunc: func(esTemplate.TemplateBuilder, string, int64, int64, string, bool) (string, error) { + return "ES version 7", nil + }, + want: "ES version 7", + wantErr: false, + }, + {name: "ES version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "false"}, + fixMappingFunc: func(esTemplate.TemplateBuilder, string, int64, int64, string, bool) (string, error) { + return "ES version 6", nil + }, + want: "ES version 6", + wantErr: false, + }, + {name: "Parse Error version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "false"}, + fixMappingFunc: func(esTemplate.TemplateBuilder, string, int64, int64, string, bool) (string, error) { + return "", errors.New("parse error") + }, + want: "", + wantErr: true, + }, {name: "Parse Error version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "true"}, + fixMappingFunc: func(esTemplate.TemplateBuilder, string, int64, int64, string, bool) (string, error) { + return "", errors.New("parse error") + }, + want: "", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + oldFixMappingFunc := fixMappingFunc + oldLoadMappingFunc := loadMappingFunc + defer func() { + fixMappingFunc = oldFixMappingFunc + loadMappingFunc = oldLoadMappingFunc + }() + + fixMappingFunc = tt.fixMappingFunc + loadMappingFunc = func(string) string { return "test" } + got, err := GetMappingAsString(&tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("getMappingAsString() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getMappingAsString() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/templatizer/main.go b/cmd/templatizer/main.go new file mode 100644 index 00000000000..87bf5017395 --- /dev/null +++ b/cmd/templatizer/main.go @@ -0,0 +1,60 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + "go.uber.org/zap" + + "github.com/jaegertracing/jaeger/cmd/templatizer/app" + "github.com/jaegertracing/jaeger/cmd/templatizer/app/renderer" + "github.com/jaegertracing/jaeger/pkg/version" +) + +var logger, _ = zap.NewDevelopment() + +func main() { + var options = app.Options{} + var command = &cobra.Command{ + Use: "jaeger-templatizer", + Short: "Jaeger templatizer prints rendered mappings as string", + Long: `Jaeger templatizer renders passed templates with provided values and prints rendered output to stdout`, + Run: func(cmd *cobra.Command, args []string) { + + if !renderer.IsValidOption(options.Mapping) { + logger.Fatal("please pass either 'jaeger-service' or 'jaeger-span' as argument") + } + + parsedMapping, err := renderer.GetMappingAsString(&options) + if err != nil { + logger.Fatal(err.Error()) + } + print(parsedMapping) + }, + } + + options.AddFlags(command) + + command.AddCommand(version.Command()) + + if err := command.Execute(); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + +} diff --git a/examples/hotrod/services/frontend/gen_assets.go b/examples/hotrod/services/frontend/gen_assets.go index b924f0754ea..46b66108f8a 100644 --- a/examples/hotrod/services/frontend/gen_assets.go +++ b/examples/hotrod/services/frontend/gen_assets.go @@ -213,7 +213,7 @@ var _escData = map[string]*_escFile{ name: "index.html", local: "examples/hotrod/services/frontend/web_assets/index.html", size: 4058, - modtime: 1601939528, + modtime: 1608397661, compressed: ` H4sIAAAAAAAC/9RXbVPjOPJ/n0/Rf23AyT/EzgMMTIgzxZJZhtmdZS7AbM1NUYcsd2yBLWUkOYFN5btf yXaCA0zd3au7zYtE6m5196+fFA1jkyajGsAwRUOBxVRpND45v7xoHx0dvG13yRNX0BR9Mue4mEllCDAp @@ -250,7 +250,7 @@ KtU3UHprcaxum88KvCzyyrNp6BVP+n8GAAD//wz8Z3XaDwAA name: "jquery-3.1.1.min.js", local: "examples/hotrod/services/frontend/web_assets/jquery-3.1.1.min.js", size: 86709, - modtime: 1598995573, + modtime: 1608397661, compressed: ` H4sIAAAAAAAC/8y9fZebONIo/v/9FG02D4PaMm1nZvbewa1wMslkN7vztpPMzO5iskeAwLgxuAGnO2PY z/47KkkgME5mn+fec34nJ21A71KpVFWql5vr2dXub0dWfrh6/7m9sldXzZUVIvXtVXHMI1qnRX7VXO3u diff --git a/go.sum b/go.sum index 67bd1db2f53..b8ae769b040 100644 --- a/go.sum +++ b/go.sum @@ -421,6 +421,8 @@ github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E= github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= @@ -796,6 +798,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= diff --git a/pkg/es/config/config.go b/pkg/es/config/config.go index 9edaac60efb..dd7ebf0a1ca 100644 --- a/pkg/es/config/config.go +++ b/pkg/es/config/config.go @@ -65,6 +65,7 @@ type Configuration struct { TLS tlscfg.Options `mapstructure:"tls"` UseReadWriteAliases bool `mapstructure:"use_aliases"` CreateIndexTemplates bool `mapstructure:"create_mappings"` + UseILM bool `mapstructure:"-"` Version uint `mapstructure:"version"` } @@ -100,6 +101,7 @@ type ClientBuilder interface { IsCreateIndexTemplates() bool GetVersion() uint TagKeysAsFields() ([]string, error) + GetUseILM() bool } // NewClient creates a new ElasticSearch client @@ -293,6 +295,11 @@ func (c *Configuration) GetUseReadWriteAliases() bool { return c.UseReadWriteAliases } +// GetUseILM indicates whether ILM should be used +func (c *Configuration) GetUseILM() bool { + return c.UseILM +} + // GetTokenFilePath returns file path containing the bearer token func (c *Configuration) GetTokenFilePath() string { return c.TokenFilePath diff --git a/pkg/es/mocks/TemplateApplier.go b/pkg/es/mocks/TemplateApplier.go new file mode 100644 index 00000000000..16c77d9097f --- /dev/null +++ b/pkg/es/mocks/TemplateApplier.go @@ -0,0 +1,43 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +// Copyright (c) 2021 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +package mocks + +import ( + io "io" + + mock "github.com/stretchr/testify/mock" +) + +// TemplateApplier is an autogenerated mock type for the TemplateApplier type +type TemplateApplier struct { + mock.Mock +} + +// Execute provides a mock function with given fields: wr, data +func (_m *TemplateApplier) Execute(wr io.Writer, data interface{}) error { + ret := _m.Called(wr, data) + + var r0 error + if rf, ok := ret.Get(0).(func(io.Writer, interface{}) error); ok { + r0 = rf(wr, data) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/pkg/es/mocks/TemplateBuilder.go b/pkg/es/mocks/TemplateBuilder.go new file mode 100644 index 00000000000..459e9c345f3 --- /dev/null +++ b/pkg/es/mocks/TemplateBuilder.go @@ -0,0 +1,52 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +// Copyright (c) 2021 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + + es "github.com/jaegertracing/jaeger/pkg/es" +) + +// TemplateBuilder is an autogenerated mock type for the TemplateBuilder type +type TemplateBuilder struct { + mock.Mock +} + +// Parse provides a mock function with given fields: text +func (_m *TemplateBuilder) Parse(text string) (es.TemplateApplier, error) { + ret := _m.Called(text) + + var r0 es.TemplateApplier + if rf, ok := ret.Get(0).(func(string) es.TemplateApplier); ok { + r0 = rf(text) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(es.TemplateApplier) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(text) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/pkg/es/textTemplate.go b/pkg/es/textTemplate.go new file mode 100644 index 00000000000..304b8b63476 --- /dev/null +++ b/pkg/es/textTemplate.go @@ -0,0 +1,39 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package es + +import ( + "io" + "text/template" +) + +// TemplateApplier applies a parsed template to input data that maps to the template's variables. +type TemplateApplier interface { + Execute(wr io.Writer, data interface{}) error +} + +// TemplateBuilder parses a given string and returns TemplateApplier +// TemplateBuilder is an abstraction to support mocking template/text +type TemplateBuilder interface { + Parse(text string) (TemplateApplier, error) +} + +// TextTemplateBuilder implements TemplateBuilder +type TextTemplateBuilder struct{} + +// Parse is a wrapper for template.Parse +func (t TextTemplateBuilder) Parse(mapping string) (TemplateApplier, error) { + return template.New("mapping").Parse(mapping) +} diff --git a/pkg/es/textTemplate_test.go b/pkg/es/textTemplate_test.go new file mode 100644 index 00000000000..29488841654 --- /dev/null +++ b/pkg/es/textTemplate_test.go @@ -0,0 +1,36 @@ +// Copyright (c) 2020 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package es + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParse(t *testing.T) { + + str := "tesing parse" + writer := new(bytes.Buffer) + testTemplateBuilder := TextTemplateBuilder{} + parsedStr, err := testTemplateBuilder.Parse(str) + require.NoError(t, err) + err = parsedStr.Execute(writer, "") + require.NoError(t, err) + assert.Equal(t, str, writer.String()) + +} diff --git a/plugin/storage/es/Dockerfile.rollover b/plugin/storage/es/Dockerfile.rollover index 2be8b3064c8..eac9401a531 100644 --- a/plugin/storage/es/Dockerfile.rollover +++ b/plugin/storage/es/Dockerfile.rollover @@ -3,8 +3,9 @@ FROM python:3-alpine3.11 # Temporary fix for https://github.com/jaegertracing/jaeger/issues/1494 RUN pip install urllib3==1.24.3 -RUN pip install elasticsearch elasticsearch-curator pathlib2 +RUN pip install elasticsearch elasticsearch-curator COPY ./mappings/* /mappings/ COPY esRollover.py /es-rollover/ +COPY templatizer /usr/bin/ ENTRYPOINT ["python3", "/es-rollover/esRollover.py"] diff --git a/plugin/storage/es/esRollover.py b/plugin/storage/es/esRollover.py index 2b940af0ebd..cb63110895d 100755 --- a/plugin/storage/es/esRollover.py +++ b/plugin/storage/es/esRollover.py @@ -7,11 +7,11 @@ import os import requests import ssl +import subprocess import sys -from pathlib2 import Path +import re from requests.auth import HTTPBasicAuth - ARCHIVE_INDEX = 'jaeger-span-archive' ROLLBACK_CONDITIONS = '{"max_age": "2d"}' UNIT = 'days' @@ -19,9 +19,12 @@ SHARDS = 5 REPLICAS = 1 + def main(): if len(sys.argv) != 3: - print('USAGE: [INDEX_PREFIX=(default "")] [ARCHIVE=(default false)] ... {} ACTION http://HOSTNAME[:PORT]'.format(sys.argv[0])) + print( + 'USAGE: [INDEX_PREFIX=(default "")] [ARCHIVE=(default false)] ... {} ACTION http://HOSTNAME[:PORT]'.format( + sys.argv[0])) print('ACTION ... one of:') print('\tinit - creates indices and aliases') print('\trollover - rollover to new write index') @@ -35,19 +38,26 @@ def main(): print('ES_TLS_CA ... Path to TLS CA file.') print('ES_TLS_CERT ... Path to TLS certificate file.') print('ES_TLS_KEY ... Path to TLS key file.') + print('ES_USE_ILM .. Use ILM to manage jaeger indices.') print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.') - print('ES_VERSION ... The major Elasticsearch version. If not specified, the value will be auto-detected from Elasticsearch.') + print( + 'ES_VERSION ... The major Elasticsearch version. If not specified, the value will be auto-detected from Elasticsearch.') print('init configuration:') print('\tSHARDS ... the number of shards per index in Elasticsearch (default {}).'.format(SHARDS)) print('\tREPLICAS ... the number of replicas per index in Elasticsearch (default {}).'.format(REPLICAS)) print('rollover configuration:') - print('\tCONDITIONS ... conditions used to rollover to a new write index (default \'{}\'.'.format(ROLLBACK_CONDITIONS)) + print('\tCONDITIONS ... conditions used to rollover to a new write index (default \'{}\'.'.format( + ROLLBACK_CONDITIONS)) print('lookback configuration:') - print('\tUNIT ... used with lookback to remove indices from read alias e.g. ..., days, weeks, months, years (default {}).'.format(UNIT)) + print( + '\tUNIT ... used with lookback to remove indices from read alias e.g. ..., days, weeks, months, years (default {}).'.format( + UNIT)) print('\tUNIT_COUNT ... count of UNITs (default {}).'.format(UNIT_COUNT)) sys.exit(1) - client = create_client(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false'))) + client = create_client(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), + os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), + str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false'))) prefix = os.getenv('INDEX_PREFIX', '') if prefix != '': prefix += '-' @@ -57,38 +67,43 @@ def main(): if str2bool(os.getenv('ARCHIVE', 'false')): write_alias = prefix + ARCHIVE_INDEX + '-write' read_alias = prefix + ARCHIVE_INDEX + '-read' - perform_action(action, client, write_alias, read_alias, prefix+'jaeger-span-archive', 'jaeger-span') + perform_action(action, client, write_alias, read_alias, prefix + 'jaeger-span-archive', 'jaeger-span', prefix) else: write_alias = prefix + 'jaeger-span-write' read_alias = prefix + 'jaeger-span-read' - perform_action(action, client, write_alias, read_alias, prefix+'jaeger-span', 'jaeger-span') + perform_action(action, client, write_alias, read_alias, prefix + 'jaeger-span', 'jaeger-span', prefix) write_alias = prefix + 'jaeger-service-write' read_alias = prefix + 'jaeger-service-read' - perform_action(action, client, write_alias, read_alias, prefix+'jaeger-service', 'jaeger-service') + perform_action(action, client, write_alias, read_alias, prefix + 'jaeger-service', 'jaeger-service', prefix) -def perform_action(action, client, write_alias, read_alias, index_to_rollover, template_name): +def perform_action(action, client, write_alias, read_alias, index_to_rollover, template_name, prefix): if action == 'init': shards = os.getenv('SHARDS', SHARDS) replicas = os.getenv('REPLICAS', REPLICAS) esVersion = get_version(client) + use_ilm = str2bool(os.getenv("ES_USE_ILM", 'false')) if esVersion == 7: - mapping = Path('./mappings/'+template_name+'-7.json').read_text() + if use_ilm: + check_if_ilm_policy_exists("jaeger-ilm-policy") else: - mapping = Path('./mappings/'+template_name+'.json').read_text() - create_index_template(fix_mapping(mapping, shards, replicas), template_name) + if use_ilm: + sys.exit("ILM is supported only for ES version 7+") + create_index_template(fix_mapping(template_name, esVersion, shards, replicas, prefix.rstrip("-"), use_ilm), + prefix + template_name) index = index_to_rollover + '-000001' create_index(client, index) if is_alias_empty(client, read_alias): - create_aliases(client, read_alias, index) + create_aliases(client, read_alias, index, use_ilm) if is_alias_empty(client, write_alias): - create_aliases(client, write_alias, index) + create_aliases(client, write_alias, index, use_ilm) elif action == 'rollover': cond = ast.literal_eval(os.getenv('CONDITIONS', ROLLBACK_CONDITIONS)) rollover(client, write_alias, read_alias, cond) elif action == 'lookback': - read_alias_lookback(client, write_alias, read_alias, os.getenv('UNIT', UNIT), int(os.getenv('UNIT_COUNT', UNIT_COUNT))) + read_alias_lookback(client, write_alias, read_alias, os.getenv('UNIT', UNIT), + int(os.getenv('UNIT_COUNT', UNIT_COUNT))) else: print('Unrecognized action {}'.format(action)) sys.exit(1) @@ -97,7 +112,9 @@ def perform_action(action, client, write_alias, read_alias, index_to_rollover, t def create_index_template(template, template_name): print('Creating index template {}'.format(template_name)) headers = {'Content-Type': 'application/json'} - s = get_request_session(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false')) + s = get_request_session(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), + os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), + os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false')) r = s.put(sys.argv[2] + '/_template/' + template_name, headers=headers, data=template) print(r.text) r.raise_for_status() @@ -112,15 +129,18 @@ def create_index(client, name): create.do_action() -def create_aliases(client, alias_name, archive_index_name): +def create_aliases(client, alias_name, archive_index_name, use_ilm): """" Create read write aliases """ ilo = curator.IndexList(client) - ilo.filter_by_regex(kind='regex', value='^'+archive_index_name+'$') - alias = curator.Alias(client=client, name=alias_name) + ilo.filter_by_regex(kind='regex', value='^' + archive_index_name + '$') for index in ilo.working_list(): print("Adding index {} to alias {}".format(index, alias_name)) + if re.search(r'write', alias_name) and use_ilm: + alias = curator.Alias(client=client, name=alias_name, extra_settings={'is_write_index': True}) + else: + alias = curator.Alias(client=client, name=alias_name) alias.add(ilo) alias.do_action() @@ -174,9 +194,15 @@ def str2bool(v): return v.lower() in ('true', '1') -def fix_mapping(mapping, shards, replicas): - mapping = mapping.replace("${__NUMBER_OF_SHARDS__}", str(shards)) - mapping = mapping.replace("${__NUMBER_OF_REPLICAS__}", str(replicas)) +def fix_mapping(template_name, esVersion, shards, replicas, esprefix, use_ilm): + output = subprocess.Popen(['templatizer', '--mapping', template_name, '--esVersion', str(esVersion), + '--shards', str(shards), '--replicas', + str(replicas), '--esPrefix', esprefix, '--useILM', str(use_ilm)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + mapping, stderr = output.communicate() + if output.returncode != 0: + sys.exit(mapping) return mapping @@ -227,6 +253,18 @@ def create_client(username, password, tls, ca, cert, key, skipHostVerify): return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context) +def check_if_ilm_policy_exists(ilm_policy): + """" + Checks whether ilm is created in Elasticsearch + """ + s = get_request_session(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), + os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), + os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false')) + r = s.get(sys.argv[2] + '/_ilm/policy/' + ilm_policy) + if r.status_code != 200: + sys.exit("ILM policy '{}' doesn't exist in Elasticsearch. Please create it and rerun init".format(ilm_policy)) + + if __name__ == "__main__": logging.getLogger().setLevel(logging.DEBUG) main() diff --git a/plugin/storage/es/factory.go b/plugin/storage/es/factory.go index 41dcf9c9970..79060eae94c 100644 --- a/plugin/storage/es/factory.go +++ b/plugin/storage/es/factory.go @@ -16,11 +16,10 @@ package es import ( + "bytes" "flag" "fmt" "io" - "strconv" - "strings" "github.com/spf13/viper" "github.com/uber/jaeger-lib/metrics" @@ -167,7 +166,10 @@ func createSpanWriter( return nil, err } - spanMapping, serviceMapping := GetSpanServiceMappings(cfg.GetNumShards(), cfg.GetNumReplicas(), client.GetVersion()) + spanMapping, serviceMapping, err := GetSpanServiceMappings(es.TextTemplateBuilder{}, cfg.GetNumShards(), cfg.GetNumReplicas(), client.GetVersion(), cfg.GetIndexPrefix(), cfg.GetUseILM()) + if err != nil { + return nil, err + } writer := esSpanStore.NewSpanWriter(esSpanStore.SpanWriterParams{ Client: client, Logger: logger, @@ -181,7 +183,7 @@ func createSpanWriter( UseReadWriteAliases: cfg.GetUseReadWriteAliases(), }) if cfg.IsCreateIndexTemplates() { - err := writer.CreateTemplates(spanMapping, serviceMapping) + err := writer.CreateTemplates(spanMapping, serviceMapping, cfg.GetIndexPrefix()) if err != nil { return nil, err } @@ -190,32 +192,69 @@ func createSpanWriter( } // GetSpanServiceMappings returns span and service mappings -func GetSpanServiceMappings(shards, replicas int64, esVersion uint) (string, string) { +func GetSpanServiceMappings(tb es.TemplateBuilder, shards, replicas int64, esVersion uint, esPrefix string, useILM bool) (string, string, error) { if esVersion == 7 { - return fixMapping(loadMapping("/jaeger-span-7.json"), shards, replicas), - fixMapping(loadMapping("/jaeger-service-7.json"), shards, replicas) + spanMapping, err := FixMapping(tb, LoadMapping("/jaeger-span-7.json"), shards, replicas, esPrefix, useILM) + if err != nil { + return "", "", err + } + serviceMapping, err := FixMapping(tb, LoadMapping("/jaeger-service-7.json"), shards, replicas, esPrefix, useILM) + if err != nil { + return "", "", err + } + return spanMapping, serviceMapping, nil + } + spanMapping, err := FixMapping(tb, LoadMapping("/jaeger-span.json"), shards, replicas, "", false) + if err != nil { + return "", "", err + } + serviceMapping, err := FixMapping(tb, LoadMapping("/jaeger-service.json"), shards, replicas, "", false) + if err != nil { + return "", "", err } - return fixMapping(loadMapping("/jaeger-span.json"), shards, replicas), - fixMapping(loadMapping("/jaeger-service.json"), shards, replicas) + return spanMapping, serviceMapping, nil } // GetDependenciesMappings returns dependencies mappings -func GetDependenciesMappings(shards, replicas int64, esVersion uint) string { +func GetDependenciesMappings(tb es.TemplateBuilder, shards, replicas int64, esVersion uint) (string, error) { if esVersion == 7 { - return fixMapping(loadMapping("/jaeger-dependencies-7.json"), shards, replicas) + return FixMapping(tb, LoadMapping("/jaeger-dependencies-7.json"), shards, replicas, "", false) } - return fixMapping(loadMapping("/jaeger-dependencies.json"), shards, replicas) + return FixMapping(tb, LoadMapping("/jaeger-dependencies.json"), shards, replicas, "", false) } -func loadMapping(name string) string { +// LoadMapping returns index mappings from go assets as strings +func LoadMapping(name string) string { s, _ := mappings.FSString(false, name) return s } -func fixMapping(mapping string, shards, replicas int64) string { - mapping = strings.Replace(mapping, "${__NUMBER_OF_SHARDS__}", strconv.FormatInt(shards, 10), 1) - mapping = strings.Replace(mapping, "${__NUMBER_OF_REPLICAS__}", strconv.FormatInt(replicas, 10), 1) - return mapping +// FixMapping parses the index mappings with given values and returns parsed template as string +func FixMapping(tb es.TemplateBuilder, mapping string, shards, replicas int64, esPrefix string, useILM bool) (string, error) { + + tmpl, err := tb.Parse(mapping) + if err != nil { + return "", err + } + writer := new(bytes.Buffer) + + if esPrefix != "" { + esPrefix += "-" + } + values := struct { + NumberOfShards int64 + NumberOfReplicas int64 + ESPrefix string + UseILM bool + }{shards, replicas, esPrefix, useILM} + + err = tmpl.Execute(writer, values) + if err != nil { + + return "", err + } + + return writer.String(), nil } var _ io.Closer = (*Factory)(nil) diff --git a/plugin/storage/es/factory_test.go b/plugin/storage/es/factory_test.go index e4f37956719..a29c9b46611 100644 --- a/plugin/storage/es/factory_test.go +++ b/plugin/storage/es/factory_test.go @@ -16,13 +16,13 @@ package es import ( + "bytes" "context" "errors" "io/ioutil" "os" - "strconv" - "strings" "testing" + "text/template" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -45,6 +45,14 @@ type mockClientBuilder struct { createTemplateError error } +var mockErrTextTemplateBuilder = func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")) + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb +} + func (m *mockClientBuilder) NewClient(logger *zap.Logger, metricsFactory metrics.Factory) (es.Client, error) { if m.err == nil { c := &mocks.Client{} @@ -90,7 +98,6 @@ func TestElasticsearchFactory(t *testing.T) { _, err = f.CreateArchiveSpanWriter() assert.NoError(t, err) - assert.NoError(t, f.Close()) } @@ -164,36 +171,45 @@ func TestTagKeysAsFields(t *testing.T) { } func TestFactory_LoadMapping(t *testing.T) { - spanMapping5, serviceMapping5 := GetSpanServiceMappings(10, 0, 5) - spanMapping6, serviceMapping6 := GetSpanServiceMappings(10, 0, 6) - spanMapping7, serviceMapping7 := GetSpanServiceMappings(10, 0, 7) - dependenciesMapping6 := GetDependenciesMappings(10, 0, 6) - dependenciesMapping7 := GetDependenciesMappings(10, 0, 7) tests := []struct { - name string - toTest string + name string + esPrefix string + useILM bool }{ - {name: "/jaeger-span.json", toTest: spanMapping5}, - {name: "/jaeger-service.json", toTest: serviceMapping5}, - {name: "/jaeger-span.json", toTest: spanMapping6}, - {name: "/jaeger-service.json", toTest: serviceMapping6}, - {name: "/jaeger-span-7.json", toTest: spanMapping7}, - {name: "/jaeger-service-7.json", toTest: serviceMapping7}, - {name: "/jaeger-dependencies.json", toTest: dependenciesMapping6}, - {name: "/jaeger-dependencies-7.json", toTest: dependenciesMapping7}, + {name: "/jaeger-span.json"}, + {name: "/jaeger-service.json"}, + {name: "/jaeger-span.json"}, + {name: "/jaeger-service.json"}, + {name: "/jaeger-span-7.json", esPrefix: "test", useILM: true}, + {name: "/jaeger-service-7.json", esPrefix: "test", useILM: true}, + {name: "/jaeger-dependencies.json"}, + {name: "/jaeger-dependencies-7.json"}, } for _, test := range tests { - mapping := loadMapping(test.name) + mapping := LoadMapping(test.name) + writer := new(bytes.Buffer) f, err := os.Open("mappings/" + test.name) require.NoError(t, err) b, err := ioutil.ReadAll(f) require.NoError(t, err) assert.Equal(t, string(b), mapping) - - expectedMapping := string(b) - expectedMapping = strings.Replace(expectedMapping, "${__NUMBER_OF_SHARDS__}", strconv.FormatInt(10, 10), 1) - expectedMapping = strings.Replace(expectedMapping, "${__NUMBER_OF_REPLICAS__}", strconv.FormatInt(0, 10), 1) - assert.Equal(t, expectedMapping, fixMapping(mapping, 10, 0)) + tempMapping, err := template.New("mapping").Parse(mapping) + require.NoError(t, err) + esPrefixTemplateVal := test.esPrefix + if esPrefixTemplateVal != "" { + esPrefixTemplateVal += "-" + } + values := struct { + NumberOfShards int64 + NumberOfReplicas int64 + ESPrefix string + UseILM bool + }{10, 0, esPrefixTemplateVal, test.useILM} + err = tempMapping.Execute(writer, values) + require.NoError(t, err) + actualMapping, err := FixMapping(es.TextTemplateBuilder{}, mapping, 10, 0, test.esPrefix, test.useILM) + require.NoError(t, err) + assert.Equal(t, writer.String(), actualMapping) } } @@ -252,3 +268,203 @@ func TestNewOptions(t *testing.T) { assert.Equal(t, archiveCfg, o.others[archiveNamespace].Configuration) assert.Equal(t, archiveNamespace, o.others[archiveNamespace].namespace) } + +func TestFixMapping(t *testing.T) { + tests := []struct { + name string + templateBuilderMockFunc func() *mocks.TemplateBuilder + err string + }{ + { + name: "templateRenderSuccess", + templateBuilderMockFunc: func() *mocks.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(nil) + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "", + }, + { + name: "templateRenderFailure", + templateBuilderMockFunc: func() *mocks.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template exec error")) + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "template exec error", + }, + { + name: "templateLoadError", + templateBuilderMockFunc: func() *mocks.TemplateBuilder { + tb := mocks.TemplateBuilder{} + tb.On("Parse", mock.Anything).Return(nil, errors.New("template load error")) + return &tb + }, + err: "template load error", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := FixMapping(test.templateBuilderMockFunc(), "test", 3, 5, "test", true) + if test.err != "" { + assert.EqualError(t, err, test.err) + } else { + assert.NoError(t, err) + } + + }) + } +} + +func TestGetSpanServiceMappings(t *testing.T) { + type args struct { + shards int64 + replicas int64 + esVersion uint + esPrefix string + useILM bool + } + tests := []struct { + name string + args args + mockNewTextTemplateBuilder func() es.TemplateBuilder + err string + }{ + { + name: "ES Version 7", + args: args{ + shards: 3, + replicas: 3, + esVersion: 7, + esPrefix: "test", + useILM: true, + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(nil) + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "", + }, + { + name: "ES Version 7 Service Error", + args: args{ + shards: 3, + replicas: 3, + esVersion: 7, + esPrefix: "test", + useILM: true, + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(nil).Once() + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "template load error", + }, + + { + name: "ES Version < 7", + args: args{ + shards: 3, + replicas: 3, + esVersion: 6, + esPrefix: "test", + useILM: true, + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(nil) + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "", + }, + { + name: "ES Version < 7 Service Error", + args: args{ + shards: 3, + replicas: 3, + esVersion: 6, + esPrefix: "test", + useILM: true, + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(nil).Once() + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "template load error", + }, + { + name: "ES Version < 7 Span Error", + args: args{ + shards: 3, + replicas: 3, + esVersion: 6, + esPrefix: "test", + useILM: true, + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")) + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "template load error", + }, + { + name: "ES Version 7 Span Error", + args: args{ + shards: 3, + replicas: 3, + esVersion: 7, + esPrefix: "test", + useILM: true, + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "template load error", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, err := GetSpanServiceMappings(test.mockNewTextTemplateBuilder(), test.args.shards, test.args.replicas, + test.args.esVersion, test.args.esPrefix, + test.args.useILM) + if test.err != "" { + assert.EqualError(t, err, test.err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestGetDependenciesMappings(t *testing.T) { + + _, err := GetDependenciesMappings(mockErrTextTemplateBuilder(), 5, 5, 7) + assert.EqualError(t, err, "template load error") + _, err = GetDependenciesMappings(mockErrTextTemplateBuilder(), 5, 5, 6) + assert.EqualError(t, err, "template load error") +} diff --git a/plugin/storage/es/mappings/gen_assets.go b/plugin/storage/es/mappings/gen_assets.go index 77195053de8..02e4fce3ece 100644 --- a/plugin/storage/es/mappings/gen_assets.go +++ b/plugin/storage/es/mappings/gen_assets.go @@ -213,7 +213,7 @@ var _escData = map[string]*_escFile{ name: ".nocover", local: "plugin/storage/es/mappings/.nocover", size: 43, - modtime: 1597437395, + modtime: 1608397661, compressed: ` H4sIAAAAAAAC/youSSzJzFYoSEzOTkxPVcjILy4pVkgsLcnXTU/NSy1KLElNUUjLzEkt1uMCBAAA//8y IKK1KwAAAA== @@ -223,96 +223,100 @@ IKK1KwAAAA== "/jaeger-dependencies-7.json": { name: "jaeger-dependencies-7.json", local: "plugin/storage/es/mappings/jaeger-dependencies-7.json", - size: 283, - modtime: 1597437395, + size: 279, + modtime: 1610693565, compressed: ` -H4sIAAAAAAAC/2zPz0vDQBDF8Xv+imXxVNrFi5fcqlYU/EWK52GbfU1HknHdmYBQ8r9LRA/S3t/nC+9Y -OedZEr4oRzMUUV87v3iP6FBWCRmSIC1DVwu/nNcKM5ZOfT3jPx5kHHYo9LEnPcSS5szFkej57el609DL -HW3v183tlmhanmcFuec2nsJm8/r4cLM+oUPMmaULAjUk2jP6pKHngc3XV5f/tgWfI9Q0tLE9IEDiroev -rYyonPvp+t/efGyqpuo7AAD//66cHf8bAQAA +H4sIAAAAAAAC/2yPQWrDMBBF9z6FmGVIRDfd+BAttAcwivTtTJGnqmYMBaO7F5l00dDt478Hfx+cI5aE +76kEM1RRGh2dPgIW1EtCgSRIZOjlROe+VpixLEpjl391L9t6RZ0+50lvoaae2XfnXw78Or8f0LV2/l+q +KJljeNTe7vhRXEMpLIsXqCFNMyMn9ZlXNhqfn/5sK742qKmPId7gIeGaQaPVDYNzR5fuvX6qDW34CQAA +//8AfUo/FwEAAA== `, }, "/jaeger-dependencies.json": { name: "jaeger-dependencies.json", local: "plugin/storage/es/mappings/jaeger-dependencies.json", - size: 277, - modtime: 1597437395, + size: 273, + modtime: 1610693565, compressed: ` -H4sIAAAAAAAC/2zPzUoDMRTF8f08Rbi4Km1w4ya7qhUFv5ji+pJOTqeRTIy5d0Ao8+4y4kbG/fn94Zwb -Y0gxlOQV5Ayt3j161E1AQQ7IXYRsVrSedwLVmHshNzNjKOaAL5vH4YDKH0eWk69ByJmLM/Pz29P1ruWX -O97fb9vbPfO0/p9VlBQ7v4Tt7vXx4Wa7oIMvJebeZogi8DEiBbEpDlHJXV3+2VZ8jhAV2/nuBIvsDwnk -tI5ojPnp0m9vPjY1U/MdAAD//5ZQx/QVAQAA +H4sIAAAAAAAC/2yPQWrDMBBF9z7FMMuQiG660SFaaA9gFOvbmSJPVc0YCsZ3LzbZNGT7+O/BXzsidsy1 +JAdH4tNXwoR2yajQDB0Edjnxed8Z3EUn47hrRCya8Rt0ma9o/ffY2y21bBxpXSm8Hfh9/Dwgbdv5udRQ +iwzpUfu440dxTrWKTkFhjtyPgpItFJnFOb6+/Ns2/CwwtzCk4YYATdcCjt4WdERHl++9/dTWbd1fAAAA +//8zWep3EQEAAA== `, }, "/jaeger-service-7.json": { name: "jaeger-service-7.json", local: "plugin/storage/es/mappings/jaeger-service-7.json", - size: 878, - modtime: 1597437395, + size: 1148, + modtime: 1610634921, compressed: ` -H4sIAAAAAAAC/8ySwW7aQBCG736K1agnBFZViR72RluqVmppBcopikaDPdibeNeb3YEEIb97ZGTAhHDL -IReP5H++35/t3SVKgXE5P6MnEQ4uglYwuCcuOIwih43JeDSAYbsYWcS4IoJuuQOZurVdcsB6hbGkkLcN -n3aIs5u/36Zz/PcTF78m8x8LxGb4NhbYVyajS3A+/f/n9/fJBWrJe+OK1HEUznFluMpjWhlrBPT489lu -4Mc1R4lpRlnJKTtaVgxawpoTpfa90PWdXizfOrImQ2HrKxKOoG/3iVK7brbfw5NDoSKiJX9gu6yrPL+r -FMjWM2h44O1THXIYnqemcHVgpGW9YdBfxl97cdPfBU9SoiXJStAgVKQDOMZN8oroOftQZxzjh9DuXNJr -+vt51/1NH2rPQQzHkxx0B3RGlvvK13Wvqh41oX0Miande7Qmh2uTNMlLAAAA//8YcMrbbgMAAA== +H4sIAAAAAAAC/8ySQY/TMBCF7/kVozmuWgshLYfcOSDBglhxQsiaOpPU4NjGdrsbRf7vKMbbZgvdEwdO +kWbmvXzzPHMDgNp2/Cg9pcTBRmwBb+YZxNv7T4F7/Qg5fyceOGwjh6NWvL3BTQMwz1vQPYgvkd+9/wA5 +L15kNEVeTBZrAHzZKTB1CC3Mizg/ubLtql3klLQdIrbVrrAKexh3HKTrZdxT6MrvZhB3pfyxvy9FyMXw +L6LA3mhFVXbWfa71S+VI3ms7CMsxcSd7zaaLwuhRJ2xvXz2bDfzzwDFFoUjtWbClnWFsUzhw2e6PyAA2 +aHTPalLLYM2tOFoalwrW0LQZt94ZrSbcnIeCM8YdOcgS/TL+cuQPQSfGos+XeZe1sa57Dr2bLI1aycSj +N5SW5/1aAVa00ZOViYYoR/JP2tqrls+rAJgmz9jiD54eXOhWa/1OdLAusKSdOzK2r2/frNp5PYue0l6O +lNQeW0w0iBs8tXNzoVgx++AUx/hfYFcWcQ2/fL/VY/PBeQ5JczzDYX3iu3I2qy2v4l5FPWHi8htK2tl/ +4Xq6utzk5lcAAAD//16uKiV8BAAA `, }, "/jaeger-service.json": { name: "jaeger-service.json", local: "plugin/storage/es/mappings/jaeger-service.json", - size: 1060, - modtime: 1597437395, + size: 1056, + modtime: 1610693565, compressed: ` -H4sIAAAAAAAC/8yTT2/UMBDF7/kU1ojTamshpHLwrUARSFDQVpwQGs3Gs1mD7RjbKayqfHfk4pKkW+2J -Q3PIn/F78/xz7NtGCMjsgqXMoASsvhN3HM8SxxvT8tkK1kWSOGfjuwSqOIQA4zX/ln5wW47Y7zDtKeoE -Sjy7Rbz68vHV5QY/vcXrdxebN9eI4/pxW+RgTUvHxs3l5w/vX18cWR2FYHwnPafMGneGrU7SGmcyqPPn -C23knwOnnGRL7Z4le9paBpXjwEc9OUp98ORMC2pHNnEjxF0y1MQJHTXvaLAZ7yulRtZOn0LA3zA9NStX -RRECahbeL30C9fWfeWpTVj6Qx0xdQkdhHnE3Wif3sF5+6iEwKPjBh1991LB+OG4630dG2vY3DOrF+cuF -YFzqIVDeo6Pc7kFBpk6uYCYYm0d8C4oQ+5ZTemIgdVbyFFB9+9bMukE9HbMNEGIfOGbDabENqvCKHC/R -TmGdQJrhQAmkbHr//7o382e5j83Y/AkAAP//qd2MzCQEAAA= +H4sIAAAAAAAC/8xTy47bIBTd+yvQXUYJqiqlCz4ildplVaEbc23TAqaA00aR/73CQ8aPibKaxXjhx+E8 +OBhuFWOQyHqDiUAw2P1CaikcIoWLrumwg32mREpJuzaCyArGQDtF/7gb7JmC7BsZOwwqgmC3G+OnCf7a +fJ9ANo77x6JA3ugat7JvBd4KLXqvXcsdxURKNpqMitxoqxOI46cVN9CfgWKKvMa6I04Oz4ZApDDQG08K +XF0dWl2DaNBEqhibkqEkzrWlogYHk+QdyRgaM38yBi9hajbLV6nCGJQseV/2COLHq3i2yavu0cmEbZQW +/TJiGi2T2+L5h149gYDfdP3bBwX77bhuXR9I4rm/EIjPxy8rwrjmg8fUSYup7kBAwpbvYEEYqwe6VQsf ++ppi/GBFyqz4s0Ll7We1cINyMhYbwIfeU0ia4mobFOIJLa2rPav1pNKiDuRATLp37+deLZ/5PlZj9T8A +AP//IuFOPCAEAAA= `, }, "/jaeger-span-7.json": { name: "jaeger-span-7.json", local: "plugin/storage/es/mappings/jaeger-span-7.json", - size: 3420, - modtime: 1597437395, + size: 3680, + modtime: 1610634921, compressed: ` -H4sIAAAAAAAC/+xWXW+UQBR951eQG5+aLTEm9YG3amtsYqtp65Mxk7twYaedL2fuVjcN/93A0hYKbE2k -xhhflix3zuHcmXsO3EZxDNLk9EM4ZCZvAqQx7F0hleT3g0OzvweLuF4WiFmaMkBao+5wiVnrJXlhCxFW -6PMa/+JWiLPPp2+Oz8XHd+Li/eH50YUQ1WIc5skpmeEQeH786cPJ28MBVKNz0pSJocCUi0KSykOipJYM -6cHL3lpP39YUOCQZZitKyOBSEaTs1xTFccMLLd9DY/nGoJaZYNJOIVOA9EtTiePb9lrvh0MjGMsgNLo7 -bFtrKft34xh44whSuKbNd+tzWPSrsjTWk8ClvSFIXx287pSr7lpwyCuhkbMVpMBYJntwX66iR4iOZudt -RiH8FbJbLcmU/Ob6tT1N560jz5LCgzhgjxmdHHXlTkudlHkvERx6Mnzh0MxIGualq7cBWVpzhprmE8no -+VKOMyprSpiGnEqlZBgD5sjU1VFYr5EhBXI2Wwm9BQ6Y8/W2w1/XUigsRxVIw3WQDRHKjgO2mdIV3YYB -pAWqQIuelwYjuaWSmgKjdlM+6jYxNMk2z6awA4E7Re4U2hSvaTO8+5Tln7T9oKsGcYNqTX/saYzlZUP7 -PM+Lpv5V00l8l3m9yZuco0D+Rmb02OizJjLjZNrb5RVlDLug/6f0n5xSTwV5Mhk9W0R6Ksa6nm+sh18G -s/IPX+q/S7/jOB55dNyfveXPdm4jPpxtT0d9N2fQzT1xE5+s9W8VVdHPAAAA//+SuQbQXA0AAA== +H4sIAAAAAAAC/+xXwW7bMAy95ysEHovEGAZ0B5+3Q4G1G9buNAwGY9OOWlnSJCZtYPjfBytO6sR2OmDu +MAw7BZb4nh4pPsauZkKA1Bk9JRaZyWkPsYCLqhLRh9vPjnL5JOr6Hqkgt/AW9eIC5jMhqmohZC6ir56u +Pl6Lum6IUEn01DA0vELAGRpHmDWBDbDeM5LOWipPzFIXHuKWKoiM9LpckktMnvgVuiwcVYnoJix/ym/D +oqgD4QDIkVUyxVPYl3b5FFiitVIXkSbPlCW5JJX5SMlSMsSXb45iHf1Yk2cfpZiuKCKNS0UQs1tTCKuq +Xr2EmIOSOaXbtAltiyYEaCybZ2jrJVW5sEbJdAvzfYgzSpkNuSTUvAk+U+tHJ5kgQOu9mEOpQ8rQpvpc +72yrsZRpwlRahdzc6rf28L3O5pos6oSx8EmJdo9t91rK41UhgLeWIIYH2j4alx1SandloY2jBJdmQxC/ +vXzX2a67sWCRV0mJnK4gBsYiuoDDdj07QXQ0W2dS8v6vkN1qicbkh9/vbaNZZyw5luSfxQE7TOnqfVfu +uNRRmQeJYNGR5luLekJSPy1dUwZkafRNMMpUIhkd38lhRmV0AeOQa6mU9EPADJm6OnLjSmSIgaxJV0m5 +A/aYs/Uuw1/XkissBhVIzc0o6COUGQbsxl1XdDsMIM5ReZofeanXkjsqWZJnLO2Yj7pJ9E2yG7Vj2J7A +syLPCg2bD7Ttr75k+Rdt38sqIDao1vTHTmMs7gLt65w3G3uqxyfxfuYddd5oH3lyG5nSqdEnnciMo9Pe +LO8pZTgH/d+l/2SXOsrJkU7p1Uako3wo6+nauv9mMCl//0/9d+nPXMeJR4f9eRT+avc24MPJajrouykH +3dQdN/LKGj4sZvXsZwAAAP//iNEyH2AOAAA= `, }, "/jaeger-span.json": { name: "jaeger-span.json", local: "plugin/storage/es/mappings/jaeger-span.json", - size: 3830, - modtime: 1597437395, + size: 3826, + modtime: 1610693565, compressed: ` -H4sIAAAAAAAC/+xW0W/TPhB+z18RnX5PUxf9hDQe8jbYEJPYQNt4Qsi6JpfUm2Mb+zqopv7vKE1Lm9ZJ -QGoQEvShbWx/391n333xcxTHwFRZhUyQxnDygFSSO/UW9ekJTOp5T8xSlx7Senkcg9Q5fUv0vJqSE6YQ -foYu95DG/z0LcfPx+tXlrXj/Rty9Pb+9uBNiOQnDHFklMzwE3l5+eHf1+vwAWqG1UpeJJs+Ui0KSyn2i -ZCUZ0rP/W2sdfZmTZ59kmM0oIY1TRZCym9MBJ7kkX2isZAZpgcpTFMeryLCOuJUucipwrlhsRuoxVGr7 -GMfQBMu3ZPVnLSWOYR1LbPbdQ/rpB3hLU++8RS0YSy8qtLshVrPr5PbH6xNdWIIUHmnx1bgcJvvzstTG -kcCpeSJIX5y9bC1YtteDRZ6JCjmbQQqMZXICOwuWUQDXUmGdycj7P0zIOqukT9D63+doh211KDunb52x -5FiSb9UAO8zo6qKtqU9Pj5YdHWDRkeY7i3oEcj8Obb1ByNLoG6zo+EkzOr6XXczK6BL6gddSKenD8Lw2 -xlZWhXEVMqRA1mQzUTXgYIR83uj+1cwKhWVHPlJzbdBhnDJdsMYy20LanjfZa9lAUTeEsiLPWNnudm0L -C3ViY93dDIF0B1IeSHs1/UiL0Piwz/yE1wRUrlBPqOb026Mylvcr8jHjRn3Py6FXxMaA9+q1p/I8uSeZ -0aGJjPCqYOx5HZnpA2UMQwT/6vuvrm9HBTnSGY1uyY6K8G4cuylCd5oR4oSuIccIM3hkB13f1fF7oNFP -ONjZR971jk4+vrGOU6u9N/jmt/5eRsvoewAAAP//W45CgfYOAAA= +H4sIAAAAAAAC/+xWTW/bPAy++1cIPBap8eIFuoPPu+ywDlh7GwaDsWlHrb4mMd2Cwv998EeX2JWdDYiH +AVsOSUzpIfmI5GM9J0IAk3YKmSATcPWAVJO/Dg7N9RVs2vVAzNLUAbJ2uxAgTUnfUrPXW/K5rfKwQ18G +yMTzs0hvO/OH6q4ziqbZxEGenJIFTmEfB/MUqNE5aerUUGAq80qSKkOqpJYM2c1/o72evuwpcEgLLHaU +ksGtIsjY7+mVT/JpeTCoZQFZhSpQIkQXGYaIR9p5SRXuFecvltaGSh0fhYA+WHl01n4GKkLAECt/OfMA +2acf4KOb9tQdmpyxDrlGdxqiWx2Sm9rbah4cQQaPdPhqfQmb6bqsjfWU49Y+EWT/37wZbWjG+8Eh73KN +XOwgA8Y6vYKTDU0SwY1YOG8LCuEPIzJklS4RGv59Tk68dUU5qb7z1pFnSWHUA+yxoHdvx5yW+CxwOeEB +Dj0ZvnNoVnAe1nHbHhCytOYWNV0+aUbP93LOs7KmhmXge6mUDHF42YriKKvKeo0MGZCzxS7XPTgaodz3 +vH81s0phPZOPNNyKcxyn7Bysl8wxkbHmbSYjG2nq3qHUFBi1mx/XMbHYJPbSPe8hku6ZlM+k3S0/0iFm +P68zP6E1EZYd6gnVnn57VMb6vnO+Ztxk6bk594p4EeBJvy50XiD/JAt6LSIrvCoYF15HdvtABcM5B//6 ++6/ub08VeTIFrS7Jnqr4aVx6KGJ3mhXixK4hlwhztmSvpn5u4ieg1SscnewLn/rMJF9eWNfp1cUbfP/b +fjdJk3wPAAD//5ROSoHyDgAA `, }, diff --git a/plugin/storage/es/mappings/jaeger-dependencies-7.json b/plugin/storage/es/mappings/jaeger-dependencies-7.json index 1284a756a19..af6be27c337 100644 --- a/plugin/storage/es/mappings/jaeger-dependencies-7.json +++ b/plugin/storage/es/mappings/jaeger-dependencies-7.json @@ -1,8 +1,8 @@ { "index_patterns": "*jaeger-dependencies-*", "settings":{ - "index.number_of_shards": ${__NUMBER_OF_SHARDS__}, - "index.number_of_replicas": ${__NUMBER_OF_REPLICAS__}, + "index.number_of_shards": {{ .NumberOfShards }}, + "index.number_of_replicas": {{ .NumberOfReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true }, diff --git a/plugin/storage/es/mappings/jaeger-dependencies.json b/plugin/storage/es/mappings/jaeger-dependencies.json index 3ed1e9b3f59..dfa5a49e959 100644 --- a/plugin/storage/es/mappings/jaeger-dependencies.json +++ b/plugin/storage/es/mappings/jaeger-dependencies.json @@ -1,8 +1,8 @@ { "template": "*jaeger-dependencies-*", "settings":{ - "index.number_of_shards": ${__NUMBER_OF_SHARDS__}, - "index.number_of_replicas": ${__NUMBER_OF_REPLICAS__}, + "index.number_of_shards": {{ .NumberOfShards }}, + "index.number_of_replicas": {{ .NumberOfReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true }, diff --git a/plugin/storage/es/mappings/jaeger-service-7.json b/plugin/storage/es/mappings/jaeger-service-7.json index d27627e26ce..f37d3f85115 100644 --- a/plugin/storage/es/mappings/jaeger-service-7.json +++ b/plugin/storage/es/mappings/jaeger-service-7.json @@ -1,10 +1,21 @@ { - "index_patterns": "*jaeger-service-*", + "index_patterns": "*{{ .ESPrefix }}jaeger-service-*", + {{- if .UseILM }} + "aliases": { + "{{ .ESPrefix }}jaeger-service-read" : {} + }, + {{- end }} "settings":{ - "index.number_of_shards": ${__NUMBER_OF_SHARDS__}, - "index.number_of_replicas": ${__NUMBER_OF_REPLICAS__}, + "index.number_of_shards": {{ .NumberOfShards }}, + "index.number_of_replicas": {{ .NumberOfReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true + {{ if .UseILM }} + ,"lifecycle": { + "name": "jaeger-ilm-policy", + "rollover_alias": "{{ .ESPrefix }}jaeger-service-write" + } + {{- end }} }, "mappings":{ "dynamic_templates":[ diff --git a/plugin/storage/es/mappings/jaeger-service.json b/plugin/storage/es/mappings/jaeger-service.json index 20acb88e41e..750aa8236ea 100644 --- a/plugin/storage/es/mappings/jaeger-service.json +++ b/plugin/storage/es/mappings/jaeger-service.json @@ -1,8 +1,8 @@ { "template": "*jaeger-service-*", "settings":{ - "index.number_of_shards": ${__NUMBER_OF_SHARDS__}, - "index.number_of_replicas": ${__NUMBER_OF_REPLICAS__}, + "index.number_of_shards": {{ .NumberOfShards }}, + "index.number_of_replicas": {{ .NumberOfReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true, "index.mapper.dynamic":false diff --git a/plugin/storage/es/mappings/jaeger-span-7.json b/plugin/storage/es/mappings/jaeger-span-7.json index ad8b96c2727..9a95e463f86 100644 --- a/plugin/storage/es/mappings/jaeger-span-7.json +++ b/plugin/storage/es/mappings/jaeger-span-7.json @@ -1,10 +1,21 @@ { - "index_patterns": "*jaeger-span-*", + "index_patterns": "*{{ .ESPrefix }}jaeger-span-*", + {{- if .UseILM }} + "aliases": { + "{{ .ESPrefix }}jaeger-span-read": {} + }, + {{- end }} "settings":{ - "index.number_of_shards": ${__NUMBER_OF_SHARDS__}, - "index.number_of_replicas": ${__NUMBER_OF_REPLICAS__}, + "index.number_of_shards": {{ .NumberOfShards }}, + "index.number_of_replicas": {{ .NumberOfReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true + {{ if .UseILM }} + ,"lifecycle": { + "name": "jaeger-ilm-policy", + "rollover_alias": "{{ .ESPrefix }}jaeger-span-write" + } + {{ end }} }, "mappings":{ "dynamic_templates":[ diff --git a/plugin/storage/es/mappings/jaeger-span.json b/plugin/storage/es/mappings/jaeger-span.json index 46052729306..b3c5da90a12 100644 --- a/plugin/storage/es/mappings/jaeger-span.json +++ b/plugin/storage/es/mappings/jaeger-span.json @@ -1,8 +1,8 @@ { "template": "*jaeger-span-*", "settings":{ - "index.number_of_shards": ${__NUMBER_OF_SHARDS__}, - "index.number_of_replicas": ${__NUMBER_OF_REPLICAS__}, + "index.number_of_shards": {{ .NumberOfShards }}, + "index.number_of_replicas": {{ .NumberOfReplicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true, "index.mapper.dynamic":false diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index a18f3d0e4d4..b7c768f2cca 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -53,11 +53,11 @@ const ( suffixTagsFile = suffixTagsAsFields + ".config-file" suffixTagDeDotChar = suffixTagsAsFields + ".dot-replacement" suffixReadAlias = ".use-aliases" + suffixUseILM = ".use-ilm" suffixCreateIndexTemplate = ".create-index-templates" suffixEnabled = ".enabled" suffixVersion = ".version" suffixMaxDocCount = ".max-doc-count" - // default number of documents to return from a query (elasticsearch allowed limit) // see search.max_buckets and index.max_result_window defaultMaxDocCount = 10_000 @@ -240,6 +240,12 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { "Use read and write aliases for indices. Use this option with Elasticsearch rollover "+ "API. It requires an external component to create aliases before startup and then performing its management. "+ "Note that "+nsConfig.namespace+suffixMaxSpanAge+" will influence trace search window start times.") + flagSet.Bool( + nsConfig.namespace+suffixUseILM, + nsConfig.UseILM, + "(experimental) Option to enable ILM for jaeger span & service indices. Use this option with "+nsConfig.namespace+suffixReadAlias+". "+ + "It requires an external component to create aliases before startup and then performing its management. "+ + "ILM policy must be manually created in ES before startup. Supported only for elasticsearch version 7+.") flagSet.Bool( nsConfig.namespace+suffixCreateIndexTemplate, nsConfig.CreateIndexTemplates, @@ -300,6 +306,7 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { cfg.Version = uint(v.GetInt(cfg.namespace + suffixVersion)) cfg.MaxDocCount = v.GetInt(cfg.namespace + suffixMaxDocCount) + cfg.UseILM = v.GetBool(cfg.namespace + suffixUseILM) if v.IsSet(cfg.namespace + suffixMaxNumSpans) { maxNumSpans := v.GetInt(cfg.namespace + suffixMaxNumSpans) diff --git a/plugin/storage/es/options_test.go b/plugin/storage/es/options_test.go index 73bb1476e93..65096e8bf94 100644 --- a/plugin/storage/es/options_test.go +++ b/plugin/storage/es/options_test.go @@ -67,6 +67,7 @@ func TestOptionsWithFlags(t *testing.T) { "--es.tags-as-fields.include=test,tags", "--es.tags-as-fields.config-file=./file.txt", "--es.tags-as-fields.dot-replacement=!", + "--es.use-ilm=true", }) opts.InitFromViper(v) @@ -84,7 +85,6 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, "./file.txt", primary.Tags.File) assert.Equal(t, "test,tags", primary.Tags.Include) assert.Equal(t, "20060102", primary.IndexDateLayout) - aux := opts.Get("es.aux") assert.Equal(t, []string{"3.3.3.3", "4.4.4.4"}, aux.Servers) assert.Equal(t, "hello", aux.Username) @@ -98,6 +98,7 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, "./file.txt", aux.Tags.File) assert.Equal(t, "test,tags", aux.Tags.Include) assert.Equal(t, "2006.01.02", aux.IndexDateLayout) + assert.True(t, primary.UseILM) } func TestMaxNumSpansUsage(t *testing.T) { diff --git a/plugin/storage/es/spanstore/writer.go b/plugin/storage/es/spanstore/writer.go index a0a8d77cd92..8bcdc67ca48 100644 --- a/plugin/storage/es/spanstore/writer.go +++ b/plugin/storage/es/spanstore/writer.go @@ -101,12 +101,15 @@ func NewSpanWriter(p SpanWriterParams) *SpanWriter { } // CreateTemplates creates index templates. -func (s *SpanWriter) CreateTemplates(spanTemplate, serviceTemplate string) error { - _, err := s.client.CreateTemplate("jaeger-span").Body(spanTemplate).Do(context.Background()) +func (s *SpanWriter) CreateTemplates(spanTemplate, serviceTemplate, indexPrefix string) error { + if indexPrefix != "" { + indexPrefix += "-" + } + _, err := s.client.CreateTemplate(indexPrefix + "jaeger-span").Body(spanTemplate).Do(context.Background()) if err != nil { return err } - _, err = s.client.CreateTemplate("jaeger-service").Body(serviceTemplate).Do(context.Background()) + _, err = s.client.CreateTemplate(indexPrefix + "jaeger-service").Body(serviceTemplate).Do(context.Background()) if err != nil { return err } diff --git a/plugin/storage/es/spanstore/writer_test.go b/plugin/storage/es/spanstore/writer_test.go index b92cc98a0f8..6c40fbd8371 100644 --- a/plugin/storage/es/spanstore/writer_test.go +++ b/plugin/storage/es/spanstore/writer_test.go @@ -190,6 +190,7 @@ func TestCreateTemplates(t *testing.T) { err string spanTemplateService func() *mocks.TemplateCreateService serviceTemplateService func() *mocks.TemplateCreateService + indexPrefix string }{ { spanTemplateService: func() *mocks.TemplateCreateService { @@ -204,6 +205,20 @@ func TestCreateTemplates(t *testing.T) { tService.On("Do", context.Background()).Return(nil, nil) return tService }, + }, { + spanTemplateService: func() *mocks.TemplateCreateService { + tService := &mocks.TemplateCreateService{} + tService.On("Body", mock.Anything).Return(tService) + tService.On("Do", context.Background()).Return(nil, nil) + return tService + }, + serviceTemplateService: func() *mocks.TemplateCreateService { + tService := &mocks.TemplateCreateService{} + tService.On("Body", mock.Anything).Return(tService) + tService.On("Do", context.Background()).Return(nil, nil) + return tService + }, + indexPrefix: "test", }, { err: "span-template-error", @@ -239,9 +254,13 @@ func TestCreateTemplates(t *testing.T) { for _, test := range tests { withSpanWriter(func(w *spanWriterTest) { - w.client.On("CreateTemplate", "jaeger-span").Return(test.spanTemplateService()) - w.client.On("CreateTemplate", "jaeger-service").Return(test.serviceTemplateService()) - err := w.writer.CreateTemplates(mock.Anything, mock.Anything) + prefix := "" + if test.indexPrefix != "" { + prefix = test.indexPrefix + "-" + } + w.client.On("CreateTemplate", prefix+"jaeger-span").Return(test.spanTemplateService()) + w.client.On("CreateTemplate", prefix+"jaeger-service").Return(test.serviceTemplateService()) + err := w.writer.CreateTemplates(mock.Anything, mock.Anything, test.indexPrefix) if test.err != "" { assert.Error(t, err, test.err) } diff --git a/plugin/storage/es/templatizer b/plugin/storage/es/templatizer new file mode 100755 index 00000000000..b391f493511 Binary files /dev/null and b/plugin/storage/es/templatizer differ diff --git a/plugin/storage/es/templatizer-darwin-amd64 b/plugin/storage/es/templatizer-darwin-amd64 new file mode 100755 index 00000000000..799616e455a Binary files /dev/null and b/plugin/storage/es/templatizer-darwin-amd64 differ diff --git a/plugin/storage/integration/elasticsearch_test.go b/plugin/storage/integration/elasticsearch_test.go index 7eb1df8bac1..ac93d2659dd 100644 --- a/plugin/storage/integration/elasticsearch_test.go +++ b/plugin/storage/integration/elasticsearch_test.go @@ -31,6 +31,7 @@ import ( "go.uber.org/zap" "github.com/jaegertracing/jaeger/model" + esTemplate "github.com/jaegertracing/jaeger/pkg/es" eswrapper "github.com/jaegertracing/jaeger/pkg/es/wrapper" "github.com/jaegertracing/jaeger/pkg/testutils" "github.com/jaegertracing/jaeger/plugin/storage/es" @@ -107,7 +108,10 @@ func (s *ESStorageIntegration) initSpanstore(allTagsAsFields, archive bool) erro return err } client := eswrapper.WrapESClient(s.client, bp, esVersion) - spanMapping, serviceMapping := es.GetSpanServiceMappings(5, 1, client.GetVersion()) + spanMapping, serviceMapping, err := es.GetSpanServiceMappings(esTemplate.TextTemplateBuilder{}, 5, 1, client.GetVersion(), indexPrefix, false) + if err != nil { + return err + } w := spanstore.NewSpanWriter( spanstore.SpanWriterParams{ Client: client, @@ -118,7 +122,7 @@ func (s *ESStorageIntegration) initSpanstore(allTagsAsFields, archive bool) erro TagDotReplacement: tagKeyDeDotChar, Archive: archive, }) - err = w.CreateTemplates(spanMapping, serviceMapping) + err = w.CreateTemplates(spanMapping, serviceMapping, indexPrefix) if err != nil { return err } @@ -134,7 +138,10 @@ func (s *ESStorageIntegration) initSpanstore(allTagsAsFields, archive bool) erro MaxDocCount: defaultMaxDocCount, }) dependencyStore := dependencystore.NewDependencyStore(client, s.logger, indexPrefix, indexDateLayout, defaultMaxDocCount) - depMapping := es.GetDependenciesMappings(5, 1, client.GetVersion()) + depMapping, err := es.GetDependenciesMappings(esTemplate.TextTemplateBuilder{}, 5, 1, client.GetVersion()) + if err != nil { + return err + } err = dependencyStore.CreateTemplates(depMapping) if err != nil { return err @@ -194,6 +201,21 @@ func TestElasticsearchStorage_Archive(t *testing.T) { testElasticsearchStorage(t, false, true) } +func TestElasticsearchStorage_IndexTemplates(t *testing.T) { + if os.Getenv("STORAGE") != "elasticsearch" { + t.Skip("Integration test against ElasticSearch skipped; set STORAGE env var to elasticsearch to run this") + } + if err := healthCheck(); err != nil { + t.Fatal(err) + } + s := &ESStorageIntegration{} + require.NoError(t, s.initializeES(true, false)) + serviceTemplateExists, _ := s.client.IndexTemplateExists(indexPrefix + "-jaeger-service").Do(context.Background()) + spanTemplateExists, _ := s.client.IndexTemplateExists(indexPrefix + "-jaeger-span").Do(context.Background()) + assert.True(t, serviceTemplateExists) + assert.True(t, spanTemplateExists) +} + func (s *StorageIntegration) testArchiveTrace(t *testing.T) { defer s.cleanUp(t) tID := model.NewTraceID(uint64(11), uint64(22)) diff --git a/plugin/storage/integration/es_index_cleaner_test.go b/plugin/storage/integration/es_index_cleaner_test.go index d70adec0ae9..1eec135f032 100644 --- a/plugin/storage/integration/es_index_cleaner_test.go +++ b/plugin/storage/integration/es_index_cleaner_test.go @@ -34,7 +34,7 @@ const ( serviceIndexName = "jaeger-service-2019-01-01" indexCleanerImage = "jaegertracing/jaeger-es-index-cleaner:latest" rolloverImage = "jaegertracing/jaeger-es-rollover:latest" - rolloverNowEnvVar = "CONDITIONS='{\"max_age\":\"0s\"}'" + rolloverNowEnvVar = `CONDITIONS='{"max_age":"0s"}'` ) func TestIndexCleaner_doNotFailOnEmptyStorage(t *testing.T) { diff --git a/plugin/storage/integration/es_index_rollover_test.go b/plugin/storage/integration/es_index_rollover_test.go new file mode 100644 index 00000000000..5a3f7f34896 --- /dev/null +++ b/plugin/storage/integration/es_index_rollover_test.go @@ -0,0 +1,171 @@ +// Copyright (c) 2019 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build index_rollover + +package integration + +import ( + "context" + "fmt" + "os/exec" + "strconv" + "testing" + + "github.com/olivere/elastic" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + indexILMName = "jaeger-ilm-policy" + rolloverImage = "jaegertracing/jaeger-es-rollover:latest" +) + +func TestIndexRollover_FailIfILMNotPresent(t *testing.T) { + client, err := createESClient() + require.NoError(t, err) + esVersion, err := getVersion(client) + require.NoError(t, err) + if esVersion != 7 { + t.Skip("Integration test - " + t.Name() + " against ElasticSearch skipped for ES version " + fmt.Sprint(esVersion)) + } + // make sure ES is clean + cleanES(t, client, "jaeger-ilm-policy") + envVars := []string{"ES_USE_ILM=true"} + err = runEsRollover("init", envVars) + assert.EqualError(t, err, "exit status 1") + indices, err := client.IndexNames() + require.NoError(t, err) + assert.Empty(t, indices) +} + +func TestIndexRollover_CreateIndicesWithILM(t *testing.T) { + client, err := createESClient() + require.NoError(t, err) + + esVersion, err := getVersion(client) + require.NoError(t, err) + + if esVersion != 7 { + cleanES(t, client, "") + err := runEsRollover("init", []string{"ES_USE_ILM=true"}) + assert.EqualError(t, err, "exit status 1") + indices, err1 := client.IndexNames() + require.NoError(t, err1) + assert.Empty(t, indices) + + } else { + envVars := []string{"ES_USE_ILM=true"} + expectedIndices := []string{"jaeger-span-000001", "jaeger-service-000001"} + t.Run(fmt.Sprintf("%s_no_prefix", "CreateIndicesWithILM"), func(t *testing.T) { + runIndexRolloverWithILMTest(t, client, "", expectedIndices, envVars) + }) + t.Run(fmt.Sprintf("%s_prefix", "CreateIndicesWithILM"), func(t *testing.T) { + runIndexRolloverWithILMTest(t, client, indexPrefix, expectedIndices, append(envVars, "INDEX_PREFIX="+indexPrefix)) + }) + } +} + +func runIndexRolloverWithILMTest(t *testing.T, client *elastic.Client, prefix string, expectedIndices, envVars []string) { + writeAliases := []string{"jaeger-service-write", "jaeger-span-write"} + + // make sure ES is cleaned before test + cleanES(t, client, "jaeger-ilm-policy") + // make sure ES is cleaned after test + defer cleanES(t, client, "jaeger-ilm-policy") + err := createILMPolicy(client, "jaeger-ilm-policy") + require.NoError(t, err) + + if prefix != "" { + prefix = prefix + "-" + } + var expected, expectedWriteAliases, actualWriteAliases []string + for _, index := range expectedIndices { + expected = append(expected, prefix+index) + } + for _, alias := range writeAliases { + expectedWriteAliases = append(expectedWriteAliases, prefix+alias) + } + + // Run rollover with given EnvVars + err = runEsRollover("init", envVars) + require.NoError(t, err) + + indices, err := client.IndexNames() + require.NoError(t, err) + + //Get ILM Policy Attached + settings, err := client.IndexGetSettings(expected...).FlatSettings(true).Do(context.Background()) + require.NoError(t, err) + //Check ILM Policy is attached and Get rollover alias attached + for _, v := range settings { + assert.Equal(t, indexILMName, v.Settings["index.lifecycle.name"]) + actualWriteAliases = append(actualWriteAliases, v.Settings["index.lifecycle.rollover_alias"].(string)) + } + //Check indices created + assert.ElementsMatch(t, indices, expected, fmt.Sprintf("indices found: %v, expected: %v", indices, expected)) + //Check rollover alias is write alias + assert.ElementsMatch(t, actualWriteAliases, expectedWriteAliases, fmt.Sprintf("aliases found: %v, expected: %v", actualWriteAliases, expectedWriteAliases)) +} + +func createESClient() (*elastic.Client, error) { + return elastic.NewClient( + elastic.SetURL(queryURL), + elastic.SetSniff(false)) +} + +func runEsRollover(action string, envs []string) error { + var dockerEnv string + for _, e := range envs { + dockerEnv += fmt.Sprintf(" -e %s", e) + } + args := fmt.Sprintf("docker run %s --rm --net=host %s %s http://%s", dockerEnv, rolloverImage, action, queryHostPort) + cmd := exec.Command("/bin/sh", "-c", args) + out, err := cmd.CombinedOutput() + fmt.Println(string(out)) + return err +} + +func getVersion(client *elastic.Client) (uint, error) { + pingResult, _, err := client.Ping(queryURL).Do(context.Background()) + if err != nil { + return 0, err + } + esVersion, err := strconv.Atoi(string(pingResult.Version.Number[0])) + if err != nil { + return 0, err + } + return uint(esVersion), nil +} + +func createILMPolicy(client *elastic.Client, policyName string) error { + _, err := client.XPackIlmPutLifecycle().Policy(policyName).BodyString(`{"policy": {"phases": {"hot": {"min_age": "0ms","actions": {"rollover": {"max_age": "1d"},"set_priority": {"priority": 100}}}}}}`).Do(context.Background()) + return err +} + +func cleanES(t *testing.T, client *elastic.Client, policyName string) { + _, err := client.DeleteIndex("*").Do(context.Background()) + require.NoError(t, err) + esVersion, err := getVersion(client) + require.NoError(t, err) + if esVersion == 7 { + _, err = client.XPackIlmDeleteLifecycle().Policy(policyName).Do(context.Background()) + if err != nil && !elastic.IsNotFound(err) { + assert.Fail(t, "Not able to clean up ILM Policy") + } + } + _, err = client.IndexDeleteTemplate("*").Do(context.Background()) + require.NoError(t, err) +} diff --git a/scripts/travis/es-integration-test.sh b/scripts/travis/es-integration-test.sh index 6ff5575410d..0f7551cfe0f 100755 --- a/scripts/travis/es-integration-test.sh +++ b/scripts/travis/es-integration-test.sh @@ -61,6 +61,7 @@ run_integration_test() { local cid=$(setup_es ${es_version}) STORAGE=elasticsearch make storage-integration-test make index-cleaner-integration-test + make index-rollover-integration-test make es-otel-exporter-integration-test teardown_es ${cid} }