3085 - Change type of logs from nested to object to reduce fixedbit memory consumption

cmd/es-rollover/app/flags.go

@@ -10,29 +10,31 @@ import (
 )
 
 const (
-	indexPrefix      = "index-prefix"
-	archive          = "archive"
-	username         = "es.username"
-	password         = "es.password"
-	useILM           = "es.use-ilm"
-	ilmPolicyName    = "es.ilm-policy-name"
-	timeout          = "timeout"
-	skipDependencies = "skip-dependencies"
-	adaptiveSampling = "adaptive-sampling"
+	indexPrefix            = "index-prefix"
+	archive                = "archive"
+	username               = "es.username"
+	password               = "es.password"
+	useILM                 = "es.use-ilm"
+	ilmPolicyName          = "es.ilm-policy-name"
+	timeout                = "timeout"
+	skipDependencies       = "skip-dependencies"
+	adaptiveSampling       = "adaptive-sampling"
+	disableLogsFieldSearch = "disable-logs-field-search"
 )
 
 // Config holds the global configurations for the es rollover, common to all actions
 type Config struct {
-	IndexPrefix      string
-	Archive          bool
-	Username         string
-	Password         string
-	TLSEnabled       bool
-	ILMPolicyName    string
-	UseILM           bool
-	Timeout          int
-	SkipDependencies bool
-	AdaptiveSampling bool
+	IndexPrefix            string
+	Archive                bool
+	Username               string
+	Password               string
+	TLSEnabled             bool
+	ILMPolicyName          string
+	UseILM                 bool
+	Timeout                int
+	SkipDependencies       bool
+	AdaptiveSampling       bool
+	DisableLogsFieldSearch bool
 }
 
 // AddFlags adds flags
@@ -46,6 +48,7 @@ func AddFlags(flags *flag.FlagSet) {
 	flags.Int(timeout, 120, "Number of seconds to wait for master node response")
 	flags.Bool(skipDependencies, false, "Disable rollover for dependencies index")
 	flags.Bool(adaptiveSampling, false, "Enable rollover for adaptive sampling index")
+	flags.Bool(disableLogsFieldSearch, false, "Set to `true` to set type logs.field mapping to object")
 }
 
 // InitFromViper initializes config from viper.Viper.
@@ -62,4 +65,5 @@ func (c *Config) InitFromViper(v *viper.Viper) {
 	c.Timeout = v.GetInt(timeout)
 	c.SkipDependencies = v.GetBool(skipDependencies)
 	c.AdaptiveSampling = v.GetBool(adaptiveSampling)
+	c.DisableLogsFieldSearch = v.GetBool(disableLogsFieldSearch)
 }

cmd/es-rollover/app/flags_test.go

@@ -32,6 +32,7 @@ func TestBindFlags(t *testing.T) {
 		"--es.ilm-policy-name=jaeger-ilm",
 		"--skip-dependencies=true",
 		"--adaptive-sampling=true",
+		"--disable-logs-field-search=true",
 	})
 	require.NoError(t, err)
 
@@ -44,4 +45,5 @@ func TestBindFlags(t *testing.T) {
 	assert.Equal(t, "jaeger-ilm", c.ILMPolicyName)
 	assert.True(t, c.SkipDependencies)
 	assert.True(t, c.AdaptiveSampling)
+	assert.True(t, c.DisableLogsFieldSearch)
 }

cmd/es-rollover/app/init/action.go

@@ -40,6 +40,7 @@ func (c Action) getMapping(version uint, templateName string) (string, error) {
 		UseILM:                       c.Config.UseILM,
 		ILMPolicyName:                c.Config.ILMPolicyName,
 		EsVersion:                    version,
+		DisableLogsFieldSearch:       c.Config.DisableLogsFieldSearch,
 	}
 	return mappingBuilder.GetMapping(templateName)
 }

cmd/esmapping-generator/app/flags.go

@@ -9,23 +9,25 @@ import (
 
 // Options represent configurable parameters for jaeger-esmapping-generator
 type Options struct {
-	Mapping       string
-	EsVersion     uint
-	Shards        int64
-	Replicas      int64
-	IndexPrefix   string
-	UseILM        string // using string as util is being used in python and using bool leads to type issues.
-	ILMPolicyName string
+	Mapping                string
+	EsVersion              uint
+	Shards                 int64
+	Replicas               int64
+	IndexPrefix            string
+	UseILM                 string // using string as util is being used in python and using bool leads to type issues.
+	ILMPolicyName          string
+	DisableLogsFieldSearch string // using string as util is being used in python and using bool leads to type issues.
 }
 
 const (
-	mappingFlag       = "mapping"
-	esVersionFlag     = "es-version"
-	shardsFlag        = "shards"
-	replicasFlag      = "replicas"
-	indexPrefixFlag   = "index-prefix"
-	useILMFlag        = "use-ilm"
-	ilmPolicyNameFlag = "ilm-policy-name"
+	mappingFlag            = "mapping"
+	esVersionFlag          = "es-version"
+	shardsFlag             = "shards"
+	replicasFlag           = "replicas"
+	indexPrefixFlag        = "index-prefix"
+	useILMFlag             = "use-ilm"
+	ilmPolicyNameFlag      = "ilm-policy-name"
+	disableLogsFieldSearch = "disable-logs-field-search"
 )
 
 // AddFlags adds flags for esmapping-generator main program
@@ -65,6 +67,11 @@ func (o *Options) AddFlags(command *cobra.Command) {
 		ilmPolicyNameFlag,
 		"jaeger-ilm-policy",
 		"The name of the ILM policy to use if ILM is active")
+	command.Flags().StringVar(
+		&o.DisableLogsFieldSearch,
+		disableLogsFieldSearch,
+		"false",
+		"Set to `true` to set type logs.field mapping to object")
 
 	// mark mapping flag as mandatory
 	command.MarkFlagRequired(mappingFlag)

cmd/esmapping-generator/app/flags_test.go

@@ -25,6 +25,7 @@ func TestOptionsWithDefaultFlags(t *testing.T) {
 	assert.Equal(t, "", o.IndexPrefix)
 	assert.Equal(t, "false", o.UseILM)
 	assert.Equal(t, "jaeger-ilm-policy", o.ILMPolicyName)
+	assert.Equal(t, "false", o.DisableLogsFieldSearch)
 }
 
 func TestOptionsWithFlags(t *testing.T) {
@@ -40,6 +41,7 @@ func TestOptionsWithFlags(t *testing.T) {
 		"--index-prefix=test",
 		"--use-ilm=true",
 		"--ilm-policy-name=jaeger-test-policy",
+		"--disable-logs-field-search=true",
 	})
 	require.NoError(t, err)
 	assert.Equal(t, "jaeger-span", o.Mapping)
@@ -49,6 +51,7 @@ func TestOptionsWithFlags(t *testing.T) {
 	assert.Equal(t, "test", o.IndexPrefix)
 	assert.Equal(t, "true", o.UseILM)
 	assert.Equal(t, "jaeger-test-policy", o.ILMPolicyName)
+	assert.Equal(t, "true", o.DisableLogsFieldSearch)
 }
 
 func TestMain(m *testing.M) {

cmd/esmapping-generator/app/renderer/render.go

@@ -24,15 +24,20 @@ func GetMappingAsString(builder es.TemplateBuilder, opt *app.Options) (string, e
 	if err != nil {
 		return "", err
 	}
+	disableLogsFieldSearch, err := strconv.ParseBool(opt.DisableLogsFieldSearch)
+	if err != nil {
+		return "", err
+	}
 
 	mappingBuilder := mappings.MappingBuilder{
-		TemplateBuilder: builder,
-		Shards:          opt.Shards,
-		Replicas:        opt.Replicas,
-		EsVersion:       opt.EsVersion,
-		IndexPrefix:     opt.IndexPrefix,
-		UseILM:          enableILM,
-		ILMPolicyName:   opt.ILMPolicyName,
+		TemplateBuilder:        builder,
+		Shards:                 opt.Shards,
+		Replicas:               opt.Replicas,
+		EsVersion:              opt.EsVersion,
+		IndexPrefix:            opt.IndexPrefix,
+		UseILM:                 enableILM,
+		ILMPolicyName:          opt.ILMPolicyName,
+		DisableLogsFieldSearch: disableLogsFieldSearch,
 	}
 	return mappingBuilder.GetMapping(opt.Mapping)
 }

cmd/esmapping-generator/app/renderer/render_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/jaegertracing/jaeger/cmd/esmapping-generator/app"
 	"github.com/jaegertracing/jaeger/pkg/es/mocks"
 	"github.com/jaegertracing/jaeger/pkg/testutils"
+	"github.com/jaegertracing/jaeger/plugin/storage/es/mappings"
 )
 
 func TestIsValidOption(t *testing.T) {
@@ -36,22 +37,86 @@ func TestIsValidOption(t *testing.T) {
 
 func Test_getMappingAsString(t *testing.T) {
 	tests := []struct {
-		name    string
-		args    app.Options
-		want    string
-		wantErr error
+		name          string
+		args          app.Options
+		wantedMapping *mappings.MappingBuilder
+		want          string
+		wantErr       error
 	}{
 		{
-			name: "ES version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true", ILMPolicyName: "jaeger-test-policy"},
+			name: "ES version 7",
+			args: app.Options{
+				Mapping:                "jaeger-span",
+				EsVersion:              7,
+				Shards:                 5,
+				Replicas:               1,
+				IndexPrefix:            "test-",
+				UseILM:                 "true",
+				ILMPolicyName:          "jaeger-test-policy",
+				DisableLogsFieldSearch: "true",
+			},
+			wantedMapping: &mappings.MappingBuilder{
+				EsVersion:              7,
+				Shards:                 5,
+				Replicas:               1,
+				IndexPrefix:            "test-",
+				UseILM:                 true,
+				ILMPolicyName:          "jaeger-test-policy",
+				DisableLogsFieldSearch: true,
+			},
 			want: "ES version 7",
 		},
 		{
-			name: "Parse Error version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true", ILMPolicyName: "jaeger-test-policy"},
+			name: "Parse Error version 7",
+			args: app.Options{
+				Mapping:                "jaeger-span",
+				EsVersion:              7,
+				Shards:                 5,
+				Replicas:               1,
+				IndexPrefix:            "test-",
+				UseILM:                 "true",
+				ILMPolicyName:          "jaeger-test-policy",
+				DisableLogsFieldSearch: "false",
+			},
+			wantedMapping: &mappings.MappingBuilder{
+				EsVersion:              7,
+				Shards:                 5,
+				Replicas:               1,
+				IndexPrefix:            "test-",
+				UseILM:                 false,
+				ILMPolicyName:          "jaeger-test-policy",
+				DisableLogsFieldSearch: false,
+			},
 			wantErr: errors.New("parse error"),
 		},
 		{
-			name: "Parse bool error", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "foo", ILMPolicyName: "jaeger-test-policy"},
-			wantErr: errors.New("strconv.ParseBool: parsing \"foo\": invalid syntax"),
+			name: "Parse bool error for UseILM",
+			args: app.Options{
+				Mapping:       "jaeger-span",
+				EsVersion:     7,
+				Shards:        5,
+				Replicas:      1,
+				IndexPrefix:   "test-",
+				UseILM:        "foo",
+				ILMPolicyName: "jaeger-test-policy",
+			},
+			wantedMapping: &mappings.MappingBuilder{},
+			wantErr:       errors.New("strconv.ParseBool: parsing \"foo\": invalid syntax"),
+		},
+		{
+			name: "Parse bool error for DisableLogsFieldSearch",
+			args: app.Options{
+				Mapping:                "jaeger-span",
+				EsVersion:              7,
+				Shards:                 5,
+				Replicas:               1,
+				IndexPrefix:            "test-",
+				UseILM:                 "false",
+				ILMPolicyName:          "jaeger-test-policy",
+				DisableLogsFieldSearch: "foo",
+			},
+			wantedMapping: &mappings.MappingBuilder{},
+			wantErr:       errors.New("strconv.ParseBool: parsing \"foo\": invalid syntax"),
 		},
 	}
 	for _, tt := range tests {
@@ -60,11 +125,17 @@ func Test_getMappingAsString(t *testing.T) {
 			mockTemplateApplier := &mocks.TemplateApplier{}
 			mockTemplateApplier.On("Execute", mock.Anything, mock.Anything).Return(
 				func(wr io.Writer, data any) error {
+					if actualMapping, ok := data.(*mappings.MappingBuilder); ok {
+						require.Equal(t, tt.wantedMapping, actualMapping)
+					} else {
+						require.Fail(t, "unexpected mapping type")
+					}
 					wr.Write([]byte(tt.want))
 					return nil
 				},
 			)
 			mockTemplateBuilder := &mocks.TemplateBuilder{}
+			tt.wantedMapping.TemplateBuilder = mockTemplateBuilder
 			mockTemplateBuilder.On("Parse", mock.Anything).Return(mockTemplateApplier, tt.wantErr)
 
 			// Test

pkg/es/config/config.go

@@ -75,6 +75,7 @@ type Configuration struct {
 	Version                        uint           `mapstructure:"version"`
 	LogLevel                       string         `mapstructure:"log_level"`
 	SendGetBodyAs                  string         `mapstructure:"send_get_body_as"`
+	DisableLogsFieldSearch         bool           `mapstructure:"disable_logs_field_search"`
 }
 
 // TagsAsFields holds configuration for tag schema.

plugin/storage/es/factory.go

@@ -236,6 +236,7 @@ func createSpanReader(
 		Logger:                        logger,
 		MetricsFactory:                mFactory,
 		Tracer:                        tp.Tracer("esSpanStore.SpanReader"),
+		DisableLogsFieldSearch:        cfg.DisableLogsFieldSearch,
 	}), nil
 }
 
@@ -322,6 +323,7 @@ func mappingBuilderFromConfig(cfg *config.Configuration) mappings.MappingBuilder
 		PrioritySpanTemplate:         cfg.PrioritySpanTemplate,
 		PriorityServiceTemplate:      cfg.PriorityServiceTemplate,
 		PriorityDependenciesTemplate: cfg.PriorityDependenciesTemplate,
+		DisableLogsFieldSearch:       cfg.DisableLogsFieldSearch,
 	}
 }