Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

3085 - Change type of logs from nested to object to reduce fixedbit memory consumption #4697

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
42 changes: 23 additions & 19 deletions cmd/es-rollover/app/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,29 +10,31 @@ import (
)

const (
indexPrefix = "index-prefix"
archive = "archive"
username = "es.username"
password = "es.password"
useILM = "es.use-ilm"
ilmPolicyName = "es.ilm-policy-name"
timeout = "timeout"
skipDependencies = "skip-dependencies"
adaptiveSampling = "adaptive-sampling"
indexPrefix = "index-prefix"
archive = "archive"
username = "es.username"
password = "es.password"
useILM = "es.use-ilm"
ilmPolicyName = "es.ilm-policy-name"
timeout = "timeout"
skipDependencies = "skip-dependencies"
adaptiveSampling = "adaptive-sampling"
disableLogsFieldSearch = "disable-logs-field-search"
)

// Config holds the global configurations for the es rollover, common to all actions
type Config struct {
IndexPrefix string
Archive bool
Username string
Password string
TLSEnabled bool
ILMPolicyName string
UseILM bool
Timeout int
SkipDependencies bool
AdaptiveSampling bool
IndexPrefix string
Archive bool
Username string
Password string
TLSEnabled bool
ILMPolicyName string
UseILM bool
Timeout int
SkipDependencies bool
AdaptiveSampling bool
DisableLogsFieldSearch bool
}

// AddFlags adds flags
Expand All @@ -46,6 +48,7 @@ func AddFlags(flags *flag.FlagSet) {
flags.Int(timeout, 120, "Number of seconds to wait for master node response")
flags.Bool(skipDependencies, false, "Disable rollover for dependencies index")
flags.Bool(adaptiveSampling, false, "Enable rollover for adaptive sampling index")
flags.Bool(disableLogsFieldSearch, false, "Set to `true` to set type logs.field mapping to object")
}

// InitFromViper initializes config from viper.Viper.
Expand All @@ -62,4 +65,5 @@ func (c *Config) InitFromViper(v *viper.Viper) {
c.Timeout = v.GetInt(timeout)
c.SkipDependencies = v.GetBool(skipDependencies)
c.AdaptiveSampling = v.GetBool(adaptiveSampling)
c.DisableLogsFieldSearch = v.GetBool(disableLogsFieldSearch)
}
2 changes: 2 additions & 0 deletions cmd/es-rollover/app/flags_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ func TestBindFlags(t *testing.T) {
"--es.ilm-policy-name=jaeger-ilm",
"--skip-dependencies=true",
"--adaptive-sampling=true",
"--disable-logs-field-search=true",
})
require.NoError(t, err)

Expand All @@ -44,4 +45,5 @@ func TestBindFlags(t *testing.T) {
assert.Equal(t, "jaeger-ilm", c.ILMPolicyName)
assert.True(t, c.SkipDependencies)
assert.True(t, c.AdaptiveSampling)
assert.True(t, c.DisableLogsFieldSearch)
}
1 change: 1 addition & 0 deletions cmd/es-rollover/app/init/action.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ func (c Action) getMapping(version uint, templateName string) (string, error) {
UseILM: c.Config.UseILM,
ILMPolicyName: c.Config.ILMPolicyName,
EsVersion: version,
DisableLogsFieldSearch: c.Config.DisableLogsFieldSearch,
}
return mappingBuilder.GetMapping(templateName)
}
Expand Down
35 changes: 21 additions & 14 deletions cmd/esmapping-generator/app/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,25 @@ import (

// Options represent configurable parameters for jaeger-esmapping-generator
type Options struct {
Mapping string
EsVersion uint
Shards int64
Replicas int64
IndexPrefix string
UseILM string // using string as util is being used in python and using bool leads to type issues.
ILMPolicyName string
Mapping string
EsVersion uint
Shards int64
Replicas int64
IndexPrefix string
UseILM string // using string as util is being used in python and using bool leads to type issues.
ILMPolicyName string
DisableLogsFieldSearch string // using string as util is being used in python and using bool leads to type issues.
}

const (
mappingFlag = "mapping"
esVersionFlag = "es-version"
shardsFlag = "shards"
replicasFlag = "replicas"
indexPrefixFlag = "index-prefix"
useILMFlag = "use-ilm"
ilmPolicyNameFlag = "ilm-policy-name"
mappingFlag = "mapping"
esVersionFlag = "es-version"
shardsFlag = "shards"
replicasFlag = "replicas"
indexPrefixFlag = "index-prefix"
useILMFlag = "use-ilm"
ilmPolicyNameFlag = "ilm-policy-name"
disableLogsFieldSearch = "disable-logs-field-search"
)

// AddFlags adds flags for esmapping-generator main program
Expand Down Expand Up @@ -65,6 +67,11 @@ func (o *Options) AddFlags(command *cobra.Command) {
ilmPolicyNameFlag,
"jaeger-ilm-policy",
"The name of the ILM policy to use if ILM is active")
command.Flags().StringVar(
&o.DisableLogsFieldSearch,
disableLogsFieldSearch,
"false",
"Set to `true` to set type logs.field mapping to object")

// mark mapping flag as mandatory
command.MarkFlagRequired(mappingFlag)
Expand Down
3 changes: 3 additions & 0 deletions cmd/esmapping-generator/app/flags_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ func TestOptionsWithDefaultFlags(t *testing.T) {
assert.Equal(t, "", o.IndexPrefix)
assert.Equal(t, "false", o.UseILM)
assert.Equal(t, "jaeger-ilm-policy", o.ILMPolicyName)
assert.Equal(t, "false", o.DisableLogsFieldSearch)
}

func TestOptionsWithFlags(t *testing.T) {
Expand All @@ -40,6 +41,7 @@ func TestOptionsWithFlags(t *testing.T) {
"--index-prefix=test",
"--use-ilm=true",
"--ilm-policy-name=jaeger-test-policy",
"--disable-logs-field-search=true",
})
require.NoError(t, err)
assert.Equal(t, "jaeger-span", o.Mapping)
Expand All @@ -49,6 +51,7 @@ func TestOptionsWithFlags(t *testing.T) {
assert.Equal(t, "test", o.IndexPrefix)
assert.Equal(t, "true", o.UseILM)
assert.Equal(t, "jaeger-test-policy", o.ILMPolicyName)
assert.Equal(t, "true", o.DisableLogsFieldSearch)
}

func TestMain(m *testing.M) {
Expand Down
19 changes: 12 additions & 7 deletions cmd/esmapping-generator/app/renderer/render.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,20 @@ func GetMappingAsString(builder es.TemplateBuilder, opt *app.Options) (string, e
if err != nil {
return "", err
}
disableLogsFieldSearch, err := strconv.ParseBool(opt.DisableLogsFieldSearch)
if err != nil {
return "", err
}

mappingBuilder := mappings.MappingBuilder{
TemplateBuilder: builder,
Shards: opt.Shards,
Replicas: opt.Replicas,
EsVersion: opt.EsVersion,
IndexPrefix: opt.IndexPrefix,
UseILM: enableILM,
ILMPolicyName: opt.ILMPolicyName,
TemplateBuilder: builder,
Shards: opt.Shards,
Replicas: opt.Replicas,
EsVersion: opt.EsVersion,
IndexPrefix: opt.IndexPrefix,
UseILM: enableILM,
ILMPolicyName: opt.ILMPolicyName,
DisableLogsFieldSearch: disableLogsFieldSearch,
}
return mappingBuilder.GetMapping(opt.Mapping)
}
Expand Down
87 changes: 79 additions & 8 deletions cmd/esmapping-generator/app/renderer/render_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/jaegertracing/jaeger/cmd/esmapping-generator/app"
"github.com/jaegertracing/jaeger/pkg/es/mocks"
"github.com/jaegertracing/jaeger/pkg/testutils"
"github.com/jaegertracing/jaeger/plugin/storage/es/mappings"
)

func TestIsValidOption(t *testing.T) {
Expand All @@ -36,22 +37,86 @@ func TestIsValidOption(t *testing.T) {

func Test_getMappingAsString(t *testing.T) {
tests := []struct {
name string
args app.Options
want string
wantErr error
name string
args app.Options
wantedMapping *mappings.MappingBuilder
want string
wantErr error
}{
{
name: "ES version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true", ILMPolicyName: "jaeger-test-policy"},
name: "ES version 7",
args: app.Options{
Mapping: "jaeger-span",
EsVersion: 7,
Shards: 5,
Replicas: 1,
IndexPrefix: "test-",
UseILM: "true",
ILMPolicyName: "jaeger-test-policy",
DisableLogsFieldSearch: "true",
},
wantedMapping: &mappings.MappingBuilder{
EsVersion: 7,
Shards: 5,
Replicas: 1,
IndexPrefix: "test-",
UseILM: true,
ILMPolicyName: "jaeger-test-policy",
DisableLogsFieldSearch: true,
},
want: "ES version 7",
},
{
name: "Parse Error version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true", ILMPolicyName: "jaeger-test-policy"},
name: "Parse Error version 7",
args: app.Options{
Mapping: "jaeger-span",
EsVersion: 7,
Shards: 5,
Replicas: 1,
IndexPrefix: "test-",
UseILM: "true",
ILMPolicyName: "jaeger-test-policy",
DisableLogsFieldSearch: "false",
},
wantedMapping: &mappings.MappingBuilder{
EsVersion: 7,
Shards: 5,
Replicas: 1,
IndexPrefix: "test-",
UseILM: false,
ILMPolicyName: "jaeger-test-policy",
DisableLogsFieldSearch: false,
},
wantErr: errors.New("parse error"),
},
{
name: "Parse bool error", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "foo", ILMPolicyName: "jaeger-test-policy"},
wantErr: errors.New("strconv.ParseBool: parsing \"foo\": invalid syntax"),
name: "Parse bool error for UseILM",
args: app.Options{
Mapping: "jaeger-span",
EsVersion: 7,
Shards: 5,
Replicas: 1,
IndexPrefix: "test-",
UseILM: "foo",
ILMPolicyName: "jaeger-test-policy",
},
wantedMapping: &mappings.MappingBuilder{},
wantErr: errors.New("strconv.ParseBool: parsing \"foo\": invalid syntax"),
},
{
name: "Parse bool error for DisableLogsFieldSearch",
args: app.Options{
Mapping: "jaeger-span",
EsVersion: 7,
Shards: 5,
Replicas: 1,
IndexPrefix: "test-",
UseILM: "false",
ILMPolicyName: "jaeger-test-policy",
DisableLogsFieldSearch: "foo",
},
wantedMapping: &mappings.MappingBuilder{},
wantErr: errors.New("strconv.ParseBool: parsing \"foo\": invalid syntax"),
},
}
for _, tt := range tests {
Expand All @@ -60,11 +125,17 @@ func Test_getMappingAsString(t *testing.T) {
mockTemplateApplier := &mocks.TemplateApplier{}
mockTemplateApplier.On("Execute", mock.Anything, mock.Anything).Return(
func(wr io.Writer, data any) error {
if actualMapping, ok := data.(*mappings.MappingBuilder); ok {
require.Equal(t, tt.wantedMapping, actualMapping)
} else {
require.Fail(t, "unexpected mapping type")
}
wr.Write([]byte(tt.want))
return nil
},
)
mockTemplateBuilder := &mocks.TemplateBuilder{}
tt.wantedMapping.TemplateBuilder = mockTemplateBuilder
mockTemplateBuilder.On("Parse", mock.Anything).Return(mockTemplateApplier, tt.wantErr)

// Test
Expand Down
1 change: 1 addition & 0 deletions pkg/es/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ type Configuration struct {
Version uint `mapstructure:"version"`
LogLevel string `mapstructure:"log_level"`
SendGetBodyAs string `mapstructure:"send_get_body_as"`
DisableLogsFieldSearch bool `mapstructure:"disable_logs_field_search"`
}

// TagsAsFields holds configuration for tag schema.
Expand Down
2 changes: 2 additions & 0 deletions plugin/storage/es/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,7 @@ func createSpanReader(
Logger: logger,
MetricsFactory: mFactory,
Tracer: tp.Tracer("esSpanStore.SpanReader"),
DisableLogsFieldSearch: cfg.DisableLogsFieldSearch,
}), nil
}

Expand Down Expand Up @@ -322,6 +323,7 @@ func mappingBuilderFromConfig(cfg *config.Configuration) mappings.MappingBuilder
PrioritySpanTemplate: cfg.PrioritySpanTemplate,
PriorityServiceTemplate: cfg.PriorityServiceTemplate,
PriorityDependenciesTemplate: cfg.PriorityDependenciesTemplate,
DisableLogsFieldSearch: cfg.DisableLogsFieldSearch,
}
}

Expand Down
Loading