From d5adcf3a93b234df7a398c0e69b38f26400c06bc Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 11 May 2023 17:48:19 +0200 Subject: [PATCH 1/5] Add synthetic mode as default when creating a datastream --- internal/packages/archetype/data_stream_manifest.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/packages/archetype/data_stream_manifest.go b/internal/packages/archetype/data_stream_manifest.go index 27d418cc07..41da08c69d 100644 --- a/internal/packages/archetype/data_stream_manifest.go +++ b/internal/packages/archetype/data_stream_manifest.go @@ -27,4 +27,6 @@ streams:{{if eq .Manifest.Type "logs" }} title: Period default: 10s {{- end}} +elasticsearch: + source_mode: synthetic ` From 41f308f1279f2d3122043f6cd2bdb1bb476963de Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 12 May 2023 13:49:15 +0200 Subject: [PATCH 2/5] Add new question for synthetic source --- cmd/create_data_stream.go | 32 ++++++++++++++----- .../archetype/data_stream_manifest.go | 4 +++ internal/packages/packages.go | 31 ++++++++++-------- 3 files changed, 45 insertions(+), 22 deletions(-) diff --git a/cmd/create_data_stream.go b/cmd/create_data_stream.go index d9ec2779f7..60ece660b6 100644 --- a/cmd/create_data_stream.go +++ b/cmd/create_data_stream.go @@ -19,9 +19,10 @@ const createDataStreamLongDescription = `Use this command to create a new data s The command can extend the package with a new data stream using embedded data stream template and wizard.` type newDataStreamAnswers struct { - Name string - Title string - Type string + Name string + Title string + Type string + Synthetic bool } func createDataStreamCommandAction(cmd *cobra.Command, args []string) error { @@ -61,6 +62,14 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error { }, Validate: survey.Required, }, + { + Name: "synthetic", + Prompt: &survey.Confirm{ + Message: "Enable synthetic source:", + Default: true, + }, + Validate: survey.Required, + }, } var answers newDataStreamAnswers err = survey.Ask(qs, &answers) @@ -79,12 +88,19 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error { } func createDataStreamDescriptorFromAnswers(answers newDataStreamAnswers, packageRoot string) archetype.DataStreamDescriptor { + manifest := packages.DataStreamManifest{ + Name: answers.Name, + Title: answers.Title, + Type: answers.Type, + } + if answers.Synthetic { + elasticsearch := packages.Elasticsearch{ + SourceMode: "synthetic", + } + manifest.Elasticsearch = &elasticsearch + } return archetype.DataStreamDescriptor{ - Manifest: packages.DataStreamManifest{ - Name: answers.Name, - Title: answers.Title, - Type: answers.Type, - }, + Manifest: manifest, PackageRoot: packageRoot, } } diff --git a/internal/packages/archetype/data_stream_manifest.go b/internal/packages/archetype/data_stream_manifest.go index 41da08c69d..2f9574aae3 100644 --- a/internal/packages/archetype/data_stream_manifest.go +++ b/internal/packages/archetype/data_stream_manifest.go @@ -27,6 +27,10 @@ streams:{{if eq .Manifest.Type "logs" }} title: Period default: 10s {{- end}} +{{ if .Manifest.Elasticsearch }} +{{ if eq .Manifest.Elasticsearch.SourceMode "synthetic" }} elasticsearch: source_mode: synthetic +{{- end}} +{{- end}} ` diff --git a/internal/packages/packages.go b/internal/packages/packages.go index 376f792284..ecdb85180e 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -132,22 +132,25 @@ type PackageManifest struct { Categories []string `config:"categories" json:"categories" yaml:"categories"` } +type Elasticsearch struct { + IndexTemplate *struct { + IngestPipeline *struct { + Name string `config:"name" json:"name" yaml:"name"` + } `config:"ingest_pipeline" json:"ingest_pipeline" yaml:"ingest_pipeline"` + } `config:"index_template" json:"index_template" yaml:"index_template"` + SourceMode string `config:"source_mode" json:"source_mode" yaml:"source_mode"` +} + // DataStreamManifest represents the structure of a data stream's manifest type DataStreamManifest struct { - Name string `config:"name" json:"name" yaml:"name"` - Title string `config:"title" json:"title" yaml:"title"` - Type string `config:"type" json:"type" yaml:"type"` - Dataset string `config:"dataset" json:"dataset" yaml:"dataset"` - Hidden bool `config:"hidden" json:"hidden" yaml:"hidden"` - Release string `config:"release" json:"release" yaml:"release"` - Elasticsearch *struct { - IndexTemplate *struct { - IngestPipeline *struct { - Name string `config:"name" json:"name" yaml:"name"` - } `config:"ingest_pipeline" json:"ingest_pipeline" yaml:"ingest_pipeline"` - } `config:"index_template" json:"index_template" yaml:"index_template"` - } `config:"elasticsearch" json:"elasticsearch" yaml:"elasticsearch"` - Streams []struct { + Name string `config:"name" json:"name" yaml:"name"` + Title string `config:"title" json:"title" yaml:"title"` + Type string `config:"type" json:"type" yaml:"type"` + Dataset string `config:"dataset" json:"dataset" yaml:"dataset"` + Hidden bool `config:"hidden" json:"hidden" yaml:"hidden"` + Release string `config:"release" json:"release" yaml:"release"` + Elasticsearch *Elasticsearch `config:"elasticsearch" json:"elasticsearch" yaml:"elasticsearch"` + Streams []struct { Input string `config:"input" json:"input" yaml:"input"` Vars []Variable `config:"vars" json:"vars" yaml:"vars"` } `config:"streams" json:"streams" yaml:"streams"` From f3a68b18969d11bf03665861b178b3da997962ae Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 1 Jun 2023 12:59:35 +0200 Subject: [PATCH 3/5] Set synthetic source mode just for metrics and the survey question --- cmd/create_data_stream.go | 25 +++++++++++++------ .../archetype/data_stream_manifest.go | 2 +- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cmd/create_data_stream.go b/cmd/create_data_stream.go index 60ece660b6..7e2a1c4a8d 100644 --- a/cmd/create_data_stream.go +++ b/cmd/create_data_stream.go @@ -62,14 +62,6 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error { }, Validate: survey.Required, }, - { - Name: "synthetic", - Prompt: &survey.Confirm{ - Message: "Enable synthetic source:", - Default: true, - }, - Validate: survey.Required, - }, } var answers newDataStreamAnswers err = survey.Ask(qs, &answers) @@ -77,6 +69,23 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error { return errors.Wrap(err, "prompt failed") } + if answers.Type == "metrics" { + qs := []*survey.Question{ + { + Name: "synthetic", + Prompt: &survey.Confirm{ + Message: "Enable synthetic source:", + Default: true, + }, + Validate: survey.Required, + }, + } + err = survey.Ask(qs, &answers) + if err != nil { + return errors.Wrap(err, "prompt failed") + } + } + descriptor := createDataStreamDescriptorFromAnswers(answers, packageRoot) err = archetype.CreateDataStream(descriptor) if err != nil { diff --git a/internal/packages/archetype/data_stream_manifest.go b/internal/packages/archetype/data_stream_manifest.go index 2f9574aae3..f7a0e04160 100644 --- a/internal/packages/archetype/data_stream_manifest.go +++ b/internal/packages/archetype/data_stream_manifest.go @@ -26,11 +26,11 @@ streams:{{if eq .Manifest.Type "logs" }} type: text title: Period default: 10s -{{- end}} {{ if .Manifest.Elasticsearch }} {{ if eq .Manifest.Elasticsearch.SourceMode "synthetic" }} elasticsearch: source_mode: synthetic {{- end}} {{- end}} +{{- end}} ` From 68fb6c4e3580bee7da9228a465e25bf3eb969e60 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 6 Jun 2023 13:00:57 +0200 Subject: [PATCH 4/5] Add index_mode --- cmd/create_data_stream.go | 44 +++++++++++++++---- .../archetype/data_stream_manifest.go | 7 ++- internal/packages/packages.go | 1 + 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/cmd/create_data_stream.go b/cmd/create_data_stream.go index 7e2a1c4a8d..95f22235c0 100644 --- a/cmd/create_data_stream.go +++ b/cmd/create_data_stream.go @@ -19,10 +19,11 @@ const createDataStreamLongDescription = `Use this command to create a new data s The command can extend the package with a new data stream using embedded data stream template and wizard.` type newDataStreamAnswers struct { - Name string - Title string - Type string - Synthetic bool + Name string + Title string + Type string + Synthetic bool + TimeSeries bool } func createDataStreamCommandAction(cmd *cobra.Command, args []string) error { @@ -74,7 +75,7 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error { { Name: "synthetic", Prompt: &survey.Confirm{ - Message: "Enable synthetic source:", + Message: "Enable synthetic source?", Default: true, }, Validate: survey.Required, @@ -85,6 +86,22 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error { return errors.Wrap(err, "prompt failed") } } + if answers.Synthetic { + qs := []*survey.Question{ + { + Name: "timeSeries", + Prompt: &survey.Confirm{ + Message: "Enable time series?", + Default: false, + }, + Validate: survey.Required, + }, + } + err = survey.Ask(qs, &answers) + if err != nil { + return errors.Wrap(err, "prompt failed") + } + } descriptor := createDataStreamDescriptorFromAnswers(answers, packageRoot) err = archetype.CreateDataStream(descriptor) @@ -102,12 +119,21 @@ func createDataStreamDescriptorFromAnswers(answers newDataStreamAnswers, package Title: answers.Title, Type: answers.Type, } - if answers.Synthetic { - elasticsearch := packages.Elasticsearch{ - SourceMode: "synthetic", + + if !answers.Synthetic && !answers.TimeSeries { + return archetype.DataStreamDescriptor{ + Manifest: manifest, + PackageRoot: packageRoot, } - manifest.Elasticsearch = &elasticsearch } + elasticsearch := packages.Elasticsearch{} + if answers.Synthetic { + elasticsearch.SourceMode = "synthetic" + } + if answers.TimeSeries { + elasticsearch.IndexMode = "time_series" + } + manifest.Elasticsearch = &elasticsearch return archetype.DataStreamDescriptor{ Manifest: manifest, PackageRoot: packageRoot, diff --git a/internal/packages/archetype/data_stream_manifest.go b/internal/packages/archetype/data_stream_manifest.go index f7a0e04160..ef132f7fe8 100644 --- a/internal/packages/archetype/data_stream_manifest.go +++ b/internal/packages/archetype/data_stream_manifest.go @@ -27,9 +27,12 @@ streams:{{if eq .Manifest.Type "logs" }} title: Period default: 10s {{ if .Manifest.Elasticsearch }} -{{ if eq .Manifest.Elasticsearch.SourceMode "synthetic" }} elasticsearch: - source_mode: synthetic +{{ if .Manifest.Elasticsearch.SourceMode }} + source_mode: {{ .Manifest.Elasticsearch.SourceMode }} +{{- end}} +{{ if .Manifest.Elasticsearch.IndexMode }} + index_mode: {{ .Manifest.Elasticsearch.IndexMode }} {{- end}} {{- end}} {{- end}} diff --git a/internal/packages/packages.go b/internal/packages/packages.go index ecdb85180e..ff50962396 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -139,6 +139,7 @@ type Elasticsearch struct { } `config:"ingest_pipeline" json:"ingest_pipeline" yaml:"ingest_pipeline"` } `config:"index_template" json:"index_template" yaml:"index_template"` SourceMode string `config:"source_mode" json:"source_mode" yaml:"source_mode"` + IndexMode string `config:"index_mode" json:"index_mode" yaml:"index_mode"` } // DataStreamManifest represents the structure of a data stream's manifest From 85ad5b1c14c219856495d025005c146a13a0725f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 8 Jun 2023 13:23:11 +0200 Subject: [PATCH 5/5] Update questions --- cmd/create_data_stream.go | 52 +++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/cmd/create_data_stream.go b/cmd/create_data_stream.go index 95f22235c0..e4ec6689e7 100644 --- a/cmd/create_data_stream.go +++ b/cmd/create_data_stream.go @@ -19,11 +19,11 @@ const createDataStreamLongDescription = `Use this command to create a new data s The command can extend the package with a new data stream using embedded data stream template and wizard.` type newDataStreamAnswers struct { - Name string - Title string - Type string - Synthetic bool - TimeSeries bool + Name string + Title string + Type string + SyntheticAndTimeSeries bool + Synthetic bool } func createDataStreamCommandAction(cmd *cobra.Command, args []string) error { @@ -73,9 +73,9 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error { if answers.Type == "metrics" { qs := []*survey.Question{ { - Name: "synthetic", + Name: "syntheticAndTimeSeries", Prompt: &survey.Confirm{ - Message: "Enable synthetic source?", + Message: "Enable time series and synthetic source?", Default: true, }, Validate: survey.Required, @@ -85,21 +85,22 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error { if err != nil { return errors.Wrap(err, "prompt failed") } - } - if answers.Synthetic { - qs := []*survey.Question{ - { - Name: "timeSeries", - Prompt: &survey.Confirm{ - Message: "Enable time series?", - Default: false, + + if !answers.SyntheticAndTimeSeries { + qs := []*survey.Question{ + { + Name: "synthetic", + Prompt: &survey.Confirm{ + Message: "Enable synthetic source?", + Default: true, + }, + Validate: survey.Required, }, - Validate: survey.Required, - }, - } - err = survey.Ask(qs, &answers) - if err != nil { - return errors.Wrap(err, "prompt failed") + } + err = survey.Ask(qs, &answers) + if err != nil { + return errors.Wrap(err, "prompt failed") + } } } @@ -120,17 +121,16 @@ func createDataStreamDescriptorFromAnswers(answers newDataStreamAnswers, package Type: answers.Type, } - if !answers.Synthetic && !answers.TimeSeries { + if !answers.SyntheticAndTimeSeries && !answers.Synthetic { return archetype.DataStreamDescriptor{ Manifest: manifest, PackageRoot: packageRoot, } } - elasticsearch := packages.Elasticsearch{} - if answers.Synthetic { - elasticsearch.SourceMode = "synthetic" + elasticsearch := packages.Elasticsearch{ + SourceMode: "synthetic", } - if answers.TimeSeries { + if answers.SyntheticAndTimeSeries { elasticsearch.IndexMode = "time_series" } manifest.Elasticsearch = &elasticsearch