Create input packages

cmd/create_data_stream.go

@@ -40,6 +40,15 @@ func createDataStreamCommandAction(cmd *cobra.Command, args []string) error {
 		return errors.New("package root not found, you can only create new data stream in the package context")
 	}
 
+	manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot)
+	if err != nil {
+		return fmt.Errorf("reading package manifest failed (path: %s): %w", packageRoot, err)
+	}
+
+	if manifest.Type == "input" {
+		return fmt.Errorf("data-streams are not supported in input packages")
+	}
+
 	qs := []*survey.Question{
 		{
 			Name: "name",

cmd/create_package.go

@@ -28,6 +28,7 @@ const (
 
 type newPackageAnswers struct {
 	Name                string
+	Type                string
 	Version             string
 	SourceLicense       string `survey:"source_license"`
 	Title               string
@@ -36,12 +37,31 @@ type newPackageAnswers struct {
 	KibanaVersion       string `survey:"kibana_version"`
 	ElasticSubscription string `survey:"elastic_subscription"`
 	GithubOwner         string `survey:"github_owner"`
+	DataStreamType      string `survey:"datastream_type"`
 }
 
 func createPackageCommandAction(cmd *cobra.Command, args []string) error {
 	cmd.Println("Create a new package")
 
 	qs := []*survey.Question{
+		{
+			Name: "type",
+			Prompt: &survey.Select{
+				Message: "Package type:",
+				Options: []string{"input", "integration"},
+				Default: "integration",
+			},
+			Validate: survey.Required,
+		},
+	}
+
+	var answers newPackageAnswers
+	err := survey.Ask(qs, &answers)
+	if err != nil {
+		return fmt.Errorf("prompt failed: %w", err)
+	}
+
+	qs = []*survey.Question{
 		{
 			Name: "name",
 			Prompt: &survey.Input{
@@ -132,8 +152,23 @@ func createPackageCommandAction(cmd *cobra.Command, args []string) error {
 		},
 	}
 
-	var answers newPackageAnswers
-	err := survey.Ask(qs, &answers)
+	if answers.Type == "input" {
+		inputQs := []*survey.Question{
+			{
+				Name: "datastream_type",
+				Prompt: &survey.Select{
+					Message: "Input Data Stream type:",
+					Options: []string{"logs", "metrics"},
+					Default: "logs",
+				},
+				Validate: survey.Required,
+			},
+		}
+
+		qs = append(qs, inputQs...)
+	}
+
+	err = survey.Ask(qs, &answers)
 	if err != nil {
 		return fmt.Errorf("prompt failed: %w", err)
 	}
@@ -153,11 +188,16 @@ func createPackageDescriptorFromAnswers(answers newPackageAnswers) archetype.Pac
 	if answers.SourceLicense != noLicenseValue {
 		sourceLicense = answers.SourceLicense
 	}
+
+	inputDataStreamType := ""
+	if answers.Type == "input" {
+		inputDataStreamType = answers.DataStreamType
+	}
 	return archetype.PackageDescriptor{
 		Manifest: packages.PackageManifest{
 			Name:    answers.Name,
 			Title:   answers.Title,
-			Type:    "integration",
+			Type:    answers.Type,
 			Version: answers.Version,
 			Source: packages.Source{
 				License: sourceLicense,
@@ -177,5 +217,6 @@ func createPackageDescriptorFromAnswers(answers newPackageAnswers) archetype.Pac
 			Description: answers.Description,
 			Categories:  answers.Categories,
 		},
+		InputDataStreamType: inputDataStreamType,
 	}
 }

internal/packages/archetype/_static/dataStream-agent-stream.yml.tmpl

@@ -0,0 +1,14 @@
+{{if eq .Manifest.Type "logs"}}paths:
+{{ "{{#each paths as |path i|}}" }}
+  - {{ "{{path}}" }}
+{{ "{{/each}}" }}
+exclude_files: [".gz$"]
+processors:
+  - add_locale: ~
+{{else}}metricsets: ["sample_metricset"]
+hosts:
+{{ "{{#each hosts}}" }}
+  - {{ "{{this}}" }}
+{{ "{{/each}}" }}
+period: {{ "{{period}}" }}
+{{end}}

internal/packages/archetype/_static/dataStream-elasticsearch-ingest-pipeline.yml.tmpl

@@ -0,0 +1,10 @@
+---
+description: Pipeline for processing sample logs
+processors:
+- set:
+    field: sample_field
+    value: "1"
+on_failure:
+- set:
+    field: error.message
+    value: {{ "'{{ _ingest.on_failure_message }}'" }}

internal/packages/archetype/data_stream_manifest.go → internal/packages/archetype/_static/dataStream-manifest.yml.tmpl

@@ -1,10 +1,4 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package archetype
-
-const dataStreamManifestTemplate = `title: "{{.Manifest.Title}}"
+title: "{{.Manifest.Title}}"
 type: {{.Manifest.Type}}
 streams:{{if eq .Manifest.Type "logs" }}
   - input: logfile
@@ -35,5 +29,4 @@ elasticsearch:
   index_mode: {{ .Manifest.Elasticsearch.IndexMode }}
 {{- end}}
 {{- end}}
-{{- end}}
-`
+{{- end}}

internal/packages/archetype/_static/fields-base.yml.tmpl

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

internal/packages/archetype/_static/input-package-agent-config.yml.tmpl

@@ -0,0 +1,16 @@
+data_stream:
+  dataset: {{ "{{data_stream.dataset}}" }}
+{{if eq .InputDataStreamType "logs"}}paths:
+{{ "{{#each paths as |path i|}}" }}
+  - {{"{{path}}"}}
+{{ "{{/each}}" }}
+exclude_files: [".gz$"]
+processors:
+  - add_locale: ~
+{{else}}metricsets: ["sample_metricset"]
+hosts:
+{{ "{{#each hosts}}" }}
+  -  {{ "{{this}}" }}
+{{ "{{/each}}" }}
+period: {{ "{{period}}" }}
+{{end}}

internal/packages/archetype/_static/package-changelog.yml.tmpl

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "{{.Manifest.Version}}"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link

internal/packages/archetype/package_docs_readme.go → internal/packages/archetype/_static/package-docs-readme.md.tmpl

@@ -1,10 +1,4 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package archetype
-
-const packageDocsReadme = `<!-- Use this template language as a starting point, replacing {placeholder text} with details about the integration. -->
+<!-- Use this template language as a starting point, replacing {placeholder text} with details about the integration. -->
 <!-- Find more detailed documentation guidelines in https://github.com/elastic/integrations/blob/main/docs/documentation_guidelines.md -->
 
 # {{.Manifest.Title}}
@@ -57,12 +51,12 @@ For step-by-step instructions on how to set up an integration, see the
 <!-- Repeat for each data stream of the current type -->
 <!-- ### {Data stream name}
 
-The ` + "`{data stream name}`" + ` data stream provides events from {source} of the following types: {list types}. -->
+The `{data stream name}` data stream provides events from {source} of the following types: {list types}. -->
 
 <!-- Optional -->
 <!-- #### Example
 
-An example event for ` + "`{data stream name}`" + ` looks as following:
+An example event for `{data stream name}` looks as following:
 
 {code block with example} -->
 
@@ -76,15 +70,15 @@ An example event for ` + "`{data stream name}`" + ` looks as following:
 <!-- Repeat for each data stream of the current type -->
 <!-- ### {Data stream name}
 
-The ` + "`{data stream name}`" + ` data stream provides events from {source} of the following types: {list types}. -->
+The `{data stream name}` data stream provides events from {source} of the following types: {list types}. -->
 
 <!-- Optional -->
 <!-- #### Example
 
-An example event for ` + "`{data stream name}`" + ` looks as following:
+An example event for `{data stream name}` looks as following:
 
 {code block with example} -->
 
 <!-- #### Exported fields
 
-{insert table} -->`
+{insert table} -->

internal/packages/archetype/package_manifest.go → internal/packages/archetype/_static/package-manifest.yml.tmpl

@@ -1,10 +1,4 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package archetype
-
-const packageManifestTemplate = `format_version: 2.8.0
+format_version: 2.10.0
 name: {{.Manifest.Name}}
 title: "{{.Manifest.Title}}"
 version: {{.Manifest.Version}}
@@ -31,13 +25,50 @@ icons:
     size: 32x32
     type: image/svg+xml
 policy_templates:
+{{- if eq .Manifest.Type "integration" }}
   - name: sample
     title: Sample logs
     description: Collect sample logs
     inputs:
       - type: logfile
         title: Collect sample logs from instances
         description: Collecting sample logs
+{{ else -}}
+{{ if eq .InputDataStreamType "logs"}}
+  - name: sample
+    type: logs
+    title: Sample logs
+    description: Collect sample logs
+    input: logfile
+    template_path: input.yml.hbs
+    vars:
+      - name: paths
+        type: text
+        title: Paths
+        multi: true
+        default:
+          - /var/log/*.log
+{{ else }}
+  - name: sample
+    type: metrics
+    title: Sample metrics
+    description: Collect sample metrics
+    input: sample/metrics
+    template_path: input.yml.hbs
+    vars:
+      - name: period
+        type: text
+        title: Period
+        default: 10s
+      - name: hosts
+        type: text
+        title: Hosts
+        multi: true
+        required: true
+        show_user: true
+        default:
+          - localhost
+{{ end }}
+{{ end -}}
 owner:
-  github: {{.Manifest.Owner.Github}}
-`
+  github: {{.Manifest.Owner.Github}}