Support "input packages"

.ci/Jenkinsfile

@@ -104,7 +104,7 @@ pipeline {
                     'check-packages-with-kind': generateTestCommandStage(command: 'test-check-packages-with-kind', artifacts: ['build/test-results/*.xml', 'build/kubectl-dump.txt', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
                     'check-packages-other': generateTestCommandStage(command: 'test-check-packages-other', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
                     'check-packages-with-custom-agent': generateTestCommandStage(command: 'test-check-packages-with-custom-agent', artifacts: ['build/test-results/*.xml', 'build/elastic-stack-dump/check-*/logs/*.log', 'build/elastic-stack-dump/check-*/logs/fleet-server-internal/*'], junitArtifacts: true, publishCoverage: true),
-                    'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log', 'build/integrations/*.sig']),
+                    'build-zip': generateTestCommandStage(command: 'test-build-zip', artifacts: ['build/elastic-stack-dump/build-zip/logs/*.log', 'build/packages/*.sig']),
                     'profiles-command': generateTestCommandStage(command: 'test-profiles-command')
                   ]
 

.ci/package-storage-publish.groovy

@@ -60,14 +60,14 @@ pipeline {
             }
           }
         }
-        stash(allowEmpty: true, name: 'build-package', includes: "${BASE_DIR}/build/integrations/*.zip", useDefaultExcludes: false)
+        stash(allowEmpty: true, name: 'build-package', includes: "${BASE_DIR}/build/packages/*.zip", useDefaultExcludes: false)
       }
     }
     stage('Sign and publish package') {
       steps {
         cleanup(source: 'build-package')
         dir("${BASE_DIR}") {
-          packageStoragePublish('build/integrations')
+          packageStoragePublish('build/packages')
         }
       }
     }

docs/howto/dependency_management.md

@@ -42,7 +42,7 @@ The tool will try to download and cache locally referenced schemas (e.g. `git@0b
 Cached files are stored in a dedicated directory - `~/.elastic-package/cache/fields/`. It's assumed that schema (versioned) files
 do not change.
 
-To verify if building process went well, you can open `build` directory and compare fields (e.g. `./build/integrations/nginx/1.2.3/access/fields/ecs.yml`):
+To verify if building process went well, you can open `build` directory and compare fields (e.g. `./build/packages/nginx/1.2.3/access/fields/ecs.yml`):
 
 ```yaml
 - description: |-

internal/builder/packages.go

@@ -9,15 +9,16 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/elastic/package-spec/code/go/pkg/validator"
 	"github.com/pkg/errors"
 
+	"github.com/elastic/package-spec/code/go/pkg/validator"
+
 	"github.com/elastic/elastic-package/internal/files"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
 )
 
-const buildIntegrationsFolder = "integrations"
+const builtPackagesFolder = "packages"
 
 type BuildOptions struct {
 	PackageRoot string
@@ -102,7 +103,7 @@ func buildPackagesRootDirectory() (string, error) {
 		return "", errors.Wrap(err, "can't locate build directory")
 	}
 	if !found {
-		buildDir, err = createBuildDirectory(buildIntegrationsFolder) // TODO add support for other package types
+		buildDir, err = createBuildDirectory(builtPackagesFolder)
 		if err != nil {
 			return "", errors.Wrap(err, "can't create new build directory")
 		}
@@ -118,7 +119,7 @@ func FindBuildPackagesDirectory() (string, bool, error) {
 	}
 
 	if found {
-		path := filepath.Join(buildDir, buildIntegrationsFolder) // TODO add support for other package types
+		path := filepath.Join(buildDir, builtPackagesFolder)
 		fileInfo, err := os.Stat(path)
 		if errors.Is(err, os.ErrNotExist) {
 			return "", false, nil

internal/packages/packages.go

@@ -85,7 +85,12 @@ type Conditions struct {
 
 // PolicyTemplate is a configuration of inputs responsible for collecting log or metric data.
 type PolicyTemplate struct {
-	Inputs []Input `config:"inputs" json:"inputs" yaml:"inputs"`
+	Inputs []Input `config:"inputs,omitempty" json:"inputs,omitempty" yaml:"inputs,omitempty"`
+
+	// For purposes of "input packages"
+	Input        string `config:"input,omitempty" json:"input,omitempty" yaml:"input,omitempty"`
+	Type         string `config:"type,omitempty" json:"type,omitempty" yaml:"type,omitempty"`
+	TemplatePath string `config:"template_path,omitempty" json:"template_path,omitempty" yaml:"template_path,omitempty"`
 }
 
 // Owner defines package owners, either a single person or a team.
@@ -273,7 +278,7 @@ func isPackageManifest(path string) (bool, error) {
 	if err != nil {
 		return false, errors.Wrapf(err, "reading package manifest failed (path: %s)", path)
 	}
-	return m.Type == "integration" && m.Version != "", nil // TODO add support for other package types
+	return (m.Type == "integration" || m.Type == "input") && m.Version != "", nil
 }
 
 func isDataStreamManifest(path string) (bool, error) {

scripts/test-build-zip.sh

@@ -38,7 +38,7 @@ done
 cd -
 
 # Remove unzipped built packages, leave .zip files
-rm -r build/integrations/*/
+rm -r build/packages/*/
 
 # Boot up the stack
 eval "$(elastic-package stack shellinit)"

test/packages/other/sql_input/agent/input/input.yml.hbs

@@ -0,0 +1,9 @@
+metricsets: ["query"]
+period: {{period}}
+hosts:
+{{#each hosts}}
+  - {{this}}
+{{/each}}
+driver: {{driver}}
+sql_query: {{sql_query}
+sql_response_format: {{sql_response_format}}

test/packages/other/sql_input/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.2.0"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/325

test/packages/other/sql_input/docs/README.md

@@ -0,0 +1 @@
+# SQL Input

test/packages/other/sql_input/fields/input.yml

@@ -0,0 +1,4 @@
+- name: input.name
+  type: constant_keyword
+  description: Sample field to be added.
+  value: logs

test/packages/other/sql_input/manifest.yml

@@ -0,0 +1,67 @@
+format_version: 1.0.0
+name: sql_input
+title: SQL Input
+description: >-
+  Execute custom queries against an SQL database and store the results in Elasticsearch.
+type: input
+version: 0.2.0
+license: basic
+categories:
+  - custom
+  - datastore
+policy_templates:
+  - name: sql_query
+    type: metrics
+    title: SQL Query
+    description: Query the database to capture metrics.
+    input: sql
+    template_path: input.yml.hbs
+    vars:
+      - name: hosts
+        type: text
+        title: Hosts
+        multi: true
+        required: true
+        show_user: true
+        default:
+          - http://127.0.0.1
+      - name: period
+        type: text
+        title: Period
+        multi: false
+        required: true
+        show_user: true
+        default: 10s
+      - name: driver
+        type: text
+        title: Driver
+        description: "Supported database drivers: mssql, mysql, oracle, postgres"
+        multi: false
+        required: true
+        show_user: true
+        default: "mysql"
+      - name: sql_query
+        type: text
+        title: Query
+        multi: false
+        required: true
+        show_user: true
+        default: "SHOW GLOBAL STATUS LIKE 'Innodb_system%'"
+      - name: sql_response_format
+        type: text
+        title: Response format
+        description: "Supported response formats: variables, table"
+        multi: false
+        required: true
+        show_user: false
+        default: "variables"
+icons:
+  - src: "/img/sample-logo.svg"
+    type: "image/svg+xml"
+screenshots:
+  - src: "/img/sample-screenshot.png"
+    title: "Sample screenshot"
+    size: "600x600"
+    type: "image/png"
+owner:
+  github: elastic/integrations

test/packages/parallel/gcp/data_stream/compute/fields/agent.yml

@@ -5,49 +5,11 @@
   footnote: 'Examples: If Metricbeat is running on an GCP Compute VM and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.'
   type: group
   fields:
-    - name: account.id
-      level: extended
-      type: keyword
-      ignore_above: 1024
-      description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
-
-        Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.'
-      example: 666777888999
-    - name: availability_zone
-      level: extended
-      type: keyword
-      ignore_above: 1024
-      description: Availability zone in which this host is running.
-      example: us-east-1c
-    - name: instance.id
-      level: extended
-      type: keyword
-      ignore_above: 1024
-      description: Instance ID of the host machine.
-      example: i-1234567890abcdef0
     - name: instance.name
       level: extended
       type: keyword
       ignore_above: 1024
       description: Instance name of the host machine.
-    - name: machine.type
-      level: extended
-      type: keyword
-      ignore_above: 1024
-      description: Machine type of the host machine.
-      example: t2.medium
-    - name: provider
-      level: extended
-      type: keyword
-      ignore_above: 1024
-      description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
-      example: aws
-    - name: region
-      level: extended
-      type: keyword
-      ignore_above: 1024
-      description: Region in which this host is running.
-      example: us-east-1
     - name: project.id
       type: keyword
       description: Name of the project in Google Cloud.

test/packages/parallel/gcp/docs/compute.md

@@ -101,14 +101,14 @@ An example event for `compute` looks as following:
 | cloud | Fields related to the cloud or infrastructure the events are coming from. | group |
 | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword |
 | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword |
-| cloud.availability_zone | Availability zone in which this host is running. | keyword |
+| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword |
 | cloud.image.id | Image ID for the cloud instance. | keyword |
 | cloud.instance.id | Instance ID of the host machine. | keyword |
 | cloud.instance.name | Instance name of the host machine. | keyword |
 | cloud.machine.type | Machine type of the host machine. | keyword |
 | cloud.project.id | Name of the project in Google Cloud. | keyword |
 | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword |
-| cloud.region | Region in which this host is running. | keyword |
+| cloud.region | Region in which this host, resource, or service is located. | keyword |
 | container.id | Unique container id. | keyword |
 | container.image.name | Name of the image the container was built on. | keyword |
 | container.labels | Image labels. | object |