Introduce datasources in package to configure inputs and streams

In elastic/beats#15940 datasources, inputs and streams are introduced into the agent config. To make it possible to configure these in the UI and through the API, some changes to the manifest definitions of a package and datasets are needed.

**Package definition**

**Further changes**

* Rename `agent/input` to `agent/stream` as a stream is configured there.

dev/package-examples/nginx-1.2.0/dataset/error/agent/config/input.yml → dev/package-examples/nginx-1.2.0/dataset/access/agent/stream/stream.yml

@@ -1,8 +1,10 @@
-type: log
+
+input: log
+paths:
 {{#each paths}}
-paths: "{{this}}"
+  - "{{this}}"
 {{/each}}
-ingest_pipeline: {{pipeline}}
+
 exclude_files: [".gz$"]
 
 processors:

dev/package-examples/nginx-1.2.0/dataset/access/manifest.yml

@@ -8,18 +8,22 @@ type: logs
 # The ingest pipeline which should be used.
 ingest_pipeline: default
 
-vars:
-  - name: paths
-    # Should we define this as array? How will the UI best make sense of it?
-    type: textarea
-    default:
-      - /var/log/nginx/access.log*
-    # I suggest to use ECS fields for this config options here: https://github.com/elastic/ecs/blob/master/schemas/os.yml
-    # This would need to be based on a predefined definition on what can be filtered on
-    os.darwin:
-      - /usr/local/var/log/nginx/access.log*
-    os.windows:
-      - c:/programdata/nginx/logs/*access.log*
+# List of supported inputs
+inputs:
+  - type: log
+    vars:
+      - name: paths
+        required: true
+        # Should we define this as array? How will the UI best make sense of it?
+        type: textarea
+        default:
+          - /var/log/nginx/access.log*
+        # I suggest to use ECS fields for this config options here: https://github.com/elastic/ecs/blob/master/schemas/os.yml
+        # This would need to be based on a predefined definition on what can be filtered on
+        os.darwin:
+          - /usr/local/var/log/nginx/access.log*
+        os.windows:
+          - c:/programdata/nginx/logs/*access.log*
 
 requirements:
   elasticsearch.processors:

dev/package-examples/nginx-1.2.0/dataset/error/agent/stream/stream.yml

@@ -0,0 +1,23 @@
+
+# The selected input has to be passed to the stream config when processed.
+
+{{#if input == log}}
+input: log
+
+{{#each paths}}
+paths: "{{this}}"
+{{/each}}
+exclude_files: [".gz$"]
+
+processors:
+  - add_locale: ~
+{{/if}}
+
+
+# This is an example stream config on how multiple inputs could be supported
+
+{{#if input == syslog}}
+input: syslog
+
+# TODO: would need some more config options
+{{/if}}

dev/package-examples/nginx-1.2.0/dataset/error/manifest.yml

@@ -1,11 +1,27 @@
 title: Nginx Error Logs
 type: logs
 ingest_pipeline: pipeline
-vars:
-  - name: paths
-    default:
-      - /var/log/nginx/error.log*
-    os.darwin:
-      - /usr/local/var/log/nginx/error.log*
-    os.windows:
-      - c:/programdata/nginx/logs/error.log*
+
+
+# This is an example that multiple inputs are supported by one dataset
+inputs:
+  - type: log
+    vars:
+      - name: paths
+        required: true
+        default:
+          - /var/log/nginx/error.log*
+        os.darwin:
+          - /usr/local/var/log/nginx/error.log*
+        os.windows:
+          - c:/programdata/nginx/logs/error.log*
+
+  - type: syslog
+    vars:
+      # Are udp and tcp syslog input two different inputs?
+      - name: protocol.udp.host
+        required: true
+        default:
+          - "localhost:9000"
+
+

dev/package-examples/nginx-1.2.0/dataset/stubstatus/agent/input/input.yml → dev/package-examples/nginx-1.2.0/dataset/stubstatus/agent/stream/stream.yml

@@ -1,4 +1,4 @@
-type: metric/nginx
+input: metrics/nginx
 metricsets: ["stubstatus"]
 period: {{period}}
 enabled: true

dev/package-examples/nginx-1.2.0/dataset/stubstatus/manifest.yml

@@ -10,17 +10,10 @@ compatibility: linux, freebsd
 # Each input can be in its own release status
 release: beta
 
-vars:
-  - name: hosts
-    description: Nginx hosts
-    default:
-      ["http://127.0.0.1"]
-    required: true
-  - name: period
-    description: "Collection period. Valid values: 10s, 5m, 2h"
-    default: "10s"
-  - name: username
-    type: text
-  - name: password
-    # This is the html input type?
-    type: password
+# List of inputs this dataset supports
+inputs:
+  - type: "metric/nginx"
+
+    # Only the variables have to be repeated that are not specified as part of the input
+    vars:
+      # All variables are specified in the input already

dev/package-examples/nginx-1.2.0/manifest.yml

@@ -12,3 +12,45 @@ requirement:
     versions: ">7.1.0 <7.6.0"
   elasticsearch:
     versions: ">7.0.1"
+
+datasources:
+  -
+    # Do we need a name for the data source?
+    name: nginx
+
+    # List of inputs this datasource supports
+    inputs:
+      -
+        # An id can be given, in case the type used here is not unique
+        # This is for selection in the stream
+        # id: nginx
+        type: metrics/nginx
+
+        # Common configuration options for this input
+        vars:
+          - name: hosts
+            description: Nginx hosts
+            default:
+              ["http://127.0.0.1"]
+            # All the config options that are required should be shown in the UI
+            required: true
+          - name: period
+            description: "Collection period. Valid values: 10s, 5m, 2h"
+            default: "10s"
+          - name: username
+            type: text
+          - name: password
+            # This is the html input type?
+            type: password
+
+      -
+        type: logs
+
+        # Common configuration options for this input
+        vars:
+
+      -
+        type: syslog
+
+        # Common configuration options for this input
+        vars:

dev/package-examples/system-0.9.0/dataset/cpu/agent/stream/stream.yml

@@ -0,0 +1,6 @@
+input: metrics/system
+enabled: false # default true
+metricset: cpu
+period: 10s
+dataset: system.cpu
+metrics: ["percentages", "normalized_percentages"]

dev/package-examples/system-0.9.0/dataset/cpu/manifest.yml

@@ -5,4 +5,5 @@ release: beta
 # Needs to describe the type of this input
 type: metrics
 
-
+inputs:
+  - type: metrics/system

dev/package-examples/system-0.9.0/dataset/load/agent/stream/stream.yml

@@ -0,0 +1,4 @@
+inputy: metric/system
+enabled: true
+metricsets:
+  - load

dev/package-examples/system-0.9.0/dataset/load/manifest.yml

@@ -5,4 +5,5 @@ release: beta
 # Needs to describe the type of this input
 type: metrics
 
-
+inputs:
+  - type: system/metrics

dev/package-examples/system-0.9.0/dataset/memory/agent/stream.yml/stream.yml

@@ -0,0 +1,4 @@
+input: metric/system
+enabled: true
+metricsets:
+  - memory

dev/package-examples/system-0.9.0/dataset/memory/manifest.yml

@@ -5,4 +5,6 @@ release: beta
 # Needs to describe the type of this input
 type: metrics
 
+inputs:
+  - type: system/metrics
 

dev/package-examples/system-0.9.0/manifest.yml

@@ -10,3 +10,10 @@ release: ga
 requirement:
   kibana:
     versions: "<8.0.0"
+
+
+datasources:
+  - inputs:
+      type: metrics/system
+  - inputs:
+      type: log