Init commit of linux system tests (elastic#1340)

* init commit of linux system tests

* fix changelog version

* format code

* use import ECS values

* format, again

* add ecs fields to the rest of the data_streams

* remove error ecs fields

packages/linux/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@1.10

packages/linux/_dev/deploy/docker/Dockerfile

@@ -0,0 +1,3 @@
+FROM ubuntu:latest
+WORKDIR /home
+CMD sleep 600

packages/linux/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,6 @@
+version: '2.3'
+services:
+  linux:
+    build: .
+    volumes:
+      - ${SERVICE_LOGS_DIR}:/home/examples

packages/linux/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.4.2"
+  changes:
+    - description: Add system tests
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1340
 - version: "0.4.1"
   changes:
     - description: Fix event.module in network_summary

packages/linux/data_stream/conntrack/_dev/test/system/test-default-config.yml

@@ -0,0 +1,3 @@
+vars: ~
+data_stream:
+  vars: ~

packages/linux/data_stream/conntrack/fields/ecs.yml

@@ -0,0 +1,9 @@
+- name: ecs.version
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  external: ecs

packages/linux/data_stream/entropy/_dev/test/system/test-default-config.yml

@@ -0,0 +1,3 @@
+vars: ~
+data_stream:
+  vars: ~

packages/linux/data_stream/entropy/fields/ecs.yml

@@ -0,0 +1,9 @@
+- name: ecs.version
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  external: ecs

packages/linux/data_stream/iostat/_dev/test/system/test-default-config.yml

@@ -0,0 +1,3 @@
+vars: ~
+data_stream:
+  vars: ~

packages/linux/data_stream/iostat/fields/ecs.yml

@@ -0,0 +1,9 @@
+- name: ecs.version
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  external: ecs

packages/linux/data_stream/ksm/_dev/test/system/test-default-config.yml

@@ -0,0 +1,3 @@
+vars: ~
+data_stream:
+  vars: ~

packages/linux/data_stream/ksm/fields/ecs.yml

@@ -0,0 +1,9 @@
+- name: ecs.version
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  external: ecs

packages/linux/data_stream/memory/_dev/test/system/test-default-config.yml

@@ -0,0 +1,3 @@
+vars:
+data_stream:
+  vars: ~

packages/linux/data_stream/memory/fields/ecs.yml

@@ -0,0 +1,9 @@
+- name: ecs.version
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  external: ecs

packages/linux/data_stream/network_summary/_dev/test/system/test-default-config.yml

@@ -0,0 +1,3 @@
+vars: ~
+data_stream:
+  vars: ~

packages/linux/data_stream/network_summary/fields/ecs.yml

@@ -0,0 +1,9 @@
+- name: ecs.version
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  external: ecs

packages/linux/data_stream/pageinfo/fields/ecs.yml

@@ -0,0 +1,9 @@
+- name: ecs.version
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  external: ecs

packages/linux/data_stream/raid/fields/ecs.yml

@@ -0,0 +1,9 @@
+- name: ecs.version
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  external: ecs

packages/linux/data_stream/service/fields/ecs.yml

@@ -1,3 +1,12 @@
+- name: ecs.version
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  external: ecs
 - name: process
   title: Process
   group: 2

packages/linux/data_stream/socket/fields/ecs.yml

@@ -1,3 +1,12 @@
+- name: ecs.version
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  external: ecs
 - name: network
   title: Network
   group: 2

packages/linux/data_stream/users/fields/ecs.yml

@@ -1,3 +1,12 @@
+- name: ecs.version
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  external: ecs
 - name: source
   title: Source
   group: 2

packages/linux/docs/README.md

@@ -72,7 +72,9 @@ entropy will be out of a total pool size of 4096.
 | data_stream.dataset | Data stream dataset. | constant_keyword |
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |
+| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |
 | event.dataset | Event dataset | constant_keyword |
+| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long |
 | event.module | Event module | constant_keyword |
 | host.architecture | Operating system architecture. | keyword |
 | host.containerized | If the host is a container. | boolean |
@@ -90,6 +92,8 @@ entropy will be out of a total pool size of 4096.
 | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword |
 | host.os.version | Operating system version as a raw string. | keyword |
 | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |
+| service.address | Service address | keyword |
+| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |
 | system.entropy.available_bits | The available bits of entropy | long |
 | system.entropy.pct | The percentage of available entropy, relative to the pool size of 4096 | scaled_float |
 
@@ -120,7 +124,9 @@ operating system. These events are global and sorted by protocol.
 | data_stream.dataset | Data stream dataset. | constant_keyword |
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |
+| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |
 | event.dataset | Event dataset | constant_keyword |
+| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long |
 | event.module | Event module | constant_keyword |
 | host.architecture | Operating system architecture. | keyword |
 | host.containerized | If the host is a container. | boolean |
@@ -138,6 +144,8 @@ operating system. These events are global and sorted by protocol.
 | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword |
 | host.os.version | Operating system version as a raw string. | keyword |
 | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |
+| service.address | Service address | keyword |
+| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |
 | system.network_summary.icmp.* | ICMP counters | object |
 | system.network_summary.ip.* | IP counters | object |
 | system.network_summary.tcp.* | TCP counters | object |
@@ -174,7 +182,9 @@ This data stream is available on:
 | data_stream.dataset | Data stream dataset. | constant_keyword |
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |
+| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |
 | event.dataset | Event dataset | constant_keyword |
+| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long |
 | event.module | Event module | constant_keyword |
 | host.architecture | Operating system architecture. | keyword |
 | host.containerized | If the host is a container. | boolean |
@@ -192,6 +202,8 @@ This data stream is available on:
 | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword |
 | host.os.version | Operating system version as a raw string. | keyword |
 | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |
+| service.address | Service address | keyword |
+| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |
 | system.raid.blocks.synced | Number of blocks on the device that are in sync, in 1024-byte blocks. | long |
 | system.raid.blocks.total | Number of blocks the device holds, in 1024-byte blocks. | long |
 | system.raid.disks.active | Number of active disks. | long |
@@ -234,14 +246,16 @@ This data stream is available on:
 | data_stream.dataset | Data stream dataset. | constant_keyword |
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |
+| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |
 | event.dataset | Event dataset | constant_keyword |
+| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long |
 | event.module | Event module | constant_keyword |
 | host.architecture | Operating system architecture. | keyword |
 | host.containerized | If the host is a container. | boolean |
 | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword |
 | host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword |
 | host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword |
-| host.ip | Host ip addresses. | ip |
+| host.ip | Host ip address. | ip |
 | host.mac | Host mac address. | keyword |
 | host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword |
 | host.os.build | OS build information. | keyword |
@@ -252,13 +266,15 @@ This data stream is available on:
 | host.os.name | Operating system name, without the version. | keyword |
 | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword |
 | host.os.version | Operating system version as a raw string. | keyword |
-| host.type | Type of host. | keyword |
+| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |
 | process.exit_code | Identifier of the group of processes the process belongs to. | long |
 | process.name | Process name. Sometimes called program name or similar. | keyword |
 | process.pgid | Identifier of the group of processes the process belongs to. | long |
 | process.pid | Process id. | long |
 | process.ppid | Parent process' pid. | long |
 | process.working_directory | The working directory of the process. | keyword |
+| service.address | Service address | keyword |
+| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |
 | system.service.exec_code | The SIGCHLD code from the service's main process | keyword |
 | system.service.load_state | The load state of the service | keyword |
 | system.service.name | The name of the service | keyword |
@@ -308,7 +324,9 @@ missing short-lived connections.
 | data_stream.dataset | Data stream dataset. | constant_keyword |
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |
+| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |
 | event.dataset | Event dataset | constant_keyword |
+| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long |
 | event.module | Event module | constant_keyword |
 | host.architecture | Operating system architecture. | keyword |
 | host.containerized | If the host is a container. | boolean |
@@ -331,6 +349,8 @@ missing short-lived connections.
 | process.executable | Absolute path to the process executable. | keyword |
 | process.name | Process name. Sometimes called program name or similar. | keyword |
 | process.pid | Process id. | long |
+| service.address | Service address | keyword |
+| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |
 | system.socket.local.ip | Local IP address. This can be an IPv4 or IPv6 address. | ip |
 | system.socket.local.port | Local port. | long |
 | system.socket.process.cmdline | Full command line | keyword |
@@ -368,7 +388,9 @@ The linux/users data stream reports logged in users and associated sessions via
 | data_stream.dataset | Data stream dataset. | constant_keyword |
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |
+| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |
 | event.dataset | Event dataset | constant_keyword |
+| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long |
 | event.module | Event module | constant_keyword |
 | host.architecture | Operating system architecture. | keyword |
 | host.containerized | If the host is a container. | boolean |
@@ -386,6 +408,8 @@ The linux/users data stream reports logged in users and associated sessions via
 | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword |
 | host.os.version | Operating system version as a raw string. | keyword |
 | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |
+| service.address | Service address | keyword |
+| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |
 | source.ip | IP address of the source (IPv4 or IPv6). | ip |
 | source.port | Port of the source. | long |
 | system.users.id | The ID of the session | keyword |

packages/linux/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: linux
 title: Linux
-version: 0.4.1
+version: 0.4.2
 license: basic
 description: Linux Integration
 type: integration