[TSDB] [Postgres] TSDB Enablement of Datastream - Activity (#5457)

* Add dimension mappings for activity datastream.

packages/postgresql/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.15.2"
+  changes:
+    - description: Added changes to support TSDB Enablement for the activity datastream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/5457
 - version: "1.15.1"
   changes:
     - description: Add new dimension fields to the `database` and `bgwriter` datastreams.

packages/postgresql/data_stream/activity/fields/agent.yml

@@ -9,6 +9,7 @@
       level: extended
       type: keyword
       ignore_above: 1024
+      dimension: true
       description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment.
 
         Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.'
@@ -17,6 +18,7 @@
       level: extended
       type: keyword
       ignore_above: 1024
+      dimension: true
       description: Availability zone in which this host is running.
       example: us-east-1c
     - name: instance.id
@@ -25,6 +27,7 @@
       ignore_above: 1024
       description: Instance ID of the host machine.
       example: i-1234567890abcdef0
+      dimension: true
     - name: instance.name
       level: extended
       type: keyword
@@ -42,10 +45,12 @@
       ignore_above: 1024
       description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean.
       example: aws
+      dimension: true
     - name: region
       level: extended
       type: keyword
       ignore_above: 1024
+      dimension: true
       description: Region in which this host is running.
       example: us-east-1
     - name: project.id
@@ -67,6 +72,7 @@
       type: keyword
       ignore_above: 1024
       description: Unique container id.
+      dimension: true
     - name: image.name
       level: extended
       type: keyword
@@ -134,6 +140,7 @@
       level: core
       type: keyword
       ignore_above: 1024
+      dimension: true
       description: 'Name of the host.
 
         It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.'
@@ -196,3 +203,11 @@
       description: >
         OS codename, if any.
 
+- name: agent
+  title: Agent
+  type: group
+  fields:
+    - name: id
+      type: keyword
+      ignore_above: 1024
+      dimension: true

packages/postgresql/data_stream/activity/fields/ecs.yml

@@ -6,5 +6,6 @@
   name: event.duration
 - external: ecs
   name: service.address
+  dimension: true
 - external: ecs
-  name: service.type
+  name: service.type

packages/postgresql/data_stream/activity/fields/fields.yml

@@ -1,27 +1,53 @@
 - name: postgresql.activity
   type: group
   fields:
+    - name: backend_type
+      type: keyword
+      # Reason to add as a dimension field:  Multiple events having various backend_type (autovacuum launcher, autovacuum worker, etc) returned
+      dimension: true
+      description: |
+        Type of backend
+    - name: wait_event
+      type: keyword
+      # Reason to add as a dimension field: Multiple events having various wait_event https://www.postgresql.org/docs/current/monitoring-stats.html#WAIT-EVENT-TABLE
+      dimension: true
+      description: |
+        Wait event name if backend is currently waiting, otherwise NULL
+    - name: wait_event_type
+      type: keyword
+      # Reason to add as a dimension field: Multiple events having various wait_event_type https://www.postgresql.org/docs/current/monitoring-stats.html#WAIT-EVENT-TABLE
+      dimension: true
+      description: |
+        The type of event for which the backend is waiting, if any; otherwise NULL.
     - name: database.oid
       type: long
       description: |
         OID of the database this backend is connected to.
     - name: database.name
       type: keyword
+      dimension: true
+      # Reason to add as a dimension field: capture metrics of one or more database using a connection is possible.
       description: |
         Name of the database this backend is connected to.
     - name: pid
       type: long
+      # Reason to add as a dimension field: Observed events for multiple pids
+      dimension: true
       description: |
         Process ID of this backend.
     - name: user.id
       type: long
+      # Reason to add as a dimension field: databases can be multi-user, the activity of multiple users must be supported.
+      dimension: true
       description: |
         OID of the user logged into this backend.
     - name: user.name
       type: keyword
       description: |
         Name of the user logged into this backend.
     - name: application_name
+      # Reason to add as a dimension field: One or more client applications can connect.
+      dimension: true
       type: keyword
       description: |
         Name of the application that is connected to this backend.
@@ -73,5 +99,7 @@
           * disabled: This state is reported if track_activities is disabled in this backend.
     - name: query
       type: keyword
+      dimension: true
+      # Reason to add as a dimension field: pg_stat_bgwriter and pg_stat_database returns same fields. The length of the query is very small and hence content hashing is not needed.
       description: |
         Text of this backend's most recent query. If state is active this field shows the currently executing query. In all other states, it shows the last query that was executed.

packages/postgresql/data_stream/activity/manifest.yml

@@ -11,4 +11,4 @@ streams:
         multi: false
         required: true
         show_user: true
-        default: 10s
+        default: 10s

packages/postgresql/docs/README.md

@@ -242,6 +242,7 @@ An example event for `activity` looks as following:
 | Field | Description | Type |
 |---|---|---|
 | @timestamp | Event timestamp. | date |
+| agent.id |  | keyword |
 | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword |
 | cloud.availability_zone | Availability zone in which this host is running. | keyword |
 | cloud.image.id | Image ID for the cloud instance. | keyword |
@@ -282,6 +283,7 @@ An example event for `activity` looks as following:
 | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |
 | postgresql.activity.application_name | Name of the application that is connected to this backend. | keyword |
 | postgresql.activity.backend_start | Time when this process was started, i.e., when the client connected to the server. | date |
+| postgresql.activity.backend_type | Type of backend | keyword |
 | postgresql.activity.client.address | IP address of the client connected to this backend. | keyword |
 | postgresql.activity.client.hostname | Host name of the connected client, as reported by a reverse DNS lookup of client_addr. | keyword |
 | postgresql.activity.client.port | TCP port number that the client is using for communication with this backend, or -1 if a Unix socket is used. | long |
@@ -295,6 +297,8 @@ An example event for `activity` looks as following:
 | postgresql.activity.transaction_start | Time when this process' current transaction was started. | date |
 | postgresql.activity.user.id | OID of the user logged into this backend. | long |
 | postgresql.activity.user.name | Name of the user logged into this backend. | keyword |
+| postgresql.activity.wait_event | Wait event name if backend is currently waiting, otherwise NULL | keyword |
+| postgresql.activity.wait_event_type | The type of event for which the backend is waiting, if any; otherwise NULL. | keyword |
 | postgresql.activity.waiting | True if this backend is currently waiting on a lock. | boolean |
 | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword |
 | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword |

packages/postgresql/kibana/dashboard/postgresql-4288b790-b79f-11e9-a579-f5c0a5d81340.json

@@ -882,8 +882,8 @@
                                                         "query": ""
                                                     },
                                                     "isBucketed": false,
-                                                    "label": "Average of postgresql.statement.query.calls",
-                                                    "operationType": "average",
+                                                    "label": "Maximum of postgresql.statement.query.calls",
+                                                    "operationType": "max",
                                                     "params": {
                                                         "emptyAsNull": true
                                                     },

packages/postgresql/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: postgresql
 title: PostgreSQL
-version: "1.15.1"
+version: "1.15.2"
 license: basic
 description: Collect logs and metrics from PostgreSQL servers with Elastic Agent.
 type: integration