merge branch main

packages/apache_tomcat/_dev/build/docs/README.md

@@ -6,7 +6,7 @@
 
 Use the Apache Tomcat integration to:
 
-- Collect metrics related to the cache, request and session and collect logs related to access, catalina, and localhost.
+- Collect metrics related to the cache, memory, request and session and collect logs related to access, catalina, and localhost.
 - Create visualizations to monitor, measure and analyze the usage trend and key data, and derive business insights.
 - Create alerts to reduce the MTTD and also the MTTR by referencing relevant logs when troubleshooting an issue.
 
@@ -16,13 +16,14 @@ The Apache Tomcat integration collects logs and metrics data.
 
 Logs help you keep a record of events that happen on your machine. The `Log` data streams collected by Apache Tomcat integration are `access`, `catalina`, and `localhost`, so that users can keep track of the IP addresses of the clients, bytes returned to the client or sent by clients, etc., so that users could monitor and troubleshoot the performance of Java applications.
 
-Metrics give you insight into the statistics of the Apache Tomcat. The `Metric` data streams collected by the Apache Tomcat integration are `cache`, `request` and `session`, so that the user can monitor and troubleshoot the performance of the Apache Tomcat instance.
+Metrics give you insight into the statistics of the Apache Tomcat. The `Metric` data streams collected by the Apache Tomcat integration are `cache`, `memory`, `request` and `session`, so that the user can monitor and troubleshoot the performance of the Apache Tomcat instance.
 
 Data streams:
 - `access`: Collects information related to overall performance of Java applications.
 - `cache`: Collects information related to the overall cache of the Apache Tomcat instance.
 - `catalina`: Collects information related to the startup and shutdown of the Apache Tomcat application server, the deployment of new applications, or the failure of one or more subsystems.
 - `localhost`: Collects information related to Web application activity which is related to HTTP transactions between the application server and the client.
+- `memory`: Collects information related to heap memory, non-heap memory and garbage collection of the Tomcat instance.
 - `request`: Collects information related to requests of the Apache Tomcat instance.
 - `session`: Collects information related to overall created, active and expired sessions of the Tomcat instance.
 
@@ -166,6 +167,14 @@ This is the `Cache` data stream. This data stream collects metrics related to th
 
 {{fields "cache"}}
 
+### Memory
+
+This is the `memory` data stream. This data stream collects metrics related to the heap memory, non-heap memory, garbage collection time and count.
+
+{{event "memory"}}
+
+{{fields "memory"}}
+
 ### Request
 
 This is the `Request` data stream. This data stream collects metrics related to request count, and amount of data received and sent.

packages/apache_tomcat/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.9.0"
+  changes:
+    - description: Apache Tomcat integration package with "memory" data stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6527
 - version: "0.8.0"
   changes:
     - description: Update the processor description link.

packages/apache_tomcat/data_stream/memory/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,2 @@
+dynamic_fields:
+  event.ingested: ".*"

packages/apache_tomcat/data_stream/memory/_dev/test/pipeline/test-gc-metrics.json

@@ -0,0 +1,19 @@
+{
+    "events": [
+        {
+            "prometheus": {
+                "labels": {
+                    "host": "localhost",
+                    "name": "GarbageCollector",
+                    "instance": "127.0.0.1:9090",
+                    "job": "prometheus"
+                },
+                "metrics": {
+                    "java_lang_G1_Old_Generation_CollectionCount": 0,
+                    "java_lang_G1_Old_Generation_Valid": 1,
+                    "java_lang_G1_Old_Generation_CollectionTime": 0
+                }
+            }
+        }
+    ]
+}

packages/apache_tomcat/data_stream/memory/_dev/test/pipeline/test-gc-metrics.json-expected.json

@@ -0,0 +1,33 @@
+{
+    "expected": [
+        {
+            "apache_tomcat": {
+                "memory": {
+                    "doc_type": "gc",
+                    "gc": {
+                        "collection": {
+                            "count": 0,
+                            "time": {
+                                "ms": 0
+                            }
+                        },
+                        "valid": 1
+                    }
+                }
+            },
+            "ecs": {
+                "version": "8.7.0"
+            },
+            "event": {
+                "category": [
+                    "web"
+                ],
+                "kind": "metric",
+                "module": "apache_tomcat",
+                "type": [
+                    "info"
+                ]
+            }
+        }
+    ]
+}

packages/apache_tomcat/data_stream/memory/_dev/test/pipeline/test-memory-metrics.json

@@ -0,0 +1,26 @@
+{
+    "events": [
+        {
+            "prometheus": {
+                "labels": {
+                    "host": "localhost",
+                    "name": "Memory",
+                    "instance": "127.0.0.1:9090",
+                    "job": "prometheus"
+                },
+                "metrics": {
+                    "java_lang_Memory_ObjectPendingFinalizationCount": 0,
+                    "java_lang_Memory_HeapMemoryUsage_used": 4.5216344e+07,
+                    "java_lang_Memory_NonHeapMemoryUsage_used": 3.6318104e+07,
+                    "java_lang_Memory_Verbose": 1,
+                    "java_lang_Memory_NonHeapMemoryUsage_init": 7.667712e+06,
+                    "java_lang_Memory_HeapMemoryUsage_committed": 5.38968064e+08,
+                    "java_lang_Memory_NonHeapMemoryUsage_max": -1,
+                    "java_lang_Memory_HeapMemoryUsage_max": 1.073741824e+09,
+                    "java_lang_Memory_HeapMemoryUsage_init": 5.36870912e+08,
+                    "java_lang_Memory_NonHeapMemoryUsage_committed": 3.9518208e+07
+                }
+            }
+        }
+    ]
+}

packages/apache_tomcat/data_stream/memory/_dev/test/pipeline/test-memory-metrics.json-expected.json

@@ -0,0 +1,56 @@
+{
+    "expected": [
+        {
+            "apache_tomcat": {
+                "memory": {
+                    "doc_type": "memory",
+                    "heap": {
+                        "committed": {
+                            "bytes": 5.38968064E8
+                        },
+                        "init": {
+                            "bytes": 5.36870912E8
+                        },
+                        "max": {
+                            "bytes": 1.073741824E9
+                        },
+                        "used": {
+                            "bytes": 4.5216344E7
+                        }
+                    },
+                    "non_heap": {
+                        "committed": {
+                            "bytes": 3.9518208E7
+                        },
+                        "init": {
+                            "bytes": 7667712.0
+                        },
+                        "max": {
+                            "bytes": -1
+                        },
+                        "used": {
+                            "bytes": 3.6318104E7
+                        }
+                    },
+                    "object_pending_finalization": {
+                        "count": 0
+                    },
+                    "verbose": true
+                }
+            },
+            "ecs": {
+                "version": "8.7.0"
+            },
+            "event": {
+                "category": [
+                    "web"
+                ],
+                "kind": "metric",
+                "module": "apache_tomcat",
+                "type": [
+                    "info"
+                ]
+            }
+        }
+    ]
+}

packages/apache_tomcat/data_stream/memory/_dev/test/system/test-default-config.yml

@@ -0,0 +1,4 @@
+vars:
+  hosts:
+    - http://{{Hostname}}:{{Port}}/metrics
+input: prometheus/metrics

packages/apache_tomcat/data_stream/memory/agent/stream/stream.yml.hbs

@@ -0,0 +1,28 @@
+metricsets: ["collector"]
+hosts:
+{{#each hosts}}
+  - {{this}}
+{{/each}}
+metrics_filters:
+  include: ["java_lang_Memory_*","java_lang_G1_Old_Generation_*"]
+period: {{period}}
+{{#if username}}
+username: "{{username}}"
+{{/if}}
+{{#if password}}
+password: "{{password}}"
+{{/if}}
+{{#if ssl}}
+{{ssl}}
+{{/if}}
+tags:
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/apache_tomcat/data_stream/memory/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,111 @@
+---
+description: Pipeline for processing Apache Tomcat Memory metrics.
+processors:
+  - set:
+      field: ecs.version
+      value: 8.7.0
+  - set:
+      field: event.kind
+      value: metric
+  - set:
+      field: event.module
+      value: apache_tomcat
+  - set:
+      field: event.type
+      value: [info]
+  - set:
+      field: event.category
+      value: [web]
+  - set:
+      field: apache_tomcat.memory.verbose
+      value: true
+      if: ctx.prometheus?.metrics?.java_lang_Memory_Verbose == 1
+  - set:
+      field: apache_tomcat.memory.verbose
+      value: false
+      if: ctx.prometheus?.metrics?.java_lang_Memory_Verbose == 0
+  - rename:
+      field: prometheus.metrics.java_lang_Memory_HeapMemoryUsage_max
+      target_field: apache_tomcat.memory.heap.max.bytes
+      ignore_missing: true
+  - rename:
+      field: prometheus.metrics.java_lang_Memory_HeapMemoryUsage_init
+      target_field: apache_tomcat.memory.heap.init.bytes
+      ignore_missing: true
+  - rename:
+      field: prometheus.metrics.java_lang_Memory_HeapMemoryUsage_used
+      target_field: apache_tomcat.memory.heap.used.bytes
+      ignore_missing: true
+  - rename:
+      field: prometheus.metrics.java_lang_Memory_HeapMemoryUsage_committed
+      target_field: apache_tomcat.memory.heap.committed.bytes
+      ignore_missing: true
+  - rename:
+      field: prometheus.metrics.java_lang_Memory_NonHeapMemoryUsage_max
+      target_field: apache_tomcat.memory.non_heap.max.bytes
+      ignore_missing: true
+  - rename:
+      field: prometheus.metrics.java_lang_Memory_NonHeapMemoryUsage_init
+      target_field: apache_tomcat.memory.non_heap.init.bytes
+      ignore_missing: true
+  - rename:
+      field: prometheus.metrics.java_lang_Memory_NonHeapMemoryUsage_used
+      target_field: apache_tomcat.memory.non_heap.used.bytes
+      ignore_missing: true
+  - rename:
+      field: prometheus.metrics.java_lang_Memory_NonHeapMemoryUsage_committed
+      target_field: apache_tomcat.memory.non_heap.committed.bytes
+      ignore_missing: true
+  - rename:
+      field: prometheus.metrics.java_lang_Memory_ObjectPendingFinalizationCount
+      target_field: apache_tomcat.memory.object_pending_finalization.count
+      ignore_missing: true
+  - rename:
+      field: prometheus.metrics.java_lang_G1_Old_Generation_CollectionCount
+      target_field: apache_tomcat.memory.gc.collection.count
+      ignore_missing: true
+  - rename:
+      field: prometheus.metrics.java_lang_G1_Old_Generation_CollectionTime
+      target_field: apache_tomcat.memory.gc.collection.time.ms
+      ignore_missing: true
+  - rename:
+      field: prometheus.metrics.java_lang_G1_Old_Generation_Valid
+      target_field: apache_tomcat.memory.gc.valid
+      ignore_missing: true
+  - set:
+      field: apache_tomcat.memory.doc_type
+      value: memory
+      if: ctx.apache_tomcat?.memory?.heap != null || ctx.apache_tomcat?.memory?.non_heap != null
+  - set:
+      field: apache_tomcat.memory.doc_type
+      value: gc
+      if: ctx.apache_tomcat?.memory?.gc != null
+  - remove:
+      field:
+        - prometheus
+      ignore_missing: true
+  - script:
+      description: Drops null/empty values recursively.
+      lang: painless
+      source: |
+        boolean drop(Object o) {
+            if (o == null || o == "") {
+            return true;
+            } else if (o instanceof Map) {
+            ((Map) o).values().removeIf(v -> drop(v));
+            return (((Map) o).size() == 0);
+            } else if (o instanceof List) {
+            ((List) o).removeIf(v -> drop(v));
+            return (((List) o).length == 0);
+            }
+            return false;
+        }
+        drop(ctx);
+on_failure:
+  - set:
+      field: error.message
+      value: "{{{_ingest.on_failure_message}}}"
+  - append:
+      field: event.kind
+      value: pipeline_error
+      allow_duplicates: false

packages/apache_tomcat/data_stream/memory/fields/base-fields.yml

@@ -0,0 +1,15 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: tags
+  type: keyword
+  description: List of keywords used to tag each event.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

packages/apache_tomcat/data_stream/memory/fields/ecs.yml

@@ -0,0 +1,39 @@
+- external: ecs
+  name: agent.id
+  dimension: true
+- external: ecs
+  name: cloud.account.id
+  dimension: true
+- external: ecs
+  name: cloud.availability_zone
+  dimension: true
+- external: ecs
+  name: cloud.instance.id
+  dimension: true
+- external: ecs
+  name: cloud.provider
+  dimension: true
+- external: ecs
+  name: cloud.region
+  dimension: true
+- external: ecs
+  name: container.id
+  dimension: true
+- external: ecs
+  name: ecs.version
+- external: ecs
+  name: error.message
+- external: ecs
+  name: event.category
+- external: ecs
+  name: event.type
+- external: ecs
+  name: event.kind
+- external: ecs
+  name: event.module
+- external: ecs
+  name: host.name
+  dimension: true
+- external: ecs
+  name: service.address
+  dimension: true