revert unwanted changes

elastic · May 8, 2024 · f256f7e · f256f7e
1 parent b80ebf2
commit f256f7e
Show file tree

Hide file tree

Showing 17 changed files with 130 additions and 125 deletions.
diff --git a/packages/mongodb_atlas/_dev/build/docs/README.md b/packages/mongodb_atlas/_dev/build/docs/README.md
@@ -6,7 +6,7 @@
 
 Use the MongoDB Atlas integration to:
 
-- Collect MongoDB Atlas mongod audit and mongod database logs along with hardware and process metrics for comprehensive monitoring and analysis.
+- Collect MongoDB Atlas mongod audit logs, mongod database logs, hardware and process metrics for comprehensive monitoring and analysis.
 - Create informative visualizations to track usage trends, measure key metrics, and derive actionable business insights.
 - Set up alerts to minimize Mean Time to Detect (MTTD) and Mean Time to Resolve (MTTR) by quickly referencing relevant logs during troubleshooting.
 
@@ -22,7 +22,7 @@ Data streams:
 - `hardware`: This data stream collects all the Atlas Search hardware and status data series within the provided time range for one process in the specified project.
 - `mongod_audit`: The auditing facility allows administrators and users to track system activity for deployments with multiple users and applications. Mongod Audit logs capture events related to database operations such as insertions, updates, deletions, user authentication, etc., occurring within the mongod instances.
 - `mongod_database`: This data stream collects a running log of events, including entries such as incoming connections, commands run, and issues encountered. Generally, database log messages are useful for diagnosing issues, monitoring your deployment, and tuning performance.
-- `process`: This data stream collects host metrics per process for all the hosts of the specified group. Metrics like measurements for the host, such as CPU usage, number of I/O operations and memory are available on this data stream.
+- `process`: This data stream collects host metrics per process for all the hosts in the specified group. Metrics like measurements for the host, such as CPU usage, number of I/O operations and memory usage are available in this data stream.
 
 Note:
 - Users can monitor and see the logs and metrics inside the ingested documents for MongoDB Atlas in the `logs-*` index pattern from `Discover`.
@@ -101,7 +101,7 @@ This is the `mongod_database` data stream. This datastream collects a running lo
 ## Metrics reference
 
 ### Hardware
-This data stream collects hardware and status metrics per process of the specified group. Metrics like measurements for the hardware and status, such as CPU usage and JVM memory usage are available on this data stream.
+This data stream collects hardware and status metrics for each process in the specified group. It includes measurements such as CPU usage, memory consumption, JVM memory usage, disk usage, etc.
 
 {{event "hardware"}}
 
@@ -112,4 +112,4 @@ This data stream collects host metrics per process for all the hosts of the spec
 
 {{event "process"}}
 
-{{fields "process"}}
+{{fields "process"}}
diff --git a/packages/mongodb_atlas/changelog.yml b/packages/mongodb_atlas/changelog.yml
@@ -1,7 +1,7 @@
 # newer versions go on top
 - version: "0.0.4"
   changes:
-    - description: MongoDB Atlas integration package with "hardware" data stream.
+    - description: Add "hardware" data stream to MongoDB Atlas package.
       type: enhancement
       link: https://github.com/elastic/integrations/pull/9689
 - version: "0.0.3"

diff --git a/packages/mongodb_atlas/data_stream/hardware/_dev/test/pipeline/test-hardware-metrics.json b/packages/mongodb_atlas/data_stream/hardware/_dev/test/pipeline/test-hardware-metrics.json
@@ -20,4 +20,4 @@
             "processId": "hostname1"
         }
     ]
-}
+}
diff --git a/...db_atlas/data_stream/hardware/_dev/test/pipeline/test-hardware-metrics.json-expected.json b/...db_atlas/data_stream/hardware/_dev/test/pipeline/test-hardware-metrics.json-expected.json
@@ -6,11 +6,12 @@
             },
             "event": {
                 "category": [
-                    "driver"
+                    "database"
                 ],
                 "kind": "event",
                 "module": "mongodb_atlas",
                 "type": [
+                    "access",
                     "info"
                 ]
             },
@@ -56,19 +57,21 @@
                             }
                         }
                     },
-                    "jvm": {
-                        "memory": {
-                            "heap": {
-                                "available": {
-                                    "mb": 511.0000000000001
-                                },
-                                "used": {
-                                    "mb": 132.40000000000003
+                    "status": {
+                        "jvm": {
+                            "memory": {
+                                "heap": {
+                                    "available": {
+                                        "mb": 511.0000000000001
+                                    },
+                                    "used": {
+                                        "mb": 132.40000000000003
+                                    }
                                 }
                             }
-                        }
-                    },
-                    "page_faults": 0.17222000007921603
+                        },
+                        "page_faults": 0.17222000007921603
+                    }
                 },
                 "process_id": "hostname1"
             }

diff --git a/packages/mongodb_atlas/data_stream/hardware/_dev/test/system/test-default-config.yml b/packages/mongodb_atlas/data_stream/hardware/_dev/test/system/test-default-config.yml
@@ -1,13 +1,9 @@
 vars:
-  url:
-    - http://{{Hostname}}:{{Port}}
-  public_key:
-    - admin
-  private_key:
-    - MongoDB@123
+  url: http://{{Hostname}}:{{Port}}
+  public_key: admin
+  private_key: MongoDB@123
 data_stream:
   vars:
-    groupId:
-      - mongodb-group1
+    groupId: mongodb-group1
 input: cel
 service: mongodbatlas
diff --git a/packages/mongodb_atlas/data_stream/hardware/elasticsearch/ingest_pipeline/default.yml b/packages/mongodb_atlas/data_stream/hardware/elasticsearch/ingest_pipeline/default.yml
@@ -12,10 +12,10 @@ processors:
       value: mongodb_atlas
   - set:
       field: event.category
-      value: ["driver"]
+      value: ["database"]
   - set:
       field: event.type
-      value: ["info"]
+      value: ["access","info"]
   - rename:
       field: groupId
       target_field: mongodb_atlas.group_id
@@ -58,15 +58,15 @@ processors:
       ignore_missing: true
   - rename:
       field: status.JVM_MAX_MEMORY
-      target_field: mongodb_atlas.hardware.jvm.memory.heap.available.mb
+      target_field: mongodb_atlas.hardware.status.jvm.memory.heap.available.mb
       ignore_missing: true
   - rename:
       field: status.JVM_CURRENT_MEMORY
-      target_field: mongodb_atlas.hardware.jvm.memory.heap.used.mb
+      target_field: mongodb_atlas.hardware.status.jvm.memory.heap.used.mb
       ignore_missing: true
   - rename:
       field: status.PAGE_FAULTS
-      target_field: mongodb_atlas.hardware.page_faults
+      target_field: mongodb_atlas.hardware.status.page_faults
       ignore_missing: true
   - rename:
       field: processId

diff --git a/packages/mongodb_atlas/data_stream/hardware/fields/fields.yml b/packages/mongodb_atlas/data_stream/hardware/fields/fields.yml
@@ -2,7 +2,7 @@
   type: group
   fields:
     - name: group_id
-      description: Identifier for the project of the event.
+      description: Unique identifier that identifies the project.
       type: keyword
     - name: hardware
       type: group
@@ -12,7 +12,7 @@
           fields:
             - name: disk_usage.bytes
               type: long
-              metric_type: counter
+              metric_type: gauge
               unit: byte
               description: Total bytes of disk space that search processes use.
             - name: process
@@ -36,17 +36,17 @@
                   fields:
                     - name: resident.bytes
                       type: long
-                      metric_type: counter
+                      metric_type: gauge
                       unit: byte
                       description: Total bytes of resident memory that search processes occupy.
                     - name: shared.bytes
                       type: long
-                      metric_type: counter
+                      metric_type: gauge
                       unit: byte
                       description: Total bytes of shared memory that search processes occupy.
                     - name: virtual.bytes
                       type: long
-                      metric_type: counter
+                      metric_type: gauge
                       unit: byte
                       description: Total bytes of virtual memory that search processes occupy.
                 - name: normalized
@@ -65,27 +65,30 @@
                           metric_type: gauge
                           unit: percent
                           description: Percentage of time that the CPU spent servicing user calls for the search process.
-        - name: jvm
+        - name: status
           type: group
           fields:
-            - name: memory
+            - name: jvm
               type: group
               fields:
-                - name: heap
+                - name: memory
                   type: group
                   fields:
-                    - name: available.mb
-                      type: long
-                      metric_type: counter
-                      description: Total amount of available memory in the JVM heap.
-                    - name: used.mb
-                      type: long
-                      metric_type: gauge
-                      description: Amount of memory that the JVM heap is currently using.
-        - name: page_faults
-          type: double
-          metric_type: gauge
-          description: Average rate of page faults on this process per second over the selected sample period.
+                    - name: heap
+                      type: group
+                      fields:
+                        - name: available.mb
+                          type: long
+                          metric_type: gauge
+                          description: Total amount of available memory in the JVM heap.
+                        - name: used.mb
+                          type: long
+                          metric_type: gauge
+                          description: Amount of memory that the JVM heap is currently using.
+            - name: page_faults
+              type: double
+              metric_type: gauge
+              description: Average rate of page faults on this process per second over the selected sample period.
     - name: process_id
       type: keyword
-      description: Combination of hostname and Internet Assigned Numbers Authority (IANA) port that serves the MongoDB process.
+      description: Combination of hostname and MongoDB process port that serves the MongoDB process.
diff --git a/packages/mongodb_atlas/data_stream/hardware/manifest.yml b/packages/mongodb_atlas/data_stream/hardware/manifest.yml
@@ -3,6 +3,7 @@ type: logs
 streams:
   - input: cel
     title: MongoDB Atlas Hardware metrics
+    enabled: false
     description: Collect Hardware metrics from MongoDB Atlas.
     template_path: input.yml.hbs
     vars:
@@ -13,7 +14,7 @@ streams:
         default: 10m
         multi: false
         required: true
-        show_user: false
+        show_user: true
       - name: http_client_timeout
         type: text
         title: HTTP Client Timeout

diff --git a/packages/mongodb_atlas/data_stream/hardware/sample_event.json b/packages/mongodb_atlas/data_stream/hardware/sample_event.json
@@ -1,8 +1,8 @@
 {
-    "@timestamp": "2024-04-24T07:36:16.449Z",
+    "@timestamp": "2024-05-08T05:28:35.903Z",
     "agent": {
-        "ephemeral_id": "1998d9c0-3d6f-4f52-b342-826fc26ce89c",
-        "id": "d4d946c7-17c9-4ce6-b388-89e75a08f407",
+        "ephemeral_id": "f1da46ba-c948-41e5-8858-28b1db234a9c",
+        "id": "130eb953-a957-4fbb-ba6f-5bd31442e2f2",
         "name": "docker-fleet-agent",
         "type": "filebeat",
         "version": "8.13.0"
@@ -16,20 +16,21 @@
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "d4d946c7-17c9-4ce6-b388-89e75a08f407",
+        "id": "130eb953-a957-4fbb-ba6f-5bd31442e2f2",
         "snapshot": false,
         "version": "8.13.0"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
-            "driver"
+            "database"
         ],
         "dataset": "mongodb_atlas.hardware",
-        "ingested": "2024-04-24T07:36:26Z",
+        "ingested": "2024-05-08T05:28:45Z",
         "kind": "event",
         "module": "mongodb_atlas",
         "type": [
+            "access",
             "info"
         ]
     },
@@ -42,10 +43,10 @@
         "hostname": "docker-fleet-agent",
         "id": "8259e024976a406e8a54cdbffeb84fec",
         "ip": [
-            "192.168.248.7"
+            "192.168.253.7"
         ],
         "mac": [
-            "02-42-C0-A8-F8-07"
+            "02-42-C0-A8-FD-07"
         ],
         "name": "docker-fleet-agent",
         "os": {
@@ -69,7 +70,9 @@
                     "bytes": 175143048.53333333
                 }
             },
-            "page_faults": 0.16428448420436206
+            "status": {
+                "page_faults": 0.16428448420436206
+            }
         },
         "process_id": "hostname1"
     },

diff --git a/packages/mongodb_atlas/data_stream/process/_dev/test/pipeline/test-process-metrics.json b/packages/mongodb_atlas/data_stream/process/_dev/test/pipeline/test-process-metrics.json
@@ -1,8 +1,8 @@
 {
     "events": [
         {
-            "groupId": "646f4379c47da356740d14ad",
-            "hostId": "atlas-ccx4uc-shard-00-00.q5ljb.mongodb.net:27017",
+            "groupId": "group1",
+            "hostId": "host1",
             "response": {
                 "ASSERT_MSG": 0.213805548,
                 "ASSERT_REGULAR": 0.33244343545,
@@ -138,7 +138,7 @@
                 "TICKETS_AVAILABLE_READS": 127,
                 "TICKETS_AVAILABLE_WRITE": 126
             },
-            "processId": "atlas-ccx4uc-shard-00-00.q5ljb.mongodb.net:27017"
+            "processId": "hostname1"
         }
     ]
-}
+}
diff --git a/...godb_atlas/data_stream/process/_dev/test/pipeline/test-process-metrics.json-expected.json b/...godb_atlas/data_stream/process/_dev/test/pipeline/test-process-metrics.json-expected.json
@@ -15,11 +15,11 @@
                 ]
             },
             "group": {
-                "id": "646f4379c47da356740d14ad"
+                "id": "group1"
             },
             "mongodb_atlas": {
-                "group_id": "646f4379c47da356740d14ad",
-                "host_id": "atlas-ccx4uc-shard-00-00.q5ljb.mongodb.net:27017",
+                "group_id": "group1",
+                "host_id": "host1",
                 "process": {
                     "assert": {
                         "msg": 0.213805548,
@@ -475,7 +475,7 @@
                         }
                     }
                 },
-                "process_id": "atlas-ccx4uc-shard-00-00.q5ljb.mongodb.net:27017"
+                "process_id": "hostname1"
             }
         }
     ]

diff --git a/packages/mongodb_atlas/data_stream/process/_dev/test/system/test-default-config.yml b/packages/mongodb_atlas/data_stream/process/_dev/test/system/test-default-config.yml
@@ -1,13 +1,9 @@
 vars:
-  url:
-    - http://{{Hostname}}:{{Port}}
-  public_key:
-    - admin
-  private_key:
-    - MongoDB@123
+  url: http://{{Hostname}}:{{Port}}
+  public_key: admin
+  private_key: MongoDB@123
 data_stream:
   vars:
-    groupId:
-      - mongodb-group1
+    groupId: mongodb-group1
 input: cel
 service: mongodbatlas
diff --git a/packages/mongodb_atlas/data_stream/process/fields/fields.yml b/packages/mongodb_atlas/data_stream/process/fields/fields.yml
@@ -2,7 +2,7 @@
   type: group
   fields:
     - name: group_id
-      description: Identifier for the project of the event.
+      description: Unique identifier that identifies the project.
       type: keyword
     - name: host_id
       description: Unique identifier of the host for the MongoDB process.
@@ -831,5 +831,5 @@
                   type: long
                   metric_type: gauge
     - name: process_id
-      description: Combination of hostname and Internet Assigned Numbers Authority (IANA) port that serves the MongoDB process.
+      description: Combination of hostname and MongoDB process port that serves the MongoDB process.
       type: keyword