Migrate obs-infraobs-integrations to package-spec v3 #3 (#8203)

* [oracle]: migration with yq * [oracle_weblogic]: migration with yq * [postgresql]: migration with yq * [rabbitmq]: migration with yq * [redisenterprise]: migration with yq * [salesforce]: migration with yq * [spring_boot]: migration with yq * [redis]: migration with yq * [system]: migration with yq * [tomcat]: migration with yq * [traefik]: migration with yq * [vsphere]: migration with yq * [websphere_application_server]: migration with yq * [zookeeper]: migration with yq * [oracle] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 1.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -skip-format -fix-dotted-yaml-keys -add-owner-type packages/oracle * [oracle_weblogic] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 1.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/oracle_weblogic * [php_fpm] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 2.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/php_fpm * [postgresql] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 1.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/postgresql * [prometheus_input] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 2.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/prometheus_input * [prometheus_input]: fix test-default-config.yml * [rabbitmq] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 1.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/rabbitmq * [redisenterprise] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 1.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/redisenterprise * [spring_boot] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 1.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/spring_boot * [sql] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 2.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/sql_input * [statsd_input] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 2.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/statsd_input * [system] - change to format_version 3.0.0 The format_version in the package manifest changed from 1.0.0 to 3.0.0. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/system * [traefik] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 1.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/traefik * [vsphere] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 1.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/vsphere * [websphere_application_server] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 1.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/websphere_application_server * [zookeeper] - removed dotted YAML keys from manifest The format_version in the package manifest changed from 1.0.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest. [git-generate] go run github.com/andrewkroh/go-examples/ecs-update@latest -v -format-version=3.0.0 -fix-dotted-yaml-keys -add-owner-type packages/zookeeper * Add validation.yml * Fix PR number in changelog * revert changes for packages for which v3 migration failed * Fix PR number in changelog * [oracle]: fix tests * [oracle]: add validation * [oracle_weblogic]: fix tests * [oracle_weblogic]: fix readme * [spring_boot]: fix tests * remove release: beta from oracle and redisenterprise * Remove system package changes * Update packages/oracle/data_stream/database_audit/elasticsearch/ingest_pipeline/default.yml * Revert "remove release: beta from oracle and redisenterprise" This reverts commit 3e1b8aa. * improve healthcheck for spring_boot to fix ci * test commit * Revert "test commit" This reverts commit d951b2e. * play with stack version * [statsd_input]: revert package-spec v3 changes * [sql_input]: revert package-spec v3 changes
elastic · Oct 25, 2023 · da0fc7a · da0fc7a
1 parent 487d594
commit da0fc7a
Show file tree

Hide file tree

Showing 138 changed files with 1,603 additions and 1,224 deletions.
diff --git a/packages/oracle/changelog.yml b/packages/oracle/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: 1.22.0
+  changes:
+    - description: Update the package format_version to 3.0.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8203
 - version: 1.21.0
   changes:
     - description: Adapt fields for changes in file system info

diff --git a/...ata_stream/database_audit/_dev/test/pipeline/test-oracle-database-audit.log-expected.json b/...ata_stream/database_audit/_dev/test/pipeline/test-oracle-database-audit.log-expected.json
diff --git a/packages/oracle/data_stream/database_audit/elasticsearch/ingest_pipeline/default.yml b/packages/oracle/data_stream/database_audit/elasticsearch/ingest_pipeline/default.yml
@@ -12,10 +12,10 @@ processors:
       value: event
   - set:
       field: event.category
-      value: database
+      value: [database]
   - set:
       field: event.type
-      value: access
+      value: [access]
   - set:
       field: event.outcome
       value: success
@@ -138,9 +138,13 @@ processors:
       field: tmp_timestamp
       patterns:
         - "%{ISO8601_TIMEZONE:event.timezone}$"
-  - rename:
+  - set:
+      field: user.roles
+      value: ["{{{oracle.database_audit.privilege}}}"]
+      ignore_empty_value: true
+      ignore_failure: true
+  - remove:
       field: oracle.database_audit.privilege
-      target_field: user.roles
       ignore_missing: true
   - rename:
       field: LENGTH

diff --git a/packages/oracle/data_stream/database_audit/fields/fields.yml b/packages/oracle/data_stream/database_audit/fields/fields.yml
@@ -89,11 +89,6 @@
       description: >
         The related OS user.
 
-    - name: os_userid
-      type: keyword
-      description: >
-        The related OS user.
-
     - name: returncode
       type: keyword
       description: >

diff --git a/packages/oracle/data_stream/database_audit/sample_event.json b/packages/oracle/data_stream/database_audit/sample_event.json
@@ -28,13 +28,17 @@
     "event": {
         "action": "database_audit",
         "agent_id_status": "verified",
-        "category": "database",
+        "category": [
+            "database"
+        ],
         "dataset": "oracle.database_audit",
         "ingested": "2023-10-05T12:18:05Z",
         "kind": "event",
         "outcome": "success",
         "timezone": "-04:00",
-        "type": "access"
+        "type": [
+            "access"
+        ]
     },
     "host": {
         "architecture": "x86_64",
@@ -107,6 +111,8 @@
         "oracle-database_audit"
     ],
     "user": {
-        "roles": "SYSDBA"
+        "roles": [
+            "SYSDBA"
+        ]
     }
 }
diff --git a/packages/oracle/data_stream/memory/fields/fields.yml b/packages/oracle/data_stream/memory/fields/fields.yml
@@ -1,6 +1,5 @@
 - name: oracle.memory
   type: group
-  release: beta
   fields:
     - name: pga
       type: group

diff --git a/packages/oracle/data_stream/performance/fields/fields.yml b/packages/oracle/data_stream/performance/fields/fields.yml
@@ -1,6 +1,5 @@
 - name: oracle.performance
   type: group
-  release: beta
   fields:
     - name: machine
       type: keyword

diff --git a/packages/oracle/data_stream/sysmetric/fields/fields.yml b/packages/oracle/data_stream/sysmetric/fields/fields.yml
@@ -1,6 +1,5 @@
 - name: oracle.sysmetric
   type: group
-  release: beta
   fields:
     - name: long_table_scans_per_sec
       type: double

diff --git a/packages/oracle/data_stream/system_statistics/fields/fields.yml b/packages/oracle/data_stream/system_statistics/fields/fields.yml
@@ -1,6 +1,5 @@
 - name: oracle.system_statistics
   type: group
-  release: beta
   fields:
     - name: parallel_operations_not_downgraded
       type: double

diff --git a/packages/oracle/data_stream/tablespace/fields/fields.yml b/packages/oracle/data_stream/tablespace/fields/fields.yml
@@ -1,6 +1,5 @@
 - name: oracle.tablespace
   type: group
-  release: beta
   fields:
     - name: name
       type: keyword

diff --git a/packages/oracle/docs/README.md b/packages/oracle/docs/README.md
@@ -194,13 +194,17 @@ An example event for `database_audit` looks as following:
     "event": {
         "action": "database_audit",
         "agent_id_status": "verified",
-        "category": "database",
+        "category": [
+            "database"
+        ],
         "dataset": "oracle.database_audit",
         "ingested": "2023-10-05T12:18:05Z",
         "kind": "event",
         "outcome": "success",
         "timezone": "-04:00",
-        "type": "access"
+        "type": [
+            "access"
+        ]
     },
     "host": {
         "architecture": "x86_64",
@@ -273,7 +277,9 @@ An example event for `database_audit` looks as following:
         "oracle-database_audit"
     ],
     "user": {
-        "roles": "SYSDBA"
+        "roles": [
+            "SYSDBA"
+        ]
     }
 }
 ```

diff --git a/packages/oracle/manifest.yml b/packages/oracle/manifest.yml
@@ -1,16 +1,17 @@
-format_version: 1.0.0
+format_version: "3.0.0"
 name: oracle
 title: "Oracle"
-version: 1.21.0
-license: basic
+version: "1.22.0"
 description: Collect Oracle Audit Log, Performance metrics, Tablespace metrics, Sysmetrics metrics, System statistics metrics, memory metrics from Oracle database.
 type: integration
 categories:
   - observability
   - datastore
-release: ga
 conditions:
-  kibana.version: "^8.8.0"
+  kibana:
+    version: "^8.8.0"
+  elastic:
+    subscription: basic
 screenshots:
   - src: /img/Oracle-overview-dashboard.png
     title: Oracle overview dashboard
@@ -63,3 +64,4 @@ policy_templates:
         description: Collecting performance metrics, tablespace metrics, sysmetrics, system statistics metrics and memory metrics from Oracle database instances
 owner:
   github: elastic/obs-infraobs-integrations
+  type: elastic
diff --git a/packages/oracle/validation.yml b/packages/oracle/validation.yml
@@ -0,0 +1,3 @@
+errors:
+  exclude_checks:
+    - SVR00002
diff --git a/packages/oracle_weblogic/changelog.yml b/packages/oracle_weblogic/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.3.0"
+  changes:
+    - description: Update the package format_version to 3.0.0.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8203
 - version: "1.2.0"
   changes:
     - description: Enable time series data streams for the metrics datasets. This dramatically reduces storage for metrics and is expected to progressively improve query performance. For more details, see https://www.elastic.co/guide/en/elasticsearch/reference/current/tsds.html.

diff --git a/packages/oracle_weblogic/data_stream/access/_dev/test/pipeline/test-access.log-expected.json b/packages/oracle_weblogic/data_stream/access/_dev/test/pipeline/test-access.log-expected.json
@@ -9,19 +9,21 @@
                 "category": [
                     "log"
                 ],
-                "ingested": "2023-01-18T08:48:25.790100846Z",
+                "ingested": "2023-10-13T11:27:51.595108679Z",
                 "kind": "event",
                 "module": "oracle_weblogic",
-                "original": "172.17.32.1 - - [26/Mar/2022:23:40:05 +0530] \"GET /medrec/javax.faces.resource/welcome.jpg.xhtml?ln=img\u0026amp;con=bayland HTTP/1.1\" 200 844071 ",
+                "original": "172.17.32.1 - - [26/Mar/2022:23:40:05 +0530] \"GET /medrec/javax.faces.resource/welcome.jpg.xhtml?ln=img&amp;con=bayland HTTP/1.1\" 200 844071 ",
                 "type": "access"
             },
             "host": {
-                "ip": "172.17.32.1"
+                "ip": [
+                    "172.17.32.1"
+                ]
             },
             "http": {
                 "request": {
                     "method": "GET",
-                    "referrer": "/medrec/javax.faces.resource/welcome.jpg.xhtml?ln=img\u0026amp;con=bayland"
+                    "referrer": "/medrec/javax.faces.resource/welcome.jpg.xhtml?ln=img&amp;con=bayland"
                 },
                 "response": {
                     "bytes": 844071,
@@ -48,14 +50,16 @@
                 "category": [
                     "log"
                 ],
-                "ingested": "2023-01-18T08:48:25.790115221Z",
+                "ingested": "2023-10-13T11:27:51.595116845Z",
                 "kind": "event",
                 "module": "oracle_weblogic",
                 "original": "172.17.32.1 - - [26/Mar/2022:23:40:05 +0530] \"GET /favicon.ico HTTP/1.1\" 404 1164 ",
                 "type": "access"
             },
             "host": {
-                "ip": "172.17.32.1"
+                "ip": [
+                    "172.17.32.1"
+                ]
             },
             "http": {
                 "request": {
@@ -87,14 +91,16 @@
                 "category": [
                     "log"
                 ],
-                "ingested": "2023-01-18T08:48:25.790116721Z",
+                "ingested": "2023-10-13T11:27:51.595117970Z",
                 "kind": "event",
                 "module": "oracle_weblogic",
                 "original": "0:0:0:0:0:0:0:1 - - [26/Mar/2022:23:40:39 +0530] \"GET /favicon.ico HTTP/1.1\" 404 1164 ",
                 "type": "access"
             },
             "host": {
-                "ip": "0:0:0:0:0:0:0:1"
+                "ip": [
+                    "0:0:0:0:0:0:0:1"
+                ]
             },
             "http": {
                 "request": {
@@ -126,14 +132,16 @@
                 "category": [
                     "log"
                 ],
-                "ingested": "2023-01-18T08:48:25.790118096Z",
+                "ingested": "2023-10-13T11:27:51.595118887Z",
                 "kind": "event",
                 "module": "oracle_weblogic",
                 "original": "172.17.32.1 - - [26/Mar/2022:23:43:44 +0530] \"GET /medrec/index.xhtml HTTP/1.1\" 200 8863 ",
                 "type": "access"
             },
             "host": {
-                "ip": "172.17.32.1"
+                "ip": [
+                    "172.17.32.1"
+                ]
             },
             "http": {
                 "request": {
@@ -165,19 +173,21 @@
                 "category": [
                     "log"
                 ],
-                "ingested": "2023-01-18T08:48:25.790119263Z",
+                "ingested": "2023-10-13T11:27:51.595119762Z",
                 "kind": "event",
                 "module": "oracle_weblogic",
-                "original": "172.17.32.1 - - [26/Mar/2022:23:43:45 +0530] \"GET /medrec/javax.faces.resource/admin.jpg.xhtml?ln=img\u0026amp;con=bayland HTTP/1.1\" 200 8116 ",
+                "original": "172.17.32.1 - - [26/Mar/2022:23:43:45 +0530] \"GET /medrec/javax.faces.resource/admin.jpg.xhtml?ln=img&amp;con=bayland HTTP/1.1\" 200 8116 ",
                 "type": "access"
             },
             "host": {
-                "ip": "172.17.32.1"
+                "ip": [
+                    "172.17.32.1"
+                ]
             },
             "http": {
                 "request": {
                     "method": "GET",
-                    "referrer": "/medrec/javax.faces.resource/admin.jpg.xhtml?ln=img\u0026amp;con=bayland"
+                    "referrer": "/medrec/javax.faces.resource/admin.jpg.xhtml?ln=img&amp;con=bayland"
                 },
                 "response": {
                     "bytes": 8116,
@@ -204,19 +214,21 @@
                 "category": [
                     "log"
                 ],
-                "ingested": "2023-01-18T08:48:25.790120388Z",
+                "ingested": "2023-10-13T11:27:51.595120637Z",
                 "kind": "event",
                 "module": "oracle_weblogic",
-                "original": "172.17.32.1 - - [26/Mar/2022:23:43:45 +0530] \"GET /medrec/javax.faces.resource/patient.jpg.xhtml?ln=img\u0026amp;con=bayland HTTP/1.1\" 200 10096 ",
+                "original": "172.17.32.1 - - [26/Mar/2022:23:43:45 +0530] \"GET /medrec/javax.faces.resource/patient.jpg.xhtml?ln=img&amp;con=bayland HTTP/1.1\" 200 10096 ",
                 "type": "access"
             },
             "host": {
-                "ip": "172.17.32.1"
+                "ip": [
+                    "172.17.32.1"
+                ]
             },
             "http": {
                 "request": {
                     "method": "GET",
-                    "referrer": "/medrec/javax.faces.resource/patient.jpg.xhtml?ln=img\u0026amp;con=bayland"
+                    "referrer": "/medrec/javax.faces.resource/patient.jpg.xhtml?ln=img&amp;con=bayland"
                 },
                 "response": {
                     "bytes": 10096,
@@ -243,19 +255,21 @@
                 "category": [
                     "log"
                 ],
-                "ingested": "2023-01-18T08:48:25.790121555Z",
+                "ingested": "2023-10-13T11:27:51.595121429Z",
                 "kind": "event",
                 "module": "oracle_weblogic",
-                "original": "172.17.32.1 - - [26/Mar/2022:23:43:45 +0530] \"GET /medrec/javax.faces.resource/physician.jpg.xhtml?ln=img\u0026amp;con=bayland HTTP/1.1\" 200 10169 ",
+                "original": "172.17.32.1 - - [26/Mar/2022:23:43:45 +0530] \"GET /medrec/javax.faces.resource/physician.jpg.xhtml?ln=img&amp;con=bayland HTTP/1.1\" 200 10169 ",
                 "type": "access"
             },
             "host": {
-                "ip": "172.17.32.1"
+                "ip": [
+                    "172.17.32.1"
+                ]
             },
             "http": {
                 "request": {
                     "method": "GET",
-                    "referrer": "/medrec/javax.faces.resource/physician.jpg.xhtml?ln=img\u0026amp;con=bayland"
+                    "referrer": "/medrec/javax.faces.resource/physician.jpg.xhtml?ln=img&amp;con=bayland"
                 },
                 "response": {
                     "bytes": 10169,

diff --git a/packages/oracle_weblogic/data_stream/access/_dev/test/pipeline/test-common-config.yml b/packages/oracle_weblogic/data_stream/access/_dev/test/pipeline/test-common-config.yml
@@ -1,5 +1,5 @@
 dynamic_fields:
-  event.ingested: ".*"
+  "event.ingested": ".*"
 fields:
   tags:
     - preserve_original_event
diff --git a/packages/oracle_weblogic/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/oracle_weblogic/data_stream/access/elasticsearch/ingest_pipeline/default.yml
@@ -54,6 +54,11 @@ processors:
     target_field: host.ip
     ignore_missing: true
     ignore_failure: true
+- set:
+    field: host.ip
+    value: ["{{{host.ip}}}"]
+    ignore_empty_value: true
+    ignore_failure: true
 - convert:
     field: http.response.status_code
     type: long

diff --git a/packages/oracle_weblogic/data_stream/access/sample_event.json b/packages/oracle_weblogic/data_stream/access/sample_event.json
@@ -36,7 +36,9 @@
         "architecture": "x86_64",
         "containerized": true,
         "hostname": "docker-fleet-agent",
-        "ip": "172.17.32.1",
+        "ip": [
+            "172.17.32.1"
+        ],
         "mac": [
             "02:42:ac:12:00:07"
         ],