Add Lumos Package

.github/CODEOWNERS

@@ -219,6 +219,7 @@
 /packages/lmd @elastic/ml-ui @elastic/sec-applied-ml
 /packages/log @elastic/elastic-agent-data-plane
 /packages/logstash @elastic/stack-monitoring
+/packages/lumos @elastic/security-service-integrations
 /packages/lyve_cloud @elastic/security-service-integrations
 /packages/m365_defender @elastic/security-service-integrations
 /packages/mattermost @elastic/security-service-integrations

packages/lumos/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: "git@v8.11.0"

packages/lumos/_dev/build/docs/README.md

@@ -0,0 +1,31 @@
+# Lumos Integration
+
+The Lumos integration uses [Lumos' API](https://api.lumos.com/) to retrieve Activity Logs and ingest them into Elasticsearch. This allows you to search, observe, and visualize the Activity Logs through Elasticsearch.
+
+The Elastic agent running this integration interacts with Lumos' infrastructure using their APIs to retrieve [Activity Logs](https://api.lumos.com/activity_logs) for a Lumos tenant.
+
+## Configuration
+
+### Enabling the integration in Elastic
+
+1. In Kibana go to **Management > Integrations**
+2. In the "Search for integrations" search bar type **Lumos**.
+3. Click on "Lumos" integration from the search results.
+4. Click on **Add Lumos** button to add Lumos integration.
+
+### Configure Lumos Activity Logs data stream
+
+1. In Lumos go to **Settings > API Tokens**
+2. Click on "Add API Token", enter a name and description
+3. Copy the key starting with `lsk_`
+4. While adding Lumos integration in Elastic, paste your key into the `API Token` field
+
+## Logs
+
+### Activity Logs
+
+Activity Logs summarize the history of changes and events occurring within Lumos.
+
+{{fields "activity_logs"}}
+
+{{event "activity_logs"}}

packages/lumos/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3.0'
+services:
+  lumos:
+    image: docker.elastic.co/observability/stream:v0.11.0
+    hostname: lumos
+    ports:
+      - 8080
+    volumes:
+      - ./files:/files:ro
+    environment:
+      PORT: 8080
+    command:
+      - http-server
+      - --addr=:8080
+      - --config=/files/config.yml

packages/lumos/_dev/deploy/docker/files/config.yml

@@ -0,0 +1,42 @@
+rules:
+  - path: /activity_logs
+    methods: ["GET"]
+    request_headers:
+      authorization: Bearer xoxp-1234567890
+    responses:
+      - status_code: 200
+        body: |-
+          {
+              "items": [
+                  {
+                      "actor": {
+                          "actor_type": "Lumos user",
+                          "email": "wile.e.coyote@lumos.com",
+                          "family_name": "Wile",
+                          "given_name": "Coyote"
+                      },
+                      "event_began_at": "2024-03-12T16:09:14",
+                      "event_hash": "630b90cedc35a8a5f43361534099bee51e032f42dd442085fc76ef094d228f543c78fbe59c132df992cf71a6b8496504e8ebbc6020fbae1f34206676985412e7",
+                      "event_metadata": {},
+                      "event_type": "SOD_POLICY_DELETED",
+                      "event_type_user_friendly": "A user deleted a SOD Policy",
+                      "outcome": "Succeeded",
+                      "targets": [
+                          {
+                              "name": "Untitled Rule",
+                              "target_type": "SOD Policy"
+                          }
+                      ]
+                  }
+              ],
+              "limit": 50,
+              "links": {
+                  "first": "/activity_logs?offset=0",
+                  "last": "/activity_logs?offset=1",
+                  "next": null,
+                  "prev": null,
+                  "self": "/activity_logs"
+              },
+              "offset": 0,
+              "total": 1
+          }

packages/lumos/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.1.0"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9276

packages/lumos/data_stream/activity_logs/_dev/test/pipeline/test-activity-logs.log

@@ -0,0 +1,2 @@
+{"actor":{"actor_type":"Lumos user","email":"wile.e.coyote@lumos.com","family_name":"Wile","given_name":"Coyote"},"event_began_at":"2024-03-12T16:09:14","event_hash":"630b90cedc35a8a5f43361534099bee51e032f42dd442085fc76ef094d228f543c78fbe59c132df992cf71a6b8496504e8ebbc6020fbae1f34206676985412e7","event_metadata":{},"event_type":"SOD_POLICY_DELETED","event_type_user_friendly":"A user deleted a SOD Policy","outcome":"Succeeded","targets":[{"name":"Untitled Rule","target_type":"SOD Policy"}]}
+{"actor":{"actor_type":"Lumos user","email":"wile.e.coyote@lumos.com","family_name":"Wile","given_name":"Coyote"},"event_began_at":"2024-03-12T16:09:14","event_hash":"630b90cedc35a8a5f43361534099bee51e032f42dd442085fc76ef094d228f543c78fbe59c132df992cf71a6b8496504e8ebbc6020fbae1f34206676985412e7","event_metadata":{},"event_type":"SOD_POLICY_DELETED","event_type_user_friendly":"A user deleted a SOD Policy","outcome":"Succeeded","targets":[]}

packages/lumos/data_stream/activity_logs/_dev/test/pipeline/test-activity-logs.log-expected.json

@@ -0,0 +1,60 @@
+{
+    "expected": [
+        {
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "SOD_POLICY_DELETED",
+                "id": "630b90cedc35a8a5f43361534099bee51e032f42dd442085fc76ef094d228f543c78fbe59c132df992cf71a6b8496504e8ebbc6020fbae1f34206676985412e7",
+                "outcome": "success",
+                "type": "info",
+                "kind": "event"
+            },
+            "lumos": {
+                "activity_logs": {
+                    "actor": {
+                        "actor_type": "Lumos user",
+                        "email": "wile.e.coyote@lumos.com",
+                        "family_name": "Wile",
+                        "given_name": "Coyote"
+                    },
+                    "event_began_at": "2024-03-12T16:09:14",
+                    "event_type_user_friendly": "A user deleted a SOD Policy",
+                    "targets": [
+                        {
+                            "name": "Untitled Rule",
+                            "target_type": "SOD Policy"
+                        }
+                    ]
+                }
+            },
+            "message": "{\"actor\":{\"actor_type\":\"Lumos user\",\"email\":\"wile.e.coyote@lumos.com\",\"family_name\":\"Wile\",\"given_name\":\"Coyote\"},\"event_began_at\":\"2024-03-12T16:09:14\",\"event_hash\":\"630b90cedc35a8a5f43361534099bee51e032f42dd442085fc76ef094d228f543c78fbe59c132df992cf71a6b8496504e8ebbc6020fbae1f34206676985412e7\",\"event_metadata\":{},\"event_type\":\"SOD_POLICY_DELETED\",\"event_type_user_friendly\":\"A user deleted a SOD Policy\",\"outcome\":\"Succeeded\",\"targets\":[{\"name\":\"Untitled Rule\",\"target_type\":\"SOD Policy\"}]}"
+        },
+        {
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "SOD_POLICY_DELETED",
+                "id": "630b90cedc35a8a5f43361534099bee51e032f42dd442085fc76ef094d228f543c78fbe59c132df992cf71a6b8496504e8ebbc6020fbae1f34206676985412e7",
+                "outcome": "success",
+                "type": "info",
+                "kind": "event"
+            },
+            "lumos": {
+                "activity_logs": {
+                    "actor": {
+                        "actor_type": "Lumos user",
+                        "email": "wile.e.coyote@lumos.com",
+                        "family_name": "Wile",
+                        "given_name": "Coyote"
+                    },
+                    "event_began_at": "2024-03-12T16:09:14",
+                    "event_type_user_friendly": "A user deleted a SOD Policy"
+                }
+            },
+            "message": "{\"actor\":{\"actor_type\":\"Lumos user\",\"email\":\"wile.e.coyote@lumos.com\",\"family_name\":\"Wile\",\"given_name\":\"Coyote\"},\"event_began_at\":\"2024-03-12T16:09:14\",\"event_hash\":\"630b90cedc35a8a5f43361534099bee51e032f42dd442085fc76ef094d228f543c78fbe59c132df992cf71a6b8496504e8ebbc6020fbae1f34206676985412e7\",\"event_metadata\":{},\"event_type\":\"SOD_POLICY_DELETED\",\"event_type_user_friendly\":\"A user deleted a SOD Policy\",\"outcome\":\"Succeeded\",\"targets\":[]}"
+        }
+    ]
+}

packages/lumos/data_stream/activity_logs/_dev/test/system/test-default-config.yml

@@ -0,0 +1,9 @@
+input: httpjson
+service: lumos
+vars:
+  api_url: http://{{Hostname}}:{{Port}}
+data_stream:
+  vars:
+    api_token: xoxp-1234567890
+assert:
+  hit_count: 1

packages/lumos/data_stream/activity_logs/agent/stream/httpjson.yml.hbs

@@ -0,0 +1,31 @@
+config_version: "2"
+interval: {{interval}}
+request.method: "GET"
+request.url: {{api_url}}/activity_logs
+
+{{#if http_client_timeout}}
+request.timeout: {{http_client_timeout}}
+{{/if}}
+
+request.transforms:
+  - set:
+      target: header.Authorization
+      value: "Bearer {{api_token}}"
+
+response.pagination:
+  - set:
+      target: url.value
+      value: '{{api_url}}[[.last_response.body.links.next]]'
+      fail_on_template_error: true
+
+response.split:
+  target: body.items
+
+cursor:
+  since:
+    value: '[[.last_event.created]]'
+
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/lumos/data_stream/activity_logs/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,68 @@
+---
+description: Pipeline for processing Lumos Activity Logs
+processors:
+  - set:
+      field: ecs.version
+      value: 8.11.0
+  - json:
+      field: message
+      target_field: lumos.activity_logs
+  - rename:
+      field: lumos.activity_logs.event_hash
+      target_field: event.id
+      ignore_missing: true
+  - rename:
+      field: lumos.activity_logs.event_type
+      target_field: event.action
+      ignore_missing: true
+  - rename:
+      field: lumos.activity_logs.outcome
+      target_field: event.outcome
+      ignore_missing: true
+  - set:
+      field: event.outcome
+      value: unknown
+      if: (ctx.event.outcome != "Failed") && (ctx.event.outcome != "Succeeded")
+  - set:
+      field: event.outcome
+      value: failure
+      if: ctx.event.outcome == "Failed"
+  - set:
+      field: event.outcome
+      value: success
+      if: ctx.event.outcome == "Succeeded"
+  - set:
+      field: event.kind
+      value: event
+  - set:
+      field: event.type
+      value: info
+  - script:
+      description: Drops null/empty values recursively
+      lang: painless
+      ignore_failure: true
+      source: |
+        boolean drop(Object o) {
+          if (o == null || o == "") {
+            return true;
+          } else if (o instanceof Map) {
+            ((Map) o).values().removeIf(v -> drop(v));
+            return (((Map) o).size() == 0);
+          } else if (o instanceof List) {
+            ((List) o).removeIf(v -> drop(v));
+            return (((List) o).length == 0);
+          }
+          return false;
+        }
+        drop(ctx);
+on_failure:
+  - append:
+      field: error.message
+      value: >-
+        Processor {{{_ingest.on_failure_processor_type}}} with tag
+        {{{_ingest.on_failure_processor_tag}}} in pipeline
+        {{{_ingest.on_failure_pipeline}}} failed with message:
+        {{{_ingest.on_failure_message}}}
+  - set:
+      field: event.kind
+      value: pipeline_error

packages/lumos/data_stream/activity_logs/fields/base-fields.yml

@@ -0,0 +1,16 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: event.module
+  type: constant_keyword
+  description: Event module
+  value: lumos
+- name: "@timestamp"
+  type: date
+  description: Event timestamp.

packages/lumos/data_stream/activity_logs/fields/ecs.yml

@@ -0,0 +1,4 @@
+- external: ecs
+  name: ecs.version
+- external: ecs
+  name: message

packages/lumos/data_stream/activity_logs/fields/fields.yml

@@ -0,0 +1,39 @@
+- name: input.type
+  type: keyword
+  description: Input type
+- name: event.id
+  type: keyword
+  description: The event hash
+- name: event.created
+  type: date
+  description: The time the event began
+- name: event.action
+  type: keyword
+  description: The activity that occurred
+- name: event.outcome
+  type: keyword
+  description: The outcome of the event, whether it succeeded or failed
+- name: lumos.activity_logs.actor.actor_type
+  type: keyword
+  description: The type of actor
+- name: lumos.activity_logs.actor.email
+  type: keyword
+  description: The email of the actor
+- name: lumos.activity_logs.actor.family_name
+  type: keyword
+  description: The family name of the actor
+- name: lumos.activity_logs.actor.given_name
+  type: keyword
+  description: The given name of the actor
+- name: lumos.activity_logs.event_type_user_friendly
+  type: keyword
+  description: The user friendly type of the event
+- name: lumos.activity_logs.event_began_at
+  type: keyword
+  description: The time the event began
+- name: lumos.activity_logs.targets
+  type: group
+- name: lumos.activity_logs.targets.target_type
+  type: keyword
+- name: lumos.activity_logs.targets.name
+  type: keyword

packages/lumos/data_stream/activity_logs/manifest.yml

@@ -0,0 +1,39 @@
+type: logs
+title: Lumos Activity Logs
+streams:
+  - input: httpjson
+    vars:
+      - name: api_token
+        type: password
+        title: API Token
+        description: The API Token used to authenticate with the Lumos API
+        multi: false
+        required: true
+        show_user: true
+        secret: true
+      - name: interval
+        type: text
+        title: Interval
+        multi: false
+        required: true
+        show_user: true
+        description: Interval at which the logs will be pulled. The value must be between 2m and 1h. Supported units for this parameter are h/m/s.
+        default: 1h
+      - name: initial_interval
+        type: text
+        title: Initial Interval
+        multi: false
+        required: true
+        show_user: false
+        description: Initial interval at which the logs will be pulled. Defaults to 24 hours. Supported units for this parameter are h/m/s.
+        default: 24h
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. \nThis executes in the agent before the logs are parsed. \nSee [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n"
+    template_path: httpjson.yml.hbs
+    title: Lumos Activity Logs
+    description: Collect Lumos Activity Logs via the API