diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7bab5f6239bf..1d50f20ebaef 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -147,6 +147,7 @@ /packages/elastic_package_registry @elastic/ecosystem /packages/elasticsearch @elastic/stack-monitoring /packages/enterprisesearch @elastic/stack-monitoring +/packages/entityanalytics_ad @elastic/security-service-integrations /packages/entityanalytics_entra_id @elastic/security-service-integrations /packages/entityanalytics_okta @elastic/security-service-integrations /packages/eset_protect @elastic/security-service-integrations diff --git a/packages/entityanalytics_ad/_dev/build/build.yml b/packages/entityanalytics_ad/_dev/build/build.yml new file mode 100644 index 000000000000..71f48ba2a9c8 --- /dev/null +++ b/packages/entityanalytics_ad/_dev/build/build.yml @@ -0,0 +1,4 @@ +dependencies: + ecs: + reference: "git@v8.11.0" + import_mappings: true diff --git a/packages/entityanalytics_ad/_dev/build/docs/README.md b/packages/entityanalytics_ad/_dev/build/docs/README.md new file mode 100644 index 000000000000..bb24074d704c --- /dev/null +++ b/packages/entityanalytics_ad/_dev/build/docs/README.md @@ -0,0 +1,69 @@ +# Active Directory Entity Analytics + +This Active Directory Entity Analytics integration allows users to securely stream User Entities data to Elastic Security via the Active Directory LDAP look-ups. When integrated with Elastic Security, this valuable data can be leveraged within Elastic for risk-scoring scenarios (e.g., context enrichments) and detecting advanced analytics (UBA) use cases. + +## Data streams + +The Active Directory Entity Analytics integration collects one type of data: user. + +**User** is used to retrieve all user entries available from an Active Directory server. + +## Requirements + +- Elastic Agent must be installed. +- You can install only one Elastic Agent per host. +- Elastic Agent is required to stream data using Entity Analytics Input and ship the data to Elastic, where the events will then be processed via the integration's ingest pipelines. + +### Installing and managing an Elastic Agent: + +You have a few options for installing and managing an Elastic Agent: + +### Install a Fleet-managed Elastic Agent (recommended): + +With this approach, you install Elastic Agent and use Fleet in Kibana to define, configure, and manage your agents in a central location. We recommend using Fleet management because it makes the management and upgrade of your agents considerably easier. + +### Install Elastic Agent in standalone mode (advanced users): + +With this approach, you install Elastic Agent and manually configure the agent locally on the system where it’s installed. You are responsible for managing and upgrading the agents. This approach is reserved for advanced users only. + +### Install Elastic Agent in a containerized environment: + +You can run Elastic Agent inside a container, either with Fleet Server or standalone. Docker images for all versions of Elastic Agent are available from the Elastic Docker registry, and we provide deployment manifests for running on Kubernetes. + +There are some minimum requirements for running Elastic Agent and for more information, refer to the link [here](https://www.elastic.co/guide/en/fleet/current/elastic-agent-installation.html). + +The minimum **kibana.version** required is **8.14.0**. + +## Setup + +### To collect data from Active Directory, follow the below steps: + +- Obtain the LDAP username, e.g. `CN=Administrator,CN=Users,DC=testserver,DC=local` and password, and LDAP host address for the Active Directory server that you will be collecting data from. +- Determine the Base DN for the directory to be used, e.g. `CN=Users,DC=testserver,DC=local`. + +### Enabling the integration in Elastic: + +1. In Kibana, go to Management > Integrations. +2. In the "Search for integrations" search bar, type Active Directory Entity Analytics. +3. Click on the "Active Directory Entity Analytics" integration from the search results. +4. Click on the Add Active Directory Entity Analytics Integration button to add the integration. +5. While adding the integration, add the user, host and base DN details obtained above. +6. Save the integration by adding other necessary parameters. + +## Usage + +The Active Directory provider periodically contacts the server, retrieving updates for users, updates its internal cache of user metadata, and ships updated user metadata to Elasticsearch. + +Fetching and shipping updates occurs in one of two processes: **full synchronizations** and **incremental updates**. Full synchronizations will send the entire list of users in state, along with write markers to indicate the start and end of the synchronization event. Incremental updates will only send data for changed users during that event. Changes on a user can come in many forms, whether it be a change to the user’s metadata, or a user was added or deleted. By default, full synchronizations occur every 24 hours and incremental updates occur every 15 minutes. These intervals may be customized to suit your use case. + +## Logs reference + +### User + +This is the `User` dataset. + +#### Example + +{{event "user"}} + +{{fields "user"}} diff --git a/packages/entityanalytics_ad/changelog.yml b/packages/entityanalytics_ad/changelog.yml new file mode 100644 index 000000000000..e67c14252ebf --- /dev/null +++ b/packages/entityanalytics_ad/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial Release. + type: enhancement + link: https://github.com/elastic/integrations/pull/9485 diff --git a/packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-common-config.yml b/packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 000000000000..37e8fa225fdc --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_duplicate_custom_fields diff --git a/packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-user.json b/packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-user.json new file mode 100644 index 000000000000..3e17d54c42db --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-user.json @@ -0,0 +1,359 @@ +{ + "events": [ + { + "@timestamp": "2024-03-27T21:30:17.067Z", + "event": { + "action": "started", + "start": "2024-03-27T21:30:17.067Z" + }, + "labels": { + "identity_source": "entity-analytics-entityanalytics_ad.user-8c3c1f67-428d-4a95-a6de-69a2b8f952c3" + } + }, + { + "@timestamp": "2024-03-27T21:30:18.978Z", + "activedirectory": { + "groups": [ + { + "cn": "Group Policy Creator Owners", + "dSCorePropagationData": [ + "2024-01-22T06:37:40Z", + "1601-01-01T00:00:01Z" + ], + "description": "Members in this group can modify group policy for the domain", + "distinguishedName": "CN=Group Policy Creator Owners,CN=Users,DC=testserver,DC=local", + "groupType": "-2147483646", + "instanceType": "4", + "isCriticalSystemObject": true, + "member": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "memberOf": "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local", + "name": "Group Policy Creator Owners", + "objectCategory": "CN=Group,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "objectClass": [ + "top", + "group" + ], + "objectGUID": "q72u7gQzkkGd1X2IFbK6Hw==", + "objectSid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPxCAIAAA==", + "sAMAccountName": "Group Policy Creator Owners", + "sAMAccountType": "268435456", + "uSNChanged": "12391", + "uSNCreated": "12354", + "whenChanged": "2024-01-22T06:37:40Z", + "whenCreated": "2024-01-22T06:37:40Z" + }, + { + "adminCount": "1", + "cn": "Domain Admins", + "dSCorePropagationData": [ + "2024-01-22T06:52:50Z", + "2024-01-22T06:37:40Z", + "1601-01-01T00:04:16Z" + ], + "description": "Designated administrators of the domain", + "distinguishedName": "CN=Domain Admins,CN=Users,DC=testserver,DC=local", + "groupType": "-2147483646", + "instanceType": "4", + "isCriticalSystemObject": true, + "member": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "memberOf": [ + "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local", + "CN=Administrators,CN=Builtin,DC=testserver,DC=local" + ], + "name": "Domain Admins", + "objectCategory": "CN=Group,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "objectClass": [ + "top", + "group" + ], + "objectGUID": "dcLZNxKP90+za+sRLhh8kA==", + "objectSid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPxAAIAAA==", + "sAMAccountName": "Domain Admins", + "sAMAccountType": "268435456", + "uSNChanged": "12770", + "uSNCreated": "12345", + "whenChanged": "2024-01-22T06:52:50Z", + "whenCreated": "2024-01-22T06:37:40Z" + }, + { + "adminCount": "1", + "cn": "Enterprise Admins", + "dSCorePropagationData": [ + "2024-01-22T06:52:50Z", + "2024-01-22T06:37:40Z", + "1601-01-01T00:04:16Z" + ], + "description": "Designated administrators of the enterprise", + "distinguishedName": "CN=Enterprise Admins,CN=Users,DC=testserver,DC=local", + "groupType": "-2147483640", + "instanceType": "4", + "isCriticalSystemObject": true, + "member": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "memberOf": [ + "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local", + "CN=Administrators,CN=Builtin,DC=testserver,DC=local" + ], + "name": "Enterprise Admins", + "objectCategory": "CN=Group,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "objectClass": [ + "top", + "group" + ], + "objectGUID": "8NElU6E6TU61DNFLIe7VHQ==", + "objectSid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPxBwIAAA==", + "sAMAccountName": "Enterprise Admins", + "sAMAccountType": "268435456", + "uSNChanged": "12773", + "uSNCreated": "12339", + "whenChanged": "2024-01-22T06:52:50Z", + "whenCreated": "2024-01-22T06:37:40Z" + }, + { + "adminCount": "1", + "cn": "Schema Admins", + "dSCorePropagationData": [ + "2024-01-22T06:52:50Z", + "2024-01-22T06:37:40Z", + "1601-01-01T00:04:16Z" + ], + "description": "Designated administrators of the schema", + "distinguishedName": "CN=Schema Admins,CN=Users,DC=testserver,DC=local", + "groupType": "-2147483640", + "instanceType": "4", + "isCriticalSystemObject": true, + "member": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "memberOf": "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local", + "name": "Schema Admins", + "objectCategory": "CN=Group,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "objectClass": [ + "top", + "group" + ], + "objectGUID": "Ti+p/M4gtECARFBF3cNSpw==", + "objectSid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPxBgIAAA==", + "sAMAccountName": "Schema Admins", + "sAMAccountType": "268435456", + "uSNChanged": "12769", + "uSNCreated": "12336", + "whenChanged": "2024-01-22T06:52:50Z", + "whenCreated": "2024-01-22T06:37:40Z" + } + ], + "id": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "user": { + "accountExpires": "2185-07-21T23:34:33.709551516Z", + "adminCount": "1", + "badPasswordTime": "133517595269561536", + "badPwdCount": "0", + "cn": "Administrator", + "codePage": "0", + "countryCode": "0", + "dSCorePropagationData": [ + "2024-01-22T06:52:50Z", + "2024-01-22T06:52:50Z", + "2024-01-22T06:37:40Z", + "1601-01-01T18:12:16Z" + ], + "description": "Built-in account for administering the computer/domain", + "distinguishedName": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "instanceType": "4", + "isCriticalSystemObject": true, + "lastLogoff": "0", + "lastLogon": "2024-02-08T06:51:02.1812823Z", + "lastLogonTimestamp": "2024-03-27T04:30:09.6399883Z", + "logonCount": "8", + "memberOf": [ + "CN=Group Policy Creator Owners,CN=Users,DC=testserver,DC=local", + "CN=Domain Admins,CN=Users,DC=testserver,DC=local", + "CN=Enterprise Admins,CN=Users,DC=testserver,DC=local", + "CN=Schema Admins,CN=Users,DC=testserver,DC=local", + "CN=Administrators,CN=Builtin,DC=testserver,DC=local" + ], + "name": "Administrator", + "objectCategory": "CN=Person,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "objectClass": [ + "top", + "person", + "organizationalPerson", + "user" + ], + "objectGUID": "kUXoCTwYv0iZNc6UadAk1w==", + "objectSid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPx9AEAAA==", + "primaryGroupID": "513", + "pwdLastSet": "2024-01-22T06:15:39.8703568Z", + "sAMAccountName": "Administrator", + "sAMAccountType": "805306368", + "uSNChanged": "25166", + "uSNCreated": "8196", + "userAccountControl": "66048", + "whenChanged": "2024-03-27T04:30:09Z", + "whenCreated": "2024-01-22T06:36:59Z" + }, + "whenChanged": "2024-03-27T04:30:09Z" + }, + "event": { + "action": "user-discovered" + }, + "labels": { + "identity_source": "entity-analytics-entityanalytics_ad.user-8c3c1f67-428d-4a95-a6de-69a2b8f952c3" + }, + "user": { + "id": "CN=Administrator,CN=Users,DC=testserver,DC=local" + } + }, + { + "@timestamp": "2024-03-27T21:30:18.979Z", + "activedirectory": { + "id": "CN=Guest,CN=Users,DC=testserver,DC=local", + "user": { + "accountExpires": "2185-07-21T23:34:33.709551516Z", + "badPasswordTime": "0", + "badPwdCount": "0", + "cn": "Guest", + "codePage": "0", + "countryCode": "0", + "dSCorePropagationData": [ + "2024-01-22T06:37:40Z", + "1601-01-01T00:00:01Z" + ], + "description": "Built-in account for guest access to the computer/domain", + "distinguishedName": "CN=Guest,CN=Users,DC=testserver,DC=local", + "instanceType": "4", + "isCriticalSystemObject": true, + "lastLogoff": "0", + "lastLogon": "2185-07-21T23:34:33.709551616Z", + "logonCount": "0", + "memberOf": "CN=Guests,CN=Builtin,DC=testserver,DC=local", + "name": "Guest", + "objectCategory": "CN=Person,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "objectClass": [ + "top", + "person", + "organizationalPerson", + "user" + ], + "objectGUID": "hSt/40XJQU6cf+J2XoYMHw==", + "objectSid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPx9QEAAA==", + "primaryGroupID": "514", + "pwdLastSet": "2185-07-21T23:34:33.709551616Z", + "sAMAccountName": "Guest", + "sAMAccountType": "805306368", + "uSNChanged": "8197", + "uSNCreated": "8197", + "userAccountControl": "66082", + "whenChanged": "2024-01-22T06:36:59Z", + "whenCreated": "2024-01-22T06:36:59Z" + }, + "whenChanged": "2024-01-22T06:36:59Z" + }, + "event": { + "action": "user-discovered" + }, + "labels": { + "identity_source": "entity-analytics-entityanalytics_ad.user-8c3c1f67-428d-4a95-a6de-69a2b8f952c3" + }, + "user": { + "id": "CN=Guest,CN=Users,DC=testserver,DC=local" + } + }, + { + "@timestamp": "2024-03-27T21:30:18.980Z", + "activedirectory": { + "groups": [ + { + "cn": "Denied RODC Password Replication Group", + "dSCorePropagationData": [ + "2024-01-22T06:37:40Z", + "1601-01-01T00:00:01Z" + ], + "description": "Members in this group cannot have their passwords replicated to any read-only domain controllers in the domain", + "distinguishedName": "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local", + "groupType": "-2147483644", + "instanceType": "4", + "isCriticalSystemObject": true, + "member": [ + "CN=Read-only Domain Controllers,CN=Users,DC=testserver,DC=local", + "CN=Group Policy Creator Owners,CN=Users,DC=testserver,DC=local", + "CN=Domain Admins,CN=Users,DC=testserver,DC=local", + "CN=Cert Publishers,CN=Users,DC=testserver,DC=local", + "CN=Enterprise Admins,CN=Users,DC=testserver,DC=local", + "CN=Schema Admins,CN=Users,DC=testserver,DC=local", + "CN=Domain Controllers,CN=Users,DC=testserver,DC=local", + "CN=krbtgt,CN=Users,DC=testserver,DC=local" + ], + "name": "Denied RODC Password Replication Group", + "objectCategory": "CN=Group,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "objectClass": [ + "top", + "group" + ], + "objectGUID": "Ij75/i03bkSTx4bqdQlZ3w==", + "objectSid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPxPAIAAA==", + "sAMAccountName": "Denied RODC Password Replication Group", + "sAMAccountType": "536870912", + "uSNChanged": "12433", + "uSNCreated": "12405", + "whenChanged": "2024-01-22T06:37:40Z", + "whenCreated": "2024-01-22T06:37:40Z" + } + ], + "id": "CN=krbtgt,CN=Users,DC=testserver,DC=local", + "user": { + "accountExpires": "2185-07-21T23:34:33.709551516Z", + "adminCount": "1", + "badPasswordTime": "0", + "badPwdCount": "0", + "cn": "krbtgt", + "codePage": "0", + "countryCode": "0", + "dSCorePropagationData": [ + "2024-01-22T06:52:50Z", + "2024-01-22T06:37:40Z", + "1601-01-01T00:04:16Z" + ], + "description": "Key Distribution Center Service Account", + "distinguishedName": "CN=krbtgt,CN=Users,DC=testserver,DC=local", + "instanceType": "4", + "isCriticalSystemObject": true, + "lastLogoff": "0", + "lastLogon": "2185-07-21T23:34:33.709551616Z", + "logonCount": "0", + "memberOf": "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local", + "msDS-SupportedEncryptionTypes": "0", + "name": "krbtgt", + "objectCategory": "CN=Person,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "objectClass": [ + "top", + "person", + "organizationalPerson", + "user" + ], + "objectGUID": "rdk2F/qu4Eud52Q5bTXc7g==", + "objectSid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPx9gEAAA==", + "primaryGroupID": "513", + "pwdLastSet": "2024-01-22T06:37:40.4305135Z", + "sAMAccountName": "krbtgt", + "sAMAccountType": "805306368", + "servicePrincipalName": "kadmin/changepw", + "showInAdvancedViewOnly": true, + "uSNChanged": "12785", + "uSNCreated": "12324", + "userAccountControl": "514", + "whenChanged": "2024-01-22T06:52:50Z", + "whenCreated": "2024-01-22T06:37:40Z" + }, + "whenChanged": "2024-01-22T06:52:50Z" + }, + "event": { + "action": "user-discovered" + }, + "labels": { + "identity_source": "entity-analytics-entityanalytics_ad.user-8c3c1f67-428d-4a95-a6de-69a2b8f952c3" + }, + "user": { + "id": "CN=krbtgt,CN=Users,DC=testserver,DC=local" + } + }, + {"@timestamp":"2024-03-27T21:30:18.980Z","event":{"action":"completed","end":"2024-03-27T21:30:18.980Z"},"labels":{"identity_source":"entity-analytics-entityanalytics_ad.user-8c3c1f67-428d-4a95-a6de-69a2b8f952c3"}} + ] +} \ No newline at end of file diff --git a/packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-user.json-expected.json b/packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-user.json-expected.json new file mode 100644 index 000000000000..aa9086576dca --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-user.json-expected.json @@ -0,0 +1,504 @@ +{ + "expected": [ + { + "@timestamp": "2024-03-27T21:30:17.067Z", + "asset": { + "category": "entity", + "type": "activedirectory_user" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "started", + "category": [ + "iam" + ], + "kind": "asset", + "start": "2024-03-27T21:30:17.067Z", + "type": [ + "info" + ] + }, + "labels": { + "identity_source": "entity-analytics-entityanalytics_ad.user-8c3c1f67-428d-4a95-a6de-69a2b8f952c3" + }, + "tags": [ + "preserve_duplicate_custom_fields" + ] + }, + { + "@timestamp": "2024-03-27T21:30:18.978Z", + "asset": { + "category": "entity", + "create_date": "2024-01-22T06:36:59.000Z", + "id": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "last_updated": "2024-03-27T04:30:09.000Z", + "name": "Administrator", + "type": "activedirectory_user" + }, + "ecs": { + "version": "8.11.0" + }, + "entityanalytics_ad": { + "groups": [ + { + "cn": "Group Policy Creator Owners", + "description": "Members in this group can modify group policy for the domain", + "distinguished_name": "CN=Group Policy Creator Owners,CN=Users,DC=testserver,DC=local", + "ds_core_propagation_data": [ + "2024-01-22T06:37:40Z", + "1601-01-01T00:00:01Z" + ], + "group_type": "-2147483646", + "instance_type": "4", + "is_critical_system_object": true, + "member": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "member_of": "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local", + "name": "Group Policy Creator Owners", + "object_category": "CN=Group,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "object_class": [ + "top", + "group" + ], + "object_guid": "q72u7gQzkkGd1X2IFbK6Hw==", + "object_sid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPxCAIAAA==", + "sam_account_name": "Group Policy Creator Owners", + "sam_account_type": "268435456", + "usn_changed": "12391", + "usn_created": "12354", + "when_changed": "2024-01-22T06:37:40Z", + "when_created": "2024-01-22T06:37:40Z" + }, + { + "admin_count": "1", + "cn": "Domain Admins", + "description": "Designated administrators of the domain", + "distinguished_name": "CN=Domain Admins,CN=Users,DC=testserver,DC=local", + "ds_core_propagation_data": [ + "2024-01-22T06:52:50Z", + "2024-01-22T06:37:40Z", + "1601-01-01T00:04:16Z" + ], + "group_type": "-2147483646", + "instance_type": "4", + "is_critical_system_object": true, + "member": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "member_of": [ + "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local", + "CN=Administrators,CN=Builtin,DC=testserver,DC=local" + ], + "name": "Domain Admins", + "object_category": "CN=Group,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "object_class": [ + "top", + "group" + ], + "object_guid": "dcLZNxKP90+za+sRLhh8kA==", + "object_sid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPxAAIAAA==", + "sam_account_name": "Domain Admins", + "sam_account_type": "268435456", + "usn_changed": "12770", + "usn_created": "12345", + "when_changed": "2024-01-22T06:52:50Z", + "when_created": "2024-01-22T06:37:40Z" + }, + { + "admin_count": "1", + "cn": "Enterprise Admins", + "description": "Designated administrators of the enterprise", + "distinguished_name": "CN=Enterprise Admins,CN=Users,DC=testserver,DC=local", + "ds_core_propagation_data": [ + "2024-01-22T06:52:50Z", + "2024-01-22T06:37:40Z", + "1601-01-01T00:04:16Z" + ], + "group_type": "-2147483640", + "instance_type": "4", + "is_critical_system_object": true, + "member": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "member_of": [ + "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local", + "CN=Administrators,CN=Builtin,DC=testserver,DC=local" + ], + "name": "Enterprise Admins", + "object_category": "CN=Group,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "object_class": [ + "top", + "group" + ], + "object_guid": "8NElU6E6TU61DNFLIe7VHQ==", + "object_sid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPxBwIAAA==", + "sam_account_name": "Enterprise Admins", + "sam_account_type": "268435456", + "usn_changed": "12773", + "usn_created": "12339", + "when_changed": "2024-01-22T06:52:50Z", + "when_created": "2024-01-22T06:37:40Z" + }, + { + "admin_count": "1", + "cn": "Schema Admins", + "description": "Designated administrators of the schema", + "distinguished_name": "CN=Schema Admins,CN=Users,DC=testserver,DC=local", + "ds_core_propagation_data": [ + "2024-01-22T06:52:50Z", + "2024-01-22T06:37:40Z", + "1601-01-01T00:04:16Z" + ], + "group_type": "-2147483640", + "instance_type": "4", + "is_critical_system_object": true, + "member": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "member_of": "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local", + "name": "Schema Admins", + "object_category": "CN=Group,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "object_class": [ + "top", + "group" + ], + "object_guid": "Ti+p/M4gtECARFBF3cNSpw==", + "object_sid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPxBgIAAA==", + "sam_account_name": "Schema Admins", + "sam_account_type": "268435456", + "usn_changed": "12769", + "usn_created": "12336", + "when_changed": "2024-01-22T06:52:50Z", + "when_created": "2024-01-22T06:37:40Z" + } + ], + "id": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "user": { + "account_expires": "2185-07-21T23:34:33.709551516Z", + "admin_count": "1", + "bad_password_time": "133517595269561536", + "bad_pwd_count": "0", + "cn": "Administrator", + "code_page": "0", + "country_code": "0", + "description": "Built-in account for administering the computer/domain", + "distinguished_name": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "ds_core_propagation_data": [ + "2024-01-22T06:52:50Z", + "2024-01-22T06:52:50Z", + "2024-01-22T06:37:40Z", + "1601-01-01T18:12:16Z" + ], + "instance_type": "4", + "is_critical_system_object": true, + "last_logoff": "0", + "last_logon": "2024-02-08T06:51:02.1812823Z", + "last_logon_timestamp": "2024-03-27T04:30:09.6399883Z", + "logon_count": "8", + "member_of": [ + "CN=Group Policy Creator Owners,CN=Users,DC=testserver,DC=local", + "CN=Domain Admins,CN=Users,DC=testserver,DC=local", + "CN=Enterprise Admins,CN=Users,DC=testserver,DC=local", + "CN=Schema Admins,CN=Users,DC=testserver,DC=local", + "CN=Administrators,CN=Builtin,DC=testserver,DC=local" + ], + "name": "Administrator", + "object_category": "CN=Person,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "object_class": [ + "top", + "person", + "organizationalPerson", + "user" + ], + "object_guid": "kUXoCTwYv0iZNc6UadAk1w==", + "object_sid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPx9AEAAA==", + "primary_group_id": "513", + "pwd_last_set": "2024-01-22T06:15:39.8703568Z", + "sam_account_name": "Administrator", + "sam_account_type": "805306368", + "user_account_control": "66048", + "usn_changed": "25166", + "usn_created": "8196", + "when_changed": "2024-03-27T04:30:09Z", + "when_created": "2024-01-22T06:36:59Z" + }, + "when_changed": "2024-03-27T04:30:09Z" + }, + "event": { + "action": "user-discovered", + "category": [ + "iam" + ], + "kind": "asset", + "type": [ + "info" + ] + }, + "labels": { + "identity_source": "entity-analytics-entityanalytics_ad.user-8c3c1f67-428d-4a95-a6de-69a2b8f952c3" + }, + "related": { + "user": [ + "Administrator", + "CN=Administrator,CN=Users,DC=testserver,DC=local", + "kUXoCTwYv0iZNc6UadAk1w==", + "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPx9AEAAA==" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "account": { + "password_change_date": "2024-01-22T06:15:39.870Z" + }, + "domain": "testserver.local", + "id": "CN=Administrator,CN=Users,DC=testserver,DC=local", + "name": "Administrator" + } + }, + { + "@timestamp": "2024-03-27T21:30:18.979Z", + "asset": { + "category": "entity", + "create_date": "2024-01-22T06:36:59.000Z", + "id": "CN=Guest,CN=Users,DC=testserver,DC=local", + "last_updated": "2024-01-22T06:36:59.000Z", + "name": "Guest", + "type": "activedirectory_user" + }, + "ecs": { + "version": "8.11.0" + }, + "entityanalytics_ad": { + "id": "CN=Guest,CN=Users,DC=testserver,DC=local", + "user": { + "account_expires": "2185-07-21T23:34:33.709551516Z", + "bad_password_time": "0", + "bad_pwd_count": "0", + "cn": "Guest", + "code_page": "0", + "country_code": "0", + "description": "Built-in account for guest access to the computer/domain", + "distinguished_name": "CN=Guest,CN=Users,DC=testserver,DC=local", + "ds_core_propagation_data": [ + "2024-01-22T06:37:40Z", + "1601-01-01T00:00:01Z" + ], + "instance_type": "4", + "is_critical_system_object": true, + "last_logoff": "0", + "last_logon": "2185-07-21T23:34:33.709551616Z", + "logon_count": "0", + "member_of": "CN=Guests,CN=Builtin,DC=testserver,DC=local", + "name": "Guest", + "object_category": "CN=Person,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "object_class": [ + "top", + "person", + "organizationalPerson", + "user" + ], + "object_guid": "hSt/40XJQU6cf+J2XoYMHw==", + "object_sid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPx9QEAAA==", + "primary_group_id": "514", + "pwd_last_set": "2185-07-21T23:34:33.709551616Z", + "sam_account_name": "Guest", + "sam_account_type": "805306368", + "user_account_control": "66082", + "usn_changed": "8197", + "usn_created": "8197", + "when_changed": "2024-01-22T06:36:59Z", + "when_created": "2024-01-22T06:36:59Z" + }, + "when_changed": "2024-01-22T06:36:59Z" + }, + "event": { + "action": "user-discovered", + "category": [ + "iam" + ], + "kind": "asset", + "type": [ + "info" + ] + }, + "labels": { + "identity_source": "entity-analytics-entityanalytics_ad.user-8c3c1f67-428d-4a95-a6de-69a2b8f952c3" + }, + "related": { + "user": [ + "Guest", + "CN=Guest,CN=Users,DC=testserver,DC=local", + "hSt/40XJQU6cf+J2XoYMHw==", + "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPx9QEAAA==" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "account": { + "password_change_date": "2185-07-21T23:34:33.709Z" + }, + "domain": "testserver.local", + "id": "CN=Guest,CN=Users,DC=testserver,DC=local", + "name": "Guest" + } + }, + { + "@timestamp": "2024-03-27T21:30:18.980Z", + "asset": { + "category": "entity", + "create_date": "2024-01-22T06:37:40.000Z", + "id": "CN=krbtgt,CN=Users,DC=testserver,DC=local", + "last_updated": "2024-01-22T06:52:50.000Z", + "name": "krbtgt", + "type": "activedirectory_user" + }, + "ecs": { + "version": "8.11.0" + }, + "entityanalytics_ad": { + "groups": [ + { + "cn": "Denied RODC Password Replication Group", + "description": "Members in this group cannot have their passwords replicated to any read-only domain controllers in the domain", + "distinguished_name": "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local", + "ds_core_propagation_data": [ + "2024-01-22T06:37:40Z", + "1601-01-01T00:00:01Z" + ], + "group_type": "-2147483644", + "instance_type": "4", + "is_critical_system_object": true, + "member": [ + "CN=Read-only Domain Controllers,CN=Users,DC=testserver,DC=local", + "CN=Group Policy Creator Owners,CN=Users,DC=testserver,DC=local", + "CN=Domain Admins,CN=Users,DC=testserver,DC=local", + "CN=Cert Publishers,CN=Users,DC=testserver,DC=local", + "CN=Enterprise Admins,CN=Users,DC=testserver,DC=local", + "CN=Schema Admins,CN=Users,DC=testserver,DC=local", + "CN=Domain Controllers,CN=Users,DC=testserver,DC=local", + "CN=krbtgt,CN=Users,DC=testserver,DC=local" + ], + "name": "Denied RODC Password Replication Group", + "object_category": "CN=Group,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "object_class": [ + "top", + "group" + ], + "object_guid": "Ij75/i03bkSTx4bqdQlZ3w==", + "object_sid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPxPAIAAA==", + "sam_account_name": "Denied RODC Password Replication Group", + "sam_account_type": "536870912", + "usn_changed": "12433", + "usn_created": "12405", + "when_changed": "2024-01-22T06:37:40Z", + "when_created": "2024-01-22T06:37:40Z" + } + ], + "id": "CN=krbtgt,CN=Users,DC=testserver,DC=local", + "user": { + "account_expires": "2185-07-21T23:34:33.709551516Z", + "admin_count": "1", + "bad_password_time": "0", + "bad_pwd_count": "0", + "cn": "krbtgt", + "code_page": "0", + "country_code": "0", + "description": "Key Distribution Center Service Account", + "distinguished_name": "CN=krbtgt,CN=Users,DC=testserver,DC=local", + "ds_core_propagation_data": [ + "2024-01-22T06:52:50Z", + "2024-01-22T06:37:40Z", + "1601-01-01T00:04:16Z" + ], + "instance_type": "4", + "is_critical_system_object": true, + "last_logoff": "0", + "last_logon": "2185-07-21T23:34:33.709551616Z", + "logon_count": "0", + "member_of": "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local", + "msds-supported_encryption_types": "0", + "name": "krbtgt", + "object_category": "CN=Person,CN=Schema,CN=Configuration,DC=testserver,DC=local", + "object_class": [ + "top", + "person", + "organizationalPerson", + "user" + ], + "object_guid": "rdk2F/qu4Eud52Q5bTXc7g==", + "object_sid": "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPx9gEAAA==", + "primary_group_id": "513", + "pwd_last_set": "2024-01-22T06:37:40.4305135Z", + "sam_account_name": "krbtgt", + "sam_account_type": "805306368", + "service_principal_name": "kadmin/changepw", + "show_in_advanced_view_only": true, + "user_account_control": "514", + "usn_changed": "12785", + "usn_created": "12324", + "when_changed": "2024-01-22T06:52:50Z", + "when_created": "2024-01-22T06:37:40Z" + }, + "when_changed": "2024-01-22T06:52:50Z" + }, + "event": { + "action": "user-discovered", + "category": [ + "iam" + ], + "kind": "asset", + "type": [ + "info" + ] + }, + "labels": { + "identity_source": "entity-analytics-entityanalytics_ad.user-8c3c1f67-428d-4a95-a6de-69a2b8f952c3" + }, + "related": { + "user": [ + "krbtgt", + "CN=krbtgt,CN=Users,DC=testserver,DC=local", + "rdk2F/qu4Eud52Q5bTXc7g==", + "AQUAAAAAAAUVAAAA0JU2Fq1k30YZ7UPx9gEAAA==" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ], + "user": { + "account": { + "password_change_date": "2024-01-22T06:37:40.430Z" + }, + "domain": "testserver.local", + "id": "CN=krbtgt,CN=Users,DC=testserver,DC=local", + "name": "krbtgt" + } + }, + { + "@timestamp": "2024-03-27T21:30:18.980Z", + "asset": { + "category": "entity", + "type": "activedirectory_user" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "completed", + "category": [ + "iam" + ], + "end": "2024-03-27T21:30:18.980Z", + "kind": "asset", + "type": [ + "info" + ] + }, + "labels": { + "identity_source": "entity-analytics-entityanalytics_ad.user-8c3c1f67-428d-4a95-a6de-69a2b8f952c3" + }, + "tags": [ + "preserve_duplicate_custom_fields" + ] + } + ] +} \ No newline at end of file diff --git a/packages/entityanalytics_ad/data_stream/user/agent/stream/entity-analytics.yml.hbs b/packages/entityanalytics_ad/data_stream/user/agent/stream/entity-analytics.yml.hbs new file mode 100644 index 000000000000..49588fee255a --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/user/agent/stream/entity-analytics.yml.hbs @@ -0,0 +1,22 @@ +provider: activedirectory +sync_interval: {{sync_interval}} +update_interval: {{update_interval}} +ad_base_dn: {{ad_base_dn}} +ad_url: {{ad_url}} +ad_user: {{ad_user}} +ad_password: {{ad_password}} +tags: + - users-entities +{{#if preserve_duplicate_custom_fields}} + - preserve_duplicate_custom_fields +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/entityanalytics_ad/data_stream/user/elasticsearch/ingest_pipeline/default.yml b/packages/entityanalytics_ad/data_stream/user/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000000..e8efd5e24857 --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/user/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,49 @@ +--- +description: Pipeline for processing User logs. +processors: + - set: + field: ecs.version + tag: set_ecs_version + value: 8.11.0 + + - pipeline: + name: '{{ IngestPipeline "entity" }}' + if: ctx.activedirectory != null + tag: pipeline_entity + ignore_missing_pipeline: true + - pipeline: + name: '{{ IngestPipeline "marker" }}' + if: ctx.activedirectory == null + tag: pipeline_marker + ignore_missing_pipeline: true + + - script: + lang: painless + description: Drops null/empty values recursively. + tag: painless_remove_null + source: |- + boolean drop(Object object) { + if (object == null || object == '') { + return true; + } else if (object instanceof Map) { + ((Map) object).values().removeIf(v -> drop(v)); + return (((Map) object).size() == 0); + } else if (object instanceof List) { + ((List) object).removeIf(v -> drop(v)); + return (((List) object).length == 0); + } + return false; + } + drop(ctx); + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null +on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: event.kind + value: pipeline_error diff --git a/packages/entityanalytics_ad/data_stream/user/elasticsearch/ingest_pipeline/entity.yml b/packages/entityanalytics_ad/data_stream/user/elasticsearch/ingest_pipeline/entity.yml new file mode 100644 index 000000000000..a7e936d505e5 --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/user/elasticsearch/ingest_pipeline/entity.yml @@ -0,0 +1,208 @@ +--- +description: Pipeline for processing User logs. +processors: + - set: + field: event.kind + tag: set_event_kind + value: asset + - set: + field: event.category + tag: set_event_category + value: ['iam'] + - set: + field: event.type + tag: set_event_type + value: ['user','info'] + - set: + field: asset.category + tag: set_asset_category + value: entity + - set: + field: asset.type + tag: set_asset_type + value: activedirectory_user + + - script: + lang: painless + description: This script processor rename the fields under the activedirectory objects. + params: + "accountExpires": "account_expires" + "adminCount": "admin_count" + "badPasswordTime": "bad_password_time" + "badPwdCount": "bad_pwd_count" + "cn": "cn" + "codePage": "code_page" + "countryCode": "country_code" + "description": "description" + "distinguishedName": "distinguished_name" + "dSCorePropagationData": "ds_core_propagation_data" + "groups": "groups" + "groupType": "group_type" + "instanceType": "instance_type" + "isCriticalSystemObject": "is_critical_system_object" + "lastLogoff": "last_logoff" + "lastLogon": "last_logon" + "lastLogonTimestamp": "last_logon_timestamp" + "logonCount": "logon_count" + "member": "member" + "memberOf": "member_of" + "msDS-SupportedEncryptionTypes": "msds-supported_encryption_types" + "name": "name" + "object_category": "object_category" + "objectCategory": "object_category" + "objectClass": "object_class" + "objectGUID": "object_guid" + "objectSid": "object_sid" + "primaryGroupID": "primary_group_id" + "pwdLastSet": "pwd_last_set" + "sAMAccountName": "sam_account_name" + "sAMAccountType": "sam_account_type" + "servicePrincipalName": "service_principal_name" + "showInAdvancedViewOnly": "show_in_advanced_view_only" + "userAccountControl": "user_account_control" + "uSNChanged": "usn_changed" + "uSNCreated": "usn_created" + "whenChanged": "when_changed" + "whenCreated": "when_created" + tag: painless_to_rename_fields_under_activedirectory_groups + source: | + def renameKeys(Map src, Map keyMap) { + def dst = new HashMap(); + for (def entry: src.entrySet()) { + def key = entry.getKey(); + def value = entry.getValue(); + if (value instanceof Map) { + if (keyMap.containsKey(key)) { + dst[keyMap[key]] = renameKeys(value, keyMap); + } else { + dst[key] = renameKeys(value, keyMap); + } + } else if (value instanceof List) { + def updatedList = []; + for (def item: value) { + if (item instanceof Map) { + updatedList.add(renameKeys(item, keyMap)); + } else { + updatedList.add(item); + } + } + if (keyMap.containsKey(key)) { + dst[keyMap[key]] = updatedList; + } else { + dst[key] = value; + } + } else { + if (keyMap.containsKey(key)) { + dst[keyMap[key]] = value; + } else { + dst[key] = value; + } + } + } + return dst; + } + + ctx.activedirectory = renameKeys(ctx.activedirectory, params) + + - date: + field: activedirectory.user.when_created + target_field: asset.create_date + tag: date_user_created + formats: + - ISO8601 + if: ctx.activedirectory?.user?.when_created != null && ctx.activedirectory.user.when_created != '' + on_failure: + - remove: + field: activedirectory.user.when_created + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + field: activedirectory.user.when_changed + target_field: asset.last_updated + tag: date_user_changed + formats: + - ISO8601 + if: ctx.activedirectory?.user?.when_changed != null && ctx.activedirectory.user.when_changed != '' + on_failure: + - remove: + field: activedirectory.user.when_changed + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + field: activedirectory.user.pwd_last_set + target_field: user.account.password_change_date + tag: date_user_password_changed + formats: + - ISO8601 + if: ctx.activedirectory?.user?.pwd_last_set != null && ctx.activedirectory.user.pwd_last_set != '' + on_failure: + - remove: + field: activedirectory.user.pwd_last_set + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + + - set: + field: asset.name + copy_from: activedirectory.user.name + ignore_empty_value: true + - set: + field: user.name + copy_from: activedirectory.user.name + ignore_empty_value: true + - gsub: + tag: gsub_user_dn + field: activedirectory.user.distinguished_name + pattern: '^.*?DC=' + replacement: '' + target_field: user.domain + if: ctx.activedirectory?.user?.distinguished_name != null + - gsub: + tag: gsub_user_domain + field: user.domain + pattern: ',DC=' + replacement: '.' + if: ctx.user?.domain != null + - set: + field: asset.id + copy_from: activedirectory.id + ignore_empty_value: true + + - append: + field: related.user + value: "{{{activedirectory.user.name}}}" + tag: append_name_into_related_user + allow_duplicates: false + if: ctx.activedirectory?.user?.name != null + - append: + field: related.user + value: "{{{activedirectory.id}}}" + tag: append_id_into_related_user + allow_duplicates: false + if: ctx.activedirectory?.id != null + - append: + field: related.user + value: "{{{activedirectory.user.object_guid}}}" + tag: append_object_guid_into_related_user + allow_duplicates: false + if: ctx.activedirectory?.user?.object_guid != null + - append: + field: related.user + value: "{{{activedirectory.user.object_sid}}}" + tag: append_object_sid_into_related_user + allow_duplicates: false + if: ctx.activedirectory?.user?.object_sid != null + + - rename: + field: activedirectory + target_field: entityanalytics_ad + +on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: event.kind + value: pipeline_error diff --git a/packages/entityanalytics_ad/data_stream/user/elasticsearch/ingest_pipeline/marker.yml b/packages/entityanalytics_ad/data_stream/user/elasticsearch/ingest_pipeline/marker.yml new file mode 100644 index 000000000000..e71d020029b6 --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/user/elasticsearch/ingest_pipeline/marker.yml @@ -0,0 +1,35 @@ +--- +description: Pipeline for processing publication markers. +processors: + - set: + field: ecs.version + tag: set_ecs_version + value: 8.11.0 + - set: + field: event.kind + tag: set_event_kind + value: asset + - set: + field: event.category + tag: set_event_category + value: ['iam'] + - set: + field: event.type + tag: set_event_type + value: ['info'] + - set: + field: asset.category + tag: set_asset_category + value: entity + - set: + field: asset.type + tag: set_asset_type + value: activedirectory_user + +on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: event.kind + value: pipeline_error diff --git a/packages/entityanalytics_ad/data_stream/user/fields/base-fields.yml b/packages/entityanalytics_ad/data_stream/user/fields/base-fields.yml new file mode 100644 index 000000000000..8bbb3d9ffd2c --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/user/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module. + value: entityanalytics_ad +- name: event.dataset + type: constant_keyword + description: Event dataset. + value: entityanalytics_ad.user +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/entityanalytics_ad/data_stream/user/fields/beats.yml b/packages/entityanalytics_ad/data_stream/user/fields/beats.yml new file mode 100644 index 000000000000..a43f4ff852ca --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/user/fields/beats.yml @@ -0,0 +1,6 @@ +- name: input.type + type: keyword + description: Type of filebeat input. +- name: tags + type: keyword + description: User defined tags. diff --git a/packages/entityanalytics_ad/data_stream/user/fields/ecs.yml b/packages/entityanalytics_ad/data_stream/user/fields/ecs.yml new file mode 100644 index 000000000000..7ab99caceaf6 --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/user/fields/ecs.yml @@ -0,0 +1,104 @@ +- name: asset + type: group + fields: + - name: category + type: keyword + - name: costCenter + type: keyword + - name: create_date + type: date + - name: id + type: keyword + - name: last_seen + type: date + - name: last_status_change_date + type: date + - name: last_updated + type: date + - name: name + type: keyword + - name: status + type: keyword + - name: type + type: keyword + - name: vendor + type: keyword +- name: labels + type: group + fields: + - name: identity_source + type: keyword +- name: user + type: group + fields: + - name: account + type: group + fields: + - name: activated_date + type: date + - name: change_date + type: date + - name: create_date + type: date + - name: password_change_date + type: date + - name: status + type: group + fields: + - name: deprovisioned + type: boolean + - name: locked_out + type: boolean + - name: password_expired + type: boolean + - name: recovery + type: boolean + - name: suspended + type: boolean + - name: geo + type: group + fields: + - name: city_name + type: keyword + - name: country_iso_code + type: keyword + - name: name + type: keyword + - name: postal_code + type: keyword + - name: region_name + type: keyword + - name: timezone + type: keyword + - name: organization + type: group + fields: + - name: name + type: keyword + - name: profile + type: group + fields: + - name: department + type: keyword + - name: first_name + type: keyword + - name: id + type: keyword + - name: job_title + type: keyword + - name: last_name + type: keyword + - name: manager + type: keyword + - name: mobile_phone + type: keyword + - name: other_identities + type: keyword + - name: primaryPhone + type: keyword + - name: secondEmail + type: keyword + - name: status + type: keyword + - name: type + type: keyword diff --git a/packages/entityanalytics_ad/data_stream/user/fields/fields.yml b/packages/entityanalytics_ad/data_stream/user/fields/fields.yml new file mode 100644 index 000000000000..75a4a0ed1e58 --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/user/fields/fields.yml @@ -0,0 +1,124 @@ +- name: entityanalytics_ad + type: group + fields: + - name: id + type: keyword + - name: user + type: group + fields: + - name: account_expires + type: keyword + - name: admin_count + type: keyword + - name: bad_password_time + type: keyword + - name: bad_pwd_count + type: keyword + - name: cn + type: keyword + - name: code_page + type: keyword + - name: country_code + type: keyword + - name: description + type: keyword + - name: distinguished_name + type: keyword + - name: ds_core_propagation_data + type: date + - name: instance_type + type: keyword + - name: is_critical_system_object + type: boolean + - name: last_logoff + type: keyword + - name: last_logon + type: date + - name: last_logon_timestamp + type: date + - name: logon_count + type: keyword + - name: member_of + type: keyword + - name: msds-supported_encryption_types + type: keyword + - name: name + type: keyword + - name: object_category + type: keyword + - name: object_class + type: keyword + - name: object_guid + type: keyword + - name: object_sid + type: keyword + - name: primary_group_id + type: keyword + - name: pwd_last_set + type: date + - name: sam_account_name + type: keyword + - name: sam_account_type + type: keyword + - name: service_principal_name + type: keyword + - name: show_in_advanced_view_only + type: boolean + - name: user_account_control + type: keyword + - name: usn_changed + type: keyword + - name: usn_created + type: keyword + - name: when_changed + type: date + - name: when_created + type: date + - name: groups + type: group + fields: + - name: admin_count + type: keyword + - name: cn + type: keyword + - name: description + type: keyword + - name: distinguished_name + type: keyword + - name: ds_core_propagation_data + type: date + - name: group_type + type: keyword + - name: instance_type + type: keyword + - name: is_critical_system_object + type: boolean + - name: member + type: keyword + - name: member_of + type: keyword + - name: name + type: keyword + - name: object_category + type: keyword + - name: object_class + type: keyword + - name: object_guid + type: keyword + - name: object_sid + type: keyword + - name: sam_account_name + type: keyword + - name: sam_account_type + type: keyword + - name: usn_changed + type: keyword + - name: usn_created + type: keyword + - name: when_changed + type: date + - name: when_created + type: date + + - name: when_changed + type: date diff --git a/packages/entityanalytics_ad/data_stream/user/manifest.yml b/packages/entityanalytics_ad/data_stream/user/manifest.yml new file mode 100644 index 000000000000..afada65a464b --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/user/manifest.yml @@ -0,0 +1,85 @@ +title: Collect User Identities logs from Active Directory +type: logs +streams: + - input: entity-analytics + title: User Identities logs + description: Collect User Identities logs from Active Directory. + template_path: entity-analytics.yml.hbs + vars: + - name: ad_base_dn + type: text + title: Active Directory Base DN + multi: false + required: true + show_user: true + description: The Base DN for the Active Directory. + - name: ad_url + type: text + title: Active Directory URL + multi: false + required: true + show_user: true + description: The URL for the Active Directory server including the appropriate LDAP scheme. + - name: ad_user + type: text + title: Active Directory User + multi: false + required: true + show_user: true + description: The Active Directory user name. + - name: ad_password + type: password + title: Active Directory User Password + multi: false + required: true + show_user: true + description: The Active Directory user's password, used for authentication. + secret: true + - name: sync_interval + type: text + title: Sync Interval + description: How often full synchronizations should occur. Must be greater than Update Interval. Expected value is a duration string (15m, 1h, 1m30, etc), defaults to 24h. Supported units for this parameter are h/m/s. + multi: false + required: true + show_user: true + default: 24h + - name: update_interval + type: text + title: Update Interval + description: How often incremental updates should occur. Must be less than Sync Interval. Expected value is a duration string (15m, 1h, 1m30, etc), defaults to 15m. Supported units for this parameter are h/m/s. + default: 15m + multi: false + required: true + show_user: true + - name: id + type: text + title: Input ID + description: Identity Source. Which will be added to every event as a label. + multi: false + required: false + show_user: false + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - entityanalytics_ad-user + - name: preserve_duplicate_custom_fields + required: true + show_user: false + title: Preserve duplicate custom fields + description: Preserve entityanalytics_ad.user fields that were copied to Elastic Common Schema (ECS) fields. + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/entityanalytics_ad/data_stream/user/sample_event.json b/packages/entityanalytics_ad/data_stream/user/sample_event.json new file mode 100644 index 000000000000..b995c0859d0f --- /dev/null +++ b/packages/entityanalytics_ad/data_stream/user/sample_event.json @@ -0,0 +1,53 @@ +{ + "@timestamp": "2024-04-02T02:44:08.198Z", + "agent": { + "ephemeral_id": "c8f2cffa-8316-41a2-8ad6-89ef2f3ecd2b", + "id": "277a9e26-8aae-4bc6-abcc-21db22ad29d7", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.13.0" + }, + "asset": { + "category": "entity", + "type": "activedirectory_user" + }, + "data_stream": { + "dataset": "entityanalytics_ad.user", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "277a9e26-8aae-4bc6-abcc-21db22ad29d7", + "snapshot": false, + "version": "8.13.0" + }, + "event": { + "action": "started", + "agent_id_status": "verified", + "category": [ + "iam" + ], + "dataset": "entityanalytics_ad.user", + "ingested": "2024-04-02T02:44:20Z", + "kind": "asset", + "start": "2024-04-02T02:44:08.198Z", + "type": [ + "info" + ] + }, + "input": { + "type": "entity-analytics" + }, + "labels": { + "identity_source": "entity-analytics-entityanalytics_ad.user-2270bd23-5392-4185-959b-b01ac2b8d89a" + }, + "tags": [ + "users-entities", + "preserve_duplicate_custom_fields", + "forwarded", + "entityanalytics_ad-user" + ] +} \ No newline at end of file diff --git a/packages/entityanalytics_ad/docs/README.md b/packages/entityanalytics_ad/docs/README.md new file mode 100644 index 000000000000..e221b8e3bbe1 --- /dev/null +++ b/packages/entityanalytics_ad/docs/README.md @@ -0,0 +1,234 @@ +# Active Directory Entity Analytics + +This Active Directory Entity Analytics integration allows users to securely stream User Entities data to Elastic Security via the Active Directory LDAP look-ups. When integrated with Elastic Security, this valuable data can be leveraged within Elastic for risk-scoring scenarios (e.g., context enrichments) and detecting advanced analytics (UBA) use cases. + +## Data streams + +The Active Directory Entity Analytics integration collects one type of data: user. + +**User** is used to retrieve all user entries available from an Active Directory server. + +## Requirements + +- Elastic Agent must be installed. +- You can install only one Elastic Agent per host. +- Elastic Agent is required to stream data using Entity Analytics Input and ship the data to Elastic, where the events will then be processed via the integration's ingest pipelines. + +### Installing and managing an Elastic Agent: + +You have a few options for installing and managing an Elastic Agent: + +### Install a Fleet-managed Elastic Agent (recommended): + +With this approach, you install Elastic Agent and use Fleet in Kibana to define, configure, and manage your agents in a central location. We recommend using Fleet management because it makes the management and upgrade of your agents considerably easier. + +### Install Elastic Agent in standalone mode (advanced users): + +With this approach, you install Elastic Agent and manually configure the agent locally on the system where it’s installed. You are responsible for managing and upgrading the agents. This approach is reserved for advanced users only. + +### Install Elastic Agent in a containerized environment: + +You can run Elastic Agent inside a container, either with Fleet Server or standalone. Docker images for all versions of Elastic Agent are available from the Elastic Docker registry, and we provide deployment manifests for running on Kubernetes. + +There are some minimum requirements for running Elastic Agent and for more information, refer to the link [here](https://www.elastic.co/guide/en/fleet/current/elastic-agent-installation.html). + +The minimum **kibana.version** required is **8.14.0**. + +## Setup + +### To collect data from Active Directory, follow the below steps: + +- Obtain the LDAP username, e.g. `CN=Administrator,CN=Users,DC=testserver,DC=local` and password, and LDAP host address for the Active Directory server that you will be collecting data from. +- Determine the Base DN for the directory to be used, e.g. `CN=Users,DC=testserver,DC=local`. + +### Enabling the integration in Elastic: + +1. In Kibana, go to Management > Integrations. +2. In the "Search for integrations" search bar, type Active Directory Entity Analytics. +3. Click on the "Active Directory Entity Analytics" integration from the search results. +4. Click on the Add Active Directory Entity Analytics Integration button to add the integration. +5. While adding the integration, add the user, host and base DN details obtained above. +6. Save the integration by adding other necessary parameters. + +## Usage + +The Active Directory provider periodically contacts the server, retrieving updates for users, updates its internal cache of user metadata, and ships updated user metadata to Elasticsearch. + +Fetching and shipping updates occurs in one of two processes: **full synchronizations** and **incremental updates**. Full synchronizations will send the entire list of users in state, along with write markers to indicate the start and end of the synchronization event. Incremental updates will only send data for changed users during that event. Changes on a user can come in many forms, whether it be a change to the user’s metadata, or a user was added or deleted. By default, full synchronizations occur every 24 hours and incremental updates occur every 15 minutes. These intervals may be customized to suit your use case. + +## Logs reference + +### User + +This is the `User` dataset. + +#### Example + +An example event for `user` looks as following: + +```json +{ + "@timestamp": "2024-04-02T02:44:08.198Z", + "agent": { + "ephemeral_id": "c8f2cffa-8316-41a2-8ad6-89ef2f3ecd2b", + "id": "277a9e26-8aae-4bc6-abcc-21db22ad29d7", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.13.0" + }, + "asset": { + "category": "entity", + "type": "activedirectory_user" + }, + "data_stream": { + "dataset": "entityanalytics_ad.user", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "277a9e26-8aae-4bc6-abcc-21db22ad29d7", + "snapshot": false, + "version": "8.13.0" + }, + "event": { + "action": "started", + "agent_id_status": "verified", + "category": [ + "iam" + ], + "dataset": "entityanalytics_ad.user", + "ingested": "2024-04-02T02:44:20Z", + "kind": "asset", + "start": "2024-04-02T02:44:08.198Z", + "type": [ + "info" + ] + }, + "input": { + "type": "entity-analytics" + }, + "labels": { + "identity_source": "entity-analytics-entityanalytics_ad.user-2270bd23-5392-4185-959b-b01ac2b8d89a" + }, + "tags": [ + "users-entities", + "preserve_duplicate_custom_fields", + "forwarded", + "entityanalytics_ad-user" + ] +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| asset.category | | keyword | +| asset.costCenter | | keyword | +| asset.create_date | | date | +| asset.id | | keyword | +| asset.last_seen | | date | +| asset.last_status_change_date | | date | +| asset.last_updated | | date | +| asset.name | | keyword | +| asset.status | | keyword | +| asset.type | | keyword | +| asset.vendor | | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| entityanalytics_ad.groups.admin_count | | keyword | +| entityanalytics_ad.groups.cn | | keyword | +| entityanalytics_ad.groups.description | | keyword | +| entityanalytics_ad.groups.distinguished_name | | keyword | +| entityanalytics_ad.groups.ds_core_propagation_data | | date | +| entityanalytics_ad.groups.group_type | | keyword | +| entityanalytics_ad.groups.instance_type | | keyword | +| entityanalytics_ad.groups.is_critical_system_object | | boolean | +| entityanalytics_ad.groups.member | | keyword | +| entityanalytics_ad.groups.member_of | | keyword | +| entityanalytics_ad.groups.name | | keyword | +| entityanalytics_ad.groups.object_category | | keyword | +| entityanalytics_ad.groups.object_class | | keyword | +| entityanalytics_ad.groups.object_guid | | keyword | +| entityanalytics_ad.groups.object_sid | | keyword | +| entityanalytics_ad.groups.sam_account_name | | keyword | +| entityanalytics_ad.groups.sam_account_type | | keyword | +| entityanalytics_ad.groups.usn_changed | | keyword | +| entityanalytics_ad.groups.usn_created | | keyword | +| entityanalytics_ad.groups.when_changed | | date | +| entityanalytics_ad.groups.when_created | | date | +| entityanalytics_ad.id | | keyword | +| entityanalytics_ad.user.account_expires | | keyword | +| entityanalytics_ad.user.admin_count | | keyword | +| entityanalytics_ad.user.bad_password_time | | keyword | +| entityanalytics_ad.user.bad_pwd_count | | keyword | +| entityanalytics_ad.user.cn | | keyword | +| entityanalytics_ad.user.code_page | | keyword | +| entityanalytics_ad.user.country_code | | keyword | +| entityanalytics_ad.user.description | | keyword | +| entityanalytics_ad.user.distinguished_name | | keyword | +| entityanalytics_ad.user.ds_core_propagation_data | | date | +| entityanalytics_ad.user.instance_type | | keyword | +| entityanalytics_ad.user.is_critical_system_object | | boolean | +| entityanalytics_ad.user.last_logoff | | keyword | +| entityanalytics_ad.user.last_logon | | date | +| entityanalytics_ad.user.last_logon_timestamp | | date | +| entityanalytics_ad.user.logon_count | | keyword | +| entityanalytics_ad.user.member_of | | keyword | +| entityanalytics_ad.user.msds-supported_encryption_types | | keyword | +| entityanalytics_ad.user.name | | keyword | +| entityanalytics_ad.user.object_category | | keyword | +| entityanalytics_ad.user.object_class | | keyword | +| entityanalytics_ad.user.object_guid | | keyword | +| entityanalytics_ad.user.object_sid | | keyword | +| entityanalytics_ad.user.primary_group_id | | keyword | +| entityanalytics_ad.user.pwd_last_set | | date | +| entityanalytics_ad.user.sam_account_name | | keyword | +| entityanalytics_ad.user.sam_account_type | | keyword | +| entityanalytics_ad.user.service_principal_name | | keyword | +| entityanalytics_ad.user.show_in_advanced_view_only | | boolean | +| entityanalytics_ad.user.user_account_control | | keyword | +| entityanalytics_ad.user.usn_changed | | keyword | +| entityanalytics_ad.user.usn_created | | keyword | +| entityanalytics_ad.user.when_changed | | date | +| entityanalytics_ad.user.when_created | | date | +| entityanalytics_ad.when_changed | | date | +| event.dataset | Event dataset. | constant_keyword | +| event.module | Event module. | constant_keyword | +| input.type | Type of filebeat input. | keyword | +| labels.identity_source | | keyword | +| tags | User defined tags. | keyword | +| user.account.activated_date | | date | +| user.account.change_date | | date | +| user.account.create_date | | date | +| user.account.password_change_date | | date | +| user.account.status.deprovisioned | | boolean | +| user.account.status.locked_out | | boolean | +| user.account.status.password_expired | | boolean | +| user.account.status.recovery | | boolean | +| user.account.status.suspended | | boolean | +| user.geo.city_name | | keyword | +| user.geo.country_iso_code | | keyword | +| user.geo.name | | keyword | +| user.geo.postal_code | | keyword | +| user.geo.region_name | | keyword | +| user.geo.timezone | | keyword | +| user.organization.name | | keyword | +| user.profile.department | | keyword | +| user.profile.first_name | | keyword | +| user.profile.id | | keyword | +| user.profile.job_title | | keyword | +| user.profile.last_name | | keyword | +| user.profile.manager | | keyword | +| user.profile.mobile_phone | | keyword | +| user.profile.other_identities | | keyword | +| user.profile.primaryPhone | | keyword | +| user.profile.secondEmail | | keyword | +| user.profile.status | | keyword | +| user.profile.type | | keyword | + diff --git a/packages/entityanalytics_ad/img/logo.svg b/packages/entityanalytics_ad/img/logo.svg new file mode 100644 index 000000000000..3a4effc7b5ce --- /dev/null +++ b/packages/entityanalytics_ad/img/logo.svg @@ -0,0 +1,1248 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/entityanalytics_ad/manifest.yml b/packages/entityanalytics_ad/manifest.yml new file mode 100644 index 000000000000..9e4f5f16520c --- /dev/null +++ b/packages/entityanalytics_ad/manifest.yml @@ -0,0 +1,30 @@ +format_version: "3.0.2" +name: entityanalytics_ad +title: Active Directory Entity Analytics +version: "0.0.1" +description: "Collect User Identities from Active Directory Entity with Elastic Agent." +type: integration +categories: + - security +conditions: + kibana: + version: "^8.14.0" + elastic: + subscription: "basic" +screenshots: [] +icons: + - src: /img/logo.svg + title: Active Directory Logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: user + title: User Identities + description: Collect user identities. + inputs: + - type: entity-analytics + title: Collect user identities + description: Collecting identities from Active Directory. +owner: + github: elastic/security-service-integrations + type: elastic diff --git a/packages/entityanalytics_ad/validation.yml b/packages/entityanalytics_ad/validation.yml new file mode 100644 index 000000000000..9dcaa3b03ff0 --- /dev/null +++ b/packages/entityanalytics_ad/validation.yml @@ -0,0 +1,5 @@ +errors: + exclude_checks: + - SVR00002 # Mandatory filters in dashboards. + - SVR00004 # References in dashboards. + - SVR00005 # Kibana version for saved tags.