Add aws-s3 system tests and add missing fields

packages/trellix_edr_cloud/_dev/deploy/tf/env.yml

@@ -0,0 +1,9 @@
+version: '2.3'
+services:
+  terraform:
+    environment:
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
+      - AWS_PROFILE=${AWS_PROFILE}
+      - AWS_REGION=${AWS_REGION:-us-east-1}

packages/trellix_edr_cloud/_dev/deploy/tf/files/test-event.log

@@ -0,0 +1 @@
+{"_ver":2107,"_serverId":"5B0539BF-0932-4BEA-BD12-EA52687E58BD","_eventType":"File Deleted","accessType":"connection_opened","_deviceId":"D435435b0-BB33-4625-891E-XXXXXXX","_parentEventId":"1XXXXX-8566-404c-87a3-a4c46017b87d","_eventId":"675XXXX-054c-48e8-9549-468dbb5ae5bc","_time":"2023-04-05T07:05:21.186Z","name":"Write Process Memory","authorName":"Example","data":"AAA9UFgBAAA=","arguments":["0x220a50d0000","0x1000","0x2"],"cmdLine":"\"C:\\Users\\XXXX\\AppData\\Local\\Microsoft\\OneDrive\\Update\\setup.exe\"/update","result":"2085503003216","fileModificationDate":"2023-04-04T12:38:42.821Z","fileType":"PE","fileCreationDate":"2023-04-04T12:38:40.984Z","fileMd5":"A7F7A4EEC248E6C1841EC6D5B735357B","fileSha1":"B8F93C2963CF1415A3D1C49668BF56665E3DC334","fileSha256":"F36CD7BAD72D6B6144234DBA8A101A529DABEDC07D48056126A1356A4EECA418","filePath":"C:\\ProgramFiles\\WindowsApps\\Deleted\\XXX.PowerAutomateDesktop_1.0.414.0_x64__8wekyb3d8e7483ce5b-4hhh-4a05-a9d8-a3e99e12498d\\kk-KZ\\PPP.Console.XX.YY.dll","fileSize":5632,"fileAttributes":32,"subsystem":3,"fileMagicBytes":"d0cf11e0a1b11ae1","direction":"outbound","dnsName":"content-autofill.example.com","pipeName":"\\\\.\\pipe\\Sessions\\3\\AppContainerNamedObjects\\S-1-15-2-3573721485-3817616455-324955835-1810672402-3651098853-3568380600-1295794929","destAddress":"81.2.69.192","destPort":443,"sourceAddress":"81.2.69.144","sourcePort":52376,"protocol":"tcp","taskName":"example ReportingTask-S-1-5-21-1323470238-68471550-93548180-1001","taskDescription":null,"dnsType":65,"dnsClass":1,"targetPid":1964,"pid": "2280","dnsNames":["XXX.YYY.cdn.live.net","ttt-XXX.YYY.net","SSS.YYY.cdn.live.net.XXX.net","aaa.dscd.XXX.net"],"action":"added","serviceName":"WD FILTER","serviceDescription":"Example Antivirus On-Access Malware Protection Mini-Filter Driver","serviceLoadOrderGroup":"FS FilterAnti-Virus","userName":"example user","userDomain":"DESKTOP-66XXX","userSid":"S-1-5-21-1323470238-68471550-93548180-1001","tagId":0,"commands":["%localappdata%\\XXXXXX\\OneDrive\\updater.exe"],"httpUrl":"https://xxxx-win.xxx.example.com:443settings/v2.0/compat/appraiser?os=windows&osver=0.0.0.1.example.ni_release.220506-1250&appver=0.0.0.1","httpRequestHeaders":"GETsettings/v2.0/compat/appraiser?os=windows&osver=0.0.0.1.amd64fre.ni_release.220506-1250&appver=0.0.0.2600HTTP/1.1\r\nUser-Agent:MSDW\r\n","serviceType":2,"integrityLevel":4,"versionInfoFilename":"example.EXE","versionInfoFileVersion":"0.0.0.1(WinBuild.160101.0800)","versionInfoProductName":"XXXX®Windows®OperatingSystem","versionInfoProductVersion":"0.0.0.1194","versionInfoVendorName":"Example Corporation","serviceStartType":0,"keyName":"HKLM\\SYSTEM\\CONTROLSET001\\SERVICES\\XXXX\\SECURITY","keyValueName":"SECURITY","keyValueType":"REG_BINARY","keyValue":"01001480CC000000D8000000140000003000000002001C000100000002801400FF010F0001010000000000010000000002009C0006000000000018009D01020001020000000000052000000021020000000014009D010200010100000000000512000000000018009D01020001020000000000052000000020020000000014009D010200010100000000000504000000000014009D01020001010000000000050600000000002800FF010F00010600000000000550000000BF5508723BE028D089794BF891896E7C4025ECF4010100000000000512000000010100000000000512000000","keyOldValue":"01001480F400000000010000140000003000000002001C000100000002801400FF010F00010100000000000100000","certs":[[{"type":"signing","issuerName":"US,\"example,Inc.\",ZZZZZ TrustedG4CodeSigningRSAXXXXXXXA3842021CA1","publicKeyHash":"47A58D30595525187338F85B7F8235FC919CE3FC"},{"type":"parent","issuerName":"US,example,www.example.com,ROOTCAA","publicKeyHash":"6837E0EBB63BF85F1186FBFE617B088865F44E42"},{"type":"parent","issuerName":"US,DigiCertInc,www.example.com,ROOTCA","publicKeyHash":"ECD7E382D2715D644CDF2E673FE7BA98AE1C0F4F"},{"type":"parent","issuerName":"US,DigiCertInc,www.example.com,ROOTCA","publicKeyHash":"45EBA2AFF492CB82312D518BA7A7219DF36DC80F"}]]}

packages/trellix_edr_cloud/_dev/deploy/tf/main.tf

@@ -0,0 +1,57 @@
+provider "aws" {
+  region = "us-east-1"
+  default_tags {
+    tags = {
+      environment  = var.ENVIRONMENT
+      repo         = var.REPO
+      branch       = var.BRANCH
+      build        = var.BUILD_ID
+      created_date = var.CREATED_DATE
+    }
+  }
+}
+
+resource "aws_s3_bucket" "bucket" {
+  bucket = "elastic-package-sentinel-one-bucket-${var.TEST_RUN_ID}"
+}
+
+resource "aws_sqs_queue" "queue" {
+  name       = "elastic-package-sentinel-one-queue-${var.TEST_RUN_ID}"
+  policy     = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": "*",
+      "Action": "sqs:SendMessage",
+      "Resource": "arn:aws:sqs:*:*:elastic-package-sentinel-one-queue-${var.TEST_RUN_ID}",
+      "Condition": {
+        "ArnEquals": { "aws:SourceArn": "${aws_s3_bucket.bucket.arn}" }
+      }
+    }
+  ]
+}
+POLICY
+}
+
+resource "aws_s3_bucket_notification" "bucket_notification" {
+  bucket = aws_s3_bucket.bucket.id
+
+  queue {
+    queue_arn = aws_sqs_queue.queue.arn
+    events    = ["s3:ObjectCreated:*"]
+  }
+}
+
+resource "aws_s3_object" "object" {
+  bucket = aws_s3_bucket.bucket.id
+  key    = "event.log"
+  source = "./files/test-event.log"
+
+  depends_on = [aws_sqs_queue.queue]
+}
+
+output "queue_url" {
+  value = aws_sqs_queue.queue.url
+}

packages/trellix_edr_cloud/_dev/deploy/tf/variables.tf

@@ -0,0 +1,26 @@
+variable "BRANCH" {
+  description = "Branch name or pull request for tagging purposes"
+  default     = "unknown-branch"
+}
+
+variable "BUILD_ID" {
+  description = "Build ID in the CI for tagging purposes"
+  default     = "unknown-build"
+}
+
+variable "CREATED_DATE" {
+  description = "Creation date in epoch time for tagging purposes"
+  default     = "unknown-date"
+}
+
+variable "ENVIRONMENT" {
+  default = "unknown-environment"
+}
+
+variable "REPO" {
+  default = "unknown-repo-name"
+}
+
+variable "TEST_RUN_ID" {
+  default = "detached"
+}

packages/trellix_edr_cloud/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.5.1"
+  changes:
+    - description: Add missing fields from beats input
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/8392
 - version: "0.5.0"
   changes:
     - description: Improve 'event.original' check to avoid errors if set.

packages/trellix_edr_cloud/data_stream/event/_dev/test/system/test-default-config.yml

@@ -0,0 +1,14 @@
+input: aws-s3
+wait_for_data_timeout: 20m
+vars:
+  access_key_id: "{{AWS_ACCESS_KEY_ID}}"
+  secret_access_key: "{{AWS_SECRET_ACCESS_KEY}}"
+  session_token: "{{AWS_SESSION_TOKEN}}"
+  queue_url: "{{TF_OUTPUT_queue_url}}"
+data_stream:
+  vars:
+    preserve_original_event: true
+    file_selectors: |
+      - regex: '^(.+?)\.log'
+assert:
+  hit_count: 1

packages/trellix_edr_cloud/data_stream/event/fields/beats.yml

@@ -7,3 +7,21 @@
 - name: tags
   type: keyword
   description: User defined tags.
+- name: aws.s3
+  type: group
+  fields:
+    - name: bucket
+      type: group
+      fields:
+        - name: name
+          type: keyword
+          description: The AWS S3 bucket name.
+        - name: arn
+          type: keyword
+          description: The AWS S3 bucket ARN.
+    - name: object
+      type: group
+      fields:
+        - name: key
+          type: keyword
+          description: The AWS S3 Object key.

packages/trellix_edr_cloud/docs/README.md

@@ -346,6 +346,9 @@ An example event for `event` looks as following:
 | Field | Description | Type |
 |---|---|---|
 | @timestamp | Event timestamp. | date |
+| aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword |
+| aws.s3.bucket.name | The AWS S3 bucket name. | keyword |
+| aws.s3.object.key | The AWS S3 Object key. | keyword |
 | data_stream.dataset | Data stream dataset. | constant_keyword |
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |

packages/trellix_edr_cloud/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.0"
 name: trellix_edr_cloud
 title: Trellix EDR Cloud
-version: "0.5.0"
+version: "0.5.1"
 description: Collect logs from Trellix EDR Cloud with Elastic Agent.
 type: integration
 categories: