[Exchange Server] GA of Integration, Add Dashbord Panel Titles & Syst…

…em Tests (#9560)

* Add Dashboard Titles

* Add Dashboard Titles

* Change Version to GA

* adjust PR in Changelog

* Add System Tests to all datstreams

* fix imap system test config

* remove Folder structure out of system tests sample logs

* Fix mapping

* Add convert for inode field

* specify numeric_keyword_fields in system tests

packages/microsoft_exchange_server/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,8 @@
+version: "3.0"
+services:
+  exchange_server:
+    image: alpine
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+      - ${SERVICE_LOGS_DIR}:/var/log
+    command: /bin/sh -c "cp /sample_logs/* /var/log/"

packages/microsoft_exchange_server/_dev/deploy/docker/sample_logs/imappop_Imap4.log

@@ -0,0 +1 @@
+2024-01-24T15:30:19.847Z,00000000000ABC12,2,1.2.3.4:143,10.11.12.13:65468,example123,118,31,34,authenticate,PLAIN,"R=OK;Msg=""Proxy:Host123.domain.tld:1993:SSL;ProxySuccess"";LiveIdAR=OK;ActivityContextData=0cb2fd35-94c0-44de-9860-134d27654078",

packages/microsoft_exchange_server/_dev/deploy/docker/sample_logs/imappop_Pop3.log

@@ -0,0 +1 @@
+2024-01-24T15:31:51.067Z,00000000000ABC12,1,1.2.3.4:110,10.11.12.13:12345,ccw.altitude,1,17,5,user,ccw.altitude,R=OK,

packages/microsoft_exchange_server/_dev/deploy/docker/sample_logs/smtp_SmtpRecive.log

@@ -0,0 +1 @@
+2024-01-25T15:14:39.031Z,NETBIOS\\Default Frontend NETBIOS,08DC1DB8591B229A,2,10.11.12.13:25,10.11.12.14:53228,<,EHLO mgt.my.domain.tld,

packages/microsoft_exchange_server/_dev/deploy/docker/sample_logs/smtp_SmtpSend.log

@@ -0,0 +1 @@
+2024-01-25T15:14:39.460Z,Inbound Proxy Internal Send Connector,08DC1DB8591B22A0,1,,10.11.12.13:2525,*,,attempting to connect

packages/microsoft_exchange_server/_dev/deploy/docker/sample_logs/test-httpproxy.log

@@ -0,0 +1 @@
+2024-01-24T15:26:47.957Z,3422ea93-768f-4cd4-8b0c-578038deb0b2,15,1,2507,35,R:{750498CA-0EBD-4E7F-B2F6-377AD1BDD198}:20373;RT:Execute;CI:{FF8D5880-5A7A-4AF7-8DDA-8F662BD6BCB6}:155680117;CID:{FF8D5880-5A7A-4AF7-8DDA-8F662BD6BCB6},Mapi,mail.domain.tld,/mapi/emsmdb/,,Negotiate,true,DOMAIN\user,domain.tld,MailboxGuid~0aa89cf8-aa07-4103-8a1d-ca9e619f223e,Microsoft Office/16.0 (Windows NT 10.0; Microsoft Outlook 16.0.16731; Pro),10.12.13.14,Host123,200,200,,POST,Proxy,Host456.domain.tld,15.01.2507.000,CrossForest,MailboxGuidWithDomain,Database~a6c4dbb1-3265-4fbf-9dc6-754dffd67275~~2024-02-23T15:26:43,,,70,132,,,0,1,,0,,0,,0,0,,0,10,0,0,0,0,3,0,0,0,2,0,10,0,4,7,7,7,10,,?MailboxId=0e36a769-e2a9-4d1d-98df-80be2753326c@domain.tld,,BeginRequest=2024-01-24T15:26:47.947Z;CorrelationID=<empty>;ProxyState-Run=None;FEAuth=BEVersion-1942063563;BeginGetRequestStream=2024-01-24T15:26:47.953Z;OnRequestStreamReady=2024-01-24T15:26:47.953Z;BeginGetResponse=2024-01-24T15:26:47.953Z;OnResponseReady=2024-01-24T15:26:47.957Z;EndGetResponse=2024-01-24T15:26:47.957Z;ProxyState-Complete=ProxyResponseData;SharedCacheGuard=0;EndRequest=2024-01-24T15:26:47.957Z;,,,|RoutingDB:0cb2fd35-94c0-44de-9860-134d27654078,,,CafeV1

packages/microsoft_exchange_server/_dev/deploy/docker/sample_logs/test-messagetracking.log

@@ -0,0 +1,4 @@
+2024-01-25T15:16:09.843Z,,,,exchange-mail,No suitable shadow servers,,SMTP,HAREDIRECTFAIL,70971234566456,<20240124222112.B4AE1234EF@host01.my.domain.com>,2fd37dca-1234-5bfb-175d-08dc1db88f52,mailuser@my.domain.com,,15054,1,,,Undelivered Mail Returned to Sender,MAILER-DAEMON@host01.my.domain.com,root@host01.my.domain.com,,Incoming,,,,S:DeliveryPriority=Normal;S:OriginalFromAddress=root@host01.my.domain.com;S:AccountForest=my.domain.com,Email,dc69df25-1234-564c-41c4-08dc1db88f7f,15.02.0330.005
+2024-01-25T15:16:09.949Z,10.11.12.14,exchange-mail.my.domain.com,10.11.12.14,exchange-mail,08DC1DB12C345BE5;2024-01-25T15:16:09.544Z;0,exchange-mail\Default exchange-mail,SMTP,RECEIVE,70912345566403,<20240123200014.123F425E28@host01.my.domain.com>,1e6eb197-c6b4-1234-1b69-56dc1db88f50,mailuser@my.domain.com,,7229,1,,,vzdump backup status (host01.my.domain.com): backup successful,root@host01.my.domain.com,root@host01.my.domain.com,0cA: ,Incoming,,10.11.12.13,10.11.12.14,S:ProxyHop1=exchange-mail.my.domain.com(10.11.12.14);S:MessageValue=MediumHigh;S:Replication=Failed;S:FirstForestHop=exchange-mail.my.domain.com;S:FromEntity=Internet;S:ProxiedClientIPAddress=10.11.12.13;S:ProxiedClientHostname=host01.my.domain.com;S:DeliveryPriority=Normal;S:AccountForest=my.domain.com,Email,05503123-c5b9-46fe-1234-56dc1db88f8f,15.02.0330.005
+2024-01-25T15:16:14.415Z,10.11.12.14,exchange-mail.my.domain.com,10.11.12.14,exchange-mail,08DC1DB12C345BE9;2024-01-25T15:16:12.885Z;0,exchange-mail\Default exchange-mail,SMTP,RECEIVE,70912345566407,<20240123200018.123C42553@pve-vhost01.my.domain.com>,c95b5dd1-f520-1234-e6dc-56dc1db8914d,mailuser@my.domain.com,,8251,1,,,vzdump backup status (pve-vhost01.my.domain.com): backup successful,root@pve-vhost01.my.domain.com,root@pve-vhost01.my.domain.com,0cA: ,Incoming,,10.11.12.15,10.11.12.14,S:ProxyHop1=exchange-mail.my.domain.com(10.11.12.14);S:MessageValue=MediumHigh;S:Replication=Failed;S:FirstForestHop=exchange-mail.my.domain.com;S:FromEntity=Internet;S:ProxiedClientIPAddress=10.11.12.15;S:ProxiedClientHostname=pve-vhost01.my.domain.com;S:DeliveryPriority=Normal;S:AccountForest=my.domain.com,Email,d6aef52d-0e05-1234-e29b-56dc1db89238,15.02.0330.005
+2024-01-07T00:00:07.463Z,192.168.0.1,exchange,192.168.0.2,exchange.example.com,;250 2.0.0OK20240107001234.567E6224C8@monitor.example.com[Hostname=exchange.example.com];ClientSubmitTime:,Intra-Organization SMTP Send Connector,SMTP,SEND,29519319995411,20240107001234.567E6224C8@monitor.example.com,0b7099ea-cb95-1234-328e-08dc5f139ac8,uwe.musterman@example.com,250 2.1.5Recipient OK,38663,1,,,ein Titel,support@example.com,support@example.com,2024-01-07T00:00:05.535Z;LSRV=exchange.example.com:TOTAL-HUB=1.921|SMR=0.127(SMRDE=0.002|SMRC=0.125(SMRCL=0.105|X-SMRCR=0.125))|CAT=1.698(CATOS=0.018(CATSM=0.017(CATSM-Malware Agent=0.017))|CATRESL=0.004|CATORES=1.567(CATRS=1.566(CATRS-ScanMail Routing Agent=0.117|CATRS-Transport Rule Agent=0.002(X-ETREX=0.002)|CATRS-Index Routing Agent=1.444))|CATORT=0.108(CATRT=0.107(CATRT-Journal Agent=0.107)))|QDM=0.010|SMSC=0.006(X-SMSDR=0.011)|SMS=0.076(SMSMBXD=0.071),Originating,,,,S:E2ELatency=1.928;S:MsgRecipCount=1;S:IncludeInSla=True;S:Microsoft.Exchange.Transport.MailRecipient.RequiredTlsAuthLevel=Opportunistic;S:IsSmtpResponseFromExternalServer=False;S:DeliveryPriority=Normal;S:AccountForest=example.com,Email,a7ae9ef9-e10c-4111-19bf-08dc0f111bee,15.01.2507.035

packages/microsoft_exchange_server/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.0.0"
+  changes:
+    - description: GA of Integration, Add Dashbord Panel Titles & added System Tests
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9560
 - version: "0.1.2"
   changes:
     - description: Fix Recipientstatus field type, add custom processor support & adjust docs

packages/microsoft_exchange_server/data_stream/httpproxy/_dev/test/system/test-default-config.yml

@@ -0,0 +1,11 @@
+service: exchange_server
+input: filestream
+data_stream:
+  vars:
+    paths:
+      - "{{SERVICE_LOGS_DIR}}/test-httpproxy.log"
+    preserve_original_event: true
+numeric_keyword_fields:
+  - log.file.device_id
+  - log.file.inode
+  - log.offset

packages/microsoft_exchange_server/data_stream/httpproxy/fields/ecs.yml

@@ -18,3 +18,7 @@
   name: tags
 - external: ecs
   name: user.name
+- external: ecs
+  name: ecs.version
+- external: ecs
+  name: log.file.path

packages/microsoft_exchange_server/data_stream/httpproxy/fields/fields.yml

@@ -136,3 +136,11 @@
   type: ip
 - name: microsoft.exchange.clientipaddress_internal
   type: ip
+- name: input.type
+  type: keyword
+- name: log.file.device_id
+  type: keyword
+- name: log.file.inode
+  type: keyword
+- name: log.offset
+  type: keyword

packages/microsoft_exchange_server/data_stream/imap4_pop3/_dev/test/system/test-default-config.yml

@@ -0,0 +1,11 @@
+service: exchange_server
+input: filestream
+data_stream:
+  vars:
+    paths:
+      - "{{SERVICE_LOGS_DIR}}/imappop_*.log"
+    preserve_original_event: true
+numeric_keyword_fields:
+  - log.file.device_id
+  - log.file.inode
+  - log.offset

packages/microsoft_exchange_server/data_stream/imap4_pop3/fields/ecs.yml

@@ -6,3 +6,5 @@
   name: source.ip
 - external: ecs
   name: tags
+- external: ecs
+  name: ecs.version

packages/microsoft_exchange_server/data_stream/imap4_pop3/fields/fields.yml

@@ -24,3 +24,11 @@
   type: keyword
 - name: microsoft.exchange.logtype
   type: keyword
+- name: input.type
+  type: keyword
+- name: log.file.device_id
+  type: keyword
+- name: log.file.inode
+  type: keyword 
+- name: log.offset
+  type: keyword

packages/microsoft_exchange_server/data_stream/messagetracking/_dev/test/system/test-default-config.yml

@@ -0,0 +1,11 @@
+service: exchange_server
+input: filestream
+data_stream:
+  vars:
+    paths:
+      - "{{SERVICE_LOGS_DIR}}/test-messagetracking.log"
+    preserve_original_event: true
+numeric_keyword_fields:
+  - log.file.device_id
+  - log.file.inode
+  - log.offset

packages/microsoft_exchange_server/data_stream/messagetracking/fields/ecs.yml

@@ -26,3 +26,7 @@
   name: network.bytes
 - external: ecs
   name: tags
+- external: ecs
+  name: ecs.version
+- external: ecs
+  name: log.file.path

packages/microsoft_exchange_server/data_stream/messagetracking/fields/fields.yml

@@ -9,7 +9,7 @@
 - name: microsoft.exchange.networkmessageid
   type: keyword
 - name: microsoft.exchange.recipientstatus
-  type: keyword 
+  type: keyword
 - name: microsoft.exchange.recipientcount
   type: long
 - name: microsoft.exchange.relatedrecipientaddress
@@ -36,3 +36,11 @@
   type: keyword
 - name: microsoft.exchange.logtype
   type: keyword
+- name: input.type
+  type: keyword
+- name: log.file.device_id
+  type: keyword
+- name: log.file.inode
+  type: keyword 
+- name: log.offset
+  type: keyword

packages/microsoft_exchange_server/data_stream/smtp/_dev/test/system/test-default-config.yml

@@ -0,0 +1,11 @@
+service: exchange_server
+input: filestream
+data_stream:
+  vars:
+    paths:
+      - "{{SERVICE_LOGS_DIR}}/smtp_*.log"
+    preserve_original_event: true
+numeric_keyword_fields:
+  - log.file.device_id
+  - log.file.inode
+  - log.offset

packages/microsoft_exchange_server/data_stream/smtp/fields/ecs.yml

@@ -4,3 +4,5 @@
   name: log.file.path
 - external: ecs
   name: tags
+- external: ecs
+  name: ecs.version

packages/microsoft_exchange_server/data_stream/smtp/fields/fields.yml

@@ -16,3 +16,11 @@
   type: keyword
 - name: microsoft.exchange.logtype
   type: keyword
+- name: input.type
+  type: keyword
+- name: log.file.device_id
+  type: keyword
+- name: log.file.inode
+  type: keyword 
+- name: log.offset
+  type: keyword