Skip to content

Commit

Permalink
Merge branch 're-enable-secret-below-8.10' of github.com:ritalwar/int…
Browse files Browse the repository at this point in the history 
…egrations into re-enable-secret-below-8.10
  • Loading branch information
@ritalwar
ritalwar committed Mar 13, 2024
2 parents b2d2ce2 + cb4a7a4 commit 6ee726d
Show file tree
Hide file tree
Showing 122 changed files with 5,162 additions and 603 deletions.
11 changes: 10 additions & 1 deletion .buildkite/scripts/run_sonar_scanner.sh
View file
Expand Up @@ -2,8 +2,17 @@
set -euo pipefail

run_sonar_scanner() {
local message=""
echo "--- Download coverage reports and merge them"
buildkite-agent artifact download build/test-coverage/coverage-*.xml .
if ! buildkite-agent artifact download build/test-coverage/coverage-*.xml . ; then
message="Could not download XML artifacts. Skip coverage."
echo "--- :boom: ${message}"
buildkite-agent annotate \
"[Code inspection] ${message}" \
--context "ctx-sonarqube-no-files" \
--style "warning"
exit 0
fi

echo "Merge all coverage reports"
.buildkite/scripts/merge_xml.sh
Expand Down
3 changes: 3 additions & 0 deletions .github/CODEOWNERS
View file
Validating CODEOWNERS rules …
Expand Up @@ -72,6 +72,8 @@
/packages/azure/data_stream/provisioning @elastic/obs-infraobs-integrations
/packages/azure/data_stream/signinlogs @elastic/obs-infraobs-integrations
/packages/azure/data_stream/springcloudlogs @elastic/obs-infraobs-integrations
/packages/azure/data_stream/application_gateway @elastic/security-service-integrations
/packages/azure/data_stream/firewall_logs @elastic/security-service-integrations
/packages/azure_app_service @elastic/obs-infraobs-integrations
/packages/azure_app_service/data_stream/app_service_logs @elastic/obs-infraobs-integrations
/packages/azure_application_insights @elastic/obs-infraobs-integrations
Expand Down Expand Up @@ -227,6 +229,7 @@
/packages/microsoft_defender_endpoint @elastic/security-service-integrations
/packages/microsoft_dhcp @elastic/sec-windows-platform
/packages/microsoft_exchange_online_message_trace @elastic/security-service-integrations
/packages/microsoft_exchange_server @elastic/sec-windows-platform
/packages/microsoft_sqlserver @elastic/obs-infraobs-integrations
/packages/mimecast @elastic/security-service-integrations
/packages/modsecurity @elastic/sec-deployment-and-devices
Expand Down
6 changes: 3 additions & 3 deletions go.mod
View file
Expand Up @@ -4,7 +4,7 @@ go 1.21.0

require (
github.com/blang/semver v3.5.1+incompatible
github.com/elastic/elastic-package v0.98.1
github.com/elastic/elastic-package v0.98.2
github.com/elastic/go-licenser v0.4.1
github.com/elastic/package-registry v1.23.1
github.com/magefile/mage v1.15.0
Expand Down Expand Up @@ -140,7 +140,7 @@ require (
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rivo/uniseg v0.4.3 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shirou/gopsutil/v3 v3.24.1 // indirect
github.com/shirou/gopsutil/v3 v3.24.2 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/spf13/afero v1.11.0 // indirect
Expand All @@ -155,7 +155,7 @@ require (
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/xlab/treeprint v1.2.0 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.elastic.co/apm/module/apmgorilla/v2 v2.4.8 // indirect
go.elastic.co/apm/module/apmhttp/v2 v2.4.8 // indirect
go.elastic.co/apm/module/apmzap/v2 v2.4.8 // indirect
Expand Down
14 changes: 7 additions & 7 deletions go.sum
View file
Expand Up @@ -97,8 +97,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0 h1:sx1lpZuTG5suJuvgix4FWQFCLFFbzkoOmPoHWYOPLCY=
github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0/go.mod h1:2/30n+2QRzRzus4TPVUV1T3U/j8g2ItUgvP0pcpjLGk=
github.com/elastic/elastic-package v0.98.1 h1:TWQPC4bmOv9EUjROT3KIoCtFGjuFD5EC+zjYfuIH7IA=
github.com/elastic/elastic-package v0.98.1/go.mod h1:qZcT49UAq2JGOEsmJxH6TYjwOOhnAUaouzSwVwsnFvU=
github.com/elastic/elastic-package v0.98.2 h1:/IXy/Ql5m2qYGMTruGSyDrZa3oW8f7D9fz8CYGi4sqY=
github.com/elastic/elastic-package v0.98.2/go.mod h1:O1ERev5BK6C7MvNnoYqghmxrOByEqnbxaZ/GkfwERX4=
github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4=
Expand Down Expand Up @@ -405,8 +405,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI=
github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU=
github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y=
github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
Expand Down Expand Up @@ -470,8 +470,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.einride.tech/aip v0.66.0 h1:XfV+NQX6L7EOYK11yoHHFtndeaWh3KbD9/cN/6iWEt8=
go.einride.tech/aip v0.66.0/go.mod h1:qAhMsfT7plxBX+Oy7Huol6YUvZ0ZzdUz26yZsQwfl1M=
go.elastic.co/apm/module/apmgorilla/v2 v2.4.8 h1:Yulr18ASd4fK3nzQsCxGgFSE4bbS8nouQlS1/ZgmDRs=
Expand Down Expand Up @@ -599,7 +599,7 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand Down
10 changes: 10 additions & 0 deletions packages/apache_tomcat/changelog.yml
View file
Expand Up @@ -4,6 +4,16 @@
- description: Enable secrets for sensitive fields. For more details, refer https://www.elastic.co/guide/en/fleet/current/agent-policy.html#agent-policy-secret-values
type: enhancement
link: https://github.com/elastic/integrations/pull/9321
- version: "1.3.3"
changes:
- description: Fix event.outcome for redirection status_codes 3xx.
type: bugfix
link: https://github.com/elastic/integrations/pull/9348
- version: "1.3.2"
changes:
- description: Fix non-matching grok patterns in access log pipeline for 302 errors.
type: bugfix
link: https://github.com/elastic/integrations/pull/9243
- version: "1.3.1"
changes:
- description: Disable secrets for older stack versions due to errors.
Expand Down
1 change: 1 addition & 0 deletions packages/apache_tomcat/data_stream/access/_dev/test/pipeline/test-access.log
View file
@@ -1,4 +1,5 @@
81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] "POST /host-manager/images/asf-logo.svg HTTP/1.1" 200 20486 81.2.69.145 + 400 "http://localhost:8080/host-manager/html" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" X-Forwarded-For="127.0.0.1, 127.0.0.2"
81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] "POST /host-manager/images/asf-logo.svg HTTP/1.1" 302 - 81.2.69.145 + 400 "http://localhost:8080/host-manager/html" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" X-Forwarded-For="127.0.0.1, 127.0.0.2"
81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] "POST /host-manager/images/asf-logo.svg HTTP/1.1" 200 20486 X 400 "http://localhost:8080/host-manager/html" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" X-Forwarded-For="127.0.0.1"
81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] "POST /host-manager/images/asf-logo.svg HTTP/1.1" 200 20486 50 "http://localhost:8080/host-manager/html" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" X-Forwarded-For=""
81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] "POST /host-manager/images/asf-logo.svg HTTP/1.1" 200 20486 81.2.69.145 40 "http://localhost:8080/host-manager/html" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36" X-Forwarded-For="127.0.0.1, 127.0.0.3"
Expand Down
79 changes: 79 additions & 0 deletions packages/apache_tomcat/data_stream/access/_dev/test/pipeline/test-access.log-expected.json
View file
Expand Up @@ -82,6 +82,85 @@
"version": "109.0.0.0"
}
},
{
"@timestamp": "2023-03-02T13:28:17.000Z",
"apache_tomcat": {
"access": {
"connection_status": "+",
"http": {
"ident": "-",
"useragent": "admin"
},
"ip": {
"local": "81.2.69.145"
},
"response_time": 400.0
}
},
"client": {
"ip": [
"127.0.0.1",
"127.0.0.2"
]
},
"ecs": {
"version": "8.7.0"
},
"event": {
"category": [
"web"
],
"kind": "event",
"module": "apache_tomcat",
"original": "81.2.69.144 - admin [02/Mar/2023:18:58:17 +0530] \"POST /host-manager/images/asf-logo.svg HTTP/1.1\" 302 - 81.2.69.145 + 400 \"http://localhost:8080/host-manager/html\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36\" X-Forwarded-For=\"127.0.0.1, 127.0.0.2\"",
"outcome": "success",
"type": [
"access"
]
},
"http": {
"request": {
"method": "POST",
"referrer": "http://localhost:8080/host-manager/html"
},
"response": {
"status_code": 302
},
"version": "1.1"
},
"related": {
"ip": [
"81.2.69.144",
"81.2.69.145",
"127.0.0.1",
"127.0.0.2"
]
},
"source": {
"ip": "81.2.69.144"
},
"tags": [
"preserve_original_event"
],
"url": {
"extension": "svg",
"original": "/host-manager/images/asf-logo.svg",
"path": "/host-manager/images/asf-logo.svg"
},
"user_agent": {
"device": {
"name": "Other"
},
"name": "Chrome",
"original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
"os": {
"full": "Windows 10",
"name": "Windows",
"version": "10"
},
"version": "109.0.0.0"
}
},
{
"@timestamp": "2023-03-02T13:28:17.000Z",
"apache_tomcat": {
Expand Down
6 changes: 3 additions & 3 deletions packages/apache_tomcat/data_stream/access/elasticsearch/ingest_pipeline/default.yml
View file
Expand Up @@ -42,7 +42,7 @@ processors:
field: _tmp.dissectgrok
tag: 'grok_parse_log_dissectgrok'
patterns:
- '^%{NUMBER:http.response.status_code} %{POSINT:destination.bytes}( %{GREEDYDATA:_tmp.grok})?$'
- '^%{NUMBER:http.response.status_code} (-|%{POSINT:destination.bytes})( %{GREEDYDATA:_tmp.grok})?$'
on_failure:
- append:
field: error.message
Expand Down Expand Up @@ -97,11 +97,11 @@ processors:
- set:
field: event.outcome
value: success
if: ctx.http?.response?.status_code != null && ctx.http.response.status_code >= 200 && ctx.http.response.status_code < 300
if: ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400
- set:
field: event.outcome
value: failure
if: ctx.http?.response?.status_code != null && ctx.http.response.status_code >= 400 && ctx.http.response.status_code < 600
if: ctx.http?.response?.status_code != null && ctx.http.response.status_code >= 400
- remove:
if: ctx.destination?.bytes == '-'
field: destination.bytes
Expand Down
16 changes: 12 additions & 4 deletions packages/carbon_black_cloud/_dev/build/docs/README.md
View file
Expand Up @@ -6,6 +6,12 @@ The VMware Carbon Black Cloud integration collects and parses data from the Carb

This module has been tested against `Alerts API (v6)`, `Audit Log Events (v3)` and `Vulnerability Assessment (v1)`.

## Version 1.21+ Update Disclaimer
Starting from version 1.21, if using multiple AWS data streams simultaneously configured to use AWS SQS, separate SQS queues should be configured per
data stream. The default values of file selector regexes have been commented out for this reason. The only reason the global queue now exists is to avoid
a breaking change while upgrading to version 1.21 and above. A separate SQS queue per data stream should help fix the data loss that's been occurring in the
older versions.

## Requirements

### In order to ingest data from the AWS S3 bucket you must:
Expand All @@ -21,21 +27,23 @@ This module has been tested against `Alerts API (v6)`, `Audit Log Events (v3)` a

### To collect data from AWS SQS, follow the below steps:
1. If data forwarding to an AWS S3 Bucket hasn't been configured, then first setup an AWS S3 Bucket as mentioned in the above documentation.
2. To setup an SQS queue, follow "Step 1: Create an Amazon SQS queue" mentioned in the [Documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ways-to-add-notification-config-to-bucket.html).
2. To set up an SQS queue, follow "Step 1: Create an Amazon SQS queue" mentioned in the [Documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ways-to-add-notification-config-to-bucket.html).
- While creating an SQS Queue, please provide the same bucket ARN that has been generated after creating an AWS S3 Bucket.
3. Setup event notification for an S3 bucket. Follow this [Link](https://docs.aws.amazon.com/AmazonS3/latest/userguide/enable-event-notifications.html).
- The user has to perform Step 3 for all the data-streams individually, and each time prefix parameter should be set the same as the S3 Bucket List Prefix as created earlier. (for example, `alert_logs/` for alert data stream.)
3. Set up event notification for an S3 bucket. Follow this [Link](https://docs.aws.amazon.com/AmazonS3/latest/userguide/enable-event-notifications.html).
- The user has to perform Step 3 for all the data streams individually, and each time prefix parameter should be set the same as the S3 Bucket List Prefix as created earlier. (for example, `alert_logs/` for the alert data stream.)
- For all the event notifications that have been created, select the event type as s3:ObjectCreated:*, select the destination type SQS Queue, and select the queue that has been created in Step 2.

**Note**:
- Credentials for the above AWS S3 and SQS input types should be configured using the [link](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-aws-s3.html#aws-credentials-config).
- Data collection via AWS S3 Bucket and AWS SQS are mutually exclusive in this case.
- When configuring SQS queues, separate queues should be used for each data stream instead of the global SQS queue from version 1.21 onwards to avoid data
loss. File selectors should not be used to filter out data stream logs using the global queue as it was in versions prior.

### In order to ingest data from the APIs you must generate API keys and API Secret Keys:
1. In Carbon Black Cloud, On the left navigation pane, click **Settings > API Access**.
2. Click Add API Key.
3. Give the API key a unique name and description.
- Select the appropriate access level type. Please check required Access Levels & Permissions for integration in below table.
- Select the appropriate access level type. Please check the required Access Levels & Permissions for integration in the table below.
**Note:** To use a custom access level, select Custom from the Access Level type drop-down menu and specify the Custom Access Level.
- Optional: Add authorized IP addresses.
- You can restrict the use of an API key to a specific set of IP addresses for security reasons.
Expand Down
13 changes: 13 additions & 0 deletions packages/carbon_black_cloud/changelog.yml
View file
@@ -1,4 +1,17 @@
# newer versions go on top
- version: "1.21.1"
changes:
- description: Fix handling of network direction.
type: bugfix
link: https://github.com/elastic/integrations/pull/9340
- version: "1.21.0"
changes:
- description: Introduced data stream specific SQS queues.
type: enhancement
link: https://github.com/elastic/integrations/pull/9303
- description: Fixed data loss issue by providing option for local SQS queues.
type: bugfix
link: https://github.com/elastic/integrations/pull/9303
- version: "1.20.0"
changes:
- description: Set sensitive values as secret and add missing mappings.
Expand Down
4 changes: 3 additions & 1 deletion packages/carbon_black_cloud/data_stream/alert/agent/stream/aws-s3.yml.hbs
View file
Expand Up @@ -15,7 +15,9 @@ bucket_list_prefix: {{bucket_list_prefix}}

{{else}}

{{#if queue_url}}
{{#if queue_url_alert}}
queue_url: {{queue_url_alert}}
{{else if queue_url}}
queue_url: {{queue_url}}
{{/if}}
{{#if visibility_timeout}}
Expand Down

0 comments on commit 6ee726d

Please sign in to comment.