Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[AWS] Add EMR logs dashboard #7075

Merged
merged 6 commits into from Jul 21, 2023
Merged

[AWS] Add EMR logs dashboard #7075

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7de320f
add dashboard
gpop63 Jul 20, 2023
9e5413f
bump package version
gpop63 Jul 20, 2023
8908a6d
add dashboard img in manifest
gpop63 Jul 20, 2023
8312497
update the PR ID in changelog
gpop63 Jul 20, 2023
a553f85
Merge remote-tracking branch 'upstream/main' into add_aws-emr_logs_da…
gpop63 Jul 20, 2023
7dd93a9
fix dashboard migration versions for CI
gpop63 Jul 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 5 additions & 0 deletions packages/aws/changelog.yml
View file
Open in desktop
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "1.50.2"
changes:
- description: Add EMR logs dashboard.
type: enhancement
link: https://github.com/elastic/integrations/pull/6895
gpop63 marked this conversation as resolved.
Show resolved Hide resolved
- version: "1.50.0"
changes:
- description: Add EMR logs data stream.
Expand Down
Binary file added packages/aws/img/filebeat-aws-emr-overview.png
View file
Open in desktop
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
317 changes: 317 additions & 0 deletions packages/aws/kibana/dashboard/aws-4c404390-2724-11ee-a3cb-33de1feb08f6.json
View file
Open in desktop
@@ -0,0 +1,317 @@
{
"attributes": {
"description": "",
"kibanaSavedObjectMeta": {
"searchSourceJSON": {
"filter": [
{
"$state": {
"store": "appState"
},
"meta": {
"alias": null,
"disabled": false,
"field": "data_stream.dataset",
"indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index",
"key": "data_stream.dataset",
"negate": false,
"params": {
"query": "aws.emr_logs"
},
"type": "phrase"
},
"query": {
"match_phrase": {
"data_stream.dataset": "aws.emr_logs"
}
}
}
],
"query": {
"language": "kuery",
"query": ""
}
}
},
"optionsJSON": {
"hidePanelTitles": false,
"syncColors": false,
"syncCursor": true,
"syncTooltips": false,
"useMargins": true
},
"panelsJSON": [
{
"embeddableConfig": {
"attributes": {
"description": "",
"references": [
{
"id": "logs-*",
"name": "indexpattern-datasource-layer-063f2b19-5cf4-4a39-a699-f00d4ab5aeae",
"type": "index-pattern"
}
],
"state": {
"adHocDataViews": {},
"datasourceStates": {
"formBased": {
"layers": {
"063f2b19-5cf4-4a39-a699-f00d4ab5aeae": {
"columnOrder": [
"82883b46-c5bf-426b-8145-8512134d9e46",
"5071a31f-c22c-4521-a95a-b091d815210f"
],
"columns": {
"5071a31f-c22c-4521-a95a-b091d815210f": {
"dataType": "number",
"isBucketed": false,
"label": "Count of records",
"operationType": "count",
"params": {
"emptyAsNull": true
},
"scale": "ratio",
"sourceField": "___records___"
},
"82883b46-c5bf-426b-8145-8512134d9e46": {
"customLabel": true,
"dataType": "string",
"isBucketed": true,
"label": "Process name",
"operationType": "terms",
"params": {
"exclude": [],
"excludeIsRegex": false,
"include": [],
"includeIsRegex": false,
"missingBucket": false,
"orderBy": {
"columnId": "5071a31f-c22c-4521-a95a-b091d815210f",
"type": "column"
},
"orderDirection": "desc",
"otherBucket": true,
"parentFormat": {
"id": "terms"
},
"size": 10
},
"scale": "ordinal",
"sourceField": "process.name"
}
},
"incompleteColumns": {},
"sampling": 1
}
}
},
"textBased": {
"layers": {}
}
},
"filters": [],
"internalReferences": [],
"query": {
"language": "kuery",
"query": ""
},
"visualization": {
"columns": [
{
"columnId": "82883b46-c5bf-426b-8145-8512134d9e46",
"isTransposed": false
},
{
"columnId": "5071a31f-c22c-4521-a95a-b091d815210f",
"isTransposed": false
}
],
"layerId": "063f2b19-5cf4-4a39-a699-f00d4ab5aeae",
"layerType": "data"
}
},
"title": "",
"type": "lens",
"visualizationType": "lnsDatatable"
},
"enhancements": {},
"hidePanelTitles": false
},
"gridData": {
"h": 15,
"i": "67059371-5528-45be-b6e0-1525e0e215cc",
"w": 24,
"x": 0,
"y": 0
},
"panelIndex": "67059371-5528-45be-b6e0-1525e0e215cc",
"title": "Top Process Names",
"type": "lens",
"version": "8.8.1"
},
{
"embeddableConfig": {
"attributes": {
"description": "",
"references": [
{
"id": "logs-*",
"name": "indexpattern-datasource-layer-c96dc367-7669-4582-8b49-d5dfaf453256",
"type": "index-pattern"
}
],
"state": {
"adHocDataViews": {},
"datasourceStates": {
"formBased": {
"layers": {
"c96dc367-7669-4582-8b49-d5dfaf453256": {
"columnOrder": [
"ecb84283-f29f-4ad8-a24d-004dc1f78612",
"ce1a3d48-fcab-4c38-ada6-6330fbd87ffa"
],
"columns": {
"ce1a3d48-fcab-4c38-ada6-6330fbd87ffa": {
"dataType": "number",
"isBucketed": false,
"label": "Count of records",
"operationType": "count",
"params": {
"emptyAsNull": true
},
"scale": "ratio",
"sourceField": "___records___"
},
"ecb84283-f29f-4ad8-a24d-004dc1f78612": {
"dataType": "string",
"isBucketed": true,
"label": "Top 5 values of log.level",
"operationType": "terms",
"params": {
"exclude": [],
"excludeIsRegex": false,
"include": [],
"includeIsRegex": false,
"missingBucket": false,
"orderBy": {
"columnId": "ce1a3d48-fcab-4c38-ada6-6330fbd87ffa",
"type": "column"
},
"orderDirection": "desc",
"otherBucket": true,
"parentFormat": {
"id": "terms"
},
"size": 5
},
"scale": "ordinal",
"sourceField": "log.level"
}
},
"incompleteColumns": {},
"sampling": 1
}
}
},
"textBased": {
"layers": {}
}
},
"filters": [],
"internalReferences": [],
"query": {
"language": "kuery",
"query": ""
},
"visualization": {
"layers": [
{
"categoryDisplay": "default",
"layerId": "c96dc367-7669-4582-8b49-d5dfaf453256",
"layerType": "data",
"legendDisplay": "default",
"metrics": [
"ce1a3d48-fcab-4c38-ada6-6330fbd87ffa"
],
"nestedLegend": false,
"numberDisplay": "percent",
"primaryGroups": [
"ecb84283-f29f-4ad8-a24d-004dc1f78612"
]
}
],
"shape": "pie"
}
},
"title": "",
"type": "lens",
"visualizationType": "lnsPie"
},
"enhancements": {},
"hidePanelTitles": false
},
"gridData": {
"h": 15,
"i": "94471afc-1bf7-4bbc-8e33-1d234b4631a8",
"w": 24,
"x": 24,
"y": 0
},
"panelIndex": "94471afc-1bf7-4bbc-8e33-1d234b4631a8",
"title": "Log Level",
"type": "lens",
"version": "8.8.1"
},
{
"embeddableConfig": {
"enhancements": {},
"hidePanelTitles": false
},
"gridData": {
"h": 15,
"i": "9d6500f1-b8a2-4b8a-a78b-32267bda7e45",
"w": 48,
"x": 0,
"y": 15
},
"panelIndex": "9d6500f1-b8a2-4b8a-a78b-32267bda7e45",
"panelRefName": "panel_9d6500f1-b8a2-4b8a-a78b-32267bda7e45",
"title": "EMR Logs",
"type": "search",
"version": "8.8.1"
}
],
"timeRestore": false,
"title": "[Logs AWS] EMR Overview",
"version": 1
},
"coreMigrationVersion": "8.8.0",
"created_at": "2023-07-20T17:49:07.190Z",
"id": "aws-4c404390-2724-11ee-a3cb-33de1feb08f6",
"managed": false,
"references": [
{
"id": "logs-*",
"name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index",
"type": "index-pattern"
},
{
"id": "logs-*",
"name": "67059371-5528-45be-b6e0-1525e0e215cc:indexpattern-datasource-layer-063f2b19-5cf4-4a39-a699-f00d4ab5aeae",
"type": "index-pattern"
},
{
"id": "logs-*",
"name": "94471afc-1bf7-4bbc-8e33-1d234b4631a8:indexpattern-datasource-layer-c96dc367-7669-4582-8b49-d5dfaf453256",
"type": "index-pattern"
},
{
"id": "aws-c1543010-2724-11ee-a3cb-33de1feb08f6",
"name": "9d6500f1-b8a2-4b8a-a78b-32267bda7e45:panel_9d6500f1-b8a2-4b8a-a78b-32267bda7e45",
"type": "search"
}
],
"type": "dashboard",
"typeMigrationVersion": "8.7.0"
}
41 changes: 41 additions & 0 deletions packages/aws/kibana/search/aws-c1543010-2724-11ee-a3cb-33de1feb08f6.json
View file
Open in desktop
@@ -0,0 +1,41 @@
{
"attributes": {
"columns": [],
"description": "",
"grid": {},
"hideChart": false,
"isTextBasedQuery": false,
"kibanaSavedObjectMeta": {
"searchSourceJSON": {
"filter": [],
"indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index",
"query": {
"language": "kuery",
"query": "data_stream.dataset : \"aws.emr_logs\" "
}
}
},
"sort": [
[
"@timestamp",
"desc"
]
],
"timeRestore": false,
"title": "[AWS] EMR Logs Search",
"usesAdHocDataView": false
},
"coreMigrationVersion": "8.8.0",
"created_at": "2023-07-20T17:42:11.089Z",
"id": "aws-c1543010-2724-11ee-a3cb-33de1feb08f6",
"managed": false,
"references": [
{
"id": "logs-*",
"name": "kibanaSavedObjectMeta.searchSourceJSON.index",
"type": "index-pattern"
}
],
"type": "search",
"typeMigrationVersion": "8.0.0"
}
7 changes: 6 additions & 1 deletion packages/aws/manifest.yml
View file
Open in desktop
@@ -1,7 +1,7 @@
format_version: 1.0.0
name: aws
title: AWS
version: 1.50.0
version: 1.50.2
license: basic
description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
type: integration
Expand Down Expand Up @@ -786,6 +786,11 @@ policy_templates:
title: Collect EMR logs from CloudWatch
description: Collecting logs from EMR using aws-cloudwatch input
input_group: logs
screenshots:
- src: /img/filebeat-aws-emr-overview.png
title: filebeat aws emr overview
size: 1714x893
type: image/png
icons:
- src: /img/logo_emr.svg
title: AWS EMR logo
Expand Down