From c5d5ce15377bca3178acf51afb9bac78fb769ac1 Mon Sep 17 00:00:00 2001 From: subham sarkar Date: Thu, 21 Sep 2023 15:17:38 +0530 Subject: [PATCH] microsoft_sqlserver/data_stream/transaction_log: Add support for `fetch_from_all_databases` for mssql (#7500) --- .../_dev/build/docs/README.md | 73 +++++++++++-------- packages/microsoft_sqlserver/changelog.yml | 5 ++ .../data_stream/performance/manifest.yml | 2 +- .../agent/stream/stream.yml.hbs | 33 ++++++--- .../data_stream/transaction_log/manifest.yml | 12 ++- packages/microsoft_sqlserver/docs/README.md | 73 +++++++++++-------- packages/microsoft_sqlserver/manifest.yml | 12 +-- 7 files changed, 126 insertions(+), 84 deletions(-) diff --git a/packages/microsoft_sqlserver/_dev/build/docs/README.md b/packages/microsoft_sqlserver/_dev/build/docs/README.md index f1029208246..acc36611542 100644 --- a/packages/microsoft_sqlserver/_dev/build/docs/README.md +++ b/packages/microsoft_sqlserver/_dev/build/docs/README.md @@ -1,55 +1,58 @@ # Microsoft SQL Server Integration -The Microsoft SQL Server integration package allows you to search, observe and visualize the SQL Server audit logs and metrics through Elasticsearch. +The Microsoft SQL Server integration package allows you to search, observe and visualize the SQL Server audit logs, as well as performance and transaction log metrics, through Elasticsearch. + +Auditing an instance of the SQL Server Database Engine or an individual database involves tracking and logging events that occur on the Database Engine. + +SQL Server audit lets you create server audits, which can contain server audit specifications for server-level events, and database audit specifications for database-level events. -Auditing an instance of the SQL Server Database Engine or an individual database involves tracking and logging events that occur on the Database Engine. -SQL Server audit lets you create server audits, which can contain server audit specifications for server level events, and database audit specifications for database level events. See: [SQL Server Audit page](https://docs.microsoft.com/en-us/sql/relational-databases/security/auditing/sql-server-audit-database-engine?view=sql-server-ver15) for more information on SQL Server auditing. `performance` metrics gathers the list of performance objects available on that server. Each server will have a different list of performance objects depending on the installed software. + `transaction_log` metrics collects all usage stats and the total space usage. ## Named Instance -Microsoft SQL Server has a feature that allows running multiple databases on the same host (or clustered hosts) with separate settings. Establish named instance connection by using the instance name along with the host name (Ex: `host/instance_name` or `host:named_instance_port`) to collect metrics. Details of the host configuration is provided below. +Microsoft SQL Server has a feature that allows running multiple databases on the same host (or clustered hosts) with separate settings. Establish a named instance connection by using the instance name along with the hostname (e.g. `host/instance_name` or `host:named_instance_port`) to collect metrics. Details of the host configuration are provided below. ### Query by Instance Name or Server Name in Kibana -The data can be visualized in Kibana by filtering based on the instance name and server name. The instance name can be filtered by `mssql.metrics.instance_name` and server name by `mssql.metrics.server_name` fields. +The data can be visualized in Kibana by filtering based on the instance name and server name. The instance name can be filtered by `mssql.metrics.instance_name` and the server name by `mssql.metrics.server_name` fields. ## Permission/Access required for tables -If you browse MSDN for the following tables, you will find a "Permissions" section which defines the permission needed for each table, e.g [sys.dm_db_log_space_usage](https://learn.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-log-space-usage-transact-sql?view=sql-server-ver15) Permissions section. +If you browse MSDN for the following tables, you will find a "Permissions" section that defines the permission needed for each table, e.g. [sys.dm_db_log_space_usage](https://learn.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-log-space-usage-transact-sql?view=sql-server-ver15) Permissions section. -### 1.transaction_log +### 1. transaction_log - [sys.databases](https://learn.microsoft.com/en-us/sql/relational-databases/system-compatibility-views/sys-sysdatabases-transact-sql?view=sql-server-ver16) - [sys.dm_db_log_space_usage](https://learn.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-log-space-usage-transact-sql?view=sql-server-ver16) - [sys.dm_db_log_stats (DB_ID)](https://learn.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-log-stats-transact-sql?view=sql-server-ver16) -### 2.performance +### 2. performance - [sys.dm_os_performance_counters](https://learn.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-performance-counters-transact-sql?view=sql-server-ver16) ## Host Configuration -Integration supports collecting metrics from single host. For multi host metrics, each host can be run as a new integration. +Integration supports collecting metrics from a single host. For multi-host metrics, each host can be run as a new integration. -As part of the input configuration, need to provide the user name, password and the host details. The host configuration supports both named instance or default(no-name) instance, as per the syntax below. +As part of the input configuration, need to provide the user name, password and host details. The host configuration supports both named instances or default(no-name) instances, as per the syntax below. ### Connecting to Default Instance (host) -* `host` ex: `localhost` (Instance name is not needed when connecting to default instance) or -* `host:port ` ex: `localhost:1433` +* `host` (e.g. `localhost` (Instance name is not needed when connecting to default instance)) +* `host:port` (e.g. `localhost:1433`) ### Connecting to Named Instance (host) -* `host/instance_name` ex: `localhost/namedinstance_01` or -* `host:named_instance_port` ex: `localhost:60873` +* `host/instance_name` (e.g. `localhost/namedinstance_01`) +* `host:named_instance_port` (e.g. `localhost:60873`) ## Compatibility -The package collects `performance` and `transaction_log` metrics, and `audit` events from the event log. Other log sources such as file are not supported. +The package collects `performance` and `transaction_log` metrics, and `audit` events from the event log. Other log sources such as files are not supported. ## Configuration @@ -61,53 +64,59 @@ See: [SQL Server Audit Action Groups and Actions](https://docs.microsoft.com/en- See: [Instructions on how to enable auditing for SQL Server](https://docs.microsoft.com/en-us/sql/relational-databases/security/auditing/create-a-server-audit-and-server-audit-specification?view=sql-server-ver15). ->Note: For the integration package to be able to read and send audit events the event target must be configured to be Windows event log. +> Note: For the integration package to be able to read and send audit events the event target must be configured to be Windows event log. -### audit events +### Audit events -Enable to collect SQL Server audit events from the specified windows event log channel. +Collects SQL Server audit events from the specified windows event log channel. -### log +### Log The SQL Server `log` contains user-defined events and certain system events you can use for troubleshooting. See: [View the SQL Server error log in SQL Server Management Studio](https://docs.microsoft.com/en-us/sql/relational-databases/performance/view-the-sql-server-error-log-sql-server-management-studio?view=sql-server-ver16) -### performance metrics +### Performance metrics -Collects the `performance` counter metrics. Dynamic counter feature provides flexibility to collect metrics by providing the counter as an input. +Collects the `performance` counter metrics. The dynamic counter feature provides flexibility to collect metrics by providing the counter as an input. This input can be a regular expression which will filter results based on pattern. -For example, if %grant% is given as input, it will enable metrics collection for all of the counters with name like 'Memory Grants Pending', 'Active memory grants count' etc. -MSSQL supports limited set of RegExp, See [here] (https://learn.microsoft.com/en-us/previous-versions/sql/sql-server-2008-r2/ms187489(v=sql.105)?redirectedfrom=MSDN) for details. +For example, if %grant% is given as input, it will enable metrics collection for all of the counters with names like 'Memory Grants Pending', 'Active memory grants count' etc. +MSSQL supports a limited set of regular expressions, See [here](https://learn.microsoft.com/en-us/previous-versions/sql/sql-server-2008-r2/ms187489(v=sql.105)?redirectedfrom=MSDN) for details. ->Note: Dynamic counters will go through some basic ingest pipeline post-processing to make counter names in lower case and remove special characters and these fields will not have any static field mappings. +> Note: Dynamic counters will go through some basic ingest pipeline post-processing to make counter names in lowercase and remove special characters and these fields will not have any static field mappings. -The feature `merge_results` has been introduced in 8.4 beats which create a single event by combining the metrics together in a single event. See [here](https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-sql.html#_example_merge_multiple_queries_to_single_event) for details. +The feature `merge_results` has been introduced in 8.4 beats which creates a single event by combining the metrics together in a single event. See [here](https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-sql.html#_example_merge_multiple_queries_to_single_event) for details. See: [Instructions about each performance counter metrics](https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-performance-counters-transact-sql?view=sql-server-ver15) -### transaction_log metrics +### Transaction log metrics Collects system level `transaction_log` metrics information for SQL Server instance. -Metrics for user level databases can be collected by providing list of user dbs for which metrics is to be collected. +Metrics for user-level databases can be collected by providing a list of user databases for which metrics are to be collected. See: [Instructions and the operations supported by transaction log](https://docs.microsoft.com/en-us/sql/relational-databases/logs/the-transaction-log-sql-server?view=sql-server-ver15) +### Fetch from all databases + +To simplify the process of fetching metrics from all databases on the server, you can enable the `Fetch from all databases` toggle when configuring the integration. This field overrides manually entered database names in the `Databases` input and instead fetches the required `transaction_log` metrics from all databases, including system and user-defined databases. + +Keep in mind that this feature is disabled by default and needs to be manually enabled to be activated. + ### Password URL encoding -When there are special characters in password, pass the special characters by using URL encoding. +When the password contains special characters, pass these special characters using URL encoding. ## Logs ### audit -The SQL Server audit dataset provides events from the configured Windows event log channel. All SQL Server audit specific fields are available in the `sqlserver.audit` field group. +The SQL Server audit dataset provides events from the configured Windows event log channel. All SQL Server audit-specific fields are available in the `sqlserver.audit` field group. {{fields "audit"}} ### log -The Microsoft SQL Server `log` dataset parses error logs created by Microsoft SQL server. +The Microsoft SQL Server `log` dataset parses error logs created by the Microsoft SQL server. {{event "log"}} @@ -117,7 +126,7 @@ The Microsoft SQL Server `log` dataset parses error logs created by Microsoft SQ ### performance -The Microsoft SQL Server `performance` dataset provides metrics from the performance counter table. All `performance` metrics will be available in `sqlserver.metrics` field group. +The Microsoft SQL Server `performance` dataset provides metrics from the performance counter table. All `performance` metrics will be available in the `sqlserver.metrics` field group. {{event "performance"}} @@ -125,7 +134,7 @@ The Microsoft SQL Server `performance` dataset provides metrics from the perform ### transaction_log -The Microsoft SQL Server `transaction_log` dataset provides metrics from the log space usage and log stats tables of the system databases. All `transaction_log` metrics will be available in `sqlserver.metrics` field group. +The Microsoft SQL Server `transaction_log` dataset provides metrics from the log space usage and log stats tables of the system databases. All `transaction_log` metrics will be available in the `sqlserver.metrics` field group. {{event "transaction_log"}} diff --git a/packages/microsoft_sqlserver/changelog.yml b/packages/microsoft_sqlserver/changelog.yml index 53178334358..f40bf88bb66 100644 --- a/packages/microsoft_sqlserver/changelog.yml +++ b/packages/microsoft_sqlserver/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.2.0" + changes: + - description: Add support for fetch_from_all_databases for Microsoft SQL server + type: enhancement + link: https://github.com/elastic/integrations/pull/7500 - version: "2.1.1" changes: - description: Add ecs mapping for error.code to avoid type conflicts diff --git a/packages/microsoft_sqlserver/data_stream/performance/manifest.yml b/packages/microsoft_sqlserver/data_stream/performance/manifest.yml index 11ed8f62bc2..1d1c443a396 100644 --- a/packages/microsoft_sqlserver/data_stream/performance/manifest.yml +++ b/packages/microsoft_sqlserver/data_stream/performance/manifest.yml @@ -21,4 +21,4 @@ streams: title: Microsoft SQL Server performance metrics description: Collect Microsoft SQL Server performance metrics elasticsearch: - index_mode: "time_series" \ No newline at end of file + index_mode: "time_series" diff --git a/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs b/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs index 71c6c2aa029..32db2e48adf 100644 --- a/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs +++ b/packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs @@ -1,19 +1,30 @@ metricsets: ["query"] -# Specify hosts in the below format. TODO:hosts need to be updated to support multiple entries. +# Specify hosts in the below format. TODO: hosts need to be updated to support multiple entries. hosts: - sqlserver://{{username}}:{{password}}@{{hosts}} period: {{period}} driver: mssql raw_data.enabled: true -# Collect the transaction logs from the system database +fetch_from_all_databases: {{fetch_from_all_databases}} + +# Collect the transaction logs from the database(s) sql_queries: -{{#if databases}} -{{#each databases as |database_name i|}} - - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', database_id FROM sys.databases WHERE name='{{database_name}}';" - response_format: table - - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_mb, l.active_log_size_mb,l.log_backup_time,l.log_since_last_log_backup_mb,l.log_since_last_checkpoint_mb,l.log_recovery_size_mb from sys.dm_db_log_stats(DB_ID('{{database_name}}')) l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('{{database_name}}') ;" - response_format: table - - query: "USE [{{database_name}}] ; SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('{{database_name}}') ;" - response_format: table -{{/each}} +{{#if fetch_from_all_databases}} +- query: "SELECT @@servername AS server_name, @@servicename AS instance_name, DB_NAME() AS 'database_name', DB_ID() AS database_id;" + response_format: table +- query: "SELECT @@servername AS server_name, @@servicename AS instance_name, DB_NAME() AS 'database_name', l.database_id, l.total_log_size_mb, l.active_log_size_mb,l.log_backup_time,l.log_since_last_log_backup_mb,l.log_since_last_checkpoint_mb,l.log_recovery_size_mb from sys.dm_db_log_stats(DB_ID()) l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID();" + response_format: table +- query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID();" + response_format: table +{{else}} + {{#if databases}} + {{#each databases as |database_name i|}} + - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', database_id FROM sys.databases WHERE name='{{database_name}}';" + response_format: table + - query: "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_mb, l.active_log_size_mb,l.log_backup_time,l.log_since_last_log_backup_mb,l.log_since_last_checkpoint_mb,l.log_recovery_size_mb from sys.dm_db_log_stats(DB_ID('{{database_name}}')) l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('{{database_name}}') ;" + response_format: table + - query: "USE [{{database_name}}]; SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', l.database_id, l.total_log_size_in_bytes As total_log_size_bytes, l.used_log_space_in_bytes As used_log_space_bytes, l.used_log_space_in_percent As used_log_space_pct, l.log_space_in_bytes_since_last_backup from sys.dm_db_log_space_usage l INNER JOIN sys.databases s ON l.database_id = s.database_id WHERE s.database_id = DB_ID('{{database_name}}') ;" + response_format: table + {{/each}} + {{/if}} {{/if}} \ No newline at end of file diff --git a/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml b/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml index b30584614f6..57f6f6a7089 100644 --- a/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml +++ b/packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml @@ -16,9 +16,17 @@ streams: multi: true required: true show_user: true - description: Default system databases are preloaded. For user defined databases please add additional rows and enter the database name. + description: Fetch the transaction_logs metrics from the provided databases. Both, user-defined and system database names can be provided as input, with the system databases already being included by default. default: ["master", "model", "tempdb", "msdb"] + - name: fetch_from_all_databases + type: bool + title: Fetch from all databases + multi: false + required: false + show_user: false + description: Option to enable fetching transaction_logs metrics from all databases, including both system and user-defined databases. This option overrides any database names provided in the 'Databases' field and instead considers all databases. + default: false title: Microsoft SQL Server transaction_log metrics description: Collect Microsoft SQL Server transaction_log metrics elasticsearch: - index_mode: "time_series" \ No newline at end of file + index_mode: "time_series" diff --git a/packages/microsoft_sqlserver/docs/README.md b/packages/microsoft_sqlserver/docs/README.md index 284d3015f1f..e907ec7b2bd 100644 --- a/packages/microsoft_sqlserver/docs/README.md +++ b/packages/microsoft_sqlserver/docs/README.md @@ -1,55 +1,58 @@ # Microsoft SQL Server Integration -The Microsoft SQL Server integration package allows you to search, observe and visualize the SQL Server audit logs and metrics through Elasticsearch. +The Microsoft SQL Server integration package allows you to search, observe and visualize the SQL Server audit logs, as well as performance and transaction log metrics, through Elasticsearch. + +Auditing an instance of the SQL Server Database Engine or an individual database involves tracking and logging events that occur on the Database Engine. + +SQL Server audit lets you create server audits, which can contain server audit specifications for server-level events, and database audit specifications for database-level events. -Auditing an instance of the SQL Server Database Engine or an individual database involves tracking and logging events that occur on the Database Engine. -SQL Server audit lets you create server audits, which can contain server audit specifications for server level events, and database audit specifications for database level events. See: [SQL Server Audit page](https://docs.microsoft.com/en-us/sql/relational-databases/security/auditing/sql-server-audit-database-engine?view=sql-server-ver15) for more information on SQL Server auditing. `performance` metrics gathers the list of performance objects available on that server. Each server will have a different list of performance objects depending on the installed software. + `transaction_log` metrics collects all usage stats and the total space usage. ## Named Instance -Microsoft SQL Server has a feature that allows running multiple databases on the same host (or clustered hosts) with separate settings. Establish named instance connection by using the instance name along with the host name (Ex: `host/instance_name` or `host:named_instance_port`) to collect metrics. Details of the host configuration is provided below. +Microsoft SQL Server has a feature that allows running multiple databases on the same host (or clustered hosts) with separate settings. Establish a named instance connection by using the instance name along with the hostname (e.g. `host/instance_name` or `host:named_instance_port`) to collect metrics. Details of the host configuration are provided below. ### Query by Instance Name or Server Name in Kibana -The data can be visualized in Kibana by filtering based on the instance name and server name. The instance name can be filtered by `mssql.metrics.instance_name` and server name by `mssql.metrics.server_name` fields. +The data can be visualized in Kibana by filtering based on the instance name and server name. The instance name can be filtered by `mssql.metrics.instance_name` and the server name by `mssql.metrics.server_name` fields. ## Permission/Access required for tables -If you browse MSDN for the following tables, you will find a "Permissions" section which defines the permission needed for each table, e.g [sys.dm_db_log_space_usage](https://learn.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-log-space-usage-transact-sql?view=sql-server-ver15) Permissions section. +If you browse MSDN for the following tables, you will find a "Permissions" section that defines the permission needed for each table, e.g. [sys.dm_db_log_space_usage](https://learn.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-log-space-usage-transact-sql?view=sql-server-ver15) Permissions section. -### 1.transaction_log +### 1. transaction_log - [sys.databases](https://learn.microsoft.com/en-us/sql/relational-databases/system-compatibility-views/sys-sysdatabases-transact-sql?view=sql-server-ver16) - [sys.dm_db_log_space_usage](https://learn.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-log-space-usage-transact-sql?view=sql-server-ver16) - [sys.dm_db_log_stats (DB_ID)](https://learn.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-log-stats-transact-sql?view=sql-server-ver16) -### 2.performance +### 2. performance - [sys.dm_os_performance_counters](https://learn.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-performance-counters-transact-sql?view=sql-server-ver16) ## Host Configuration -Integration supports collecting metrics from single host. For multi host metrics, each host can be run as a new integration. +Integration supports collecting metrics from a single host. For multi-host metrics, each host can be run as a new integration. -As part of the input configuration, need to provide the user name, password and the host details. The host configuration supports both named instance or default(no-name) instance, as per the syntax below. +As part of the input configuration, need to provide the user name, password and host details. The host configuration supports both named instances or default(no-name) instances, as per the syntax below. ### Connecting to Default Instance (host) -* `host` ex: `localhost` (Instance name is not needed when connecting to default instance) or -* `host:port ` ex: `localhost:1433` +* `host` (e.g. `localhost` (Instance name is not needed when connecting to default instance)) +* `host:port` (e.g. `localhost:1433`) ### Connecting to Named Instance (host) -* `host/instance_name` ex: `localhost/namedinstance_01` or -* `host:named_instance_port` ex: `localhost:60873` +* `host/instance_name` (e.g. `localhost/namedinstance_01`) +* `host:named_instance_port` (e.g. `localhost:60873`) ## Compatibility -The package collects `performance` and `transaction_log` metrics, and `audit` events from the event log. Other log sources such as file are not supported. +The package collects `performance` and `transaction_log` metrics, and `audit` events from the event log. Other log sources such as files are not supported. ## Configuration @@ -61,47 +64,53 @@ See: [SQL Server Audit Action Groups and Actions](https://docs.microsoft.com/en- See: [Instructions on how to enable auditing for SQL Server](https://docs.microsoft.com/en-us/sql/relational-databases/security/auditing/create-a-server-audit-and-server-audit-specification?view=sql-server-ver15). ->Note: For the integration package to be able to read and send audit events the event target must be configured to be Windows event log. +> Note: For the integration package to be able to read and send audit events the event target must be configured to be Windows event log. -### audit events +### Audit events -Enable to collect SQL Server audit events from the specified windows event log channel. +Collects SQL Server audit events from the specified windows event log channel. -### log +### Log The SQL Server `log` contains user-defined events and certain system events you can use for troubleshooting. See: [View the SQL Server error log in SQL Server Management Studio](https://docs.microsoft.com/en-us/sql/relational-databases/performance/view-the-sql-server-error-log-sql-server-management-studio?view=sql-server-ver16) -### performance metrics +### Performance metrics -Collects the `performance` counter metrics. Dynamic counter feature provides flexibility to collect metrics by providing the counter as an input. +Collects the `performance` counter metrics. The dynamic counter feature provides flexibility to collect metrics by providing the counter as an input. This input can be a regular expression which will filter results based on pattern. -For example, if %grant% is given as input, it will enable metrics collection for all of the counters with name like 'Memory Grants Pending', 'Active memory grants count' etc. -MSSQL supports limited set of RegExp, See [here] (https://learn.microsoft.com/en-us/previous-versions/sql/sql-server-2008-r2/ms187489(v=sql.105)?redirectedfrom=MSDN) for details. +For example, if %grant% is given as input, it will enable metrics collection for all of the counters with names like 'Memory Grants Pending', 'Active memory grants count' etc. +MSSQL supports a limited set of regular expressions, See [here](https://learn.microsoft.com/en-us/previous-versions/sql/sql-server-2008-r2/ms187489(v=sql.105)?redirectedfrom=MSDN) for details. ->Note: Dynamic counters will go through some basic ingest pipeline post-processing to make counter names in lower case and remove special characters and these fields will not have any static field mappings. +> Note: Dynamic counters will go through some basic ingest pipeline post-processing to make counter names in lowercase and remove special characters and these fields will not have any static field mappings. -The feature `merge_results` has been introduced in 8.4 beats which create a single event by combining the metrics together in a single event. See [here](https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-sql.html#_example_merge_multiple_queries_to_single_event) for details. +The feature `merge_results` has been introduced in 8.4 beats which creates a single event by combining the metrics together in a single event. See [here](https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-module-sql.html#_example_merge_multiple_queries_to_single_event) for details. See: [Instructions about each performance counter metrics](https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-performance-counters-transact-sql?view=sql-server-ver15) -### transaction_log metrics +### Transaction log metrics Collects system level `transaction_log` metrics information for SQL Server instance. -Metrics for user level databases can be collected by providing list of user dbs for which metrics is to be collected. +Metrics for user-level databases can be collected by providing a list of user databases for which metrics are to be collected. See: [Instructions and the operations supported by transaction log](https://docs.microsoft.com/en-us/sql/relational-databases/logs/the-transaction-log-sql-server?view=sql-server-ver15) +### Fetch from all databases + +To simplify the process of fetching metrics from all databases on the server, you can enable the `Fetch from all databases` toggle when configuring the integration. This field overrides manually entered database names in the `Databases` input and instead fetches the required `transaction_log` metrics from all databases, including system and user-defined databases. + +Keep in mind that this feature is disabled by default and needs to be manually enabled to be activated. + ### Password URL encoding -When there are special characters in password, pass the special characters by using URL encoding. +When the password contains special characters, pass these special characters using URL encoding. ## Logs ### audit -The SQL Server audit dataset provides events from the configured Windows event log channel. All SQL Server audit specific fields are available in the `sqlserver.audit` field group. +The SQL Server audit dataset provides events from the configured Windows event log channel. All SQL Server audit-specific fields are available in the `sqlserver.audit` field group. **Exported fields** @@ -266,7 +275,7 @@ The SQL Server audit dataset provides events from the configured Windows event l ### log -The Microsoft SQL Server `log` dataset parses error logs created by Microsoft SQL server. +The Microsoft SQL Server `log` dataset parses error logs created by the Microsoft SQL server. An example event for `log` looks as following: @@ -406,7 +415,7 @@ An example event for `log` looks as following: ### performance -The Microsoft SQL Server `performance` dataset provides metrics from the performance counter table. All `performance` metrics will be available in `sqlserver.metrics` field group. +The Microsoft SQL Server `performance` dataset provides metrics from the performance counter table. All `performance` metrics will be available in the `sqlserver.metrics` field group. An example event for `performance` looks as following: @@ -580,7 +589,7 @@ An example event for `performance` looks as following: ### transaction_log -The Microsoft SQL Server `transaction_log` dataset provides metrics from the log space usage and log stats tables of the system databases. All `transaction_log` metrics will be available in `sqlserver.metrics` field group. +The Microsoft SQL Server `transaction_log` dataset provides metrics from the log space usage and log stats tables of the system databases. All `transaction_log` metrics will be available in the `sqlserver.metrics` field group. An example event for `transaction_log` looks as following: diff --git a/packages/microsoft_sqlserver/manifest.yml b/packages/microsoft_sqlserver/manifest.yml index a7b5bec87a5..676be7bffd3 100644 --- a/packages/microsoft_sqlserver/manifest.yml +++ b/packages/microsoft_sqlserver/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: microsoft_sqlserver title: "Microsoft SQL Server" -version: "2.1.1" +version: "2.2.0" license: basic description: Collect events from Microsoft SQL Server with Elastic Agent type: integration @@ -21,16 +21,16 @@ screenshots: size: 600x600 type: image/png - src: /img/sqlserver-transaction-dashboard.png - title: Microsoft SQL Server transaction log Dashboard + title: Microsoft SQL Server Transaction Log Dashboard size: 600x600 type: image/png - src: /img/sqlserver-errorlog-dashboard.png - title: Microsoft SQL Server Error log Dashboard + title: Microsoft SQL Server Error Log Dashboard size: 600x600 type: image/png icons: - src: /img/microsoft-sql-server-logo.svg - title: Microsof SQL Server + title: Microsoft SQL Server size: 32x32 type: image/svg+xml policy_templates: @@ -54,7 +54,7 @@ policy_templates: show_user: true default: - localhost - description: Host Name Ex - (DefaultInstance - host or host:port) (Named Instance - host/instanceName or host:NamedInstancePort) + description: Hostname (e.g. For `Default Instance`, use the format `host` or `host:port` and for `Named Instance` use the format `host/instanceName` or `host:NamedInstancePort`) - name: password type: password title: Password @@ -70,7 +70,7 @@ policy_templates: required: true show_user: true default: domain\username - description: "Domain users: Pre-encode username when passing backslash. Eg: {domain}%5C{username} instead of {domain}\\\\{username}" + description: "Domain users: Pre-encode username when passing backslash e.g. {domain}%5C{username} instead of {domain}\\\\{username}" title: Collect Microsoft SQL Server performance and transaction_log metrics description: Collecting performance and transaction_log metrics from Microsoft SQL Server instances owner: