From d8360dd3aa72099200ceee0dfed861bd852eee38 Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Thu, 2 May 2024 13:44:33 +0200 Subject: [PATCH 01/15] Initial --- docs/sources/tempo/configuration/_index.md | 32 +++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index f1610fa606d..8a9d5ae83c7 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -365,7 +365,6 @@ metrics_generator: # Attribute Key to multiply span metrics [span_multiplier_key: | default = ""] - # Registry configuration registry: @@ -385,6 +384,31 @@ metrics_generator: # The maximum length of label values. Label values exceeding this limit will be truncated. [max_label_value_length: | default = 2048] + # WAL Storage configuration for traces + traces_storage: + + # Path to the metrics-generator WAL directory + # If set, will create separate wals per tenant by joining the path with the tenant ID + [path: | default = ""] + + # ??? + completedfilepath: "" + + # ??? + blocksfilepath: "" + + # ??? + v2_encoding: none + + # ??? + search_encoding: none + + # ??? + ingestion_time_range_slack: 0s + + # WAL Encoding Format Version + [version: | default = "vParquet3"] + # Storage and remote write configuration storage: @@ -409,6 +433,12 @@ metrics_generator: # considered in metrics generation. # This is to filter out spans that are outdated. [metrics_ingestion_time_range_slack: | default = 30s] + + # Timeout for generator requests (???) + [query_timeout: | default = 30s ] + + # Overides the key used to register the metrics-generator in the ring. + [override_ring_key: | default = "ring"] ``` ## Query-frontend From 5cb42ca6e9c6473cf148110361c2b791fcd27045 Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Thu, 2 May 2024 13:56:34 +0200 Subject: [PATCH 02/15] Coppied from other WAL --- docs/sources/tempo/configuration/_index.md | 32 ++++++++++++++-------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index 8a9d5ae83c7..e569a281a7d 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -387,7 +387,7 @@ metrics_generator: # WAL Storage configuration for traces traces_storage: - # Path to the metrics-generator WAL directory + # Where to store the traces while they are being apeended to # If set, will create separate wals per tenant by joining the path with the tenant ID [path: | default = ""] @@ -397,16 +397,26 @@ metrics_generator: # ??? blocksfilepath: "" - # ??? - v2_encoding: none - - # ??? - search_encoding: none - - # ??? - ingestion_time_range_slack: 0s - - # WAL Encoding Format Version + # WAL encoding/compression. + # options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 + [v2_encoding: | default = ??? ] # I think (zstd) ? + + # Defines the search data encoding/compression protocol. + # Options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 + [search_encoding: | default = ???] # I think (snappy) ? + + # When a span is written to the WAL it adjusts the start and end times of the block it is written to. + # This block start and end time range is then used when choosing blocks for search. + # This is also used for querying traces by ID when the start and end parameters are specified. To prevent spans too far + # in the past or future from impacting the block start and end times we use this configuration option. + # This option only allows spans that occur within the configured duration to adjust the block start and + # end times. + # This can result in trace not being found if the trace falls outside the slack configuration value as the + # start and end times of the block will not be updated in this case. + [ingestion_time_range_slack: | default = ???] # I think 2m ? + + # WAL Format Version + # Options: v2, vParquet, vParquet2, vParquet3 [version: | default = "vParquet3"] # Storage and remote write configuration From 3432de492b61b803261b638643a1f827a5f6fdca Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Thu, 2 May 2024 14:58:46 +0200 Subject: [PATCH 03/15] Seperated block --- docs/sources/tempo/configuration/_index.md | 120 +++++++++++---------- 1 file changed, 63 insertions(+), 57 deletions(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index e569a281a7d..1edfb439f42 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -30,6 +30,8 @@ The Tempo configuration options include: - [Local storage recommendations](#local-storage-recommendations) - [Storage block configuration example](#storage-block-configuration-example) - [Memberlist](#memberlist) + - [Configuration Blocks](#configuration-blocks) + - [WAL Config](#wal-config) - [Overrides](#overrides) - [Ingestion limits](#ingestion-limits) - [Standard overrides](#standard-overrides) @@ -384,41 +386,14 @@ metrics_generator: # The maximum length of label values. Label values exceeding this limit will be truncated. [max_label_value_length: | default = 2048] - # WAL Storage configuration for traces - traces_storage: + # Configuration block for the Write Ahead Log (WAL) + traces_storage: - # Where to store the traces while they are being apeended to - # If set, will create separate wals per tenant by joining the path with the tenant ID + # Path to store the wal files. + # Must be set. + # Example: "/var/tempo/generator/traces" [path: | default = ""] - # ??? - completedfilepath: "" - - # ??? - blocksfilepath: "" - - # WAL encoding/compression. - # options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 - [v2_encoding: | default = ??? ] # I think (zstd) ? - - # Defines the search data encoding/compression protocol. - # Options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 - [search_encoding: | default = ???] # I think (snappy) ? - - # When a span is written to the WAL it adjusts the start and end times of the block it is written to. - # This block start and end time range is then used when choosing blocks for search. - # This is also used for querying traces by ID when the start and end parameters are specified. To prevent spans too far - # in the past or future from impacting the block start and end times we use this configuration option. - # This option only allows spans that occur within the configured duration to adjust the block start and - # end times. - # This can result in trace not being found if the trace falls outside the slack configuration value as the - # start and end times of the block will not be updated in this case. - [ingestion_time_range_slack: | default = ???] # I think 2m ? - - # WAL Format Version - # Options: v2, vParquet, vParquet2, vParquet3 - [version: | default = "vParquet3"] - # Storage and remote write configuration storage: @@ -426,7 +401,8 @@ metrics_generator: path: # Configuration for the Prometheus Agent WAL - wal: + # https://github.com/prometheus/prometheus/v2.51.2/tsdb/agent/db.go#L62-L84 + wal: # How long to wait when flushing samples on shutdown [remote_write_flush_deadline: | default = 1m] @@ -448,7 +424,7 @@ metrics_generator: [query_timeout: | default = 30s ] # Overides the key used to register the metrics-generator in the ring. - [override_ring_key: | default = "ring"] + [override_ring_key: | default = "metrics-generator"] ``` ## Query-frontend @@ -1094,29 +1070,11 @@ storage: [queue_depth: ] # Configuration block for the Write Ahead Log (WAL) - wal: - - # where to store the head blocks while they are being appended to - # Example: "wal: /var/tempo/wal" - [path: ] - - # wal encoding/compression. - # options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 - [v2_encoding: | default = snappy] - - # Defines the search data encoding/compression protocol. - # Options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 - [search_encoding: | default = none] - - # When a span is written to the WAL it adjusts the start and end times of the block it is written to. - # This block start and end time range is then used when choosing blocks for search. - # This is also used for querying traces by ID when the start and end parameters are specified. To prevent spans too far - # in the past or future from impacting the block start and end times we use this configuration option. - # This option only allows spans that occur within the configured duration to adjust the block start and - # end times. - # This can result in trace not being found if the trace falls outside the slack configuration value as the - # start and end times of the block will not be updated in this case. - [ingestion_time_range_slack: | default = 2m] + wal: + [path: | default = "/var/tempo/wal"] + [v2_encoding: | default = snappy] + [search_encoding: | default = none] + [ingestion_time_range_slack: | default = 2m] # block configuration block: @@ -1244,6 +1202,54 @@ memberlist: ``` +## Configuration Blocks + +Defines re-used configuration blocks + +### WAL Config + +The Storage WAL configuration block. + +```yaml +# Where to store the wal files while they are being apeended to. +# Must be set. +# Example: "/var/tempo/wal +[path: | default = ""] + +# Where to store the completed wal files +# If not set (""), will join the `path` with "completed" to generate the effective path +# Example: "/var/tempo/wal/completed" +[completedfilepath: | default = join(.path, "/completed")] + +# Where to store the intermediate blocks while they are being apeended to. +# Will always join the `path` with "blocks" to generate the effective path +# Example: "/var/tempo/wal/blocks" (ignored) +[blocksfilepath: | = join(.path, "/blocks")] + +# WAL encoding/compression. +# options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 +[v2_encoding: | default = "zstd" ] + +# Defines the search data encoding/compression protocol. +# Options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 +[search_encoding: | default = "snappy"] + +# When a span is written to the WAL it adjusts the start and end times of the block it is written to. +# This block start and end time range is then used when choosing blocks for search. +# This is also used for querying traces by ID when the start and end parameters are specified. To prevent spans too far +# in the past or future from impacting the block start and end times we use this configuration option. +# This option only allows spans that occur within the configured duration to adjust the block start and +# end times. +# This can result in trace not being found if the trace falls outside the slack configuration value as the +# start and end times of the block will not be updated in this case. +[ingestion_time_range_slack: | default = unset] + +# WAL File Format Version +# Options: v2, vParquet, vParquet2, vParquet3 +[version: | default = "vParquet3"] +``` + + ## Overrides Tempo provides an overrides module for users to set global or per-tenant override settings. From d67d432d539337a6d455f0e5c7a6d915c581bb9d Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Thu, 2 May 2024 15:52:05 +0200 Subject: [PATCH 04/15] Added `metric-generator.processor.service_graphs.peer_attributes` --- docs/sources/tempo/configuration/_index.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index 1edfb439f42..7e9b601a568 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -323,6 +323,12 @@ metrics_generator: # per additional dimension instead of one. [enable_client_server_prefix: | default = false] + # Attributes that will be used to create a peer edge + # Attributes are searched in the order they are provided + # See https://pkg.go.dev/go.opentelemetry.io/otel/semconv/v1.18.0 + # Example: ["peer.service", "db.name", "db.system", "host.name"] + [peer_attributes: | default = ["peer.service", "db.name", "db.system"] ] + # Attribute Key to multiply span metrics [span_multiplier_key: | default = ""] From e2b5b8c0d2ed640e22b0e4ccc02ead50da7c1ddb Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Thu, 2 May 2024 16:52:51 +0200 Subject: [PATCH 05/15] Add missing `span_metrics` fields --- docs/sources/tempo/configuration/_index.md | 91 +++++++++++++++++++--- 1 file changed, 80 insertions(+), 11 deletions(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index 7e9b601a568..d3623764a94 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -31,6 +31,7 @@ The Tempo configuration options include: - [Storage block configuration example](#storage-block-configuration-example) - [Memberlist](#memberlist) - [Configuration Blocks](#configuration-blocks) + - [Filter Policies](#filter-policies) - [WAL Config](#wal-config) - [Overrides](#overrides) - [Ingestion limits](#ingestion-limits) @@ -338,7 +339,7 @@ metrics_generator: span_metrics: # Buckets for the latency histogram in seconds. - [histogram_buckets: | default = 0.002, 0.004, 0.008, 0.016, 0.032, 0.064, 0.128, 0.256, 0.512, 1.02, 2.05, 4.10] + [histogram_buckets: | default = 0.002, 0.004, 0.008, 0.016, 0.032, 0.064, 0.128, 0.256, 0.512, 1.02, 2.05, 4.10, 8.20, 16,40] # Configure intrinsic dimensions to add to the metrics. Intrinsic dimensions are taken # directly from the respective resource and span properties. @@ -353,26 +354,49 @@ metrics_generator: [status_code: | default = true] # Whether to add a status message. Important note: The span status message may # contain arbitrary strings and thus have a very high cardinality. - [status_message: | default = false] + [status_message: | default = true] # Additional dimensions to add to the metrics along with the intrinsic dimensions. # Dimensions are searched for in the resource and span attributes and are added to # the metrics if present. [dimensions: ] - # Custom labeling of dimensions is possible via a list of maps consisting of - # "name" , "source_labels" , "join" - # "name" appears in the metrics, "source_labels" are the actual - # attributes that will make up the value of the label and "join" is the - # separator if multiple source_labels are provided - [dimension_mappings: ] + # Custom labeling mapping + dimension_mappings: + + # The metric name + - [name: ] + + # The actual attributes that will make the value of the new label + [source_labels: ] + + # The separator used to join multiple `source_labels` + [join: ] + # Enable traces_target_info metrics [enable_target_info: ] - # Drop specific labels from traces_target_info metrics - [target_info_excluded_dimensions: ] + # Attribute Key to multiply span metrics [span_multiplier_key: | default = ""] + # subprocessor toggles for metrics categories that exist under the umbrella of Span Metrics + subprocessors: + # Toggle the `Latency` metric category + [ 0: | default = true ] + + # Toggle the `Count` metric category + [ 1: | default = true ] + + # Toggle the `Size` metric category + [ 2: | default = true ] + + + # List of policies that will be applied to spans for inclusion or exlusion. + [filter_policies: | default = []] + + # Drop specific labels from `traces_target_info` metrics + [target_info_excluded_dimensions: ] + # Registry configuration registry: @@ -1212,6 +1236,52 @@ memberlist: Defines re-used configuration blocks +### Filter Policies + +Span Filter block + +#### Filter Policy +```yaml +# Exclude filters (Postive Matching) +[include: ] + +# Exclude filters (Negative Matching) +[exclude: ] +``` + +#### Policy Match +```yaml +# How to match the value of attributes +# Options: "strict", "regex" +[match_type: ] + +# List of attributes to match +[attributes: ] + + # Attribute key + - [key: ] + + # Attribute value + [value: ] +``` + +#### Example +```yaml +exclude: + match_type: "regex" + attributes: + - key: "resource.service.name" + value: "unknown_service:myservice" +``` + +```yaml +include: + match_type: "strict" + attributes: + - key: "foo.bar" + value: "baz" +``` + ### WAL Config The Storage WAL configuration block. @@ -1255,7 +1325,6 @@ The Storage WAL configuration block. [version: | default = "vParquet3"] ``` - ## Overrides Tempo provides an overrides module for users to set global or per-tenant override settings. From 1ce273c987adc2c36d5a660edd78397d0e967fae Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Thu, 2 May 2024 17:04:15 +0200 Subject: [PATCH 06/15] Fix wrong bool `status_message` --- docs/sources/tempo/configuration/_index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index d3623764a94..1255f761e5d 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -354,7 +354,7 @@ metrics_generator: [status_code: | default = true] # Whether to add a status message. Important note: The span status message may # contain arbitrary strings and thus have a very high cardinality. - [status_message: | default = true] + [status_message: | default = false] # Additional dimensions to add to the metrics along with the intrinsic dimensions. # Dimensions are searched for in the resource and span attributes and are added to @@ -374,7 +374,7 @@ metrics_generator: [join: ] # Enable traces_target_info metrics - [enable_target_info: ] + [enable_target_info: | default = false] # Attribute Key to multiply span metrics [span_multiplier_key: | default = ""] From 5e416524769ce585a06c13abba881fc0f627be31 Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Fri, 3 May 2024 10:02:26 +0200 Subject: [PATCH 07/15] Some spelling and formating --- docs/sources/tempo/configuration/_index.md | 66 +++++++++++----------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index 1255f761e5d..1e4ecb00c64 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -30,9 +30,9 @@ The Tempo configuration options include: - [Local storage recommendations](#local-storage-recommendations) - [Storage block configuration example](#storage-block-configuration-example) - [Memberlist](#memberlist) - - [Configuration Blocks](#configuration-blocks) - - [Filter Policies](#filter-policies) - - [WAL Config](#wal-config) + - [Configuration blocks](#configuration-blocks) + - [Filter policy config](#filter-policy config) + - [WAL config](#wal-config) - [Overrides](#overrides) - [Ingestion limits](#ingestion-limits) - [Standard overrides](#standard-overrides) @@ -326,7 +326,7 @@ metrics_generator: # Attributes that will be used to create a peer edge # Attributes are searched in the order they are provided - # See https://pkg.go.dev/go.opentelemetry.io/otel/semconv/v1.18.0 + # See: https://pkg.go.dev/go.opentelemetry.io/otel/semconv/v1.18.0 # Example: ["peer.service", "db.name", "db.system", "host.name"] [peer_attributes: | default = ["peer.service", "db.name", "db.system"] ] @@ -379,7 +379,7 @@ metrics_generator: # Attribute Key to multiply span metrics [span_multiplier_key: | default = ""] - # subprocessor toggles for metrics categories that exist under the umbrella of Span Metrics + # Subprocessor toggles for metrics categories that exist under the umbrella of span metrics subprocessors: # Toggle the `Latency` metric category [ 0: | default = true ] @@ -391,8 +391,8 @@ metrics_generator: [ 2: | default = true ] - # List of policies that will be applied to spans for inclusion or exlusion. - [filter_policies: | default = []] + # List of policies that will be applied to spans for inclusion or exclusion. + [filter_policies: | default = []] # Drop specific labels from `traces_target_info` metrics [target_info_excluded_dimensions: ] @@ -419,7 +419,7 @@ metrics_generator: # Configuration block for the Write Ahead Log (WAL) traces_storage: - # Path to store the wal files. + # Path to store the WAL files. # Must be set. # Example: "/var/tempo/generator/traces" [path: | default = ""] @@ -432,7 +432,7 @@ metrics_generator: # Configuration for the Prometheus Agent WAL # https://github.com/prometheus/prometheus/v2.51.2/tsdb/agent/db.go#L62-L84 - wal: + wal: # How long to wait when flushing samples on shutdown [remote_write_flush_deadline: | default = 1m] @@ -1092,12 +1092,11 @@ storage: # the worker pool is used primarily when finding traces by id, but is also used by other pool: - # total number of workers pulling jobs from the queue (default: 400) - [max_workers: ] + # total number of workers pulling jobs from the queue + [max_workers: | default = 30] # length of job queue. imporatant for querier as it queues a job for every block it has to search - # (default: 20000) - [queue_depth: ] + [queue_depth: | default = 10000 ] # Configuration block for the Write Ahead Log (WAL) wal: @@ -1217,9 +1216,9 @@ memberlist: # Timeout for leaving memberlist cluster. [leave_timeout: | default = 5s] - # IP address to listen on for gossip messages. Multiple addresses may be - # specified. Defaults to 0.0.0.0 - [bind_addr: | default = ] + # IP address to listen on for gossip messages. + # Multiple addresses may be specified. + [bind_addr: | default = ["0.0.0.0"] ] # Port to listen on for gossip messages. [bind_port: | default = 7946] @@ -1232,31 +1231,31 @@ memberlist: ``` -## Configuration Blocks +## Configuration blocks -Defines re-used configuration blocks +Defines re-used configuration blocks. -### Filter Policies +### Filter policy config -Span Filter block +Span filter config block -#### Filter Policy +#### Filter policy ```yaml -# Exclude filters (Postive Matching) -[include: ] +# Exclude filters (postive matching) +[include: ] -# Exclude filters (Negative Matching) -[exclude: ] +# Exclude filters (negative matching) +[exclude: ] ``` -#### Policy Match +#### Policy match ```yaml # How to match the value of attributes # Options: "strict", "regex" [match_type: ] # List of attributes to match -[attributes: ] +attributes: # Attribute key - [key: ] @@ -1265,7 +1264,8 @@ Span Filter block [value: ] ``` -#### Example +#### Examples + ```yaml exclude: match_type: "regex" @@ -1282,12 +1282,12 @@ include: value: "baz" ``` -### WAL Config +### WAL config -The Storage WAL configuration block. +The storage WAL configuration block. ```yaml -# Where to store the wal files while they are being apeended to. +# Where to store the wal files while they are being appended to. # Must be set. # Example: "/var/tempo/wal [path: | default = ""] @@ -1297,7 +1297,7 @@ The Storage WAL configuration block. # Example: "/var/tempo/wal/completed" [completedfilepath: | default = join(.path, "/completed")] -# Where to store the intermediate blocks while they are being apeended to. +# Where to store the intermediate blocks while they are being appended to. # Will always join the `path` with "blocks" to generate the effective path # Example: "/var/tempo/wal/blocks" (ignored) [blocksfilepath: | = join(.path, "/blocks")] @@ -1320,7 +1320,7 @@ The Storage WAL configuration block. # start and end times of the block will not be updated in this case. [ingestion_time_range_slack: | default = unset] -# WAL File Format Version +# WAL file format version # Options: v2, vParquet, vParquet2, vParquet3 [version: | default = "vParquet3"] ``` From afdc378843aaaf6942765849ced7a156cad94ab9 Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Wed, 8 May 2024 09:09:15 +0000 Subject: [PATCH 08/15] Keep `histogram_buckets` precision to 3 --- docs/sources/tempo/configuration/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index 3d6f7aeab61..ccd9351437c 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -344,7 +344,7 @@ metrics_generator: span_metrics: # Buckets for the latency histogram in seconds. - [histogram_buckets: | default = 0.002, 0.004, 0.008, 0.016, 0.032, 0.064, 0.128, 0.256, 0.512, 1.02, 2.05, 4.10, 8.20, 16,40] + [histogram_buckets: | default = 0.002, 0.004, 0.008, 0.016, 0.032, 0.064, 0.128, 0.256, 0.512, 1.024, 2.048, 4.096, 8.192, 16.384] # Configure intrinsic dimensions to add to the metrics. Intrinsic dimensions are taken # directly from the respective resource and span properties. From 61827187004392cd6f16635f8f73c1f95e658d22 Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Wed, 8 May 2024 09:17:35 +0000 Subject: [PATCH 09/15] fix `dimension_mappings` formating --- docs/sources/tempo/configuration/_index.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index ccd9351437c..6d263e275f5 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -367,14 +367,11 @@ metrics_generator: [dimensions: ] # Custom labeling mapping - dimension_mappings: - - # The metric name + dimension_mappings: + # The new label name - [name: ] - # The actual attributes that will make the value of the new label [source_labels: ] - # The separator used to join multiple `source_labels` [join: ] @@ -388,14 +385,11 @@ metrics_generator: subprocessors: # Toggle the `Latency` metric category [ 0: | default = true ] - # Toggle the `Count` metric category [ 1: | default = true ] - # Toggle the `Size` metric category [ 2: | default = true ] - # List of policies that will be applied to spans for inclusion or exclusion. [filter_policies: | default = []] From 25d8da5e47af84a8a92ae5463300c8acfdcb8c83 Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Fri, 24 May 2024 16:29:20 +0000 Subject: [PATCH 10/15] Added kv-store --- docs/sources/tempo/configuration/_index.md | 91 +++++++++++++++++++--- 1 file changed, 79 insertions(+), 12 deletions(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index 0e45a97b92e..226f8384df7 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -31,7 +31,8 @@ The Tempo configuration options include: - [Storage block configuration example](#storage-block-configuration-example) - [Memberlist](#memberlist) - [Configuration blocks](#configuration-blocks) - - [Filter policy config](#filter-policy config) + - [Filter policy config](#filter-policy-config) + - [KVStore config](#kvstore-config) - [WAL config](#wal-config) - [Overrides](#overrides) - [Ingestion limits](#ingestion-limits) @@ -292,12 +293,36 @@ metrics_generator: # Ring configuration ring: + kvstore: + [store: | default = memberlist] + [prefix: | default = "collectors/"] - kvstore: + # Period at which to heartbeat the instance + # 0 disables heartbeat alltogether + [heartbeat_period: | default = 5s] - # The metrics-generator uses the ring to balance work across instances. The ring is stored - # in a key-vault store. - [store: | default = memberlist] + # The heartbeat timeout, after which, the instance is skipped. + # 0 disables timeout. + [heartbeat_timeout: | default = 1m] + + # Our Instance ID to register as in the ring. + [instance_id: | default = os.Hostname()] + + # Name of the network interface to read address from. + [instance_interface_names: | default = ["eth0", "en0"] ] + + # Our advertised IP address in the ring, (usefull if the local ip =/= the external ip) + # Will default to the configured `instance_id` ip address, + # if unset, will fallback to ip reported by `instance_interface_names` + # (Effected by `enable_inet6`) + [instance_addr: | default = auto(instance_id, instance_interface_names)] + + # Our advertised port in the ring + # Defaults to the configured GRPC listing port + [instance_port: | default = auto(listen_port)] + + # Enables the registering of ipv6 addresses in the ring. + [enable_inet6: | default = false] # Processor-specific configuration processor: @@ -691,13 +716,9 @@ compactor: [disabled: ] ring: - - kvstore: - - # in a high volume environment multiple compactors need to work together to keep up with incoming blocks. - # this tells the compactors to use a ring stored in memberlist to coordinate. - # Example: "store: memberlist" - [store: ] + kvstore: + [store: | default = memberlist] + [prefix: | default = "collectors/" ] compaction: @@ -1303,6 +1324,52 @@ include: value: "baz" ``` +### KVStore config + +The kvstore configuration block + +```yaml +# Set backing store to use +[store: | default = "consul"] + +# What prefix to use for keys +[prefix: | default = "ring."] + +# Store spesific configs +consul: + [host: | default = "localhost:8500"] + [acl_token: | default = "" ] + [http_client_timeout: | default = 20s] + [consistent_reads: | default = false] + [watch_rate_limit: | default = 1.0] + [watch_burst_size: | default = 1] + [cas_retry_delay: | default 1s] + +etcd: + [endpoints: | default = [] ] + [dial_timeout: | default = 10s] + [max_retries: | default = 10 ] + [tls_enabled: | default = false] + + # TLS config + [tls_cert_path: | default = ""] + [tls_key_path: | default = ""] + [tls_ca_path: | default = ""] + [tls_server_name: | default = ""] + [tls_insecure_skip_verify: | default = false] + [tls_cipher_suites: | default = ""] + [tls_min_version: | default = ""] + + [username: | default = ""] + [password: | default = ""] + +multi: + [primary: | default = ""] + [secondary: | default = ""] + [mirror_enabled: | default = false] + [mirror_timeout: | default = 2s] +``` + ### WAL config The storage WAL configuration block. From f596e4eb215ef758892d2db2e791be53c82e53ea Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Fri, 24 May 2024 16:40:11 +0000 Subject: [PATCH 11/15] Fixup Suggestions --- docs/sources/tempo/configuration/_index.md | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index 226f8384df7..6fa12a0cb2b 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -406,15 +406,6 @@ metrics_generator: # Attribute Key to multiply span metrics [span_multiplier_key: | default = ""] - # Subprocessor toggles for metrics categories that exist under the umbrella of span metrics - subprocessors: - # Toggle the `Latency` metric category - [ 0: | default = true ] - # Toggle the `Count` metric category - [ 1: | default = true ] - # Toggle the `Size` metric category - [ 2: | default = true ] - # List of policies that will be applied to spans for inclusion or exclusion. [filter_policies: | default = []] @@ -455,7 +446,7 @@ metrics_generator: path: # Configuration for the Prometheus Agent WAL - # https://github.com/prometheus/prometheus/v2.51.2/tsdb/agent/db.go#L62-L84 + # https://github.com/prometheus/prometheus/blob/v2.51.2/tsdb/agent/db.go#L62-L84 wal: # How long to wait when flushing samples on shutdown @@ -474,7 +465,7 @@ metrics_generator: # This is to filter out spans that are outdated. [metrics_ingestion_time_range_slack: | default = 30s] - # Timeout for generator requests (???) + # Timeout for metric requests [query_timeout: | default = 30s ] # Overides the key used to register the metrics-generator in the ring. From 900a6361e0a1092ca43c79f084d6adcdb5150097 Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Mon, 27 May 2024 12:58:12 +0200 Subject: [PATCH 12/15] Added `local_blocks` section with `Block` and `Search` configs --- docs/sources/tempo/configuration/_index.md | 211 +++++++++++++-------- 1 file changed, 131 insertions(+), 80 deletions(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index 6fa12a0cb2b..d01dc0f0deb 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -31,8 +31,10 @@ The Tempo configuration options include: - [Storage block configuration example](#storage-block-configuration-example) - [Memberlist](#memberlist) - [Configuration blocks](#configuration-blocks) + - [Block config](#block-config) - [Filter policy config](#filter-policy-config) - [KVStore config](#kvstore-config) + - [Search config](#search-config) - [WAL config](#wal-config) - [Overrides](#overrides) - [Ingestion limits](#ingestion-limits) @@ -412,6 +414,46 @@ metrics_generator: # Drop specific labels from `traces_target_info` metrics [target_info_excluded_dimensions: ] + local_blocks: + + # Block configuration + block: + + # Search configuration + search: + + # How often to run flush checks (??) + [flush_check_period: | default = 10s] + + # After this idle period, a trace is closed. (??) + [trace_idle_period: | default = 10s] + + # Maximum (???) (lifespan for any given block?) + [max_block_duration: | default = 1m] + + # Maximum block size before it is closed (?) + [max_block_bytes: | default = 500000000] + + # Maximum (???) + [complete_block_timeout: | default = 1h] + + # (Unused?) + # Max number of "live" traces + [max_live_traces: ] + + # Whether server spans should be filtered in or not. + # true means keep the server kinds (???) + # false means drop the server kinds (???) + [filter_server_spans: | default = true] + + # Number of blocks that are allowed to be processed concurently + [concurrent_blocks: | default = 10] + + # A tuning factor that controls whether the trace-level timestamp columns are used in a metrics query. + # If a block overlaps the time window by less than this ratio, then we skip the columns. + # A value of 1.0 will always load the columns, and 0.0 will never load any. + [time_overlap_cutoff: | default = 0.2] + # Registry configuration registry: @@ -432,7 +474,7 @@ metrics_generator: [max_label_value_length: | default = 2048] # Configuration block for the Write Ahead Log (WAL) - traces_storage: + traces_storage: # Path to store the WAL files. # Must be set. @@ -1058,7 +1100,7 @@ storage: # Cache type to use. Should be one of "redis", "memcached" # Example: "cache: memcached" - # Deprecated. See [cache](#cache) section below. + # Deprecated. See [cache](#cache) section. [cache: ] # Minimum compaction level of block to qualify for bloom filter caching. Default is 0 (disabled), meaning @@ -1072,54 +1114,19 @@ storage: [cache_max_block_age: ] # Configuration parameters that impact trace search - search: - - # Target number of bytes per GET request while scanning blocks. Default is 1MB. Reducing - # this value could positively impact trace search performance at the cost of more requests - # to object storage. - # Example: "chunk_size_bytes: 5_000_000" - [chunk_size_bytes: ] - - # Number of traces to prefetch while scanning blocks. Default is 1000. Increasing this value - # can improve trace search performance at the cost of memory. - # Example: "prefetch_trace_count: 10000" - [prefetch_trace_count: ] - - # Size of read buffers used when performing search on a vparquet block. This value times the read_buffer_count - # is the total amount of bytes used for buffering when performing search on a parquet block. - # Default: 1048576 - [read_buffer_size_bytes: ] - - # Number of read buffers used when performing search on a vparquet block. This value times the read_buffer_size_bytes - # is the total amount of bytes used for buffering when performing search on a parquet block. - # Default: 32 - [read_buffer_count: ] - - # Granular cache control settings for parquet metadata objects - # Deprecated. See [cache](#cache) section below. - cache_control: - - # Specifies if footer should be cached - [footer: | default = false] - - # Specifies if column index should be cached - [column_index: | default = false] - - # Specifies if offset index should be cached - [offset_index: | default = false] - + search: # Background cache configuration. Requires having a cache configured. - # Deprecated. See [cache](#cache) section below. + # Deprecated. See [cache](#cache) section. background_cache: # Memcached caching configuration block - # Deprecated. See [cache](#cache) section below. + # Deprecated. See [cache](#cache) section. memcached: # Redis configuration block # EXPERIMENTAL - # Deprecated. See [cache](#cache) section below. + # Deprecated. See [cache](#cache) section. redis: # the worker pool is used primarily when finding traces by id, but is also used by other @@ -1131,51 +1138,15 @@ storage: # length of job queue. imporatant for querier as it queues a job for every block it has to search [queue_depth: | default = 10000 ] - # Configuration block for the Write Ahead Log (WAL) - wal: + # configuration block for the Write Ahead Log (WAL) + wal: [path: | default = "/var/tempo/wal"] [v2_encoding: | default = snappy] [search_encoding: | default = none] [ingestion_time_range_slack: | default = 2m] # block configuration - block: - # block format version. options: v2, vParquet2, vParquet3, vParquet4 - [version: | default = vParquet3] - - # bloom filter false positive rate. lower values create larger filters but fewer false positives - [bloom_filter_false_positive: | default = 0.01] - - # maximum size of each bloom filter shard - [bloom_filter_shard_size_bytes: | default = 100KiB] - - # number of bytes per index record - [v2_index_downsample_bytes: | default = 1MiB] - - # block encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 - [v2_encoding: | default = zstd] - - # search data encoding/compression. same options as block encoding. - [search_encoding: | default = snappy] - - # number of bytes per search page - [search_page_size_bytes: | default = 1MiB] - - # an estimate of the number of bytes per row group when cutting Parquet blocks. lower values will - # create larger footers but will be harder to shard when searching. It is difficult to calculate - # this field directly and it may vary based on workload. This is roughly a lower bound. - [parquet_row_group_size_bytes: | default = 100MB] - - # Configures attributes to be stored in dedicated columns within the parquet file, rather than in the - # generic attribute key-value list. This allows for more efficient searching of these attributes. - # Up to 10 span attributes and 10 resource attributes can be configured as dedicated columns. - # Requires vParquet3 - parquet_dedicated_columns: - [ - name: , # name of the attribute - type: , # type of the attribute. options: string - scope: # scope of the attribute. options: resource, span - ] + block: ``` ## Memberlist @@ -1268,6 +1239,52 @@ memberlist: Defines re-used configuration blocks. +### Block config + +```yaml +# block format version. options: v2, vParquet2, vParquet3, vParquet4 +[version: | default = vParquet3] + +# bloom filter false positive rate. lower values create larger filters but fewer false positives +[bloom_filter_false_positive: | default = 0.01] + +# maximum size of each bloom filter shard +[bloom_filter_shard_size_bytes: | default = 100KiB] + +# number of bytes per index record +[v2_index_downsample_bytes: | default = 1MiB] + +# block encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 +[v2_encoding: | default = zstd] + +# search data encoding/compression. same options as block encoding. +[search_encoding: | default = snappy] + +# number of bytes per search page +[search_page_size_bytes: | default = 1MiB] + +# an estimate of the number of bytes per row group when cutting Parquet blocks. lower values will +# create larger footers but will be harder to shard when searching. It is difficult to calculate +# this field directly and it may vary based on workload. This is roughly a lower bound. +[parquet_row_group_size_bytes: | default = 100MB] + +# Configures attributes to be stored in dedicated columns within the parquet file, rather than in the +# generic attribute key-value list. This allows for more efficient searching of these attributes. +# Up to 10 span attributes and 10 resource attributes can be configured as dedicated columns. +# Requires vParquet3 +parquet_dedicated_columns: + + # name of the attribute + - [name: ] + + # type of the attribute. options: string + [type: ] + + # scope of the attribute. + # options: resource, span + [scope: ] +``` + ### Filter policy config Span filter config block @@ -1361,6 +1378,40 @@ multi: [mirror_timeout: | default = 2s] ``` +### Search config + +```yaml +# Target number of bytes per GET request while scanning blocks. Default is 1MB. Reducing +# this value could positively impact trace search performance at the cost of more requests +# to object storage. +[chunk_size_bytes: | default = 1000000] + +# Number of traces to prefetch while scanning blocks. Default is 1000. Increasing this value +# can improve trace search performance at the cost of memory. +[prefetch_trace_count: | default = 1000] + +# Number of read buffers used when performing search on a vparquet block. This value times the read_buffer_size_bytes +# is the total amount of bytes used for buffering when performing search on a parquet block. +[read_buffer_count: | default = 32] + +# Size of read buffers used when performing search on a vparquet block. This value times the read_buffer_count +# is the total amount of bytes used for buffering when performing search on a parquet block. +[read_buffer_size_bytes: | default = 1048576] + +# Granular cache control settings for parquet metadata objects +# Deprecated. See [Cache](#cache) section. +cache_control: + + # Specifies if footer should be cached + [footer: | default = false] + + # Specifies if column index should be cached + [column_index: | default = false] + + # Specifies if offset index should be cached + [offset_index: | default = false] +``` + ### WAL config The storage WAL configuration block. From 603041ad6f877ab425b5b407a13132bdb8cbc3f2 Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Mon, 27 May 2024 13:01:20 +0200 Subject: [PATCH 13/15] removed tabs --- docs/sources/tempo/configuration/_index.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index d01dc0f0deb..7e8b8ce2ece 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -417,42 +417,42 @@ metrics_generator: local_blocks: # Block configuration - block: + block: # Search configuration - search: + search: # How often to run flush checks (??) - [flush_check_period: | default = 10s] + [flush_check_period: | default = 10s] # After this idle period, a trace is closed. (??) - [trace_idle_period: | default = 10s] + [trace_idle_period: | default = 10s] # Maximum (???) (lifespan for any given block?) - [max_block_duration: | default = 1m] + [max_block_duration: | default = 1m] # Maximum block size before it is closed (?) - [max_block_bytes: | default = 500000000] + [max_block_bytes: | default = 500000000] # Maximum (???) - [complete_block_timeout: | default = 1h] + [complete_block_timeout: | default = 1h] # (Unused?) # Max number of "live" traces - [max_live_traces: ] + [max_live_traces: ] # Whether server spans should be filtered in or not. # true means keep the server kinds (???) # false means drop the server kinds (???) - [filter_server_spans: | default = true] + [filter_server_spans: | default = true] # Number of blocks that are allowed to be processed concurently - [concurrent_blocks: | default = 10] + [concurrent_blocks: | default = 10] # A tuning factor that controls whether the trace-level timestamp columns are used in a metrics query. # If a block overlaps the time window by less than this ratio, then we skip the columns. # A value of 1.0 will always load the columns, and 0.0 will never load any. - [time_overlap_cutoff: | default = 0.2] + [time_overlap_cutoff: | default = 0.2] # Registry configuration registry: From b44916add26920350a40b18955b4bec158b458b0 Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Mon, 3 Jun 2024 11:04:57 +0200 Subject: [PATCH 14/15] Updated local_blocks descriptions --- docs/sources/tempo/configuration/_index.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index 7e8b8ce2ece..49bf01a2aea 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -422,28 +422,28 @@ metrics_generator: # Search configuration search: - # How often to run flush checks (??) + # How often to run the flush loop to cut idle traces and blocks [flush_check_period: | default = 10s] - # After this idle period, a trace is closed. (??) + # A trace is considered complete after this period of inactivity (no new spans recieved) [trace_idle_period: | default = 10s] - # Maximum (???) (lifespan for any given block?) + # Maximum duration which the head block can be appended to, before cutting it. [max_block_duration: | default = 1m] - # Maximum block size before it is closed (?) + # Maximum size of the head block, before cutting it [max_block_bytes: | default = 500000000] - # Maximum (???) + # Duration to keep blocks in the ingester after they have been flushed [complete_block_timeout: | default = 1h] - # (Unused?) - # Max number of "live" traces + # Maximum amount of live traces + # If this value is exceeded, traces will be dropped with reason: `live_traces_exceeded` + # A value of 0 disables this limit. [max_live_traces: ] # Whether server spans should be filtered in or not. - # true means keep the server kinds (???) - # false means drop the server kinds (???) + # If enabled, only parent spans or spans with the SpanKind of `server` will be retained [filter_server_spans: | default = true] # Number of blocks that are allowed to be processed concurently From 93af96b50dbd4300f253b8d714fca4c381030cda Mon Sep 17 00:00:00 2001 From: WesselAtWork <115667066+WesselAtWork@users.noreply.github.com> Date: Thu, 20 Jun 2024 08:28:13 +0000 Subject: [PATCH 15/15] Spell Check --- docs/sources/tempo/configuration/_index.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index 49bf01a2aea..54b981859c1 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -300,7 +300,7 @@ metrics_generator: [prefix: | default = "collectors/"] # Period at which to heartbeat the instance - # 0 disables heartbeat alltogether + # 0 disables heartbeat altogether [heartbeat_period: | default = 5s] # The heartbeat timeout, after which, the instance is skipped. @@ -489,7 +489,7 @@ metrics_generator: # Configuration for the Prometheus Agent WAL # https://github.com/prometheus/prometheus/blob/v2.51.2/tsdb/agent/db.go#L62-L84 - wal: + wal: # How long to wait when flushing samples on shutdown [remote_write_flush_deadline: | default = 1m] @@ -674,7 +674,7 @@ querier: # Lookback period to include ingesters that were part of the shuffle sharded subring. [shuffle_sharding_ingesters_lookback_period: | default = 1hr] - # The query frontend sents sharded requests to ingesters and querier (/api/traces/) + # The query frontend sends sharded requests to ingesters and querier (/api/traces/) # By default, all healthy ingesters are queried for the trace id. # When true the querier will hash the trace id in the same way that distributors do and then # only query those ingesters who own the trace id hash as determined by the ring. @@ -691,7 +691,7 @@ querier: # A list of external endpoints that the querier will use to offload backend search requests. They must # take and return the same value as /api/search endpoint on the querier. This is intended to be - # used with serverless technologies for massive parrallelization of the search path. + # used with serverless technologies for massive parallelization of the search path. # The default value of "" disables this feature. [external_endpoints: | default = ] @@ -720,7 +720,7 @@ querier: google_cloud_run: # A list of external endpoints that the querier will use to offload backend search requests. They must # take and return the same value as /api/search endpoint on the querier. This is intended to be - # used with serverless technologies for massive parrallelization of the search path. + # used with serverless technologies for massive parallelization of the search path. # The default value of "" disables this feature. [external_endpoints: | default = ] @@ -1043,7 +1043,7 @@ storage: # Optional. Default is 0 (disabled) # Example: "hedge_requests_at: 500ms" # If set to a non-zero value a second request will be issued at the provided duration. Recommended to - # be set to p99 of Axure Blog Storage requests to reduce long tail latency. This setting is most impactful when + # be set to p99 of Azure Block Storage requests to reduce long tail latency. This setting is most impactful when # used with queriers and has minimal to no impact on other pieces. [hedge_requests_at: ] @@ -1135,7 +1135,7 @@ storage: # total number of workers pulling jobs from the queue [max_workers: | default = 30] - # length of job queue. imporatant for querier as it queues a job for every block it has to search + # length of job queue. important for querier as it queues a job for every block it has to search [queue_depth: | default = 10000 ] # configuration block for the Write Ahead Log (WAL) @@ -1291,7 +1291,7 @@ Span filter config block #### Filter policy ```yaml -# Exclude filters (postive matching) +# Exclude filters (positive matching) [include: ] # Exclude filters (negative matching) @@ -1343,7 +1343,7 @@ The kvstore configuration block # What prefix to use for keys [prefix: | default = "ring."] -# Store spesific configs +# Store specific configs consul: [host: | default = "localhost:8500"] [acl_token: | default = "" ] @@ -1682,7 +1682,7 @@ overrides: [per_tenant_override_config: | default = ""] # How frequent tenant-specific overrides are read from the configuration file. - [per_tenant_override_period: | default = 10s] + [per_tenant_override_period: | default = 10s] # User-configurable overrides configuration user_configurable_overrides: