From 55e06569689c4515bb71231bde9159ae6724f5e4 Mon Sep 17 00:00:00 2001 From: May Lee Date: Thu, 9 May 2024 13:43:17 -0400 Subject: [PATCH 1/3] add doc --- config/_default/menus/main.en.yaml | 55 +++++++++------- content/en/observability_pipelines/_index.md | 26 ++++---- .../en/observability_pipelines/setup_opw.md | 65 +++++++++++++++++++ 3 files changed, 109 insertions(+), 37 deletions(-) create mode 100644 content/en/observability_pipelines/setup_opw.md diff --git a/config/_default/menus/main.en.yaml b/config/_default/menus/main.en.yaml index 6bb7c1e32714..8657bef18a22 100644 --- a/config/_default/menus/main.en.yaml +++ b/config/_default/menus/main.en.yaml @@ -3820,131 +3820,136 @@ menu: identifier: observability_pipelines parent: log_management_heading weight: 10000 + - name: Setup + url: observability_pipelines/setup_opw/ + parent: observability_pipelines + identifier: observability_pipelines_setup_opw + weight: 1 - name: Log Volume Control url: observability_pipelines/log_volume_control/ parent: observability_pipelines identifier: observability_pipelines_log_volume_control - weight: 1 + weight: 2 - name: Splunk HTTP Event Collector url: observability_pipelines/log_volume_control/splunk_hec/ parent: observability_pipelines_log_volume_control identifier: observability_pipelines_log_volume_control_splunk_hec - weight: 1001 + weight: 2001 - name: Splunk Forwarders (TCP) url: observability_pipelines/log_volume_control/splunk_tcp/ parent: observability_pipelines_log_volume_control identifier: observability_pipelines_log_volume_control_splunk_tcp - weight: 1002 + weight: 2002 - name: Sumo Logic Hosted Collector url: observability_pipelines/log_volume_control/sumo_logic_hosted_collector/ parent: observability_pipelines_log_volume_control identifier: observability_pipelines_log_volume_control_sumo_logic_hosted_collector - weight: 1003 + weight: 2003 - name: Dual Ship Logs url: observability_pipelines/dual_ship_logs/ parent: observability_pipelines identifier: observability_pipelines_dual_ship_logs - weight: 2 + weight: 3 - name: Splunk HTTP Event Collector url: observability_pipelines/dual_ship_logs/splunk_hec/ parent: observability_pipelines_dual_ship_logs identifier: observability_pipelines_dual_ship_logs_splunk_hec - weight: 2001 + weight: 3001 - name: Splunk Forwarders (TCP) url: observability_pipelines/dual_ship_logs/splunk_tcp/ parent: observability_pipelines_dual_ship_logs identifier: observability_pipelines_dual_ship_logs_splunk_tcp - weight: 2002 + weight: 3002 - name: Sumo Logic Hosted Collector url: observability_pipelines/dual_ship_logs/sumo_logic_hosted_collector/ parent: observability_pipelines_dual_ship_logs identifier: observability_pipelines_dual_ship_logs_sumo_logic_hosted_collector - weight: 2003 + weight: 3003 - name: Archive Logs url: observability_pipelines/archive_logs/ parent: observability_pipelines identifier: observability_pipelines_archive_logs - weight: 3 + weight: 4 - name: Datadog Agent url: observability_pipelines/archive_logs/datadog_agent/ parent: observability_pipelines_archive_logs identifier: observability_pipelines_archive_logs_datadog_agent - weight: 3001 + weight: 4001 - name: Splunk HTTP Event Collector url: observability_pipelines/archive_logs/splunk_hec/ parent: observability_pipelines_archive_logs identifier: observability_pipelines_archive_logs_splunk_hec - weight: 3002 + weight: 4002 - name: Splunk Forwarders (TCP) url: observability_pipelines/archive_logs/splunk_tcp/ parent: observability_pipelines_archive_logs identifier: observability_pipelines_archive_logs_splunk_tcp - weight: 3003 + weight: 4003 - name: Sumo Logic Hosted Collector url: observability_pipelines/archive_logs/sumo_logic_hosted_collector/ parent: observability_pipelines_archive_logs identifier: observability_pipelines_archive_logs_sumo_logic_hosted_collector - weight: 3004 + weight: 4004 - name: Split Logs url: observability_pipelines/split_logs/ parent: observability_pipelines identifier: observability_pipelines_split_logs - weight: 4 + weight: 5 - name: Datadog Agent url: observability_pipelines/split_logs/datadog_agent/ parent: observability_pipelines_split_logs identifier: observability_pipelines_split_logs_datadog_agent - weight: 4001 + weight: 5001 - name: Splunk HTTP Event Collector url: observability_pipelines/split_logs/splunk_hec/ parent: observability_pipelines_split_logs identifier: observability_pipelines_split_logs_splunk_hec - weight: 4002 + weight: 5002 - name: Splunk Forwarders (TCP) url: observability_pipelines/split_logs/splunk_tcp/ parent: observability_pipelines_split_logs identifier: observability_pipelines_split_logs_splunk_tcp - weight: 4003 + weight: 5003 - name: Sumo Logic Hosted Collector url: observability_pipelines/split_logs/sumo_logic_hosted_collector/ parent: observability_pipelines_split_logs identifier: observability_pipelines_split_logs_sumo_logic_hosted_collector - weight: 4004 + weight: 5004 - name: Sensitive Data Redaction url: observability_pipelines/sensitive_data_redaction/ parent: observability_pipelines identifier: observability_pipelines_sensitive_data_redaction - weight: 5 + weight: 6 - name: Datadog Agent url: observability_pipelines/sensitive_data_redaction/datadog_agent/ parent: observability_pipelines_sensitive_data_redaction identifier: observability_pipelines_sensitive_data_redaction_datadog_agent - weight: 5001 + weight: 6001 - name: Splunk HTTP Event Collector url: observability_pipelines/sensitive_data_redaction/splunk_hec/ parent: observability_pipelines_sensitive_data_redaction identifier: observability_pipelines_sensitive_data_redaction_splunk_hec - weight: 5002 + weight: 6002 - name: Splunk Forwarders (TCP) url: observability_pipelines/sensitive_data_redaction/splunk_tcp/ parent: observability_pipelines_sensitive_data_redaction identifier: observability_pipelines_sensitive_data_redaction_splunk_tcp - weight: 5003 + weight: 6003 - name: Sumo Logic Hosted Collector url: observability_pipelines/sensitive_data_redaction/sumo_logic_hosted_collector/ parent: observability_pipelines_sensitive_data_redaction identifier: observability_pipelines_sensitive_data_redaction_sumo_logic_hosted_collector - weight: 5004 + weight: 6004 - name: Update Existing Pipelines url: observability_pipelines/update_existing_pipelines/ parent: observability_pipelines identifier: observability_pipelines_update_existing_pipelines - weight: 6 + weight: 7 - name: Best Practices for Scaling Observability Pipelines url: observability_pipelines/best_practices_for_scaling_observability_pipelines/ parent: observability_pipelines identifier: observability_pipelines_best_practices_for_scaling_observability_pipelines - weight: 7 + weight: 8 - name: Log Management url: logs/ pre: log diff --git a/content/en/observability_pipelines/_index.md b/content/en/observability_pipelines/_index.md index c6dda4828f1d..d0d90d5aa366 100644 --- a/content/en/observability_pipelines/_index.md +++ b/content/en/observability_pipelines/_index.md @@ -58,22 +58,24 @@ The Datadog UI provides a control plane to manage your Observability Pipelines W ## Get started -1. Navigate to [Observability Pipelines][1]. +1. [Set up environment variables for the Observability Pipelines Worker][1]. +1. Navigate to [Observability Pipelines][2]. 1. Select a use case: - - [Log volume control][2] - - [Dual ship logs][3] - - [Split logs][4] - - [Archive logs to Datadog Archives][5] - - [Sensitive data redaction][6] + - [Log volume control][3] + - [Dual ship logs][4] + - [Split logs][5] + - [Archive logs to Datadog Archives][6] + - [Sensitive data redaction][7] 1. Enable monitors. ## Further Reading {{< partial name="whats-next/whats-next.html" >}} -[1]: https://app.datadoghq.com/observability-pipelines -[2]: /observability_pipelines/log_volume_control/ -[3]: /observability_pipelines/dual_ship_logs/ -[4]: /observability_pipelines/split_logs/ -[5]: /observability_pipelines/archive_logs/ -[6]: /observability_pipelines/sensitive_data_redaction/ \ No newline at end of file +[1]: /observability_pipelines/setup_opw/ +[2]: https://app.datadoghq.com/observability-pipelines +[3]: /observability_pipelines/log_volume_control/ +[4]: /observability_pipelines/dual_ship_logs/ +[5]: /observability_pipelines/split_logs/ +[6]: /observability_pipelines/archive_logs/ +[7]: /observability_pipelines/sensitive_data_redaction/ diff --git a/content/en/observability_pipelines/setup_opw.md b/content/en/observability_pipelines/setup_opw.md new file mode 100644 index 000000000000..9af5b5990247 --- /dev/null +++ b/content/en/observability_pipelines/setup_opw.md @@ -0,0 +1,65 @@ +--- +title: Setup +kind: documentation +disable_toc: false +further_reading: +- link: "/observability_pipelines/log_volume_control/" + tag: "Documentation" + text: "Log volume control with Observability Pipelines" +- link: "/observability_pipelines/dual_ship_logs/" + tag: "Documentation" + text: "Dual ship logs with Observability Pipelines" +- link: "/observability_pipelines/archive_logs/" + tag: "Documentation" + text: "Archive logs with Observability Pipelines" +- link: "/observability_pipelines/split_logs/" + tag: "Documentation" + text: "Split logs with Observability Pipelines" +- link: "/observability_pipelines/sensitive_data_redaction/" + tag: "Documentation" + text: "Redact data redaction with Observability Pipelines" +- link: "/observability_pipelines/update_existing_pipelines/" + tag: "Documentation" + text: "Update existing pipelines" +--- + +## Overview + +Bootstrap the Observability Pipelines Worker within your infrastructure before you set up a pipeline. These environment variables are separate from the options in the pipelines configuration file. + +## Environment variables + +All configuration file paths specified remotely need to be under `DD_OP_DATA_DIR/config`. + +`api_key` +: env var: `DD_API_KEY` +: Create a [Datadog API key][1] for this environment variable. + +`pipeline_id` +: env var: `DD_OP_PIPELINE_ID` +: Create an [Observability Pipelines pipeline ID][2] for this environment variable. + +`site` +: env var: `DD_SITE` +: Your Datadog site (optional, default: `datadoghq.com`). +: See [Getting Started with Sites][3] for more information. + +`data_dir` +: env var: `DD_OP_DATA_DIR` +: The data directory (optional, default: `/var/lib/observability-pipelines-worker`). This is the file system directory that the Observability Pipelines Worker uses for local state. + +`tags: []` +: env var: `DD_OP_TAGS` +: The tags reported with internal metrics and can be used to filter Observability Pipelines instances for Remote Configuration deployments. + +`threads` +: env var: `DD_OP_THREADS` +: The number of threads to use for processing (optional, default: the number of available cores). + +## Further reading + +{{< partial name="whats-next/whats-next.html" >}} + +[1]: https://app.datadoghq.com/organization-settings/api-keys +[2]: https://app.datadoghq.com/observability-pipelines +[3]: /getting_started/site/ \ No newline at end of file From 7536d8043070d37d58ca150fd94c8bce2c060303 Mon Sep 17 00:00:00 2001 From: May Lee Date: Thu, 9 May 2024 15:58:08 -0400 Subject: [PATCH 2/3] apply suggested changes --- content/en/observability_pipelines/_index.md | 2 +- content/en/observability_pipelines/setup_opw.md | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/content/en/observability_pipelines/_index.md b/content/en/observability_pipelines/_index.md index d0d90d5aa366..90630d1b0d71 100644 --- a/content/en/observability_pipelines/_index.md +++ b/content/en/observability_pipelines/_index.md @@ -58,7 +58,7 @@ The Datadog UI provides a control plane to manage your Observability Pipelines W ## Get started -1. [Set up environment variables for the Observability Pipelines Worker][1]. +1. [Set up bootstrap options for the Observability Pipelines Worker][1]. 1. Navigate to [Observability Pipelines][2]. 1. Select a use case: - [Log volume control][3] diff --git a/content/en/observability_pipelines/setup_opw.md b/content/en/observability_pipelines/setup_opw.md index 9af5b5990247..baafd044a546 100644 --- a/content/en/observability_pipelines/setup_opw.md +++ b/content/en/observability_pipelines/setup_opw.md @@ -27,9 +27,11 @@ further_reading: Bootstrap the Observability Pipelines Worker within your infrastructure before you set up a pipeline. These environment variables are separate from the options in the pipelines configuration file. -## Environment variables +## Bootstrap Options -All configuration file paths specified remotely need to be under `DD_OP_DATA_DIR/config`. +To set bootstrap options, do one of the following: +- Use environmental variables. +- Create a `bootstrap.yaml` and start the Worker instance with `--bootstrap-config /path/to/bootstrap.yaml`. `api_key` : env var: `DD_API_KEY` @@ -47,6 +49,7 @@ All configuration file paths specified remotely need to be under `DD_OP_DATA_DIR `data_dir` : env var: `DD_OP_DATA_DIR` : The data directory (optional, default: `/var/lib/observability-pipelines-worker`). This is the file system directory that the Observability Pipelines Worker uses for local state. +: **Note**: All configuration file paths specified remotely need to be under `DD_OP_DATA_DIR/config`. `tags: []` : env var: `DD_OP_TAGS` From a811e9f7d6ef34c89abf208bd4735cfe83b44425 Mon Sep 17 00:00:00 2001 From: May Lee Date: Fri, 10 May 2024 10:56:49 -0400 Subject: [PATCH 3/3] use alert box for note --- content/en/observability_pipelines/setup_opw.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/content/en/observability_pipelines/setup_opw.md b/content/en/observability_pipelines/setup_opw.md index baafd044a546..070e41c9fc56 100644 --- a/content/en/observability_pipelines/setup_opw.md +++ b/content/en/observability_pipelines/setup_opw.md @@ -25,6 +25,8 @@ further_reading: ## Overview +
All configuration file paths specified remotely need to be under DD_OP_DATA_DIR/config
+ Bootstrap the Observability Pipelines Worker within your infrastructure before you set up a pipeline. These environment variables are separate from the options in the pipelines configuration file. ## Bootstrap Options @@ -49,7 +51,6 @@ To set bootstrap options, do one of the following: `data_dir` : env var: `DD_OP_DATA_DIR` : The data directory (optional, default: `/var/lib/observability-pipelines-worker`). This is the file system directory that the Observability Pipelines Worker uses for local state. -: **Note**: All configuration file paths specified remotely need to be under `DD_OP_DATA_DIR/config`. `tags: []` : env var: `DD_OP_TAGS`