From 08360712865e812102f87d3842079a5ab8829978 Mon Sep 17 00:00:00 2001 From: Prashant Shahi Date: Thu, 23 May 2024 18:01:23 +0530 Subject: [PATCH 1/4] docs(awss3): long term retention tutorial Signed-off-by: Prashant Shahi --- docs/tutorial/infinite-retention-aws-s3.md | 78 ++++++++++++++++++++++ sidebars.js | 1 + 2 files changed, 79 insertions(+) create mode 100644 docs/tutorial/infinite-retention-aws-s3.md diff --git a/docs/tutorial/infinite-retention-aws-s3.md b/docs/tutorial/infinite-retention-aws-s3.md new file mode 100644 index 00000000..54138f7a --- /dev/null +++ b/docs/tutorial/infinite-retention-aws-s3.md @@ -0,0 +1,78 @@ +--- +id: infinite-retention-aws-s3 +title: Infinite Retention of OpenTelemetry Data in AWS S3 +description: Infinite Retention of OpenTelemetry Data in AWS S3 +--- + +## Overview + +It is a common practice to backup any data for longer durations due to +compliance and audit purposes. You can use [AWS S3 Exporter][1] to +retain the OpenTelemetry data as long as we need. + +## Limitation of AWS S3 Exporter + +- Data in AWS S3 is inaccessible in SigNoz UI +- Need to use third-party tool like **Amazon Athena** to query data +- Manual task for adding prefixes for different types of signals like logs, + metrics, traces + +:::info +If you want to query data stored in AWS S3 using **SigNoz** and do not +have the requirement for infinite or very long retention period, then use +[SigNoz's AWS S3 Retention][2] instead. +::: + +## Prerequisites + +- Running instance of OpenTelemetry Collector (if not running already, see [Installation Page][3]) +- Access to AWS S3 Bucket either using AWS credentials as environment variables + or IAM roles for ECS tasks or EC2 instances (for more details, [see here][4]) + +## Adding AWS S3 Exporter + +In our example, we will use `awss3` exporter for retaining logs data, +where we will be using `us-east-1` region and `otel-data-backup` bucket. + +```yaml +exporters: + awss3/logs: + s3uploader: + region: 'us-east-1' + s3_bucket: 'otel-data-backup' + s3_prefix: 'logs' + s3_partition: 'minute' +service: + pipelines: + logs: + exporters: [otlp, awss3/logs] +``` + +Similarly, we can add it for `metrics` and `traces` pipelines as well with +different prefixes to retain metrics and traces data respectively. + +The above configuration needs to be added for your respective OtelCollector(s). + +- In case of **SigNoz Self-hosted**, you can add it in SigNoz OtelCollector +configuration to retain all incoming data of `logs` pipelines. +- In case of **SigNoz Cloud**, you will need to update all OtelCollector +configuration that is sending data. +- For **K8s-Infra** helm chart, you can add it in the `otelAgent` configuration. +- For any **Standalone OtelCollector** deployments that is directly sending +data to SigNoz, you can add it in its respective configuration. + +List of all supported configuration of `awss3` exporter can be found [here][4]. + +--- + +If you have any feedback or facing issues, feel free to join our slack +community to get help! + +[![SigNoz Slack community][5]][6] + +[1]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/awss3exporter +[2]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/awss3exporter#aws-credential-configuration +[3]: /docs/install/ +[4]: /docs/userguide/retention-period#configuring-cold-storage---amazon-s3 +[5]: /img/blog/common/join_slack_cta.webp +[6]: /slack diff --git a/sidebars.js b/sidebars.js index ab3cf776..144e1766 100644 --- a/sidebars.js +++ b/sidebars.js @@ -623,6 +623,7 @@ module.exports = { 'tutorial/setting-up-sso-saml-with-keycloak', 'tutorial/writing-clickhouse-queries-in-dashboard', 'tutorial/traefik-observability', + 'tutorial/infinite-retention-aws-s3', ], }, { From 63f737fdd8424eeed2042dc400f87e20422aaad4 Mon Sep 17 00:00:00 2001 From: Prashant Shahi Date: Thu, 23 May 2024 18:10:38 +0530 Subject: [PATCH 2/4] docs(awss3): update slack link Signed-off-by: Prashant Shahi --- docs/tutorial/infinite-retention-aws-s3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial/infinite-retention-aws-s3.md b/docs/tutorial/infinite-retention-aws-s3.md index 54138f7a..28dbf9eb 100644 --- a/docs/tutorial/infinite-retention-aws-s3.md +++ b/docs/tutorial/infinite-retention-aws-s3.md @@ -75,4 +75,4 @@ community to get help! [3]: /docs/install/ [4]: /docs/userguide/retention-period#configuring-cold-storage---amazon-s3 [5]: /img/blog/common/join_slack_cta.webp -[6]: /slack +[6]: https://signoz.io/slack From aaeb8769a8a77ca266cbfd4f1b142b56799c2036 Mon Sep 17 00:00:00 2001 From: Prashant Shahi Date: Sun, 26 May 2024 11:52:22 +0530 Subject: [PATCH 3/4] docs(awss3): update link refs Signed-off-by: Prashant Shahi --- docs/tutorial/infinite-retention-aws-s3.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorial/infinite-retention-aws-s3.md b/docs/tutorial/infinite-retention-aws-s3.md index 28dbf9eb..787a08c3 100644 --- a/docs/tutorial/infinite-retention-aws-s3.md +++ b/docs/tutorial/infinite-retention-aws-s3.md @@ -20,7 +20,7 @@ retain the OpenTelemetry data as long as we need. :::info If you want to query data stored in AWS S3 using **SigNoz** and do not have the requirement for infinite or very long retention period, then use -[SigNoz's AWS S3 Retention][2] instead. +[SigNoz's AWS S3 Retention][4] instead. ::: ## Prerequisites @@ -61,7 +61,7 @@ configuration that is sending data. - For any **Standalone OtelCollector** deployments that is directly sending data to SigNoz, you can add it in its respective configuration. -List of all supported configuration of `awss3` exporter can be found [here][4]. +List of all supported configuration of `awss3` exporter can be found [here][1]. --- From d55502e55c0b05b74645e1a5d69ba31ec2119909 Mon Sep 17 00:00:00 2001 From: Prashant Shahi Date: Thu, 30 May 2024 16:46:11 +0530 Subject: [PATCH 4/4] docs(awss3): update limitations Signed-off-by: Prashant Shahi --- docs/tutorial/infinite-retention-aws-s3.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/tutorial/infinite-retention-aws-s3.md b/docs/tutorial/infinite-retention-aws-s3.md index 787a08c3..32f6bc40 100644 --- a/docs/tutorial/infinite-retention-aws-s3.md +++ b/docs/tutorial/infinite-retention-aws-s3.md @@ -14,8 +14,9 @@ retain the OpenTelemetry data as long as we need. - Data in AWS S3 is inaccessible in SigNoz UI - Need to use third-party tool like **Amazon Athena** to query data -- Manual task for adding prefixes for different types of signals like logs, - metrics, traces +- Due to limitation of AWS S3 Exporter, you won't be able differentiate + different signals like logs, metrics, and traces - hence the need of + including different prefixes :::info If you want to query data stored in AWS S3 using **SigNoz** and do not