From 8adbfa191eaaa5b8a94feb4f3605d9ed39b06924 Mon Sep 17 00:00:00 2001 From: Jake Cahill <45230295+JakeSCahill@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:25:06 +0000 Subject: [PATCH 1/9] DOC-470 Debug bundle in Redpanda Console (#825) Co-authored-by: Kat Batuigas <36839689+kbatuigas@users.noreply.github.com> --- package-lock.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 452eeced75..6fbdb94c67 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2773,9 +2773,10 @@ } }, "node_modules/@redpanda-data/docs-extensions-and-macros": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/@redpanda-data/docs-extensions-and-macros/-/docs-extensions-and-macros-3.9.0.tgz", - "integrity": "sha512-BkiPDj69KUC+gAN1OdGqwJOy4MLbEEyn5+FQYalc/sIP6uQVYgoTd9neFwMmwz2L/lBNkQ12KAv0jFVuB+/plg==", + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@redpanda-data/docs-extensions-and-macros/-/docs-extensions-and-macros-3.8.0.tgz", + "integrity": "sha512-S4ssVauyULIqHFrRVX7ymIkHpNQo/f+D/4ENt4B3kKK1ISwlpcHIoAi4sVe6tdztO567oRp3ZMKAC9qx2mef3Q==", + "license": "ISC", "dependencies": { "@asciidoctor/tabs": "^1.0.0-beta.6", "@octokit/core": "^6.1.2", From 1474b389bdf82618352e049cc778a854eea0b3a5 Mon Sep 17 00:00:00 2001 From: Michele Cyran Date: Fri, 15 Nov 2024 14:55:29 -0700 Subject: [PATCH 2/9] what's new for 24.3 GA --- modules/get-started/pages/whats-new.adoc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/get-started/pages/whats-new.adoc b/modules/get-started/pages/whats-new.adoc index 7b974a62b1..54e8d57ebf 100644 --- a/modules/get-started/pages/whats-new.adoc +++ b/modules/get-started/pages/whats-new.adoc @@ -45,6 +45,14 @@ This release includes several updates to xref:get-started:licensing/overview.ado - *Unified license management in Redpanda Console*: You can now upload and apply a single license key for both Redpanda Console and the connected Redpanda cluster through the Redpanda Console UI. Any existing license key is overridden by the new one. +== Backfill partitions + +When running xref:manage:cluster-maintenance/nodewise-partition-recovery.adoc[node-wise partition recovery], it's possible that there may be more recent data (a higher offset) available in Tiered Storage. Redpanda attempts to recover partition data from object storage, recovering the latest offset available for a partition in either storage tier (local or object storage). This allows for the maximum amount of data to be recovered in all cases, even for topics with a replication factor of 1, where no replicas remain in local storage. + +== Configure access to object storage with a KMS key + +Users on AWS or GCP with strict data compliance requirements can manage and store encryption keys separately from their cloud provider with a xref:manage:tiered-storage.adoc#configure-object-storage[customer-managed Key Management Service (KMS) key]. + == New commands The following `rpk` commands are new in this version: From 3bc5cd9a8f2f5d80f4804c2346fd383862cb5705 Mon Sep 17 00:00:00 2001 From: Michele Cyran Date: Mon, 18 Nov 2024 08:15:30 -0700 Subject: [PATCH 3/9] intra-broker partition balancing out of beta --- modules/get-started/pages/whats-new.adoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/get-started/pages/whats-new.adoc b/modules/get-started/pages/whats-new.adoc index 54e8d57ebf..b2a7ab75cf 100644 --- a/modules/get-started/pages/whats-new.adoc +++ b/modules/get-started/pages/whats-new.adoc @@ -53,6 +53,10 @@ When running xref:manage:cluster-maintenance/nodewise-partition-recovery.adoc[no Users on AWS or GCP with strict data compliance requirements can manage and store encryption keys separately from their cloud provider with a xref:manage:tiered-storage.adoc#configure-object-storage[customer-managed Key Management Service (KMS) key]. +== Intra-broker partition balancing + +xref:manage:cluster-maintenance/cluster-balancing.adoc#intra-broker-partition-balancing[Intra-broker partition balancing], which balances partitions across cores within a Redpanda broker, has moved out of beta and is supported for production clusters. + == New commands The following `rpk` commands are new in this version: From 11612c80a8e5a8cfa4ff8da23ae2b9f289185ca1 Mon Sep 17 00:00:00 2001 From: Michele Cyran Date: Mon, 18 Nov 2024 14:07:24 -0700 Subject: [PATCH 4/9] add new properties --- modules/get-started/pages/whats-new.adoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/get-started/pages/whats-new.adoc b/modules/get-started/pages/whats-new.adoc index b2a7ab75cf..af6fc47ba6 100644 --- a/modules/get-started/pages/whats-new.adoc +++ b/modules/get-started/pages/whats-new.adoc @@ -72,6 +72,9 @@ The following `rpk` commands are new in this version: The following cluster properties are new in this version: +* xref:reference:properties/cluster-properties.adoc#debug_bundle_auto_removal_seconds[`debug_bundle_auto_removal_seconds`] +* xref:reference:properties/cluster-properties.adoc#debug_bundle_storage_dir[`debug_bundle_storage_dir`] * xref:reference:properties/cluster-properties.adoc#default_leaders_preference[`default_leaders_preference`] +* xref:reference:properties/cluster-properties.adoc#iceberg_enabled[`iceberg_enabled`] * xref:reference:properties/cluster-properties.adoc#rpk_path[`rpk_path`] * xref:reference:properties/cluster-properties.adoc#tombstone_retention_ms[`tombstone_retention_ms`] From 10f8984baebdb1480d9ed93c09362cede8f28adb Mon Sep 17 00:00:00 2001 From: Michele Cyran Date: Mon, 18 Nov 2024 15:34:07 -0700 Subject: [PATCH 5/9] add new iceberg properties --- modules/get-started/pages/whats-new.adoc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/modules/get-started/pages/whats-new.adoc b/modules/get-started/pages/whats-new.adoc index af6fc47ba6..5cd6b84701 100644 --- a/modules/get-started/pages/whats-new.adoc +++ b/modules/get-started/pages/whats-new.adoc @@ -75,6 +75,15 @@ The following cluster properties are new in this version: * xref:reference:properties/cluster-properties.adoc#debug_bundle_auto_removal_seconds[`debug_bundle_auto_removal_seconds`] * xref:reference:properties/cluster-properties.adoc#debug_bundle_storage_dir[`debug_bundle_storage_dir`] * xref:reference:properties/cluster-properties.adoc#default_leaders_preference[`default_leaders_preference`] +* xref:reference:properties/cluster-properties.adoc#iceberg_catalog_base_location[`iceberg_catalog_base_location`] +* xref:reference:properties/cluster-properties.adoc#iceberg_catalog_commit_interval_ms[`iceberg_catalog_commit_interval_ms`] +* xref:reference:properties/cluster-properties.adoc#iceberg_catalog_type[`iceberg_catalog_type`] * xref:reference:properties/cluster-properties.adoc#iceberg_enabled[`iceberg_enabled`] +* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_endpoint[`iceberg_rest_catalog_endpoint`] +* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_request_timeout_ms[`iceberg_rest_catalog_request_timeout_ms`] +* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_secret[`iceberg_rest_catalog_secret`] +* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_token[`iceberg_rest_catalog_token`] +* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_user_id[`iceberg_rest_catalog_user_id`] +* xref:reference:properties/cluster-properties.adoc#iceberg_translation_interval_ms_default[`iceberg_translation_interval_ms_default`] * xref:reference:properties/cluster-properties.adoc#rpk_path[`rpk_path`] * xref:reference:properties/cluster-properties.adoc#tombstone_retention_ms[`tombstone_retention_ms`] From 18b7b116e3b9ef2301c5deef6b87cc4d985a3104 Mon Sep 17 00:00:00 2001 From: Michele Cyran Date: Mon, 18 Nov 2024 16:15:10 -0700 Subject: [PATCH 6/9] edits from Matt's review --- modules/get-started/pages/whats-new.adoc | 32 +++++++++++++----------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/modules/get-started/pages/whats-new.adoc b/modules/get-started/pages/whats-new.adoc index 5cd6b84701..44ce69470e 100644 --- a/modules/get-started/pages/whats-new.adoc +++ b/modules/get-started/pages/whats-new.adoc @@ -7,17 +7,21 @@ This topic includes new content added in version {page-component-version}. For a * xref:redpanda-cloud:get-started:whats-new-cloud.adoc[] * xref:redpanda-cloud:get-started:cloud-overview.adoc#redpanda-cloud-vs-self-managed-feature-compatibility[Redpanda Cloud vs Self-Managed feature compatibility] -== Tombstone removal +== Leader pinning -Redpanda now supports the Kafka `delete.retention.ms` topic configuration. You can specify how long Redpanda keeps xref:manage:cluster-maintenance/compaction-settings.adoc#tombstone-record-removal[tombstone records] for compacted topics by setting `delete.retention.ms` at the topic level, or `tombstone_retention_ms` at the cluster level. +For a Redpanda cluster deployed across multiple availability zones (AZs), xref:develop:produce-data/leader-pinning.adoc[leader pinning] ensures that a topic's partition leaders are geographically closer to clients. Leader pinning can lower networking costs and help guarantee lower latency by routing produce and consume requests to brokers located in certain AZs. == Mountable topics For topics with Tiered Storage enabled, you can unmount a topic to safely detach it from a cluster and keep the topic data in the cluster's object storage bucket or container. You can mount the detached topic to either the same origin cluster, or a different one. This allows you to hibernate a topic and free up system resources taken up by the topic, or migrate a topic to a different cluster. See xref:manage:mountable-topics.adoc[Mountable topics] for details. -== Leader pinning +== Intra-broker partition balancing -For a Redpanda cluster deployed across multiple availability zones (AZs), xref:develop:produce-data/leader-pinning.adoc[leader pinning] ensures that a topic's partition leaders are geographically closer to clients. Leader pinning can lower networking costs and help guarantee lower latency by routing produce and consume requests to brokers located in certain AZs. +xref:manage:cluster-maintenance/cluster-balancing.adoc#intra-broker-partition-balancing[Intra-broker partition balancing], which balances partitions across cores within a Redpanda broker, has moved out of beta and is supported for production clusters. + +== Tombstone removal + +Redpanda now supports the Kafka `delete.retention.ms` topic configuration. You can specify how long Redpanda keeps xref:manage:cluster-maintenance/compaction-settings.adoc#tombstone-record-removal[tombstone records] for compacted topics by setting `delete.retention.ms` at the topic level, or `tombstone_retention_ms` at the cluster level. == Debug bundles in Redpanda Console @@ -33,6 +37,14 @@ Redpanda now supports declarative management of users and access control lists ( To learn more, see the xref:manage:kubernetes/security/authentication/k-user-controller.adoc[User custom resource documentation]. +== Backfill partitions + +When running xref:manage:cluster-maintenance/nodewise-partition-recovery.adoc[node-wise partition recovery], it's possible that there may be more recent data (a higher offset) available in Tiered Storage. Redpanda attempts to recover partition data from object storage, recovering the latest offset available for a partition in either storage tier (local or object storage). This allows for the maximum amount of data to be recovered in all cases, even for topics with a replication factor of 1, where no replicas remain in local storage. + +== Configure access to object storage with a KMS key + +Users on AWS or GCP with strict data compliance requirements can manage and store encryption keys separately from their cloud provider with a xref:manage:tiered-storage.adoc#configure-object-storage[customer-managed Key Management Service (KMS) key]. + == Licensing updates This release includes several updates to xref:get-started:licensing/overview.adoc[Redpanda's licensing system] to both improve transparency and make it easier to manage licenses across Redpanda clusters and Redpanda Console. @@ -45,17 +57,7 @@ This release includes several updates to xref:get-started:licensing/overview.ado - *Unified license management in Redpanda Console*: You can now upload and apply a single license key for both Redpanda Console and the connected Redpanda cluster through the Redpanda Console UI. Any existing license key is overridden by the new one. -== Backfill partitions - -When running xref:manage:cluster-maintenance/nodewise-partition-recovery.adoc[node-wise partition recovery], it's possible that there may be more recent data (a higher offset) available in Tiered Storage. Redpanda attempts to recover partition data from object storage, recovering the latest offset available for a partition in either storage tier (local or object storage). This allows for the maximum amount of data to be recovered in all cases, even for topics with a replication factor of 1, where no replicas remain in local storage. - -== Configure access to object storage with a KMS key - -Users on AWS or GCP with strict data compliance requirements can manage and store encryption keys separately from their cloud provider with a xref:manage:tiered-storage.adoc#configure-object-storage[customer-managed Key Management Service (KMS) key]. - -== Intra-broker partition balancing - -xref:manage:cluster-maintenance/cluster-balancing.adoc#intra-broker-partition-balancing[Intra-broker partition balancing], which balances partitions across cores within a Redpanda broker, has moved out of beta and is supported for production clusters. +- *30 day trial Enterprise license*: Starting with version 24.3, new Redpanda clusters automatically receive a trial license that's valid for 30 days, allowing unrestricted use of Enterprise features. This evaluation period begins when the cluster is created for the first time. After this period expires, inactive Enterprise features are disabled, and active features enter a restricted state. == New commands From e598d0bd0f9472574fb98470a8d9160bbf6aa929 Mon Sep 17 00:00:00 2001 From: Michele Cyran Date: Mon, 18 Nov 2024 19:16:25 -0700 Subject: [PATCH 7/9] + decreasing CPUs, per Kat/Matt [Intra-broker partition balancing] has moved out of beta and is supported for production clusters. This dynamically balances partitions across cores within a Redpanda broker. You also can decrease the number of CPUs assigned to a broker. Existing partition replicas are reassigned to the remaining CPU cores when the broker restarts with fewer available cores. --- modules/get-started/pages/whats-new.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/get-started/pages/whats-new.adoc b/modules/get-started/pages/whats-new.adoc index 44ce69470e..887a0fa3b6 100644 --- a/modules/get-started/pages/whats-new.adoc +++ b/modules/get-started/pages/whats-new.adoc @@ -17,7 +17,7 @@ For topics with Tiered Storage enabled, you can unmount a topic to safely detach == Intra-broker partition balancing -xref:manage:cluster-maintenance/cluster-balancing.adoc#intra-broker-partition-balancing[Intra-broker partition balancing], which balances partitions across cores within a Redpanda broker, has moved out of beta and is supported for production clusters. +xref:manage:cluster-maintenance/cluster-balancing.adoc#intra-broker-partition-balancing[Intra-broker partition balancing] has moved out of beta and is supported for production clusters. This dynamically balances partitions across cores within a Redpanda broker. You also can decrease the number of CPUs assigned to a broker. Existing partition replicas are reassigned to the remaining CPU cores when the broker restarts with fewer available cores. == Tombstone removal From a81491825e32e1eb1a2ab60648b2e1e024976afc Mon Sep 17 00:00:00 2001 From: Paulo Borges Date: Tue, 19 Nov 2024 13:55:44 -0300 Subject: [PATCH 8/9] Update --- package-lock.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6fbdb94c67..452eeced75 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2773,10 +2773,9 @@ } }, "node_modules/@redpanda-data/docs-extensions-and-macros": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/@redpanda-data/docs-extensions-and-macros/-/docs-extensions-and-macros-3.8.0.tgz", - "integrity": "sha512-S4ssVauyULIqHFrRVX7ymIkHpNQo/f+D/4ENt4B3kKK1ISwlpcHIoAi4sVe6tdztO567oRp3ZMKAC9qx2mef3Q==", - "license": "ISC", + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/@redpanda-data/docs-extensions-and-macros/-/docs-extensions-and-macros-3.9.0.tgz", + "integrity": "sha512-BkiPDj69KUC+gAN1OdGqwJOy4MLbEEyn5+FQYalc/sIP6uQVYgoTd9neFwMmwz2L/lBNkQ12KAv0jFVuB+/plg==", "dependencies": { "@asciidoctor/tabs": "^1.0.0-beta.6", "@octokit/core": "^6.1.2", From 0396cea6b92ba4ed0430d26fc413dfda0a040b4d Mon Sep 17 00:00:00 2001 From: Michele Cyran Date: Tue, 19 Nov 2024 11:38:20 -0700 Subject: [PATCH 9/9] remove iceberg properties --- modules/get-started/pages/whats-new.adoc | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/modules/get-started/pages/whats-new.adoc b/modules/get-started/pages/whats-new.adoc index 887a0fa3b6..ab1ebc84e4 100644 --- a/modules/get-started/pages/whats-new.adoc +++ b/modules/get-started/pages/whats-new.adoc @@ -77,15 +77,5 @@ The following cluster properties are new in this version: * xref:reference:properties/cluster-properties.adoc#debug_bundle_auto_removal_seconds[`debug_bundle_auto_removal_seconds`] * xref:reference:properties/cluster-properties.adoc#debug_bundle_storage_dir[`debug_bundle_storage_dir`] * xref:reference:properties/cluster-properties.adoc#default_leaders_preference[`default_leaders_preference`] -* xref:reference:properties/cluster-properties.adoc#iceberg_catalog_base_location[`iceberg_catalog_base_location`] -* xref:reference:properties/cluster-properties.adoc#iceberg_catalog_commit_interval_ms[`iceberg_catalog_commit_interval_ms`] -* xref:reference:properties/cluster-properties.adoc#iceberg_catalog_type[`iceberg_catalog_type`] -* xref:reference:properties/cluster-properties.adoc#iceberg_enabled[`iceberg_enabled`] -* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_endpoint[`iceberg_rest_catalog_endpoint`] -* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_request_timeout_ms[`iceberg_rest_catalog_request_timeout_ms`] -* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_secret[`iceberg_rest_catalog_secret`] -* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_token[`iceberg_rest_catalog_token`] -* xref:reference:properties/cluster-properties.adoc#iceberg_rest_catalog_user_id[`iceberg_rest_catalog_user_id`] -* xref:reference:properties/cluster-properties.adoc#iceberg_translation_interval_ms_default[`iceberg_translation_interval_ms_default`] * xref:reference:properties/cluster-properties.adoc#rpk_path[`rpk_path`] * xref:reference:properties/cluster-properties.adoc#tombstone_retention_ms[`tombstone_retention_ms`]