From 197a8f1820b1eada5ac685a0aba1fe8159ec88d3 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Fri, 7 Nov 2025 12:15:13 +0000 Subject: [PATCH 01/18] DOC-5937 trial Oracle checklist --- .../data-pipelines/prepare-dbs/oracle.md | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/oracle.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/oracle.md index 10bd21e677..502b2a57f3 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/oracle.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/oracle.md @@ -21,7 +21,37 @@ Oracle provides two main systems that Debezium can use to capture data changes: - [LogMiner](#logminer) - [XStream](#xstream) -The sections below explain how to configure each system for use with Debezium and RDI. +The sections below explain how to configure each system for use with Debezium and RDI. +The checklists summarize the steps you should follow to configure each system. +You may find it helpful to use them to track your progress as you work through the steps. + +**LogMiner** + +```checklist {id="oraclelogminerlist"} +- [ ] [Configure Oracle LogMiner](#1-configure-oracle-logminer) +- [ ] [Enable supplemental logging](#supp-logging) +- [ ] [Check the redo log sizing](#3-check-the-redo-log-sizing) +- [ ] [Set the Archive log destination](#4-set-the-archive-log-destination) +- [ ] [Create a user for the connector](#create-dbz-user) +``` + +**XStream** + +```checklist {id="oraclexstreamlist"} +- [ ] [Configure Xstream](#1-configure-xstream) +- [ ] [Create Xstream users](#2-create-xstream-users) +- [ ] [Create an Xstream outbound server](#3-create-an-xstream-outbound-server) +- [ ] [Add a custom Docker image for the Debezium server](#4-add-a-custom-docker-image-for-the-debezium-server) +- [ ] [Enable the Oracle configuration in RDI](#5-enable-the-oracle-configuration-in-rdi) +``` + +**Optional: XMLTYPE Support** + +```checklist {id="oraclexmltypelist"} +- [ ] [Create a custom Debezium Server image](#create-a-custom-debezium-server-image) +- [ ] [Configure RDI for XMLTYPE support](#configure-rdi-for-xmltype-support) +- [ ] [Test XMLTYPE support](#test-xmltype-support) +``` ## LogMiner @@ -330,7 +360,7 @@ you should also revoke `LOCK` on all tables: REVOKE LOCK ANY TABLE FROM c##dbzuser container=all; ``` -### 6. Configuration is complete +### 6. Configuration is complete {#logminer-complete} Once you have followed the steps above, your Oracle database is ready for Debezium to use. @@ -627,7 +657,7 @@ See the [Debezium Oracle documentation](https://debezium.io/documentation/reference/stable/connectors/oracle.html#oracle-connector-properties) for a full list of properties you can use in the `advanced.source` subsection. -### 6. Configuration is complete +### 6. Configuration is complete {#xstream-complete} After you have followed the steps above, your Oracle database is ready for Debezium to use. From a465e251233339812d2ee25693f2d3dc12e56f22 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Thu, 13 Nov 2025 14:57:51 +0000 Subject: [PATCH 02/18] DOC-5937 made checkboxes look more 'interactive' --- static/js/checklist.js | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/static/js/checklist.js b/static/js/checklist.js index 37ba3dc489..dee5189337 100644 --- a/static/js/checklist.js +++ b/static/js/checklist.js @@ -42,7 +42,31 @@ function createChecklistFromMarkdown(markdown, formId, preElement) { // Create select dropdown const select = document.createElement('select'); select.onchange = () => clChange(formId); - + + // Add styling to make the select element look like an interactive control + select.style.border = '1px solid #d0d0d0'; + select.style.borderRadius = '4px'; + select.style.padding = '4px 6px'; + select.style.backgroundColor = '#fafafa'; + select.style.cursor = 'pointer'; + select.style.fontFamily = 'inherit'; + select.style.fontSize = 'inherit'; + select.style.marginRight = '8px'; + select.style.transition = 'all 0.2s ease'; + + // Add hover effects + select.addEventListener('mouseenter', () => { + select.style.borderColor = '#999'; + select.style.backgroundColor = '#f0f0f0'; + select.style.boxShadow = '0 1px 3px rgba(0, 0, 0, 0.1)'; + }); + + select.addEventListener('mouseleave', () => { + select.style.borderColor = '#d0d0d0'; + select.style.backgroundColor = '#fafafa'; + select.style.boxShadow = 'none'; + }); + const options = [ { value: 'R', label: '❌' }, { value: 'G', label: '✅' }, @@ -56,7 +80,7 @@ function createChecklistFromMarkdown(markdown, formId, preElement) { option.textContent = opt.label; select.appendChild(option); }); - + li.appendChild(select); // Parse link and text from markdown From 6a81fb2a8cfd61043d768617b0f0614bc396c887 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Thu, 13 Nov 2025 15:11:52 +0000 Subject: [PATCH 03/18] DOC-5937 added hidden MongoDB checklist --- .../data-pipelines/prepare-dbs/mongodb.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md index 0b00d9d321..c7ec9cbf99 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md @@ -37,6 +37,15 @@ The following table summarizes the steps to prepare a MongoDB database for RDI. | MongoDB Atlas | **[SSL required](https://debezium.io/documentation/reference/stable/connectors/mongodb.html#mongodb-property-mongodb-ssl-enabled)**, provide root CA as `SOURCE_DB_CACERT` secret in RDI | | Network | RDI Collector must reach all MongoDB nodes on required ports | +```checklist {id="mongodblist" nointeractive="true" } +- [ ] [Configure oplog size](#1-configure-oplog-size) +- [ ] [Create a MongoDB user for RDI](#2-create-a-mongodb-user-for-rdi) +- [ ] [Connection string format](#3-connection-string-format) +- [ ] [Enable change streams and pre/post images (only if using a custom key)](#4-enable-change-streams-and-prepost-images-only-if-using-a-custom-key) +- [ ] [MongoDB Atlas specific requirements](#5-mongodb-atlas-specific-requirements) +- [ ] [Network and security](#6-network-and-security) +``` + ## 1. Configure oplog size The Debezium MongoDB connector relies on the [oplog](https://www.mongodb.com/docs/manual/core/replica-set-oplog/) to capture changes from a replica set. The oplog is a fixed-size, capped collection. When it reaches its maximum size, it overwrites the oldest entries. If the connector is stopped and restarted, it attempts to resume from its last recorded position in the oplog. If that position has been overwritten, the connector may fail to start and report an invalid resume token error. From 19d555f72588e83211a5af97f249532ca0b3b812 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Thu, 13 Nov 2025 15:14:33 +0000 Subject: [PATCH 04/18] DOC-5937 added hidden MySQL checklist --- .../data-pipelines/prepare-dbs/my-sql-mariadb.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/my-sql-mariadb.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/my-sql-mariadb.md index 2b7c097cc7..318086e652 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/my-sql-mariadb.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/my-sql-mariadb.md @@ -19,6 +19,15 @@ weight: 2 Follow the steps in the sections below to set up a MySQL or MariaDB database for CDC with Debezium. +```checklist {id="mysqlmariadblist" nointeractive="true" } +- [ ] [Create a CDC user](#1-create-a-cdc-user) +- [ ] [Enable the binlog](#2-enable-the-binlog) +- [ ] [Enable GTIDs](#3-enable-gtids) +- [ ] [Configure session timeouts](#4-configure-session-timeouts) +- [ ] [Enable query log events](#5-enable-query-log-events) +- [ ] [Check binlog_row_value_options](#6-check-binlog_row_value_options) +``` + ## 1. Create a CDC user The Debezium connector needs a user account to connect to MySQL/MariaDB. This From c1baee56bc67605cf16cf8583d1e6e8726b4a5f9 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Thu, 13 Nov 2025 15:27:31 +0000 Subject: [PATCH 05/18] DOC-5937 added PostgreSQL checklists and extra anchors in list --- .../data-pipelines/prepare-dbs/postgresql.md | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/postgresql.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/postgresql.md index 98d2e3e4df..6cf848cd6d 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/postgresql.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/postgresql.md @@ -33,22 +33,37 @@ your database then this might restrict the plug-ins you can use. If you can't us plug-in then could try the `pgoutput` decoder if you are using PostgreSQL 10 or above. If this doesn't work for you then you won't be able to use RDI with your database. -### Amazon RDS for PostgreSQL +```checklist {id="postgreslist" nointeractive="true" } +- [ ] [Install the logical decoding output plug-in](#install-the-logical-decoding-output-plug-in) +- [ ] [Configure the PostgreSQL server](#configure-the-postgresql-server) +- [ ] [Set up permissions](#set-up-permissions) +- [ ] [Set privileges for Debezium to create PostgreSQL publications with pgoutput](#set-privileges-for-debezium-to-create-postgresql-publications-with-pgoutput) +- [ ] [Configure PostgreSQL for replication with the Debezium connector host](#configure-postgresql-for-replication-with-the-debezium-connector-host) +``` + +## Amazon RDS for PostgreSQL Follow the steps below to enable CDC with [Amazon RDS for PostgreSQL](https://aws.amazon.com/rds/postgresql/): -1. Set the instance parameter `rds.logical_replication` to 1. +```checklist {id="postgresawslist" nointeractive="true" } +- [ ] [Set the instance parameter rds.logical_replication to 1](#1-set-the-instance-parameter-rdslogicalreplication-to-1) +- [ ] [Check that the wal_level parameter is set to logical](#2-check-that-the-wal_level-parameter-is-set-to-logical) +- [ ] [Set the Debezium plugin.name parameter to pgoutput](#3-set-the-debezium-pluginname-parameter-to-pgoutput) +- [ ] [Initiate logical replication from an AWS account that has the rds_replication role](#4-initiate-logical-replication-from-an-aws-account-that-has-the-rdsreplication-role) +``` + +1. Set the instance parameter `rds.logical_replication` to 1. -1. Check that the `wal_level` parameter is set to `logical` by running the query `SHOW wal_level` +1. Check that the `wal_level` parameter is set to `logical` by running the query `SHOW wal_level` as the database RDS master user. The parameter might not have this value in multi-zone replication setups. You can't change the value manually but it should change automatically when you set the `rds.logical_replication` parameter to 1. If it doesn't change then you probably just need to restart your database instance. You can restart manually or wait until a restart occurs during your maintenance window. -1. Set the Debezium `plugin.name` parameter to `pgoutput`. +1. Set the Debezium `plugin.name` parameter to `pgoutput`. -1. Initiate logical replication from an AWS account that has the `rds_replication` role. The role grants +1. Initiate logical replication from an AWS account that has the `rds_replication` role. The role grants permissions to manage logical slots and to stream data using logical slots. By default, only the master user account on AWS has the `rds_replication` role on Amazon RDS, but if you have administrator privileges, you can grant the role to other accounts using a query like the following: From fd826a217a6522524489751450725df3c5f2d20a Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Thu, 13 Nov 2025 15:32:06 +0000 Subject: [PATCH 06/18] DOC-5937 added Spanner checklists plus new list item anchors --- .../data-pipelines/prepare-dbs/spanner.md | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md index 59f93e5139..a1c4dab8ac 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md @@ -26,6 +26,15 @@ the monitored schemas and tables. Spanner is only supported with RDI deployed on Kubernetes/Helm. RDI VM mode does not support Spanner as a source database. {{< /note >}} +```checklist {id="spannerlist" nointeractive="true" } +- [ ] [Prepare for snapshot](#1-prepare-for-snapshot) +- [ ] [Prepare for streaming](#2-prepare-for-streaming) +- [ ] [Create a service account](#3-create-a-service-account) +- [ ] [Set up secrets for Kubernetes deployment (optional)](#4-set-up-secrets-for-kubernetes-deployment-optional) +- [ ] [Configure RDI for Spanner](#5-configure-rdi-for-spanner) +- [ ] [Additional Kubernetes configuration](#6-additional-kubernetes-configuration) +``` + ## 1. Prepare for snapshot During the snapshot phase, RDI executes multiple transactions to capture data at an exact point @@ -64,7 +73,13 @@ To allow RDI to access the Spanner instance, you'll need to create a service acc appropriate permissions. By default, RDI uses Google Cloud Workload Identity authentication. In this case RDI will assume the [service account is assigned to the GKE cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#enable_on_clusters_and_node_pools). Alternatively, you can provide the service account credentials as a Kubernetes secret (see step 4 for details). -1. Create the service account +```checklist {id="spanner-service-account" nointeractive="true" } +- [ ] [Create the service account](#create-the-service-account) +- [ ] [Grant required roles](#grant-required-roles) +- [ ] [Download the service account key](#download-the-service-account-key) +``` + +1. Create the service account ```bash gcloud iam service-accounts create spanner-reader-account \ @@ -73,7 +88,7 @@ service account credentials as a Kubernetes secret (see step 4 for details). --project=YOUR_PROJECT_ID ``` -1. Grant required roles +1. Grant required roles **Database Reader** (read access to Spanner data): @@ -99,7 +114,7 @@ service account credentials as a Kubernetes secret (see step 4 for details). --role="roles/spanner.viewer" ``` -1. Download the service account key +1. Download the service account key Save the credentials locally so they can be used later by RDI: From ab19fa0ab3ab487f57ba2e88ae1ef73d7ea76512 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Thu, 13 Nov 2025 15:37:23 +0000 Subject: [PATCH 07/18] DOC-5937 added SQL Server checklist --- .../data-pipelines/prepare-dbs/sql-server.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md index 3723aa0c84..f0820a2c04 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md @@ -23,6 +23,13 @@ capture. You need administrator privileges to do this. Once you enable CDC, it captures all of the INSERT, UPDATE, and DELETE operations on your chosen tables. The Debezium connector can then emit these events to RDI. +```checklist {id="sqlserverlist" nointeractive="true" } +- [ ] [Create a Debezium user](#1-create-a-debezium-user) +- [ ] [Enable CDC on the database](#2-enable-cdc-on-the-database) +- [ ] [Enable CDC for the tables you want to capture](#3-enable-cdc-for-the-tables-you-want-to-capture) +- [ ] [Check that you have access to the CDC table](#4-check-that-you-have-access-to-the-cdc-table) +``` + ## 1. Create a Debezium user It is strongly recommended to create a dedicated Debezium user for the connection between RDI From 110ba747298262f35085fba19becaacf4b022cc2 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Thu, 13 Nov 2025 15:48:36 +0000 Subject: [PATCH 08/18] DOC-5937 added AWS Aurora MySQL checklists and new anchors --- .../aws-aurora-rds/aws-aur-mysql.md | 46 +++++++++++++++---- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-mysql.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-mysql.md index 7e95a35ffd..195aa6689a 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-mysql.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-mysql.md @@ -25,6 +25,12 @@ Select the steps for your database type. tab1="AWS Aurora MySQL" tab2="AWS RDS MySQL" >}} +```checklist {id="auroramysql" nointeractive="true" } +- [ ] [Add an Aurora reader node](#add-an-aurora-reader-node) +- [ ] [Create and apply parameter group](#aurora-create-and-apply-parameter-group) +- [ ] [Create Debezium user](#aurora-create-debezium-user) +``` + ## Add an Aurora reader node RDI requires that your Aurora MySQL database has at least one replica or reader node. @@ -33,11 +39,17 @@ To add a reader node to an existing database, select **Add reader** from the **A You can also create one during database creation by selecting **Create an Aurora Replica or Reader node in a different AZ (recommended for scaled availability)** under **Availability & durability > Multi-AZ deployment**. -## Create and apply parameter group +## Create and apply parameter group RDI requires some changes to database parameters. On AWS Aurora, you change these parameters via a parameter group. -1. In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/),navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.Creating.html) with the following settings: +```checklist {id="auroramysql-param-group" nointeractive="true" } +- [ ] [Create a parameter group](#aurora-create-a-parameter-group) +- [ ] [Apply the parameter group](#aurora-apply-the-parameter-group) +- [ ] [Reboot the database instance](#aurora-reboot-the-database-instance) +``` + +1. In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/),navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.Creating.html) with the following settings: | Name | Value | | :-- | :-- | @@ -49,7 +61,7 @@ RDI requires some changes to database parameters. On AWS Aurora, you change thes Select **Create** to create the parameter group. -1. Navigate to **Parameter groups** in the console. Select the parameter group you have just created and then select **Edit**. Change the following parameters: +1. Navigate to **Parameter groups** in the console. Select the parameter group you have just created and then select **Edit**. Change the following parameters: | Name | Value | | :-- | :-- | @@ -60,22 +72,34 @@ RDI requires some changes to database parameters. On AWS Aurora, you change thes Select **Save Changes** to apply the changes to the parameter group. -1. Go back to your target database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. +1. Go back to your target database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. Select **Save changes** to apply the parameter group to the new database. -1. Reboot your database instance. See [Rebooting a DB instance within an Aurora cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-reboot-db-instance.html) for more information. +1. Reboot your database instance. See [Rebooting a DB instance within an Aurora cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-reboot-db-instance.html) for more information. + {{< embed-md "aur-rds-mysql-create-debezium-user.md" >}} -tab-sep- -## Create and apply parameter group +```checklist {id="rds-mysql-list" nointeractive="true" } +- [ ] [Create and apply parameter group](#rds-create-and-apply-parameter-group) +- [ ] [Create Debezium user](#rds-create-debezium-user) +``` + +## Create and apply parameter group RDI requires some changes to database parameters. On AWS RDS, you change these parameters via a parameter group. -1. In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/),navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_WorkingWithParamGroups.Creating.html) with the following settings: +```checklist {id="rds-mysql-param-group" nointeractive="true" } +- [ ] [Create a parameter group](#rds-create-a-parameter-group) +- [ ] [Apply the parameter group](#rds-apply-the-parameter-group) +- [ ] [Reboot the database instance](#rds-reboot-the-database-instance) +``` + +1. In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/),navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_WorkingWithParamGroups.Creating.html) with the following settings: | Name | Value | | :-- | :-- | @@ -86,7 +110,7 @@ RDI requires some changes to database parameters. On AWS RDS, you change these p Select **Create** to create the parameter group. -1. Navigate to **Parameter groups** in the console. Select the parameter group you have just created and then select **Edit**. Change the following parameters: +1. Navigate to **Parameter groups** in the console. Select the parameter group you have just created and then select **Edit**. Change the following parameters: | Name | Value | | :-- | :-- | @@ -95,11 +119,13 @@ RDI requires some changes to database parameters. On AWS RDS, you change these p Select **Save Changes** to apply the changes to the parameter group. -1. Go back to your target database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. +1. Go back to your target database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. Select **Save changes** to apply the parameter group to the new database. -1. Reboot your database instance. See [Rebooting a DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_RebootInstance.html) for more information. +1. Reboot your database instance. See [Rebooting a DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_RebootInstance.html) for more information. + + {{< embed-md "aur-rds-mysql-create-debezium-user.md" >}} From 1589216b537d23b0f6ae25426072acb4f64d594e Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Thu, 13 Nov 2025 15:53:43 +0000 Subject: [PATCH 09/18] DOC-5937 added AWS Aurora PostgreSQL checklists and new anchors --- .../aws-aurora-rds/aws-aur-pgsql.md | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-pgsql.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-pgsql.md index ff4a89d0c3..932cda21ff 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-pgsql.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-pgsql.md @@ -23,11 +23,22 @@ Follow the steps in the sections below to prepare an [AWS Aurora PostgreSQL](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/CHAP_GettingStartedAurora.CreatingConnecting.AuroraPostgreSQL.html) or [AWS RDS PostgreSQL](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_GettingStarted.CreatingConnecting.PostgreSQL.html) database to work with RDI. +```checklist {id="aurorapostgresql" nointeractive="true" } +- [ ] [Create and apply parameter group](#create-and-apply-parameter-group) +- [ ] [Create Debezium user](#create-debezium-user) +``` + ## Create and apply parameter group RDI requires some changes to database parameters. On AWS RDS and AWS Aurora, you change these parameters via a parameter group. -1. In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/), navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.CreatingCluster.html) with the following settings: +```checklist {id="aurorapostgresql-param-group" nointeractive="true" } +- [ ] [Create a parameter group](#create-a-parameter-group) +- [ ] [Apply the parameter group](#apply-the-parameter-group) +- [ ] [Reboot the database instance](#reboot-the-database-instance) +``` + +1. In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/), navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.CreatingCluster.html) with the following settings: | Name | Value | | :-- | :-- | @@ -38,7 +49,7 @@ RDI requires some changes to database parameters. On AWS RDS and AWS Aurora, you Select **Create** to create the parameter group. -1. Navigate to **Parameter groups** in the console. Select the group you have just created and then select **Edit**. Change the following parameters: +1. Navigate to **Parameter groups** in the console. Select the group you have just created and then select **Edit**. Change the following parameters: | Name | Value | | :-- | :-- | @@ -50,7 +61,7 @@ RDI requires some changes to database parameters. On AWS RDS and AWS Aurora, you Select **Save changes** to apply the parameter group to your database. -1. Reboot your database instance. See [Rebooting a DB instance within an Aurora cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-reboot-db-instance.html) or [Rebooting a DB instance (RDS)](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_RebootInstance.html) for more information. +1. Reboot your database instance. See [Rebooting a DB instance within an Aurora cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-reboot-db-instance.html) or [Rebooting a DB instance (RDS)](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_RebootInstance.html) for more information. ## Create Debezium user @@ -58,7 +69,13 @@ The Debezium connector needs a user account to connect to PostgreSQL. This user must have appropriate permissions on all databases where you want Debezium to capture changes. -1. Connect to PostgreSQL as the `postgres` user and create a new user for the connector: +```checklist {id="aurorapostgresql-create-debezium-user" nointeractive="true" } +- [ ] [Connect to PostgreSQL as the `postgres` user](#connect-to-postgresql-as-the-postgres-user) +- [ ] [Grant the user the necessary replication permissions](#grant-the-user-the-necessary-replication-permissions) +- [ ] [Grant the user access to the database](#grant-the-user-access-to-the-database) +``` + +1. Connect to PostgreSQL as the `postgres` user and create a new user for the connector: ```sql CREATE ROLE WITH LOGIN PASSWORD '' VALID UNTIL 'infinity'; @@ -66,7 +83,7 @@ to capture changes. Replace `` and `` with a username and password for the new user. -1. Grant the user the necessary replication permissions: +1. Grant the user the necessary replication permissions: ```sql GRANT rds_replication TO ; @@ -74,7 +91,7 @@ to capture changes. Replace `` with the username of the Debezium user. -1. Connect to your database as the `postgres` user and grant the new user access to one or more schemas in the database: +1. Connect to your database as the `postgres` user and grant the new user access to one or more schemas in the database: ```sql GRANT SELECT ON ALL TABLES IN SCHEMA TO ; From 1b17ad267f49b6bee7c56d3e45ad6536a285759d Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Thu, 13 Nov 2025 15:58:17 +0000 Subject: [PATCH 10/18] DOC-5937 added AWS RDS SQL Server checklists and new anchors --- .../aws-aurora-rds/aws-rds-sqlserver.md | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-rds-sqlserver.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-rds-sqlserver.md index 5d7dbca456..fa9a2ac397 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-rds-sqlserver.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-rds-sqlserver.md @@ -22,13 +22,23 @@ Follow the steps in the sections below to prepare a [Microsoft SQL Server on AWS Change Data Capture (CDC) is not supported on SQL Server Express Edition. Only the Standard, Enterprise, and Developer editions support CDC and are supported by RDI. {{< /note >}} +```checklist {id="rds-sqlserver-list" nointeractive="true" } +- [ ] [Create the Debezium user](#create-the-debezium-user) +- [ ] [Enable CDC on the database](#enable-cdc-on-the-database) +``` + ## Create the Debezium user The Debezium connector needs a user account to connect to SQL Server. This user must have appropriate permissions on all databases where you want Debezium to capture changes. -1. Connect to your database as an admin user and create a new user for the connector: +```checklist {id="rds-sqlserver-create-debezium-user" nointeractive="true" } +- [ ] [Connect to SQL Server as an admin user](#connect-to-sql-server-as-an-admin-user) +- [ ] [Grant the user the necessary permissions](#grant-the-user-the-necessary-permissions) +``` + +1. Connect to your database as an admin user and create a new user for the connector: ```sql USE master @@ -43,7 +53,7 @@ to capture changes. Replace `` and `` with a username and password for the new user and replace `` with the name of your database. -1. Grant the user the necessary permissions: +1. Grant the user the necessary permissions: ```sql USE master @@ -62,7 +72,13 @@ to capture changes. Change Data Capture (CDC) must be enabled for the database and for each table you want to capture. -1. Enable CDC for the database by running the following command: +```checklist {id="rds-sqlserver-enable-cdc" nointeractive="true" } +- [ ] [Enable CDC for the database](#enable-cdc-for-the-database) +- [ ] [Enable CDC for each table you want to capture](#enable-cdc-for-each-table-you-want-to-capture) +- [ ] [Add the Debezium user to the CDC role](#add-the-debezium-user-to-the-cdc-role) +``` + +1. Enable CDC for the database by running the following command: ```sql EXEC msdb.dbo.rds_cdc_enable_db '' @@ -71,7 +87,7 @@ Change Data Capture (CDC) must be enabled for the database and for each table yo Replace `` with the name of your database. -1. Enable CDC for each table you want to capture by running the following commands: +1. Enable CDC for each table you want to capture by running the following commands: ```sql USE @@ -92,7 +108,7 @@ Specifying a new name will create a corresponding database role that has full ac captured change data. {{< /note >}} -1. Add the Debezium user to the CDC role: +1. Add the Debezium user to the CDC role: ```sql USE From 1f3fa280d00502f7b4d45b777b615b8b7060fbb5 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Fri, 14 Nov 2025 09:33:52 +0000 Subject: [PATCH 11/18] DOC-5937 made MongoDB checklist interactive --- .../data-pipelines/prepare-dbs/mongodb.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md index c7ec9cbf99..ed7ac83831 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md @@ -25,7 +25,7 @@ This guide describes the steps required to prepare a MongoDB database as a sourc {{< note >}}The MongoDB connector is not capable of monitoring the changes of a standalone MongoDB server, since standalone servers do not have an oplog. The connector will work if the standalone server is converted to a replica set with one member.{{< /note >}} ## Summary -The following table summarizes the steps to prepare a MongoDB database for RDI. The sections below explain the steps in more detail. +The following table summarizes the considerations to prepare a MongoDB database for RDI. | Requirement | Description | |---------------------|-----------------------------------------------------------------------------| @@ -37,7 +37,12 @@ The following table summarizes the steps to prepare a MongoDB database for RDI. | MongoDB Atlas | **[SSL required](https://debezium.io/documentation/reference/stable/connectors/mongodb.html#mongodb-property-mongodb-ssl-enabled)**, provide root CA as `SOURCE_DB_CACERT` secret in RDI | | Network | RDI Collector must reach all MongoDB nodes on required ports | -```checklist {id="mongodblist" nointeractive="true" } +The following checklist shows the steps to prepare a MongoDB database for RDI, +with links to the sections that explain the steps in full detail. +You may find it helpful to track your progress with the checklist as you +complete each step. + +```checklist {id="mongodblist"} - [ ] [Configure oplog size](#1-configure-oplog-size) - [ ] [Create a MongoDB user for RDI](#2-create-a-mongodb-user-for-rdi) - [ ] [Connection string format](#3-connection-string-format) From 7b8aeb221312e4bd8cdfea75657c026f85c3292d Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Fri, 14 Nov 2025 09:43:15 +0000 Subject: [PATCH 12/18] DOC-5937 made MySQL checklist interactive plus added sublists --- .../prepare-dbs/my-sql-mariadb.md | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/my-sql-mariadb.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/my-sql-mariadb.md index 318086e652..3d9bb2ef96 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/my-sql-mariadb.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/my-sql-mariadb.md @@ -16,10 +16,12 @@ type: integration weight: 2 --- -Follow the steps in the sections below to set up a MySQL or MariaDB -database for CDC with Debezium. +The following checklist summarizes the steps to prepare a MySQL or MariaDB +database for RDI, with links to the sections that explain the steps in +full detail. You may find it helpful to track your progress with the +checklist as you complete each step. -```checklist {id="mysqlmariadblist" nointeractive="true" } +```checklist {id="mysqlmariadblist"} - [ ] [Create a CDC user](#1-create-a-cdc-user) - [ ] [Enable the binlog](#2-enable-the-binlog) - [ ] [Enable GTIDs](#3-enable-gtids) @@ -37,13 +39,21 @@ to capture changes. Run the [MySQL CLI client](https://dev.mysql.com/doc/refman/8.3/en/mysql.html) and then run the following commands: -1. Create the CDC user: +```checklist {id="mysqlmariadb-create-cdc-user" nointeractive="true" } +- [ ] [Create the CDC user](#create-the-cdc-user) +- [ ] [Grant the user the necessary permissions](#grant-the-user-the-necessary-permissions) +- [ ] [Finalize the user's permissions](#finalize-the-users-permissions) +``` + +1. + Create the CDC user: ```sql mysql> CREATE USER 'user'@'localhost' IDENTIFIED BY 'password'; ``` -1. Grant the required permissions to the user: +1. + Grant the required permissions to the user: ```sql # MySQL GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'user'@'localhost'; ``` -1. Finalize the user's permissions: +1. + Finalize the user's permissions: ```sql mysql> FLUSH PRIVILEGES; @@ -109,19 +120,28 @@ GTIDs are available in MySQL 5.6.5 and later. See the Follow the steps below to enable GTIDs. You will need access to the MySQL configuration file to do this. -1. Enable `gtid_mode`: +```checklist {id="mysqlmariadb-enable-gtids" nointeractive="true" } +- [ ] [Enable gtid_mode](#enable-gtid_mode) +- [ ] [Enable enforce_gtid_consistency](#enable-enforce_gtid_consistency) +- [ ] [Confirm the changes](#confirm-the-changes) +``` + +1. + Enable `gtid_mode`: ```sql mysql> gtid_mode=ON ``` -1. Enable `enforce_gtid_consistency`: +1. + Enable `enforce_gtid_consistency`: ```sql mysql> enforce_gtid_consistency=ON ``` -1. Confirm the changes: +1. + Confirm the changes: ```sql mysql> show global variables like '%GTID%'; From 9b306d8e6b526a4f16162be70033433b3909cbda Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Fri, 14 Nov 2025 09:50:03 +0000 Subject: [PATCH 13/18] DOC-5937 make PostrgreSQL checklist interactive plus add mini-lists --- .../data-pipelines/prepare-dbs/postgresql.md | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/postgresql.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/postgresql.md index 6cf848cd6d..01b7397734 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/postgresql.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/postgresql.md @@ -33,7 +33,12 @@ your database then this might restrict the plug-ins you can use. If you can't us plug-in then could try the `pgoutput` decoder if you are using PostgreSQL 10 or above. If this doesn't work for you then you won't be able to use RDI with your database. -```checklist {id="postgreslist" nointeractive="true" } +The following checklist summarizes the steps to prepare a PostgreSQL +database for RDI, with links to the sections that explain the steps in +full detail. You may find it helpful to track your progress with the +checklist as you complete each step. + +```checklist {id="postgreslist"} - [ ] [Install the logical decoding output plug-in](#install-the-logical-decoding-output-plug-in) - [ ] [Configure the PostgreSQL server](#configure-the-postgresql-server) - [ ] [Set up permissions](#set-up-permissions) @@ -52,18 +57,22 @@ Follow the steps below to enable CDC with [Amazon RDS for PostgreSQL](https://aw - [ ] [Initiate logical replication from an AWS account that has the rds_replication role](#4-initiate-logical-replication-from-an-aws-account-that-has-the-rdsreplication-role) ``` -1. Set the instance parameter `rds.logical_replication` to 1. +1. + Set the instance parameter `rds.logical_replication` to 1. -1. Check that the `wal_level` parameter is set to `logical` by running the query `SHOW wal_level` +1. + Check that the `wal_level` parameter is set to `logical` by running the query `SHOW wal_level` as the database RDS master user. The parameter might not have this value in multi-zone replication setups. You can't change the value manually but it should change automatically when you set the `rds.logical_replication` parameter to 1. If it doesn't change then you probably just need to restart your database instance. You can restart manually or wait until a restart occurs during your maintenance window. -1. Set the Debezium `plugin.name` parameter to `pgoutput`. +1. + Set the Debezium `plugin.name` parameter to `pgoutput`. -1. Initiate logical replication from an AWS account that has the `rds_replication` role. The role grants +1. + Initiate logical replication from an AWS account that has the `rds_replication` role. The role grants permissions to manage logical slots and to stream data using logical slots. By default, only the master user account on AWS has the `rds_replication` role on Amazon RDS, but if you have administrator privileges, you can grant the role to other accounts using a query like the following: @@ -190,24 +199,37 @@ this case, the source table already exists, so you must use a PostgreSQL replica group to share ownership between the Debezium user and the original owner. Configure the replication group using the following commands: -1. Create the replication group (the name `replication_group` here is +```checklist {id="postgrespgoutputlist" nointeractive="true" } +- [ ] [Create the replication group](#1-create-the-replication-group) +- [ ] [Add the original owner of the table to the group](#2-add-the-original-owner-of-the-table-to-the-group) +- [ ] [Add the Debezium replication user to the group](#3-add-the-debezium-replication-user-to-the-group) +- [ ] [Transfer ownership of the table to the replication group](#4-transfer-ownership-of-the-table-to-the-replication-group) +``` + +1. + Create the replication group (the name `replication_group` here is just an example): ```sql CREATE ROLE replication_group; ``` -1. Add the original owner of the table to the group: + +1. + Add the original owner of the table to the group: ```sql GRANT replication_group TO original_owner; ``` -1. Add the Debezium replication user to the group: +1. + Add the Debezium replication user to the group: ```sql GRANT replication_group TO replication_user; ``` -1. Transfer ownership of the table to `replication_group`: + +1. + Transfer ownership of the table to `replication_group`: ```sql ALTER TABLE table_name OWNER TO replication_group; From b93f69b0e98797caf474557df728045b4f55f69f Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Fri, 14 Nov 2025 09:53:59 +0000 Subject: [PATCH 14/18] DOC-5937 made Spanner checklist interactive plus added mini-lists --- .../data-pipelines/prepare-dbs/spanner.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md index a1c4dab8ac..1e5b16966d 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md @@ -26,7 +26,12 @@ the monitored schemas and tables. Spanner is only supported with RDI deployed on Kubernetes/Helm. RDI VM mode does not support Spanner as a source database. {{< /note >}} -```checklist {id="spannerlist" nointeractive="true" } +The following checklist summarizes the steps to prepare a Spanner +database for RDI, with links to the sections that explain the steps in +full detail. You may find it helpful to track your progress with the +checklist as you complete each step. + +```checklist {id="spannerlist"} - [ ] [Prepare for snapshot](#1-prepare-for-snapshot) - [ ] [Prepare for streaming](#2-prepare-for-streaming) - [ ] [Create a service account](#3-create-a-service-account) @@ -79,7 +84,8 @@ service account credentials as a Kubernetes secret (see step 4 for details). - [ ] [Download the service account key](#download-the-service-account-key) ``` -1. Create the service account +1. + Create the service account ```bash gcloud iam service-accounts create spanner-reader-account \ @@ -88,7 +94,8 @@ service account credentials as a Kubernetes secret (see step 4 for details). --project=YOUR_PROJECT_ID ``` -1. Grant required roles +1. + Grant required roles: **Database Reader** (read access to Spanner data): @@ -114,7 +121,8 @@ service account credentials as a Kubernetes secret (see step 4 for details). --role="roles/spanner.viewer" ``` -1. Download the service account key +1. + Download the service account key: Save the credentials locally so they can be used later by RDI: From e0c6dc9f8495396f89556182f6e2c47d584c5cb0 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Fri, 14 Nov 2025 09:59:58 +0000 Subject: [PATCH 15/18] DOC-5937 made SQL Server checklist interactive plus added mini-lists --- .../data-pipelines/prepare-dbs/sql-server.md | 57 +++++++++++++++---- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md index f0820a2c04..b9864c28d3 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md @@ -23,7 +23,12 @@ capture. You need administrator privileges to do this. Once you enable CDC, it captures all of the INSERT, UPDATE, and DELETE operations on your chosen tables. The Debezium connector can then emit these events to RDI. -```checklist {id="sqlserverlist" nointeractive="true" } +The following checklist summarizes the steps to prepare a SQL Server +database for RDI, with links to the sections that explain the steps in +full detail. You may find it helpful to track your progress with the +checklist as you complete each step. + +```checklist {id="sqlserverlist"} - [ ] [Create a Debezium user](#1-create-a-debezium-user) - [ ] [Enable CDC on the database](#2-enable-cdc-on-the-database) - [ ] [Enable CDC for the tables you want to capture](#3-enable-cdc-for-the-tables-you-want-to-capture) @@ -36,7 +41,13 @@ It is strongly recommended to create a dedicated Debezium user for the connectio and the source database. When using an existing user, ensure that the required permissions are granted and that the user is added to the CDC role. -1. Create a user with the Transact-SQL below: +```checklist {id="sqlserver-create-debezium-user" nointeractive="true" } +- [ ] [Create the Debezium user](#create-the-debezium-user) +- [ ] [Grant the user the necessary permissions](#grant-the-user-the-necessary-permissions) +``` + +1. + Create the Debezium user with the Transact-SQL below: ```sql USE master @@ -51,7 +62,8 @@ permissions are granted and that the user is added to the CDC role. Replace `MyUser`, `My_Password` and `MyDB` with your chosen values. -1. Grant required permissions: +1. + Grant the user the necessary permissions: ```sql USE master @@ -97,8 +109,14 @@ a CDC user, metadata tables, and other system objects. ## 3. Enable CDC for the tables you want to capture -1. You must also enable CDC on the tables you want Debezium to capture using the -following commands (again, you need administrator privileges for this): +```checklist {id="sqlserver-enable-cdc-tables" nointeractive="true" } +- [ ] [Enable CDC on the tables you want to capture](#enable-cdc-on-the-tables-you-want-to-capture) +- [ ] [Add the Debezium user to the CDC role](#add-the-debezium-user-to-the-cdc-role) +``` + +1. + You must also enable CDC on the tables you want Debezium to capture using the + following commands (again, you need administrator privileges for this): ```sql USE MyDB @@ -119,7 +137,8 @@ following commands (again, you need administrator privileges for this): captured change data. {{< /note >}} -1. Add the Debezium user to the CDC role: +1. + Add the Debezium user to the CDC role: ```sql USE MyDB @@ -134,7 +153,13 @@ You can use another stored procedure `sys.sp_cdc_help_change_data_capture` to query the CDC information for the database and check you have enabled it correctly. To do this, connect as the Debezium user you created previously (`MyUser`). -1. Run the `sys.sp_cdc_help_change_data_capture` stored procedure to query +```checklist {id="sqlserver-check-cdc-table" nointeractive="true" } +- [ ] [Run the stored procedure to query the CDC configuration](#run-the-stored-procedure-to-query-the-cdc-configuration) +- [ ] [Check the results](#check-the-results) +``` + +1. + Run the `sys.sp_cdc_help_change_data_capture` stored procedure to query the CDC configuration. For example, if your database was called `MyDB` then you would run the following: @@ -145,7 +170,8 @@ it correctly. To do this, connect as the Debezium user you created previously (` GO ``` -1. The query returns configuration information for each table in the database that +1. + The query returns configuration information for each table in the database that has CDC enabled and that contains change data that you are authorized to access. If the result is empty then you should check that you have privileges to access both the capture instance and the CDC tables. @@ -225,9 +251,17 @@ a schema change and resume CDC. See the [online schema updates](https://debezium.io/documentation/reference/stable/connectors/sqlserver.html#online-schema-updates) documentation for further details. -1. Make your changes to the source table schema. +```checklist {id="sqlserver-schema-changes" nointeractive="true" } +- [ ] [Make your changes to the source table schema](#make-your-changes-to-the-source-table-schema) +- [ ] [Create a new capture table for the updated source table](#create-a-new-capture-table-for-the-updated-source-table) +- [ ] [Drop the old capture table](#drop-the-old-capture-table) +``` + +1. + Make your changes to the source table schema. -1. Create a new capture table for the updated source table by running the `sys.sp_cdc_enable_table` stored +1. + Create a new capture table for the updated source table by running the `sys.sp_cdc_enable_table` stored procedure with a new, unique value for the parameter `@capture_instance`. For example, if the old value was `dbo_MyTable`, you could replace it with `dbo_MyTable_v2` (you can see the existing values by running stored procedure `sys.sp_cdc_help_change_data_capture`): @@ -242,7 +276,8 @@ documentation for further details. GO ``` -1. When Debezium starts streaming from the new capture table, drop the old capture table by running +1. + When Debezium starts streaming from the new capture table, drop the old capture table by running the `sys.sp_cdc_disable_table` stored procedure with the parameter `@capture_instance` set to the old capture instance name, `dbo_MyTable`: From 6f6bc4ceb897751556fa3dfe362c1205cf3b99e5 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Fri, 14 Nov 2025 10:05:46 +0000 Subject: [PATCH 16/18] DOC-5937 small formatting changes --- .../aws-aurora-rds/aws-aur-pgsql.md | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-pgsql.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-pgsql.md index 932cda21ff..ec9f69fe51 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-pgsql.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-pgsql.md @@ -35,10 +35,12 @@ RDI requires some changes to database parameters. On AWS RDS and AWS Aurora, you ```checklist {id="aurorapostgresql-param-group" nointeractive="true" } - [ ] [Create a parameter group](#create-a-parameter-group) - [ ] [Apply the parameter group](#apply-the-parameter-group) +- [ ] [Apply the parameter group to the database](#apply-the-parameter-group-to-the-database) - [ ] [Reboot the database instance](#reboot-the-database-instance) ``` -1. In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/), navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.CreatingCluster.html) with the following settings: +1. + In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/), navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.CreatingCluster.html) with the following settings: | Name | Value | | :-- | :-- | @@ -49,7 +51,8 @@ RDI requires some changes to database parameters. On AWS RDS and AWS Aurora, you Select **Create** to create the parameter group. -1. Navigate to **Parameter groups** in the console. Select the group you have just created and then select **Edit**. Change the following parameters: +1. + Navigate to **Parameter groups** in the console. Select the group you have just created and then select **Edit**. Change the following parameters: | Name | Value | | :-- | :-- | @@ -57,11 +60,13 @@ RDI requires some changes to database parameters. On AWS RDS and AWS Aurora, you Select **Save Changes** to apply the changes to the parameter group. -1. Go back to your database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. +1. + Go back to your database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. Select **Save changes** to apply the parameter group to your database. -1. Reboot your database instance. See [Rebooting a DB instance within an Aurora cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-reboot-db-instance.html) or [Rebooting a DB instance (RDS)](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_RebootInstance.html) for more information. +1. + Reboot your database instance. See [Rebooting a DB instance within an Aurora cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-reboot-db-instance.html) or [Rebooting a DB instance (RDS)](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_RebootInstance.html) for more information. ## Create Debezium user @@ -75,7 +80,8 @@ to capture changes. - [ ] [Grant the user access to the database](#grant-the-user-access-to-the-database) ``` -1. Connect to PostgreSQL as the `postgres` user and create a new user for the connector: +1. + Connect to PostgreSQL as the `postgres` user and create a new user for the connector: ```sql CREATE ROLE WITH LOGIN PASSWORD '' VALID UNTIL 'infinity'; @@ -83,7 +89,8 @@ to capture changes. Replace `` and `` with a username and password for the new user. -1. Grant the user the necessary replication permissions: +1. + Grant the user the necessary replication permissions: ```sql GRANT rds_replication TO ; @@ -91,7 +98,8 @@ to capture changes. Replace `` with the username of the Debezium user. -1. Connect to your database as the `postgres` user and grant the new user access to one or more schemas in the database: +1. + Connect to your database as the `postgres` user and grant the new user access to one or more schemas in the database: ```sql GRANT SELECT ON ALL TABLES IN SCHEMA TO ; From acd0bf8ce3fa24be3591ba5608bfc70ba175934a Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Fri, 14 Nov 2025 10:09:28 +0000 Subject: [PATCH 17/18] DOC-5937 fixes to AWS Aurora MySQL lists --- .../aws-aurora-rds/aws-aur-mysql.md | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-mysql.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-mysql.md index 195aa6689a..6a3734d4a7 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-mysql.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-mysql.md @@ -46,10 +46,12 @@ RDI requires some changes to database parameters. On AWS Aurora, you change thes ```checklist {id="auroramysql-param-group" nointeractive="true" } - [ ] [Create a parameter group](#aurora-create-a-parameter-group) - [ ] [Apply the parameter group](#aurora-apply-the-parameter-group) +- [ ] [Apply the parameter group to the database](#aurora-apply-the-parameter-group-to-the-database) - [ ] [Reboot the database instance](#aurora-reboot-the-database-instance) ``` -1. In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/),navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.Creating.html) with the following settings: +1. + In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/),navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.Creating.html) with the following settings: | Name | Value | | :-- | :-- | @@ -61,7 +63,8 @@ RDI requires some changes to database parameters. On AWS Aurora, you change thes Select **Create** to create the parameter group. -1. Navigate to **Parameter groups** in the console. Select the parameter group you have just created and then select **Edit**. Change the following parameters: +1. + Navigate to **Parameter groups** in the console. Select the parameter group you have just created and then select **Edit**. Change the following parameters: | Name | Value | | :-- | :-- | @@ -72,11 +75,13 @@ RDI requires some changes to database parameters. On AWS Aurora, you change thes Select **Save Changes** to apply the changes to the parameter group. -1. Go back to your target database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. +1. + Go back to your target database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. Select **Save changes** to apply the parameter group to the new database. -1. Reboot your database instance. See [Rebooting a DB instance within an Aurora cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-reboot-db-instance.html) for more information. +1. + Reboot your database instance. See [Rebooting a DB instance within an Aurora cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-reboot-db-instance.html) for more information. @@ -96,10 +101,12 @@ RDI requires some changes to database parameters. On AWS RDS, you change these p ```checklist {id="rds-mysql-param-group" nointeractive="true" } - [ ] [Create a parameter group](#rds-create-a-parameter-group) - [ ] [Apply the parameter group](#rds-apply-the-parameter-group) +- [ ] [Apply the parameter group to the database](#rds-apply-the-parameter-group-to-the-database) - [ ] [Reboot the database instance](#rds-reboot-the-database-instance) ``` -1. In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/),navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_WorkingWithParamGroups.Creating.html) with the following settings: +1. + In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/),navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_WorkingWithParamGroups.Creating.html) with the following settings: | Name | Value | | :-- | :-- | @@ -110,7 +117,8 @@ RDI requires some changes to database parameters. On AWS RDS, you change these p Select **Create** to create the parameter group. -1. Navigate to **Parameter groups** in the console. Select the parameter group you have just created and then select **Edit**. Change the following parameters: +1. + Navigate to **Parameter groups** in the console. Select the parameter group you have just created and then select **Edit**. Change the following parameters: | Name | Value | | :-- | :-- | @@ -119,11 +127,13 @@ RDI requires some changes to database parameters. On AWS RDS, you change these p Select **Save Changes** to apply the changes to the parameter group. -1. Go back to your target database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. +1. + Go back to your target database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. Select **Save changes** to apply the parameter group to the new database. -1. Reboot your database instance. See [Rebooting a DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_RebootInstance.html) for more information. +1. + Reboot your database instance. See [Rebooting a DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_RebootInstance.html) for more information. From ebb0e0ad2165d30be50e889ccfbfa77c58ba6df2 Mon Sep 17 00:00:00 2001 From: Andy Stark Date: Fri, 14 Nov 2025 10:11:35 +0000 Subject: [PATCH 18/18] DOC-5937 fixes for AWS Aurora SQL server lists --- .../aws-aurora-rds/aws-rds-sqlserver.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-rds-sqlserver.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-rds-sqlserver.md index fa9a2ac397..c7ca242db1 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-rds-sqlserver.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-rds-sqlserver.md @@ -38,7 +38,8 @@ to capture changes. - [ ] [Grant the user the necessary permissions](#grant-the-user-the-necessary-permissions) ``` -1. Connect to your database as an admin user and create a new user for the connector: +1. + Connect to your database as an admin user and create a new user for the connector: ```sql USE master @@ -53,7 +54,8 @@ to capture changes. Replace `` and `` with a username and password for the new user and replace `` with the name of your database. -1. Grant the user the necessary permissions: +1. + Grant the user the necessary permissions: ```sql USE master @@ -78,7 +80,8 @@ Change Data Capture (CDC) must be enabled for the database and for each table yo - [ ] [Add the Debezium user to the CDC role](#add-the-debezium-user-to-the-cdc-role) ``` -1. Enable CDC for the database by running the following command: +1. + Enable CDC for the database by running the following command: ```sql EXEC msdb.dbo.rds_cdc_enable_db '' @@ -87,7 +90,8 @@ Change Data Capture (CDC) must be enabled for the database and for each table yo Replace `` with the name of your database. -1. Enable CDC for each table you want to capture by running the following commands: +1. + Enable CDC for each table you want to capture by running the following commands: ```sql USE @@ -108,7 +112,8 @@ Specifying a new name will create a corresponding database role that has full ac captured change data. {{< /note >}} -1. Add the Debezium user to the CDC role: +1. + Add the Debezium user to the CDC role: ```sql USE