diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-mysql.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-mysql.md index 7e95a35ffd..6a3734d4a7 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-mysql.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-mysql.md @@ -25,6 +25,12 @@ Select the steps for your database type. tab1="AWS Aurora MySQL" tab2="AWS RDS MySQL" >}} +```checklist {id="auroramysql" nointeractive="true" } +- [ ] [Add an Aurora reader node](#add-an-aurora-reader-node) +- [ ] [Create and apply parameter group](#aurora-create-and-apply-parameter-group) +- [ ] [Create Debezium user](#aurora-create-debezium-user) +``` + ## Add an Aurora reader node RDI requires that your Aurora MySQL database has at least one replica or reader node. @@ -33,11 +39,19 @@ To add a reader node to an existing database, select **Add reader** from the **A You can also create one during database creation by selecting **Create an Aurora Replica or Reader node in a different AZ (recommended for scaled availability)** under **Availability & durability > Multi-AZ deployment**. -## Create and apply parameter group +## Create and apply parameter group RDI requires some changes to database parameters. On AWS Aurora, you change these parameters via a parameter group. -1. In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/),navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.Creating.html) with the following settings: +```checklist {id="auroramysql-param-group" nointeractive="true" } +- [ ] [Create a parameter group](#aurora-create-a-parameter-group) +- [ ] [Apply the parameter group](#aurora-apply-the-parameter-group) +- [ ] [Apply the parameter group to the database](#aurora-apply-the-parameter-group-to-the-database) +- [ ] [Reboot the database instance](#aurora-reboot-the-database-instance) +``` + +1. + In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/),navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.Creating.html) with the following settings: | Name | Value | | :-- | :-- | @@ -49,7 +63,8 @@ RDI requires some changes to database parameters. On AWS Aurora, you change thes Select **Create** to create the parameter group. -1. Navigate to **Parameter groups** in the console. Select the parameter group you have just created and then select **Edit**. Change the following parameters: +1. + Navigate to **Parameter groups** in the console. Select the parameter group you have just created and then select **Edit**. Change the following parameters: | Name | Value | | :-- | :-- | @@ -60,22 +75,38 @@ RDI requires some changes to database parameters. On AWS Aurora, you change thes Select **Save Changes** to apply the changes to the parameter group. -1. Go back to your target database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. +1. + Go back to your target database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. Select **Save changes** to apply the parameter group to the new database. -1. Reboot your database instance. See [Rebooting a DB instance within an Aurora cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-reboot-db-instance.html) for more information. +1. + Reboot your database instance. See [Rebooting a DB instance within an Aurora cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-reboot-db-instance.html) for more information. + {{< embed-md "aur-rds-mysql-create-debezium-user.md" >}} -tab-sep- -## Create and apply parameter group +```checklist {id="rds-mysql-list" nointeractive="true" } +- [ ] [Create and apply parameter group](#rds-create-and-apply-parameter-group) +- [ ] [Create Debezium user](#rds-create-debezium-user) +``` + +## Create and apply parameter group RDI requires some changes to database parameters. On AWS RDS, you change these parameters via a parameter group. -1. In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/),navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_WorkingWithParamGroups.Creating.html) with the following settings: +```checklist {id="rds-mysql-param-group" nointeractive="true" } +- [ ] [Create a parameter group](#rds-create-a-parameter-group) +- [ ] [Apply the parameter group](#rds-apply-the-parameter-group) +- [ ] [Apply the parameter group to the database](#rds-apply-the-parameter-group-to-the-database) +- [ ] [Reboot the database instance](#rds-reboot-the-database-instance) +``` + +1. + In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/),navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_WorkingWithParamGroups.Creating.html) with the following settings: | Name | Value | | :-- | :-- | @@ -86,7 +117,8 @@ RDI requires some changes to database parameters. On AWS RDS, you change these p Select **Create** to create the parameter group. -1. Navigate to **Parameter groups** in the console. Select the parameter group you have just created and then select **Edit**. Change the following parameters: +1. + Navigate to **Parameter groups** in the console. Select the parameter group you have just created and then select **Edit**. Change the following parameters: | Name | Value | | :-- | :-- | @@ -95,11 +127,15 @@ RDI requires some changes to database parameters. On AWS RDS, you change these p Select **Save Changes** to apply the changes to the parameter group. -1. Go back to your target database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. +1. + Go back to your target database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. Select **Save changes** to apply the parameter group to the new database. -1. Reboot your database instance. See [Rebooting a DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_RebootInstance.html) for more information. +1. + Reboot your database instance. See [Rebooting a DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_RebootInstance.html) for more information. + + {{< embed-md "aur-rds-mysql-create-debezium-user.md" >}} diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-pgsql.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-pgsql.md index ff4a89d0c3..ec9f69fe51 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-pgsql.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-aur-pgsql.md @@ -23,11 +23,24 @@ Follow the steps in the sections below to prepare an [AWS Aurora PostgreSQL](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/CHAP_GettingStartedAurora.CreatingConnecting.AuroraPostgreSQL.html) or [AWS RDS PostgreSQL](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_GettingStarted.CreatingConnecting.PostgreSQL.html) database to work with RDI. +```checklist {id="aurorapostgresql" nointeractive="true" } +- [ ] [Create and apply parameter group](#create-and-apply-parameter-group) +- [ ] [Create Debezium user](#create-debezium-user) +``` + ## Create and apply parameter group RDI requires some changes to database parameters. On AWS RDS and AWS Aurora, you change these parameters via a parameter group. -1. In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/), navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.CreatingCluster.html) with the following settings: +```checklist {id="aurorapostgresql-param-group" nointeractive="true" } +- [ ] [Create a parameter group](#create-a-parameter-group) +- [ ] [Apply the parameter group](#apply-the-parameter-group) +- [ ] [Apply the parameter group to the database](#apply-the-parameter-group-to-the-database) +- [ ] [Reboot the database instance](#reboot-the-database-instance) +``` + +1. + In the [Relational Database Service (RDS) console](https://console.aws.amazon.com/rds/), navigate to **Parameter groups > Create parameter group**. [Create a parameter group](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithParamGroups.CreatingCluster.html) with the following settings: | Name | Value | | :-- | :-- | @@ -38,7 +51,8 @@ RDI requires some changes to database parameters. On AWS RDS and AWS Aurora, you Select **Create** to create the parameter group. -1. Navigate to **Parameter groups** in the console. Select the group you have just created and then select **Edit**. Change the following parameters: +1. + Navigate to **Parameter groups** in the console. Select the group you have just created and then select **Edit**. Change the following parameters: | Name | Value | | :-- | :-- | @@ -46,11 +60,13 @@ RDI requires some changes to database parameters. On AWS RDS and AWS Aurora, you Select **Save Changes** to apply the changes to the parameter group. -1. Go back to your database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. +1. + Go back to your database on the RDS console, select **Modify** and then scroll down to **Additional Configuration**. Set the **DB Cluster Parameter Group** to the group you just created. Select **Save changes** to apply the parameter group to your database. -1. Reboot your database instance. See [Rebooting a DB instance within an Aurora cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-reboot-db-instance.html) or [Rebooting a DB instance (RDS)](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_RebootInstance.html) for more information. +1. + Reboot your database instance. See [Rebooting a DB instance within an Aurora cluster](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-reboot-db-instance.html) or [Rebooting a DB instance (RDS)](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_RebootInstance.html) for more information. ## Create Debezium user @@ -58,7 +74,14 @@ The Debezium connector needs a user account to connect to PostgreSQL. This user must have appropriate permissions on all databases where you want Debezium to capture changes. -1. Connect to PostgreSQL as the `postgres` user and create a new user for the connector: +```checklist {id="aurorapostgresql-create-debezium-user" nointeractive="true" } +- [ ] [Connect to PostgreSQL as the `postgres` user](#connect-to-postgresql-as-the-postgres-user) +- [ ] [Grant the user the necessary replication permissions](#grant-the-user-the-necessary-replication-permissions) +- [ ] [Grant the user access to the database](#grant-the-user-access-to-the-database) +``` + +1. + Connect to PostgreSQL as the `postgres` user and create a new user for the connector: ```sql CREATE ROLE WITH LOGIN PASSWORD '' VALID UNTIL 'infinity'; @@ -66,7 +89,8 @@ to capture changes. Replace `` and `` with a username and password for the new user. -1. Grant the user the necessary replication permissions: +1. + Grant the user the necessary replication permissions: ```sql GRANT rds_replication TO ; @@ -74,7 +98,8 @@ to capture changes. Replace `` with the username of the Debezium user. -1. Connect to your database as the `postgres` user and grant the new user access to one or more schemas in the database: +1. + Connect to your database as the `postgres` user and grant the new user access to one or more schemas in the database: ```sql GRANT SELECT ON ALL TABLES IN SCHEMA TO ; diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-rds-sqlserver.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-rds-sqlserver.md index 5d7dbca456..c7ca242db1 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-rds-sqlserver.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/aws-aurora-rds/aws-rds-sqlserver.md @@ -22,13 +22,24 @@ Follow the steps in the sections below to prepare a [Microsoft SQL Server on AWS Change Data Capture (CDC) is not supported on SQL Server Express Edition. Only the Standard, Enterprise, and Developer editions support CDC and are supported by RDI. {{< /note >}} +```checklist {id="rds-sqlserver-list" nointeractive="true" } +- [ ] [Create the Debezium user](#create-the-debezium-user) +- [ ] [Enable CDC on the database](#enable-cdc-on-the-database) +``` + ## Create the Debezium user The Debezium connector needs a user account to connect to SQL Server. This user must have appropriate permissions on all databases where you want Debezium to capture changes. -1. Connect to your database as an admin user and create a new user for the connector: +```checklist {id="rds-sqlserver-create-debezium-user" nointeractive="true" } +- [ ] [Connect to SQL Server as an admin user](#connect-to-sql-server-as-an-admin-user) +- [ ] [Grant the user the necessary permissions](#grant-the-user-the-necessary-permissions) +``` + +1. + Connect to your database as an admin user and create a new user for the connector: ```sql USE master @@ -43,7 +54,8 @@ to capture changes. Replace `` and `` with a username and password for the new user and replace `` with the name of your database. -1. Grant the user the necessary permissions: +1. + Grant the user the necessary permissions: ```sql USE master @@ -62,7 +74,14 @@ to capture changes. Change Data Capture (CDC) must be enabled for the database and for each table you want to capture. -1. Enable CDC for the database by running the following command: +```checklist {id="rds-sqlserver-enable-cdc" nointeractive="true" } +- [ ] [Enable CDC for the database](#enable-cdc-for-the-database) +- [ ] [Enable CDC for each table you want to capture](#enable-cdc-for-each-table-you-want-to-capture) +- [ ] [Add the Debezium user to the CDC role](#add-the-debezium-user-to-the-cdc-role) +``` + +1. + Enable CDC for the database by running the following command: ```sql EXEC msdb.dbo.rds_cdc_enable_db '' @@ -71,7 +90,8 @@ Change Data Capture (CDC) must be enabled for the database and for each table yo Replace `` with the name of your database. -1. Enable CDC for each table you want to capture by running the following commands: +1. + Enable CDC for each table you want to capture by running the following commands: ```sql USE @@ -92,7 +112,8 @@ Specifying a new name will create a corresponding database role that has full ac captured change data. {{< /note >}} -1. Add the Debezium user to the CDC role: +1. + Add the Debezium user to the CDC role: ```sql USE diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md index 0b00d9d321..ed7ac83831 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/mongodb.md @@ -25,7 +25,7 @@ This guide describes the steps required to prepare a MongoDB database as a sourc {{< note >}}The MongoDB connector is not capable of monitoring the changes of a standalone MongoDB server, since standalone servers do not have an oplog. The connector will work if the standalone server is converted to a replica set with one member.{{< /note >}} ## Summary -The following table summarizes the steps to prepare a MongoDB database for RDI. The sections below explain the steps in more detail. +The following table summarizes the considerations to prepare a MongoDB database for RDI. | Requirement | Description | |---------------------|-----------------------------------------------------------------------------| @@ -37,6 +37,20 @@ The following table summarizes the steps to prepare a MongoDB database for RDI. | MongoDB Atlas | **[SSL required](https://debezium.io/documentation/reference/stable/connectors/mongodb.html#mongodb-property-mongodb-ssl-enabled)**, provide root CA as `SOURCE_DB_CACERT` secret in RDI | | Network | RDI Collector must reach all MongoDB nodes on required ports | +The following checklist shows the steps to prepare a MongoDB database for RDI, +with links to the sections that explain the steps in full detail. +You may find it helpful to track your progress with the checklist as you +complete each step. + +```checklist {id="mongodblist"} +- [ ] [Configure oplog size](#1-configure-oplog-size) +- [ ] [Create a MongoDB user for RDI](#2-create-a-mongodb-user-for-rdi) +- [ ] [Connection string format](#3-connection-string-format) +- [ ] [Enable change streams and pre/post images (only if using a custom key)](#4-enable-change-streams-and-prepost-images-only-if-using-a-custom-key) +- [ ] [MongoDB Atlas specific requirements](#5-mongodb-atlas-specific-requirements) +- [ ] [Network and security](#6-network-and-security) +``` + ## 1. Configure oplog size The Debezium MongoDB connector relies on the [oplog](https://www.mongodb.com/docs/manual/core/replica-set-oplog/) to capture changes from a replica set. The oplog is a fixed-size, capped collection. When it reaches its maximum size, it overwrites the oldest entries. If the connector is stopped and restarted, it attempts to resume from its last recorded position in the oplog. If that position has been overwritten, the connector may fail to start and report an invalid resume token error. diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/my-sql-mariadb.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/my-sql-mariadb.md index 2b7c097cc7..3d9bb2ef96 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/my-sql-mariadb.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/my-sql-mariadb.md @@ -16,8 +16,19 @@ type: integration weight: 2 --- -Follow the steps in the sections below to set up a MySQL or MariaDB -database for CDC with Debezium. +The following checklist summarizes the steps to prepare a MySQL or MariaDB +database for RDI, with links to the sections that explain the steps in +full detail. You may find it helpful to track your progress with the +checklist as you complete each step. + +```checklist {id="mysqlmariadblist"} +- [ ] [Create a CDC user](#1-create-a-cdc-user) +- [ ] [Enable the binlog](#2-enable-the-binlog) +- [ ] [Enable GTIDs](#3-enable-gtids) +- [ ] [Configure session timeouts](#4-configure-session-timeouts) +- [ ] [Enable query log events](#5-enable-query-log-events) +- [ ] [Check binlog_row_value_options](#6-check-binlog_row_value_options) +``` ## 1. Create a CDC user @@ -28,13 +39,21 @@ to capture changes. Run the [MySQL CLI client](https://dev.mysql.com/doc/refman/8.3/en/mysql.html) and then run the following commands: -1. Create the CDC user: +```checklist {id="mysqlmariadb-create-cdc-user" nointeractive="true" } +- [ ] [Create the CDC user](#create-the-cdc-user) +- [ ] [Grant the user the necessary permissions](#grant-the-user-the-necessary-permissions) +- [ ] [Finalize the user's permissions](#finalize-the-users-permissions) +``` + +1. + Create the CDC user: ```sql mysql> CREATE USER 'user'@'localhost' IDENTIFIED BY 'password'; ``` -1. Grant the required permissions to the user: +1. + Grant the required permissions to the user: ```sql # MySQL GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'user'@'localhost'; ``` -1. Finalize the user's permissions: +1. + Finalize the user's permissions: ```sql mysql> FLUSH PRIVILEGES; @@ -100,19 +120,28 @@ GTIDs are available in MySQL 5.6.5 and later. See the Follow the steps below to enable GTIDs. You will need access to the MySQL configuration file to do this. -1. Enable `gtid_mode`: +```checklist {id="mysqlmariadb-enable-gtids" nointeractive="true" } +- [ ] [Enable gtid_mode](#enable-gtid_mode) +- [ ] [Enable enforce_gtid_consistency](#enable-enforce_gtid_consistency) +- [ ] [Confirm the changes](#confirm-the-changes) +``` + +1. + Enable `gtid_mode`: ```sql mysql> gtid_mode=ON ``` -1. Enable `enforce_gtid_consistency`: +1. + Enable `enforce_gtid_consistency`: ```sql mysql> enforce_gtid_consistency=ON ``` -1. Confirm the changes: +1. + Confirm the changes: ```sql mysql> show global variables like '%GTID%'; diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/oracle.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/oracle.md index 10bd21e677..502b2a57f3 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/oracle.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/oracle.md @@ -21,7 +21,37 @@ Oracle provides two main systems that Debezium can use to capture data changes: - [LogMiner](#logminer) - [XStream](#xstream) -The sections below explain how to configure each system for use with Debezium and RDI. +The sections below explain how to configure each system for use with Debezium and RDI. +The checklists summarize the steps you should follow to configure each system. +You may find it helpful to use them to track your progress as you work through the steps. + +**LogMiner** + +```checklist {id="oraclelogminerlist"} +- [ ] [Configure Oracle LogMiner](#1-configure-oracle-logminer) +- [ ] [Enable supplemental logging](#supp-logging) +- [ ] [Check the redo log sizing](#3-check-the-redo-log-sizing) +- [ ] [Set the Archive log destination](#4-set-the-archive-log-destination) +- [ ] [Create a user for the connector](#create-dbz-user) +``` + +**XStream** + +```checklist {id="oraclexstreamlist"} +- [ ] [Configure Xstream](#1-configure-xstream) +- [ ] [Create Xstream users](#2-create-xstream-users) +- [ ] [Create an Xstream outbound server](#3-create-an-xstream-outbound-server) +- [ ] [Add a custom Docker image for the Debezium server](#4-add-a-custom-docker-image-for-the-debezium-server) +- [ ] [Enable the Oracle configuration in RDI](#5-enable-the-oracle-configuration-in-rdi) +``` + +**Optional: XMLTYPE Support** + +```checklist {id="oraclexmltypelist"} +- [ ] [Create a custom Debezium Server image](#create-a-custom-debezium-server-image) +- [ ] [Configure RDI for XMLTYPE support](#configure-rdi-for-xmltype-support) +- [ ] [Test XMLTYPE support](#test-xmltype-support) +``` ## LogMiner @@ -330,7 +360,7 @@ you should also revoke `LOCK` on all tables: REVOKE LOCK ANY TABLE FROM c##dbzuser container=all; ``` -### 6. Configuration is complete +### 6. Configuration is complete {#logminer-complete} Once you have followed the steps above, your Oracle database is ready for Debezium to use. @@ -627,7 +657,7 @@ See the [Debezium Oracle documentation](https://debezium.io/documentation/reference/stable/connectors/oracle.html#oracle-connector-properties) for a full list of properties you can use in the `advanced.source` subsection. -### 6. Configuration is complete +### 6. Configuration is complete {#xstream-complete} After you have followed the steps above, your Oracle database is ready for Debezium to use. diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/postgresql.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/postgresql.md index 98d2e3e4df..01b7397734 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/postgresql.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/postgresql.md @@ -33,22 +33,46 @@ your database then this might restrict the plug-ins you can use. If you can't us plug-in then could try the `pgoutput` decoder if you are using PostgreSQL 10 or above. If this doesn't work for you then you won't be able to use RDI with your database. -### Amazon RDS for PostgreSQL +The following checklist summarizes the steps to prepare a PostgreSQL +database for RDI, with links to the sections that explain the steps in +full detail. You may find it helpful to track your progress with the +checklist as you complete each step. + +```checklist {id="postgreslist"} +- [ ] [Install the logical decoding output plug-in](#install-the-logical-decoding-output-plug-in) +- [ ] [Configure the PostgreSQL server](#configure-the-postgresql-server) +- [ ] [Set up permissions](#set-up-permissions) +- [ ] [Set privileges for Debezium to create PostgreSQL publications with pgoutput](#set-privileges-for-debezium-to-create-postgresql-publications-with-pgoutput) +- [ ] [Configure PostgreSQL for replication with the Debezium connector host](#configure-postgresql-for-replication-with-the-debezium-connector-host) +``` + +## Amazon RDS for PostgreSQL Follow the steps below to enable CDC with [Amazon RDS for PostgreSQL](https://aws.amazon.com/rds/postgresql/): -1. Set the instance parameter `rds.logical_replication` to 1. +```checklist {id="postgresawslist" nointeractive="true" } +- [ ] [Set the instance parameter rds.logical_replication to 1](#1-set-the-instance-parameter-rdslogicalreplication-to-1) +- [ ] [Check that the wal_level parameter is set to logical](#2-check-that-the-wal_level-parameter-is-set-to-logical) +- [ ] [Set the Debezium plugin.name parameter to pgoutput](#3-set-the-debezium-pluginname-parameter-to-pgoutput) +- [ ] [Initiate logical replication from an AWS account that has the rds_replication role](#4-initiate-logical-replication-from-an-aws-account-that-has-the-rdsreplication-role) +``` + +1. + Set the instance parameter `rds.logical_replication` to 1. -1. Check that the `wal_level` parameter is set to `logical` by running the query `SHOW wal_level` +1. + Check that the `wal_level` parameter is set to `logical` by running the query `SHOW wal_level` as the database RDS master user. The parameter might not have this value in multi-zone replication setups. You can't change the value manually but it should change automatically when you set the `rds.logical_replication` parameter to 1. If it doesn't change then you probably just need to restart your database instance. You can restart manually or wait until a restart occurs during your maintenance window. -1. Set the Debezium `plugin.name` parameter to `pgoutput`. +1. + Set the Debezium `plugin.name` parameter to `pgoutput`. -1. Initiate logical replication from an AWS account that has the `rds_replication` role. The role grants +1. + Initiate logical replication from an AWS account that has the `rds_replication` role. The role grants permissions to manage logical slots and to stream data using logical slots. By default, only the master user account on AWS has the `rds_replication` role on Amazon RDS, but if you have administrator privileges, you can grant the role to other accounts using a query like the following: @@ -175,24 +199,37 @@ this case, the source table already exists, so you must use a PostgreSQL replica group to share ownership between the Debezium user and the original owner. Configure the replication group using the following commands: -1. Create the replication group (the name `replication_group` here is +```checklist {id="postgrespgoutputlist" nointeractive="true" } +- [ ] [Create the replication group](#1-create-the-replication-group) +- [ ] [Add the original owner of the table to the group](#2-add-the-original-owner-of-the-table-to-the-group) +- [ ] [Add the Debezium replication user to the group](#3-add-the-debezium-replication-user-to-the-group) +- [ ] [Transfer ownership of the table to the replication group](#4-transfer-ownership-of-the-table-to-the-replication-group) +``` + +1. + Create the replication group (the name `replication_group` here is just an example): ```sql CREATE ROLE replication_group; ``` -1. Add the original owner of the table to the group: + +1. + Add the original owner of the table to the group: ```sql GRANT replication_group TO original_owner; ``` -1. Add the Debezium replication user to the group: +1. + Add the Debezium replication user to the group: ```sql GRANT replication_group TO replication_user; ``` -1. Transfer ownership of the table to `replication_group`: + +1. + Transfer ownership of the table to `replication_group`: ```sql ALTER TABLE table_name OWNER TO replication_group; diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md index 59f93e5139..1e5b16966d 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/spanner.md @@ -26,6 +26,20 @@ the monitored schemas and tables. Spanner is only supported with RDI deployed on Kubernetes/Helm. RDI VM mode does not support Spanner as a source database. {{< /note >}} +The following checklist summarizes the steps to prepare a Spanner +database for RDI, with links to the sections that explain the steps in +full detail. You may find it helpful to track your progress with the +checklist as you complete each step. + +```checklist {id="spannerlist"} +- [ ] [Prepare for snapshot](#1-prepare-for-snapshot) +- [ ] [Prepare for streaming](#2-prepare-for-streaming) +- [ ] [Create a service account](#3-create-a-service-account) +- [ ] [Set up secrets for Kubernetes deployment (optional)](#4-set-up-secrets-for-kubernetes-deployment-optional) +- [ ] [Configure RDI for Spanner](#5-configure-rdi-for-spanner) +- [ ] [Additional Kubernetes configuration](#6-additional-kubernetes-configuration) +``` + ## 1. Prepare for snapshot During the snapshot phase, RDI executes multiple transactions to capture data at an exact point @@ -64,7 +78,14 @@ To allow RDI to access the Spanner instance, you'll need to create a service acc appropriate permissions. By default, RDI uses Google Cloud Workload Identity authentication. In this case RDI will assume the [service account is assigned to the GKE cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#enable_on_clusters_and_node_pools). Alternatively, you can provide the service account credentials as a Kubernetes secret (see step 4 for details). -1. Create the service account +```checklist {id="spanner-service-account" nointeractive="true" } +- [ ] [Create the service account](#create-the-service-account) +- [ ] [Grant required roles](#grant-required-roles) +- [ ] [Download the service account key](#download-the-service-account-key) +``` + +1. + Create the service account ```bash gcloud iam service-accounts create spanner-reader-account \ @@ -73,7 +94,8 @@ service account credentials as a Kubernetes secret (see step 4 for details). --project=YOUR_PROJECT_ID ``` -1. Grant required roles +1. + Grant required roles: **Database Reader** (read access to Spanner data): @@ -99,7 +121,8 @@ service account credentials as a Kubernetes secret (see step 4 for details). --role="roles/spanner.viewer" ``` -1. Download the service account key +1. + Download the service account key: Save the credentials locally so they can be used later by RDI: diff --git a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md index 3723aa0c84..b9864c28d3 100644 --- a/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md +++ b/content/integrate/redis-data-integration/data-pipelines/prepare-dbs/sql-server.md @@ -23,13 +23,31 @@ capture. You need administrator privileges to do this. Once you enable CDC, it captures all of the INSERT, UPDATE, and DELETE operations on your chosen tables. The Debezium connector can then emit these events to RDI. +The following checklist summarizes the steps to prepare a SQL Server +database for RDI, with links to the sections that explain the steps in +full detail. You may find it helpful to track your progress with the +checklist as you complete each step. + +```checklist {id="sqlserverlist"} +- [ ] [Create a Debezium user](#1-create-a-debezium-user) +- [ ] [Enable CDC on the database](#2-enable-cdc-on-the-database) +- [ ] [Enable CDC for the tables you want to capture](#3-enable-cdc-for-the-tables-you-want-to-capture) +- [ ] [Check that you have access to the CDC table](#4-check-that-you-have-access-to-the-cdc-table) +``` + ## 1. Create a Debezium user It is strongly recommended to create a dedicated Debezium user for the connection between RDI and the source database. When using an existing user, ensure that the required permissions are granted and that the user is added to the CDC role. -1. Create a user with the Transact-SQL below: +```checklist {id="sqlserver-create-debezium-user" nointeractive="true" } +- [ ] [Create the Debezium user](#create-the-debezium-user) +- [ ] [Grant the user the necessary permissions](#grant-the-user-the-necessary-permissions) +``` + +1. + Create the Debezium user with the Transact-SQL below: ```sql USE master @@ -44,7 +62,8 @@ permissions are granted and that the user is added to the CDC role. Replace `MyUser`, `My_Password` and `MyDB` with your chosen values. -1. Grant required permissions: +1. + Grant the user the necessary permissions: ```sql USE master @@ -90,8 +109,14 @@ a CDC user, metadata tables, and other system objects. ## 3. Enable CDC for the tables you want to capture -1. You must also enable CDC on the tables you want Debezium to capture using the -following commands (again, you need administrator privileges for this): +```checklist {id="sqlserver-enable-cdc-tables" nointeractive="true" } +- [ ] [Enable CDC on the tables you want to capture](#enable-cdc-on-the-tables-you-want-to-capture) +- [ ] [Add the Debezium user to the CDC role](#add-the-debezium-user-to-the-cdc-role) +``` + +1. + You must also enable CDC on the tables you want Debezium to capture using the + following commands (again, you need administrator privileges for this): ```sql USE MyDB @@ -112,7 +137,8 @@ following commands (again, you need administrator privileges for this): captured change data. {{< /note >}} -1. Add the Debezium user to the CDC role: +1. + Add the Debezium user to the CDC role: ```sql USE MyDB @@ -127,7 +153,13 @@ You can use another stored procedure `sys.sp_cdc_help_change_data_capture` to query the CDC information for the database and check you have enabled it correctly. To do this, connect as the Debezium user you created previously (`MyUser`). -1. Run the `sys.sp_cdc_help_change_data_capture` stored procedure to query +```checklist {id="sqlserver-check-cdc-table" nointeractive="true" } +- [ ] [Run the stored procedure to query the CDC configuration](#run-the-stored-procedure-to-query-the-cdc-configuration) +- [ ] [Check the results](#check-the-results) +``` + +1. + Run the `sys.sp_cdc_help_change_data_capture` stored procedure to query the CDC configuration. For example, if your database was called `MyDB` then you would run the following: @@ -138,7 +170,8 @@ it correctly. To do this, connect as the Debezium user you created previously (` GO ``` -1. The query returns configuration information for each table in the database that +1. + The query returns configuration information for each table in the database that has CDC enabled and that contains change data that you are authorized to access. If the result is empty then you should check that you have privileges to access both the capture instance and the CDC tables. @@ -218,9 +251,17 @@ a schema change and resume CDC. See the [online schema updates](https://debezium.io/documentation/reference/stable/connectors/sqlserver.html#online-schema-updates) documentation for further details. -1. Make your changes to the source table schema. +```checklist {id="sqlserver-schema-changes" nointeractive="true" } +- [ ] [Make your changes to the source table schema](#make-your-changes-to-the-source-table-schema) +- [ ] [Create a new capture table for the updated source table](#create-a-new-capture-table-for-the-updated-source-table) +- [ ] [Drop the old capture table](#drop-the-old-capture-table) +``` + +1. + Make your changes to the source table schema. -1. Create a new capture table for the updated source table by running the `sys.sp_cdc_enable_table` stored +1. + Create a new capture table for the updated source table by running the `sys.sp_cdc_enable_table` stored procedure with a new, unique value for the parameter `@capture_instance`. For example, if the old value was `dbo_MyTable`, you could replace it with `dbo_MyTable_v2` (you can see the existing values by running stored procedure `sys.sp_cdc_help_change_data_capture`): @@ -235,7 +276,8 @@ documentation for further details. GO ``` -1. When Debezium starts streaming from the new capture table, drop the old capture table by running +1. + When Debezium starts streaming from the new capture table, drop the old capture table by running the `sys.sp_cdc_disable_table` stored procedure with the parameter `@capture_instance` set to the old capture instance name, `dbo_MyTable`: diff --git a/static/js/checklist.js b/static/js/checklist.js index 37ba3dc489..dee5189337 100644 --- a/static/js/checklist.js +++ b/static/js/checklist.js @@ -42,7 +42,31 @@ function createChecklistFromMarkdown(markdown, formId, preElement) { // Create select dropdown const select = document.createElement('select'); select.onchange = () => clChange(formId); - + + // Add styling to make the select element look like an interactive control + select.style.border = '1px solid #d0d0d0'; + select.style.borderRadius = '4px'; + select.style.padding = '4px 6px'; + select.style.backgroundColor = '#fafafa'; + select.style.cursor = 'pointer'; + select.style.fontFamily = 'inherit'; + select.style.fontSize = 'inherit'; + select.style.marginRight = '8px'; + select.style.transition = 'all 0.2s ease'; + + // Add hover effects + select.addEventListener('mouseenter', () => { + select.style.borderColor = '#999'; + select.style.backgroundColor = '#f0f0f0'; + select.style.boxShadow = '0 1px 3px rgba(0, 0, 0, 0.1)'; + }); + + select.addEventListener('mouseleave', () => { + select.style.borderColor = '#d0d0d0'; + select.style.backgroundColor = '#fafafa'; + select.style.boxShadow = 'none'; + }); + const options = [ { value: 'R', label: '❌' }, { value: 'G', label: '✅' }, @@ -56,7 +80,7 @@ function createChecklistFromMarkdown(markdown, formId, preElement) { option.textContent = opt.label; select.appendChild(option); }); - + li.appendChild(select); // Parse link and text from markdown