From 71414d8f7d8a13ec6fc090f9de87f030c41a79ed Mon Sep 17 00:00:00 2001 From: yux <34335406+yuxiqian@users.noreply.github.com> Date: Wed, 22 May 2024 21:41:43 +0800 Subject: [PATCH] [minor][docs] Rectify names of CDC sources for Flink and improve the content directory This closes #3316. --- README.md | 2 +- .../docs/connectors/flink-sources}/_index.md | 4 +- .../datastream-api-package-guidance.md | 2 +- .../db2-cdc.md | 4 +- .../mongodb-cdc.md | 4 +- .../mysql-cdc.md | 4 +- .../oceanbase-cdc.md | 6 +-- .../oracle-cdc.md | 4 +- .../connectors/flink-sources}/overview.md | 54 +++++++++---------- .../connectors/flink-sources}/postgres-cdc.md | 4 +- .../sqlserver-cdc.md | 4 +- .../tidb-cdc.md | 2 +- .../tutorials/_index.md | 2 +- .../build-real-time-data-lake-tutorial.md | 2 +- .../tutorials/build-streaming-etl-tutorial.md | 2 +- .../tutorials/db2-tutorial.md | 2 +- .../tutorials/mongodb-tutorial.md | 2 +- .../tutorials/oceanbase-tutorial.md | 2 +- .../tutorials/oracle-tutorial.md | 2 +- .../tutorials/polardbx-tutorial.md | 2 +- .../tutorials/sqlserver-tutorial.md | 2 +- .../tutorials/tidb-tutorial.md | 2 +- .../vitess-cdc.md | 2 +- docs/content.zh/docs/connectors/overview.md | 46 ---------------- .../_index.md | 4 +- .../{ => pipeline-connectors}/doris.md | 4 +- .../{ => pipeline-connectors}/kafka.md | 4 +- .../{ => pipeline-connectors}/mysql.md | 2 +- .../pipeline-connectors/overview.md | 45 ++++++++++++++++ .../{ => pipeline-connectors}/paimon.md | 4 +- .../{ => pipeline-connectors}/starrocks.md | 4 +- docs/content.zh/docs/deployment/kubernetes.md | 6 +-- docs/content.zh/docs/deployment/standalone.md | 6 +-- docs/content.zh/docs/deployment/yarn.md | 2 +- .../docs/connectors/flink-sources/_index.md | 23 ++++++++ .../datastream-api-package-guidance.md | 2 +- .../db2-cdc.md | 4 +- .../mongodb-cdc.md | 4 +- .../mysql-cdc.md | 4 +- .../oceanbase-cdc.md | 6 +-- .../oracle-cdc.md | 4 +- .../connectors/flink-sources}/overview.md | 50 ++++++++--------- .../connectors/flink-sources}/postgres-cdc.md | 4 +- .../sqlserver-cdc.md | 4 +- .../tidb-cdc.md | 2 +- .../tutorials/_index.md | 0 .../build-real-time-data-lake-tutorial.md | 2 +- .../tutorials/build-streaming-etl-tutorial.md | 2 +- .../tutorials/db2-tutorial.md | 2 +- .../tutorials/mongodb-tutorial.md | 2 +- .../tutorials/oceanbase-tutorial.md | 2 +- .../tutorials/oracle-tutorial.md | 2 +- .../tutorials/polardbx-tutorial.md | 2 +- .../tutorials/sqlserver-tutorial.md | 2 +- .../tutorials/tidb-tutorial.md | 2 +- .../vitess-cdc.md | 2 +- .../connectors/pipeline-connectors/_index.md | 23 ++++++++ .../{ => pipeline-connectors}/doris.md | 4 +- .../{ => pipeline-connectors}/kafka.md | 4 +- .../{ => pipeline-connectors}/mysql.md | 2 +- .../{ => pipeline-connectors}/overview.md | 21 +++----- .../{ => pipeline-connectors}/paimon.md | 4 +- .../{ => pipeline-connectors}/starrocks.md | 4 +- docs/content/docs/deployment/kubernetes.md | 6 +-- docs/content/docs/deployment/standalone.md | 6 +-- docs/content/docs/deployment/yarn.md | 2 +- 66 files changed, 243 insertions(+), 203 deletions(-) rename docs/{content/docs/connectors/legacy-flink-cdc-sources => content.zh/docs/connectors/flink-sources}/_index.md (95%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/datastream-api-package-guidance.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/db2-cdc.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/mongodb-cdc.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/mysql-cdc.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/oceanbase-cdc.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/oracle-cdc.md (99%) rename docs/{content/docs/connectors/legacy-flink-cdc-sources => content.zh/docs/connectors/flink-sources}/overview.md (74%) rename docs/{content/docs/connectors/legacy-flink-cdc-sources => content.zh/docs/connectors/flink-sources}/postgres-cdc.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/sqlserver-cdc.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tidb-cdc.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/_index.md (95%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/build-real-time-data-lake-tutorial.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/build-streaming-etl-tutorial.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/db2-tutorial.md (98%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/mongodb-tutorial.md (98%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/oceanbase-tutorial.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/oracle-tutorial.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/polardbx-tutorial.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/sqlserver-tutorial.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/tidb-tutorial.md (99%) rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/vitess-cdc.md (99%) delete mode 100644 docs/content.zh/docs/connectors/overview.md rename docs/content.zh/docs/connectors/{legacy-flink-cdc-sources => pipeline-connectors}/_index.md (95%) rename docs/content.zh/docs/connectors/{ => pipeline-connectors}/doris.md (99%) rename docs/content.zh/docs/connectors/{ => pipeline-connectors}/kafka.md (99%) rename docs/content.zh/docs/connectors/{ => pipeline-connectors}/mysql.md (99%) create mode 100644 docs/content.zh/docs/connectors/pipeline-connectors/overview.md rename docs/content.zh/docs/connectors/{ => pipeline-connectors}/paimon.md (99%) rename docs/content.zh/docs/connectors/{ => pipeline-connectors}/starrocks.md (99%) create mode 100644 docs/content/docs/connectors/flink-sources/_index.md rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/datastream-api-package-guidance.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/db2-cdc.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/mongodb-cdc.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/mysql-cdc.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/oceanbase-cdc.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/oracle-cdc.md (99%) rename docs/{content.zh/docs/connectors/legacy-flink-cdc-sources => content/docs/connectors/flink-sources}/overview.md (75%) rename docs/{content.zh/docs/connectors/legacy-flink-cdc-sources => content/docs/connectors/flink-sources}/postgres-cdc.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/sqlserver-cdc.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tidb-cdc.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/_index.md (100%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/build-real-time-data-lake-tutorial.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/build-streaming-etl-tutorial.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/db2-tutorial.md (98%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/mongodb-tutorial.md (98%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/oceanbase-tutorial.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/oracle-tutorial.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/polardbx-tutorial.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/sqlserver-tutorial.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/tutorials/tidb-tutorial.md (99%) rename docs/content/docs/connectors/{legacy-flink-cdc-sources => flink-sources}/vitess-cdc.md (99%) create mode 100644 docs/content/docs/connectors/pipeline-connectors/_index.md rename docs/content/docs/connectors/{ => pipeline-connectors}/doris.md (99%) rename docs/content/docs/connectors/{ => pipeline-connectors}/kafka.md (99%) rename docs/content/docs/connectors/{ => pipeline-connectors}/mysql.md (99%) rename docs/content/docs/connectors/{ => pipeline-connectors}/overview.md (54%) rename docs/content/docs/connectors/{ => pipeline-connectors}/paimon.md (99%) rename docs/content/docs/connectors/{ => pipeline-connectors}/starrocks.md (99%) diff --git a/README.md b/README.md index ada5496e8e..0949fd1c03 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ full database synchronization, sharding table synchronization, schema evolution 5. View job execution status through Flink WebUI or downstream database. Try it out yourself with our more detailed [tutorial](docs/content/docs/get-started/quickstart/mysql-to-doris.md). -You can also see [connector overview](docs/content/docs/connectors/overview.md) to view a comprehensive catalog of the +You can also see [connector overview](docs/content/docs/connectors/pipeline-connectors/overview.md) to view a comprehensive catalog of the connectors currently provided and understand more detailed configurations. diff --git a/docs/content/docs/connectors/legacy-flink-cdc-sources/_index.md b/docs/content.zh/docs/connectors/flink-sources/_index.md similarity index 95% rename from docs/content/docs/connectors/legacy-flink-cdc-sources/_index.md rename to docs/content.zh/docs/connectors/flink-sources/_index.md index 11aed50f2a..f214713dd8 100644 --- a/docs/content/docs/connectors/legacy-flink-cdc-sources/_index.md +++ b/docs/content.zh/docs/connectors/flink-sources/_index.md @@ -1,7 +1,7 @@ --- -title: Legacy Flink CDC Sources +title: Flink Source 连接器 bookCollapseSection: true -weight: 5 +weight: 2 --- -# Legacy CDC sources +# Flink Sources 连接器 -Legacy CDC sources is a set of source connectors for Apache Flink®, ingesting changes from different databases using change data capture (CDC). +Flink CDC sources is a set of source connectors for Apache Flink®, ingesting changes from different databases using change data capture (CDC). Some CDC sources integrate Debezium as the engine to capture data changes. So it can fully leverage the ability of Debezium. See more about what is [Debezium](https://github.com/debezium/debezium). -You can also read [tutorials]({{< ref "docs/connectors/legacy-flink-cdc-sources/tutorials/build-streaming-etl-tutorial" >}}) about how to use these sources. +You can also read [tutorials]({{< ref "docs/connectors/flink-sources/tutorials/build-streaming-etl-tutorial" >}}) about how to use these sources. {{< img src="/fig/cdc-flow.png" width="600px" alt="Flink CDC" >}} ## Supported Connectors -| Connector | Database | Driver | -|---------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------| -| [mongodb-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/mongodb-cdc" >}}) |
  • [MongoDB](https://www.mongodb.com): 3.6, 4.x, 5.0 | MongoDB Driver: 4.3.4 | -| [mysql-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/mysql-cdc" >}}) |
  • [MySQL](https://dev.mysql.com/doc" >}}): 5.6, 5.7, 8.0.x
  • [RDS MySQL](https://www.aliyun.com/product/rds/mysql): 5.6, 5.7, 8.0.x
  • [PolarDB MySQL](https://www.aliyun.com/product/polardb): 5.6, 5.7, 8.0.x
  • [Aurora MySQL](https://aws.amazon.com/cn/rds/aurora): 5.6, 5.7, 8.0.x
  • [MariaDB](https://mariadb.org): 10.x
  • [PolarDB X](https://github.com/ApsaraDB/galaxysql): 2.0.1 | JDBC Driver: 8.0.28 | -| [oceanbase-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/oceanbase-cdc" >}}) |
  • [OceanBase CE](https://open.oceanbase.com): 3.1.x, 4.x
  • [OceanBase EE](https://www.oceanbase.com/product/oceanbase): 2.x, 3.x, 4.x | OceanBase Driver: 2.4.x | -| [oracle-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/oracle-cdc" >}}) |
  • [Oracle](https://www.oracle.com/index.html): 11, 12, 19, 21 | Oracle Driver: 19.3.0.0 | -| [postgres-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/postgres-cdc" >}}) |
  • [PostgreSQL](https://www.postgresql.org): 9.6, 10, 11, 12, 13, 14 | JDBC Driver: 42.5.1 | -| [sqlserver-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/sqlserver-cdc" >}}) |
  • [Sqlserver](https://www.microsoft.com/sql-server): 2012, 2014, 2016, 2017, 2019 | JDBC Driver: 9.4.1.jre8 | -| [tidb-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/tidb-cdc" >}}) |
  • [TiDB](https://www.pingcap.com/): 5.1.x, 5.2.x, 5.3.x, 5.4.x, 6.0.0 | JDBC Driver: 8.0.27 | -| [db2-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/db2-cdc" >}}) |
  • [Db2](https://www.ibm.com/products/db2): 11.5 | Db2 Driver: 11.5.0.0 | -| [vitess-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/vitess-cdc" >}}) |
  • [Vitess](https://vitess.io/): 8.0.x, 9.0.x | MySql JDBC Driver: 8.0.26 | +| Connector | Database | Driver | +|----------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------| +| [mongodb-cdc]({{< ref "docs/connectors/flink-sources/mongodb-cdc" >}}) |
  • [MongoDB](https://www.mongodb.com): 3.6, 4.x, 5.0 | MongoDB Driver: 4.3.4 | +| [mysql-cdc]({{< ref "docs/connectors/flink-sources/mysql-cdc" >}}) |
  • [MySQL](https://dev.mysql.com/doc): 5.6, 5.7, 8.0.x
  • [RDS MySQL](https://www.aliyun.com/product/rds/mysql): 5.6, 5.7, 8.0.x
  • [PolarDB MySQL](https://www.aliyun.com/product/polardb): 5.6, 5.7, 8.0.x
  • [Aurora MySQL](https://aws.amazon.com/cn/rds/aurora): 5.6, 5.7, 8.0.x
  • [MariaDB](https://mariadb.org): 10.x
  • [PolarDB X](https://github.com/ApsaraDB/galaxysql): 2.0.1 | JDBC Driver: 8.0.28 | +| [oceanbase-cdc]({{< ref "docs/connectors/flink-sources/oceanbase-cdc" >}}) |
  • [OceanBase CE](https://open.oceanbase.com): 3.1.x, 4.x
  • [OceanBase EE](https://www.oceanbase.com/product/oceanbase): 2.x, 3.x, 4.x | OceanBase Driver: 2.4.x | +| [oracle-cdc]({{< ref "docs/connectors/flink-sources/oracle-cdc" >}}) |
  • [Oracle](https://www.oracle.com/index.html): 11, 12, 19, 21 | Oracle Driver: 19.3.0.0 | +| [postgres-cdc]({{< ref "docs/connectors/flink-sources/postgres-cdc" >}}) |
  • [PostgreSQL](https://www.postgresql.org): 9.6, 10, 11, 12, 13, 14 | JDBC Driver: 42.5.1 | +| [sqlserver-cdc]({{< ref "docs/connectors/flink-sources/sqlserver-cdc" >}}) |
  • [Sqlserver](https://www.microsoft.com/sql-server): 2012, 2014, 2016, 2017, 2019 | JDBC Driver: 9.4.1.jre8 | +| [tidb-cdc]({{< ref "docs/connectors/flink-sources/tidb-cdc" >}}) |
  • [TiDB](https://www.pingcap.com/): 5.1.x, 5.2.x, 5.3.x, 5.4.x, 6.0.0 | JDBC Driver: 8.0.27 | +| [db2-cdc]({{< ref "docs/connectors/flink-sources/db2-cdc" >}}) |
  • [Db2](https://www.ibm.com/products/db2): 11.5 | Db2 Driver: 11.5.0.0 | +| [vitess-cdc]({{< ref "docs/connectors/flink-sources/vitess-cdc" >}}) |
  • [Vitess](https://vitess.io/): 8.0.x, 9.0.x | MySql JDBC Driver: 8.0.26 | ## Supported Flink Versions The following table shows the version mapping between Flink® CDC Connectors and Flink®: @@ -72,17 +72,17 @@ The following table shows the version mapping between Flink® CDC Con The following table shows the current features of the connector: -| Connector | No-lock Read | Parallel Read | Exactly-once Read | Incremental Snapshot Read | -|---------------------------------------------------------------------------------------|--------------|---------------|-------------------|---------------------------| -| [mongodb-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/mongodb-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | -| [mysql-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/mysql-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | -| [oracle-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/oracle-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | -| [postgres-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/postgres-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | -| [sqlserver-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/sqlserver-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | -| [oceanbase-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/oceanbase-cdc" >}}) | ❌ | ❌ | ❌ | ❌ | -| [tidb-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/tidb-cdc" >}}) | ✅ | ❌ | ✅ | ❌ | -| [db2-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/db2-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | -| [vitess-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/vitess-cdc" >}}) | ✅ | ❌ | ✅ | ❌ | +| Connector | No-lock Read | Parallel Read | Exactly-once Read | Incremental Snapshot Read | +|----------------------------------------------------------------------------|--------------|---------------|-------------------|---------------------------| +| [mongodb-cdc]({{< ref "docs/connectors/flink-sources/mongodb-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | +| [mysql-cdc]({{< ref "docs/connectors/flink-sources/mysql-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | +| [oracle-cdc]({{< ref "docs/connectors/flink-sources/oracle-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | +| [postgres-cdc]({{< ref "docs/connectors/flink-sources/postgres-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | +| [sqlserver-cdc]({{< ref "docs/connectors/flink-sources/sqlserver-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | +| [oceanbase-cdc]({{< ref "docs/connectors/flink-sources/oceanbase-cdc" >}}) | ❌ | ❌ | ❌ | ❌ | +| [tidb-cdc]({{< ref "docs/connectors/flink-sources/tidb-cdc" >}}) | ✅ | ❌ | ✅ | ❌ | +| [db2-cdc]({{< ref "docs/connectors/flink-sources/db2-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | +| [vitess-cdc]({{< ref "docs/connectors/flink-sources/vitess-cdc" >}}) | ✅ | ❌ | ✅ | ❌ | ## Usage for Table/SQL API diff --git a/docs/content/docs/connectors/legacy-flink-cdc-sources/postgres-cdc.md b/docs/content.zh/docs/connectors/flink-sources/postgres-cdc.md similarity index 99% rename from docs/content/docs/connectors/legacy-flink-cdc-sources/postgres-cdc.md rename to docs/content.zh/docs/connectors/flink-sources/postgres-cdc.md index 45919dbb8f..9a523e09ca 100644 --- a/docs/content/docs/connectors/legacy-flink-cdc-sources/postgres-cdc.md +++ b/docs/content.zh/docs/connectors/flink-sources/postgres-cdc.md @@ -1,9 +1,9 @@ --- title: "Postgres" -weight: 6 +weight: 5 type: docs aliases: -- /connectors/legacy-flink-cdc-sources/postgres-cdc +- /connectors/flink-sources/postgres-cdc --- - -# Connectors - -Flink CDC 提供了多个Source和Sink连接器来与外部系统交互。您可以直接使用这些连接器,只需将 JAR 文件添加到您的 Flink CDC 环境中,并在您的 YAML Pipeline定义中指定所需的连接器。 - -## Supported Connectors - -| Connector | Supported Type | External System | -|------------------------------------------------------|----------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Apache Doris]({{< ref "docs/connectors/doris" >}}) | Sink |
  • [Apache Doris](https://doris.apache.org/): 1.2.x, 2.x.x | -| [MySQL]({{< ref "docs/connectors/mysql" >}}) | Source |
  • [MySQL](https://dev.mysql.com/doc): 5.6, 5.7, 8.0.x
  • [RDS MySQL](https://www.aliyun.com/product/rds/mysql): 5.6, 5.7, 8.0.x
  • [PolarDB MySQL](https://www.aliyun.com/product/polardb): 5.6, 5.7, 8.0.x
  • [Aurora MySQL](https://aws.amazon.com/cn/rds/aurora): 5.6, 5.7, 8.0.x
  • [MariaDB](https://mariadb.org): 10.x
  • [PolarDB X](https://github.com/ApsaraDB/galaxysql): 2.0.1 | -| [StarRocks]({{< ref "docs/connectors/starrocks" >}}) | Sink |
  • [StarRocks](https://www.starrocks.io/): 2.x, 3.x | - -## Develop Your Own Connector - -如果现有的连接器无法满足您的需求,您可以自行开发自己的连接器,以将您的外部系统集成到 Flink CDC 数据管道中。查阅 [Flink CDC APIs]({{< ref "docs/developer-guide/understand-flink-cdc-api" >}}) 了解如何开发您自己的连接器。 - -## Legacy Flink CDC Sources -在 3.0 版本之前引入的 Flink CDC Source 仍然可以作为普通的 Flink 连接器使用。您可以在 [Legacy CDC Sources/概览]({{< ref "docs/connectors/legacy-flink-cdc-sources/overview" >}}) 中找到更多详细信息。 - -{{< top >}} diff --git a/docs/content.zh/docs/connectors/legacy-flink-cdc-sources/_index.md b/docs/content.zh/docs/connectors/pipeline-connectors/_index.md similarity index 95% rename from docs/content.zh/docs/connectors/legacy-flink-cdc-sources/_index.md rename to docs/content.zh/docs/connectors/pipeline-connectors/_index.md index e636a211d0..589eecdbd4 100644 --- a/docs/content.zh/docs/connectors/legacy-flink-cdc-sources/_index.md +++ b/docs/content.zh/docs/connectors/pipeline-connectors/_index.md @@ -1,7 +1,7 @@ --- -title: Legacy CDC Sources +title: Pipeline 连接器 bookCollapseSection: true -weight: 5 +weight: 1 --- + +# Pipeline Connectors + +Flink CDC 提供了可用于 YAML 作业的 Pipeline Source 和 Sink 连接器来与外部系统交互。您可以直接使用这些连接器,只需将 JAR 文件添加到您的 Flink CDC 环境中,并在您的 YAML Pipeline 定义中指定所需的连接器。 + +## Supported Connectors + +| 连接器 | 类型 | 支持的外部系统 | +|--------------------------------------------------------------------------|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Apache Doris]({{< ref "docs/connectors/pipeline-connectors/doris" >}}) | Sink |
  • [Apache Doris](https://doris.apache.org/): 1.2.x, 2.x.x | +| [Kafka]({{< ref "docs/connectors/pipeline-connectors/kafka" >}}) | Sink |
  • [Kafka](https://kafka.apache.org/) | +| [MySQL]({{< ref "docs/connectors/pipeline-connectors/mysql" >}}) | Source |
  • [MySQL](https://dev.mysql.com/doc): 5.6, 5.7, 8.0.x
  • [RDS MySQL](https://www.aliyun.com/product/rds/mysql): 5.6, 5.7, 8.0.x
  • [PolarDB MySQL](https://www.aliyun.com/product/polardb): 5.6, 5.7, 8.0.x
  • [Aurora MySQL](https://aws.amazon.com/cn/rds/aurora): 5.6, 5.7, 8.0.x
  • [MariaDB](https://mariadb.org): 10.x
  • [PolarDB X](https://github.com/ApsaraDB/galaxysql): 2.0.1 | +| [Paimon]({{< ref "docs/connectors/pipeline-connectors/paimon" >}}) | Sink |
  • [Paimon](https://paimon.apache.org/): 0.6, 0.7, 0.8 | +| [StarRocks]({{< ref "docs/connectors/pipeline-connectors/starrocks" >}}) | Sink |
  • [StarRocks](https://www.starrocks.io/): 2.x, 3.x | + +## Develop Your Own Connector + +如果现有的连接器无法满足您的需求,您可以自行开发自己的连接器,以将您的外部系统集成到 Flink CDC 数据管道中。查阅 [Flink CDC APIs]({{< ref "docs/developer-guide/understand-flink-cdc-api" >}}) 了解如何开发您自己的连接器。 + +{{< top >}} diff --git a/docs/content.zh/docs/connectors/paimon.md b/docs/content.zh/docs/connectors/pipeline-connectors/paimon.md similarity index 99% rename from docs/content.zh/docs/connectors/paimon.md rename to docs/content.zh/docs/connectors/pipeline-connectors/paimon.md index 8ca1aadd83..912f967498 100644 --- a/docs/content.zh/docs/connectors/paimon.md +++ b/docs/content.zh/docs/connectors/pipeline-connectors/paimon.md @@ -1,9 +1,9 @@ --- title: "Paimon" -weight: 6 +weight: 3 type: docs aliases: -- /connectors/paimon +- /connectors/pipeline-connectors/paimon --- diff --git a/docs/content/docs/connectors/legacy-flink-cdc-sources/datastream-api-package-guidance.md b/docs/content/docs/connectors/flink-sources/datastream-api-package-guidance.md similarity index 99% rename from docs/content/docs/connectors/legacy-flink-cdc-sources/datastream-api-package-guidance.md rename to docs/content/docs/connectors/flink-sources/datastream-api-package-guidance.md index 7cfead63a9..7569fe6e9c 100644 --- a/docs/content/docs/connectors/legacy-flink-cdc-sources/datastream-api-package-guidance.md +++ b/docs/content/docs/connectors/flink-sources/datastream-api-package-guidance.md @@ -3,7 +3,7 @@ title: "DataStream API Package Guidance" weight: 998 type: docs aliases: -- /connectors/legacy-flink-cdc-sources/datastream-api-package-guidance +- /connectors/flink-sources/datastream-api-package-guidance --- -# Legacy CDC sources +# Flink CDC sources -Legacy CDC sources is a set of source connectors for Apache Flink®, ingesting changes from different databases using change data capture (CDC). +Flink CDC sources is a set of source connectors for Apache Flink®, ingesting changes from different databases using change data capture (CDC). Some CDC sources integrate Debezium as the engine to capture data changes. So it can fully leverage the ability of Debezium. See more about what is [Debezium](https://github.com/debezium/debezium). -You can also read [tutorials]({{< ref "docs/connectors/legacy-flink-cdc-sources/tutorials/build-streaming-etl-tutorial" >}}) about how to use these sources. +You can also read [tutorials]({{< ref "docs/connectors/flink-sources/tutorials/build-streaming-etl-tutorial" >}}) about how to use these sources. {{< img src="/fig/cdc-flow.png" width="600px" alt="Flink CDC" >}} ## Supported Connectors -| Connector | Database | Driver | -|---------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------| -| [mongodb-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/mongodb-cdc" >}}) |
  • [MongoDB](https://www.mongodb.com): 3.6, 4.x, 5.0 | MongoDB Driver: 4.3.4 | -| [mysql-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/mysql-cdc" >}}) |
  • [MySQL](https://dev.mysql.com/doc): 5.6, 5.7, 8.0.x
  • [RDS MySQL](https://www.aliyun.com/product/rds/mysql): 5.6, 5.7, 8.0.x
  • [PolarDB MySQL](https://www.aliyun.com/product/polardb): 5.6, 5.7, 8.0.x
  • [Aurora MySQL](https://aws.amazon.com/cn/rds/aurora): 5.6, 5.7, 8.0.x
  • [MariaDB](https://mariadb.org): 10.x
  • [PolarDB X](https://github.com/ApsaraDB/galaxysql): 2.0.1 | JDBC Driver: 8.0.28 | -| [oceanbase-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/oceanbase-cdc" >}}) |
  • [OceanBase CE](https://open.oceanbase.com): 3.1.x, 4.x
  • [OceanBase EE](https://www.oceanbase.com/product/oceanbase): 2.x, 3.x, 4.x | OceanBase Driver: 2.4.x | -| [oracle-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/oracle-cdc" >}}) |
  • [Oracle](https://www.oracle.com/index.html): 11, 12, 19, 21 | Oracle Driver: 19.3.0.0 | -| [postgres-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/postgres-cdc" >}}) |
  • [PostgreSQL](https://www.postgresql.org): 9.6, 10, 11, 12, 13, 14 | JDBC Driver: 42.5.1 | -| [sqlserver-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/sqlserver-cdc" >}}) |
  • [Sqlserver](https://www.microsoft.com/sql-server): 2012, 2014, 2016, 2017, 2019 | JDBC Driver: 9.4.1.jre8 | -| [tidb-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/tidb-cdc" >}}) |
  • [TiDB](https://www.pingcap.com/): 5.1.x, 5.2.x, 5.3.x, 5.4.x, 6.0.0 | JDBC Driver: 8.0.27 | -| [db2-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/db2-cdc" >}}) |
  • [Db2](https://www.ibm.com/products/db2): 11.5 | Db2 Driver: 11.5.0.0 | -| [vitess-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/vitess-cdc" >}}) |
  • [Vitess](https://vitess.io/): 8.0.x, 9.0.x | MySql JDBC Driver: 8.0.26 | +| Connector | Database | Driver | +|----------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------| +| [mongodb-cdc]({{< ref "docs/connectors/flink-sources/mongodb-cdc" >}}) |
  • [MongoDB](https://www.mongodb.com): 3.6, 4.x, 5.0 | MongoDB Driver: 4.3.4 | +| [mysql-cdc]({{< ref "docs/connectors/flink-sources/mysql-cdc" >}}) |
  • [MySQL](https://dev.mysql.com/doc): 5.6, 5.7, 8.0.x
  • [RDS MySQL](https://www.aliyun.com/product/rds/mysql): 5.6, 5.7, 8.0.x
  • [PolarDB MySQL](https://www.aliyun.com/product/polardb): 5.6, 5.7, 8.0.x
  • [Aurora MySQL](https://aws.amazon.com/cn/rds/aurora): 5.6, 5.7, 8.0.x
  • [MariaDB](https://mariadb.org): 10.x
  • [PolarDB X](https://github.com/ApsaraDB/galaxysql): 2.0.1 | JDBC Driver: 8.0.28 | +| [oceanbase-cdc]({{< ref "docs/connectors/flink-sources/oceanbase-cdc" >}}) |
  • [OceanBase CE](https://open.oceanbase.com): 3.1.x, 4.x
  • [OceanBase EE](https://www.oceanbase.com/product/oceanbase): 2.x, 3.x, 4.x | OceanBase Driver: 2.4.x | +| [oracle-cdc]({{< ref "docs/connectors/flink-sources/oracle-cdc" >}}) |
  • [Oracle](https://www.oracle.com/index.html): 11, 12, 19, 21 | Oracle Driver: 19.3.0.0 | +| [postgres-cdc]({{< ref "docs/connectors/flink-sources/postgres-cdc" >}}) |
  • [PostgreSQL](https://www.postgresql.org): 9.6, 10, 11, 12, 13, 14 | JDBC Driver: 42.5.1 | +| [sqlserver-cdc]({{< ref "docs/connectors/flink-sources/sqlserver-cdc" >}}) |
  • [Sqlserver](https://www.microsoft.com/sql-server): 2012, 2014, 2016, 2017, 2019 | JDBC Driver: 9.4.1.jre8 | +| [tidb-cdc]({{< ref "docs/connectors/flink-sources/tidb-cdc" >}}) |
  • [TiDB](https://www.pingcap.com/): 5.1.x, 5.2.x, 5.3.x, 5.4.x, 6.0.0 | JDBC Driver: 8.0.27 | +| [db2-cdc]({{< ref "docs/connectors/flink-sources/db2-cdc" >}}) |
  • [Db2](https://www.ibm.com/products/db2): 11.5 | Db2 Driver: 11.5.0.0 | +| [vitess-cdc]({{< ref "docs/connectors/flink-sources/vitess-cdc" >}}) |
  • [Vitess](https://vitess.io/): 8.0.x, 9.0.x | MySql JDBC Driver: 8.0.26 | ## Supported Flink Versions The following table shows the version mapping between Flink® CDC Connectors and Flink®: @@ -74,15 +74,15 @@ The following table shows the current features of the connector: | Connector | No-lock Read | Parallel Read | Exactly-once Read | Incremental Snapshot Read | |---------------------------------------------------------------------------------------|--------------|---------------|-------------------|---------------------------| -| [mongodb-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/mongodb-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | -| [mysql-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/mysql-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | -| [oracle-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/oracle-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | -| [postgres-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/postgres-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | -| [sqlserver-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/sqlserver-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | -| [oceanbase-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/oceanbase-cdc" >}}) | ❌ | ❌ | ❌ | ❌ | -| [tidb-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/tidb-cdc" >}}) | ✅ | ❌ | ✅ | ❌ | -| [db2-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/db2-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | -| [vitess-cdc]({{< ref "docs/connectors/legacy-flink-cdc-sources/vitess-cdc" >}}) | ✅ | ❌ | ✅ | ❌ | +| [mongodb-cdc]({{< ref "docs/connectors/flink-sources/mongodb-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | +| [mysql-cdc]({{< ref "docs/connectors/flink-sources/mysql-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | +| [oracle-cdc]({{< ref "docs/connectors/flink-sources/oracle-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | +| [postgres-cdc]({{< ref "docs/connectors/flink-sources/postgres-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | +| [sqlserver-cdc]({{< ref "docs/connectors/flink-sources/sqlserver-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | +| [oceanbase-cdc]({{< ref "docs/connectors/flink-sources/oceanbase-cdc" >}}) | ❌ | ❌ | ❌ | ❌ | +| [tidb-cdc]({{< ref "docs/connectors/flink-sources/tidb-cdc" >}}) | ✅ | ❌ | ✅ | ❌ | +| [db2-cdc]({{< ref "docs/connectors/flink-sources/db2-cdc" >}}) | ✅ | ✅ | ✅ | ✅ | +| [vitess-cdc]({{< ref "docs/connectors/flink-sources/vitess-cdc" >}}) | ✅ | ❌ | ✅ | ❌ | ## Usage for Table/SQL API diff --git a/docs/content.zh/docs/connectors/legacy-flink-cdc-sources/postgres-cdc.md b/docs/content/docs/connectors/flink-sources/postgres-cdc.md similarity index 99% rename from docs/content.zh/docs/connectors/legacy-flink-cdc-sources/postgres-cdc.md rename to docs/content/docs/connectors/flink-sources/postgres-cdc.md index 45919dbb8f..9a523e09ca 100644 --- a/docs/content.zh/docs/connectors/legacy-flink-cdc-sources/postgres-cdc.md +++ b/docs/content/docs/connectors/flink-sources/postgres-cdc.md @@ -1,9 +1,9 @@ --- title: "Postgres" -weight: 6 +weight: 5 type: docs aliases: -- /connectors/legacy-flink-cdc-sources/postgres-cdc +- /connectors/flink-sources/postgres-cdc --- diff --git a/docs/content/docs/connectors/doris.md b/docs/content/docs/connectors/pipeline-connectors/doris.md similarity index 99% rename from docs/content/docs/connectors/doris.md rename to docs/content/docs/connectors/pipeline-connectors/doris.md index a7598bf523..b5048eaf91 100644 --- a/docs/content/docs/connectors/doris.md +++ b/docs/content/docs/connectors/pipeline-connectors/doris.md @@ -1,9 +1,9 @@ --- title: "Doris" -weight: 3 +weight: 5 type: docs aliases: -- /connectors/doris +- /connectors/pipeline-connectors/doris ---