Simple, efficient, real-time, stable, scalable, highly available, AI, open source
English | 简体中文
- mysql
- TODO sqlserver
- TODO mongo
- TODO oracle
- mysql
- starrocks
- doris
- kafka json
- TODO kafka canal
- kafka aliyun_dts_canal
Download the latest release and extract it.
CREATE USER 'qin_cdc'@'%' IDENTIFIED BY 'xxxxxx';
GRANT SELECT, REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO 'qin_cdc'@'%';
mysql-to-starrocks.toml
# name is required and must be globally unique when multiple instances are running
name = "mysql2starrocks"
[input]
type = "mysql"
[input.config.source]
host = "127.0.0.1"
port = 3306
username = "root"
password = "root"
[input.config.source.options]
#start-gtid = "3ba13781-44eb-2157-88a5-0dc879ec2221:1-123456"
#server-id = 1001
[[transforms]]
type = "rename-column"
[transforms.config]
match-schema = "mysql_test"
match-table = "tb1"
columns = ["col_1", "col_2"]
rename-as = ["col_11", "col_22"]
[[transforms]]
type = "delete-column"
[transforms.config]
match-schema = "mysql_test"
match-table = "tb1"
columns = ["phone"]
[output]
type = "starrocks"
[output.config.target]
host = "127.0.0.1"
port = 9030
load-port = 8040 # support fe httpPort:8030 or be httpPort:8040
username = "root"
password = ""
[input.config.target.options]
batch-size = 1000
batch-interval-ms = 1000
parallel-workers = 4
[[output.config.routers]]
source-schema = "mysql_test"
source-table = "tb1"
target-schema = "sr_test"
target-table = "ods_tb1"
[[output.config.routers]]
source-schema = "mysql_test"
source-table = "tb2"
target-schema = "sr_test"
target-table = "ods_tb2"
# mapper column, optional, if empty, same name mapping
# [output.config.routers.columns-mapper]
# source-columns = []
# target-columns = []
[sr@ ~]$ ./qin-cdc-linux-xxxxxx -h
[sr@ ~]$ ./qin-cdc-linux-xxxxxx -config mysql-to-starrocks.toml -log-file mysql2starrocks.log -level info -daemon
[sr@ ~]$ tail -f mysql2starrocks.log
- Intelligent data synchronization and migration
- Data security and monitoring
- Intelligent operation and maintenance management
- User experience optimization
- Automated data mapping
- Natural language processing