-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
⛽ feat: add Kafka, Zookeeper and MySQL example to the repository (#7)
* 🐳 feat: add docker-compose file to the examples * 📝 docs: add Kafka Zookeeper Avro sample connector configuration documentation
- Loading branch information
1 parent
cae0c9f
commit 9ae5770
Showing
2 changed files
with
159 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
|
||
# Kafka ZooKeeper Debezium AVRO Integration | ||
|
||
For this example, the quickstart example from the Debezium documentation with AVRO activated is used. | ||
|
||
## Run the stack and its services | ||
|
||
```bash | ||
docker compose up -d | ||
# If you have older version of Docker Compose (v1), it is required to execute the following command: | ||
# docker-compose up -d | ||
``` | ||
|
||
## Interacting with the containers | ||
|
||
### Connect to MySQL | ||
|
||
```bash | ||
docker exec -it mysql_cli bash | ||
exec mysql -h mysql -P 3306 -u root -pdebezium | ||
``` | ||
|
||
**Note**: the password of the mysql root user is `debezium`. | ||
|
||
### Debezium | ||
|
||
#### Debezium Version | ||
|
||
```bash | ||
curl -H "Accept:application/json" localhost:8083/ | ||
``` | ||
|
||
#### Debezium Connectors list | ||
|
||
```bash | ||
curl -H "Accept:application/json" localhost:8083/connectors/ | ||
``` | ||
|
||
#### Debezium sample connector | ||
|
||
To add the new connector, execute the following command in your terminal: | ||
|
||
```bash | ||
curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" localhost:8083/connectors/ -d '{ | ||
"name":"inventory-connector", | ||
"config":{ | ||
"connector.class":"io.debezium.connector.mysql.MySqlConnector", | ||
"tasks.max":"1", | ||
"database.hostname":"mysql", | ||
"database.port":"3306", | ||
"database.user":"debezium", | ||
"database.password":"dbz", | ||
"database.server.id":"184054", | ||
"topic.prefix":"dbserver1", | ||
"database.include.list":"inventory", | ||
"schema.history.internal.kafka.bootstrap.servers":"kafka:9092", | ||
"schema.history.internal.kafka.topic":"schemahistory.inventory", | ||
"key.converter":"io.confluent.connect.avro.AvroConverter", | ||
"value.converter":"io.confluent.connect.avro.AvroConverter", | ||
"key.converter.schema.registry.url":"http://schema_registry:8081", | ||
"value.converter.schema.registry.url":"http://schema_registry:8081" | ||
} | ||
}' | ||
``` | ||
|
||
#### Debezium Connector Information | ||
|
||
```bash | ||
curl -i -X GET -H "Accept:application/json" localhost:8083/connectors/<connector-name> | ||
# curl -i -X GET -H "Accept:application/json" localhost:8083/connectors/inventory-connector | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,88 @@ | ||
version: '3' | ||
|
||
services: | ||
zookeeper: | ||
image: quay.io/debezium/zookeeper:2.5 | ||
container_name: zookeeper | ||
ports: | ||
- 2181:2181 | ||
- 2888:2888 | ||
- 3888:3888 | ||
|
||
kafka: | ||
image: quay.io/debezium/kafka:2.5 | ||
container_name: kafka | ||
ports: | ||
- 9092:9092 | ||
environment: | ||
ZOOKEEPER_CONNECT: zookeeper:2181 | ||
depends_on: | ||
- zookeeper | ||
|
||
mysql: | ||
image: quay.io/debezium/example-mysql:2.5 | ||
container_name: mysql | ||
ports: | ||
- 3306:3306 | ||
environment: | ||
MYSQL_ROOT_PASSWORD: debezium | ||
MYSQL_USER: mysqluser | ||
MYSQL_PASSWORD: mysqlpw | ||
|
||
mysql_cli: | ||
image: mysql:8.0 | ||
container_name: mysql_cli | ||
environment: | ||
MYSQL_ALLOW_EMPTY_PASSWORD: true | ||
depends_on: | ||
- mysql | ||
|
||
connect: | ||
image: quay.io/debezium/connect:2.5 | ||
container_name: connect | ||
ports: | ||
- 8083:8083 | ||
environment: | ||
GROUP_ID: 1 | ||
# Kafka config | ||
CONFIG_STORAGE_TOPIC: my_connect_configs | ||
OFFSET_STORAGE_TOPIC: my_connect_offsets | ||
STATUS_STORAGE_TOPIC: my_connect_statuses | ||
BOOTSTRAP_SERVERS: kafka:9092 | ||
# Avro config | ||
KEY_CONVERTER: io.confluent.connect.avro.AvroConverter | ||
VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter | ||
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema_registry:8081 | ||
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema_registry:8081 | ||
KAFKA_CONNECT_PLUGINS_DIR: /kafka/connect | ||
volumes: | ||
- ./jar_files:/kafka/connect/avro_jar_files | ||
depends_on: | ||
- kafka | ||
- schema_registry | ||
- mysql | ||
|
||
schema_registry: | ||
image: confluentinc/cp-schema-registry:7.1.10 | ||
container_name: schema_registry | ||
ports: | ||
- 8081:8081 | ||
environment: | ||
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181 | ||
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092 | ||
SCHEMA_REGISTRY_HOST_NAME: schema_registry | ||
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 | ||
depends_on: | ||
- zookeeper | ||
|
||
kafdrop: | ||
image: obsidiandynamics/kafdrop:4.0.1 | ||
container_name: kafdrop | ||
ports: | ||
- 9000:9000 | ||
environment: | ||
KAFKA_BROKERCONNECT: kafka:9092 | ||
depends_on: | ||
- kafka | ||
|
||
|