From c39fe1c85ca175a028521f8bf77ef9a906e24a6f Mon Sep 17 00:00:00 2001 From: Markus Gulden <> Date: Tue, 1 Jan 2019 15:40:15 +0100 Subject: [PATCH] BAEL-2321 --- .../04_Custom/connect-distributed.properties | 88 ----------------- .../04_Custom/connect-mongodb-sink.json | 30 +++--- .../04_Custom/connect-mqtt-source.json | 8 +- .../04_Custom/docker-compose.yaml | 94 +++++++++++++++++++ 4 files changed, 110 insertions(+), 110 deletions(-) delete mode 100644 libraries-data/src/main/kafka-connect/04_Custom/connect-distributed.properties create mode 100644 libraries-data/src/main/kafka-connect/04_Custom/docker-compose.yaml diff --git a/libraries-data/src/main/kafka-connect/04_Custom/connect-distributed.properties b/libraries-data/src/main/kafka-connect/04_Custom/connect-distributed.properties deleted file mode 100644 index 5b91baddbd7f..000000000000 --- a/libraries-data/src/main/kafka-connect/04_Custom/connect-distributed.properties +++ /dev/null @@ -1,88 +0,0 @@ -## -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -## - -# This file contains some of the configurations for the Kafka Connect distributed worker. This file is intended -# to be used with the examples, and some settings may differ from those used in a production system, especially -# the `bootstrap.servers` and those specifying replication factors. - -# A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. -bootstrap.servers=localhost:9092 - -# unique name for the cluster, used in forming the Connect cluster group. Note that this must not conflict with consumer group IDs -group.id=connect-cluster - -# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will -# need to configure these based on the format they want their data in when loaded from or stored into Kafka -key.converter=org.apache.kafka.connect.json.JsonConverter -value.converter=org.apache.kafka.connect.json.JsonConverter -# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply -# it to -key.converter.schemas.enable=true -value.converter.schemas.enable=true - -# Topic to use for storing offsets. This topic should have many partitions and be replicated and compacted. -# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create -# the topic before starting Kafka Connect if a specific topic configuration is needed. -# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. -# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able -# to run this example on a single-broker cluster and so here we instead set the replication factor to 1. -offset.storage.topic=connect-offsets -offset.storage.replication.factor=1 -#offset.storage.partitions=25 - -# Topic to use for storing connector and task configurations; note that this should be a single partition, highly replicated, -# and compacted topic. Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create -# the topic before starting Kafka Connect if a specific topic configuration is needed. -# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. -# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able -# to run this example on a single-broker cluster and so here we instead set the replication factor to 1. -config.storage.topic=connect-configs -config.storage.replication.factor=1 - -# Topic to use for storing statuses. This topic can have multiple partitions and should be replicated and compacted. -# Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create -# the topic before starting Kafka Connect if a specific topic configuration is needed. -# Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. -# Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able -# to run this example on a single-broker cluster and so here we instead set the replication factor to 1. -status.storage.topic=connect-status -status.storage.replication.factor=1 -#status.storage.partitions=5 - -# Flush much faster than normal, which is useful for testing/debugging -offset.flush.interval.ms=10000 - -# These are provided to inform the user about the presence of the REST host and port configs -# Hostname & Port for the REST API to listen on. If this is set, it will bind to the interface used to listen to requests. -#rest.host.name= -#rest.port=8083 - -# The Hostname & Port that will be given out to other workers to connect to i.e. URLs that are routable from other servers. -#rest.advertised.host.name= -#rest.advertised.port= - -# Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins -# (connectors, converters, transformations). The list should consist of top level directories that include -# any combination of: -# a) directories immediately containing jars with plugins and their dependencies -# b) uber-jars with plugins and their dependencies -# c) directories immediately containing the package directory structure of classes of plugins and their dependencies -# Examples: -# plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors, -# Replace the relative path below with an absolute path if you are planning to start Kafka Connect from within a -# directory other than the home directory of Confluent Platform. -plugin.path=./share/java diff --git a/libraries-data/src/main/kafka-connect/04_Custom/connect-mongodb-sink.json b/libraries-data/src/main/kafka-connect/04_Custom/connect-mongodb-sink.json index 333768e4b7f8..852f400fc67a 100644 --- a/libraries-data/src/main/kafka-connect/04_Custom/connect-mongodb-sink.json +++ b/libraries-data/src/main/kafka-connect/04_Custom/connect-mongodb-sink.json @@ -1,22 +1,14 @@ { - "firstName": "John", - "lastName": "Smith", - "age": 25, - "address": { - "streetAddress": "21 2nd Street", - "city": "New York", - "state": "NY", - "postalCode": "10021" - }, - "phoneNumber": [{ - "type": "home", - "number": "212 555-1234" - }, { - "type": "fax", - "number": "646 555-4567" - } - ], - "gender": { - "type": "male" + "name": "mongodb-sink", + "config": { + "connector.class": "at.grahsl.kafka.connect.mongodb.MongoDbSinkConnector", + "tasks.max": 1, + "topics": "connect-custom", + "mongodb.connection.uri": "mongodb://mongo-db/test?retryWrites=true", + "mongodb.collection": "MyCollection", + "key.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter.schemas.enable": false, + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": false } } diff --git a/libraries-data/src/main/kafka-connect/04_Custom/connect-mqtt-source.json b/libraries-data/src/main/kafka-connect/04_Custom/connect-mqtt-source.json index 02d87c5ad7ba..c76d326c0ad7 100644 --- a/libraries-data/src/main/kafka-connect/04_Custom/connect-mqtt-source.json +++ b/libraries-data/src/main/kafka-connect/04_Custom/connect-mqtt-source.json @@ -3,9 +3,11 @@ "config": { "connector.class": "io.confluent.connect.mqtt.MqttSourceConnector", "tasks.max": 1, - "mqtt.server.uri": "ws://broker.hivemq.com:8000/mqtt", + "mqtt.server.uri": "tcp://mosquitto:1883", "mqtt.topics": "baeldung", "kafka.topic": "connect-custom", - "value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter" + "value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter", + "confluent.topic.bootstrap.servers": "kafka:9092", + "confluent.topic.replication.factor": 1 } -} +} \ No newline at end of file diff --git a/libraries-data/src/main/kafka-connect/04_Custom/docker-compose.yaml b/libraries-data/src/main/kafka-connect/04_Custom/docker-compose.yaml new file mode 100644 index 000000000000..26cd65333589 --- /dev/null +++ b/libraries-data/src/main/kafka-connect/04_Custom/docker-compose.yaml @@ -0,0 +1,94 @@ +version: '3.3' + +services: + mosquitto: + image: eclipse-mosquitto:1.5.5 + hostname: mosquitto + container_name: mosquitto + expose: + - "1883" + ports: + - "1883:1883" + zookeeper: + image: zookeeper:3.4.9 + restart: unless-stopped + hostname: zookeeper + container_name: zookeeper + ports: + - "2181:2181" + environment: + ZOO_MY_ID: 1 + ZOO_PORT: 2181 + ZOO_SERVERS: server.1=zookeeper:2888:3888 + volumes: + - ./zookeeper/data:/data + - ./zookeeper/datalog:/datalog + kafka: + image: confluentinc/cp-kafka:5.1.0 + hostname: kafka + container_name: kafka + ports: + - "9092:9092" + environment: + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 + KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" + KAFKA_BROKER_ID: 1 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + volumes: + - ./kafka/data:/var/lib/kafka/data + depends_on: + - zookeeper + kafka-connect: + image: confluentinc/cp-kafka-connect:5.1.0 + hostname: kafka-connect + container_name: kafka-connect + ports: + - "8083:8083" + environment: + CONNECT_BOOTSTRAP_SERVERS: "kafka:9092" + CONNECT_REST_ADVERTISED_HOST_NAME: connect + CONNECT_REST_PORT: 8083 + CONNECT_GROUP_ID: compose-connect-group + CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs + CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets + CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status + CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter + CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter + CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1" + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1" + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1" + CONNECT_PLUGIN_PATH: '/usr/share/java,/etc/kafka-connect/jars' + CONNECT_CONFLUENT_TOPIC_REPLICATION_FACTOR: 1 + volumes: + - /tmp/custom/jars:/etc/kafka-connect/jars + depends_on: + - zookeeper + - kafka + - mosquitto + mongo-db: + image: mongo:4.0.5 + hostname: mongo-db + container_name: mongo-db + expose: + - "27017" + ports: + - "27017:27017" + command: --bind_ip_all --smallfiles + volumes: + - ./mongo-db:/data + mongoclient: + image: mongoclient/mongoclient:2.2.0 + container_name: mongoclient + hostname: mongoclient + depends_on: + - mongo-db + ports: + - 3000:3000 + environment: + MONGO_URL: "mongodb://mongo-db:27017" + PORT: 3000 + expose: + - "3000" \ No newline at end of file