Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
138 lines (130 sloc) 5.23 KB
version: '3'
services:
zookeeper:
image: 'confluentinc/cp-zookeeper:5.0.0'
restart: always
hostname: zookeeper
ports:
- "2181:2181"
environment:
ZOO_MY_ID: 1
ZOO_PORT: 2181
ZOO_SERVERS: server.1=zookeeper:2888:3888
COMPONENT: zookeeper
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
broker:
image: 'confluentinc/cp-kafka:5.0.0'
hostname: broker
stop_grace_period: 120s
depends_on:
- zookeeper
ports:
- "9092:9092"
- "8500:8500"
environment:
COMPONENT: kafka
KAFKA_BROKER_ID: 1001
KAFKA_RESERVED_BROKER_MAX_ID: 10000
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:9092'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_JMX_OPTS: '-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=broker -Djava.net.preferIPv4Stack=true -Dcom.sun.management.jmxremote.rmi.port=8500'
schema_registry:
image: 'confluentinc/cp-schema-registry:5.0.0'
hostname: schema_registry
depends_on:
- zookeeper
- broker
ports:
- "8081:8081"
environment:
TZ: Australia/Sydney
SCHEMA_REGISTRY_HOST_NAME: schema_registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
SCHEMA_REGISTRY_KAFKASTORE_TIMEOUT_MS: 10000
SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: 'INFO'
# This "container" is a workaround to pre-create topics for the Kafka Auction example application
# until we have a more elegant way to do that.
kafka-create-topics:
image: confluentinc/cp-kafka:5.0.0
depends_on:
- broker
- zookeeper
hostname: kafka-create-topics
# We defined a dependency on "broker", but `depends_on` will NOT wait for the
# dependencies to be "ready" before starting the "kafka-create-topics"
# container; it waits only until the dependencies have started. Hence we
# must control startup order more explicitly.
# See https://docs.docker.com/compose/startup-order/
command: "bash -c 'echo Waiting for Kafka to be ready... && \
cub zk-ready zookeeper:2181 120 && \
cub kafka-ready -b broker:9092 1 120 && \
kafka-topics --create --topic auction_account_transactions_projection --if-not-exists --zookeeper zookeeper:2181 --partitions 10 --replication-factor 1 && \
kafka-topics --create --topic auction_avro_auction-aggregate --if-not-exists --zookeeper zookeeper:2181 --partitions 10 --replication-factor 1 && \
kafka-topics --create --topic auction_avro_account-aggregate --if-not-exists --zookeeper zookeeper:2181 --partitions 10 --replication-factor 1 && \
sleep infinity'"
environment:
# The following settings are listed here only to satisfy the image's requirements.
# We override the image's `command` anyways, hence this container will not start a broker.
KAFKA_BROKER_ID: ignored
KAFKA_ZOOKEEPER_CONNECT: ignored
extra_hosts:
- "moby:127.0.0.1"
mongo:
image: mongo
restart: always
hostname: mongo
ports:
- "27017:27017"
mongo-express:
image: mongo-express
hostname: mongo-express
restart: always
ports:
- "8088:8081"
connect:
image: confluentinc/cp-kafka-connect:4.0.0
hostname: connect
ports:
- "8083:8083"
depends_on:
- zookeeper
- broker
- schema_registry
environment:
CONNECT_BOOTSTRAP_SERVERS: 'broker:9092'
CONNECT_REST_ADVERTISED_HOST_NAME: connect
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: compose-connect-group
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081'
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081'
CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'
CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
CONNECT_PLUGIN_PATH: /etc/kafka-connect/jars
volumes:
- ./mongodb_connect_jars:/etc/kafka-connect/jars
connect-configure:
image: broadinstitute/python-requests
hostname: connect-configure
depends_on:
- connect
- kafka-create-topics
volumes:
- ./scripts:/scripts
command: /scripts/configure_kafka_connect.py
You can’t perform that action at this time.