-
Notifications
You must be signed in to change notification settings - Fork 893
/
docker-compose-ccloud.yml
115 lines (111 loc) 路 5.74 KB
/
docker-compose-ccloud.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
# Use this Docker Compose if you want to run these components locally, connecting to a CCloud instance
# See docker-compose-all-local.yml if you want to run *everything* locally (i.e. no CCloud)
# N.B. This assumes that you have a /u01/connectors folder and within it the Connectors that you plan to use.
# You can get these from hub.confluent.io and confluent-hub install <connector>
# You can also download the source code and put the built object in this path.
---
version: '2'
services:
schema-registry:
image: "confluentinc/cp-schema-registry:5.2.1"
ports:
- '8081:8081'
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: SASL_SSL://CCLOUD_BROKER_HOST:9092
SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_SSL
SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required username="CCLOUD_USERNAME" password="CCLOUD_PASSWORD";'
SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: PLAIN
SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
ksql-server:
image: confluentinc/cp-ksql-server:5.2.1
depends_on:
- schema-registry
environment:
KSQL_LISTENERS: http://0.0.0.0:8088
KSQL_KSQL_SCHEMA_REGISTRY_URL: "https://CCLOUD_SCHEMA_REGISTRY"
KSQL_KSQL_SCHEMA_REGISTRY_BASIC_AUTH_CREDENTIALS_SOURCE: "USER_INFO"
KSQL_KSQL_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO: "SR_API_KEY:SR_API_SECRET"
KSQL_KSQL_SERVICE_ID: confluent_rmoff_01
KSQL_BOOTSTRAP_SERVERS: CCLOUD_BROKER_HOST:9092
KSQL_KSQL_SERVER_UI_ENABLED: false
KSQL_APPLICATION_ID: rmoff-gcp-pipeline-demo
KSQL_KSQL_STREAMS_REPLICATION_FACTOR: 3
KSQL_KSQL_SINK_REPLICAS: 3
KSQL_LISTENERS: http://0.0.0.0:8088
KSQL_CACHE_MAX_BYTES_BUFFERING: 0
KSQL_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: HTTPS
KSQL_SECURITY_PROTOCOL: SASL_SSL
KSQL_SASL_MECHANISM: PLAIN
KSQL_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required username="CCLOUD_USERNAME" password="CCLOUD_PASSWORD";'
ksql-cli:
image: confluentinc/cp-ksql-cli:5.2.1
depends_on:
- ksql-server
entrypoint: /bin/sh
tty: true
kafka-connect:
image: confluentinc/cp-kafka-connect:5.2.1
ports:
- "8083:8083"
depends_on:
- schema-registry
environment:
CONNECT_BOOTSTRAP_SERVERS: "CCLOUD_BROKER_HOST:9092"
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: compose-connect-group
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "https://CCLOUD_SCHEMA_REGISTRY"
CONNECT_VALUE_CONVERTER_BASIC_AUTH_CREDENTIALS_SOURCE: "USER_INFO"
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO: "SR_API_KEY:SR_API_SECRET"
CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "https://CCLOUD_SCHEMA_REGISTRY"
CONNECT_KEY_CONVERTER_BASIC_AUTH_CREDENTIALS_SOURCE: "USER_INFO"
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO: "SR_API_KEY:SR_API_SECRET"
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect-cp"
CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "3"
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "3"
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "3"
CONNECT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "https"
CONNECT_SASL_MECHANISM: "PLAIN"
CONNECT_REQUEST_TIMEOUT_MS: "20000"
CONNECT_RETRY_BACKOFF_MS: "500"
CONNECT_SECURITY_PROTOCOL: "SASL_SSL"
CONNECT_CONSUMER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "https"
CONNECT_CONSUMER_SASL_MECHANISM: "PLAIN"
CONNECT_CONSUMER_REQUEST_TIMEOUT_MS: "20000"
CONNECT_CONSUMER_RETRY_BACKOFF_MS: "500"
CONNECT_CONSUMER_SECURITY_PROTOCOL: "SASL_SSL"
CONNECT_PRODUCER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "https"
CONNECT_PRODUCER_SASL_MECHANISM: "PLAIN"
CONNECT_PRODUCER_REQUEST_TIMEOUT_MS: "20000"
CONNECT_PRODUCER_RETRY_BACKOFF_MS: "500"
CONNECT_PRODUCER_SECURITY_PROTOCOL: "SASL_SSL"
CONNECT_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required username="CCLOUD_USERNAME" password="CCLOUD_PASSWORD";'
CONNECT_CONSUMER_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required username="CCLOUD_USERNAME" password="CCLOUD_PASSWORD";'
CONNECT_PRODUCER_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required username="CCLOUD_USERNAME" password="CCLOUD_PASSWORD";'
CONNECT_PLUGIN_PATH: /usr/share/java,/usr/share/confluent-hub-components,/u01/connectors
# In the command section, $ are replaced with $$ to avoid the error 'Invalid interpolation format for "command" option'
command:
- bash
- -c
- |
echo "Installing Connector"
confluent-hub install --no-prompt cjmatta/kafka-connect-irc:5.0.0
confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:0.1.1
#
echo "Launching Kafka Connect worker"
/etc/confluent/docker/run &
#
sleep infinity
volumes:
- /u01/connectors:/u01/connectors
- $PWD/creds/gcp_creds.json:/root/creds/gcp_creds.json