Skip to content

fix: remove dependency on systemd; make sure kafka can be shutdown gracefully #689

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 17, 2024
Merged

fix: remove dependency on systemd; make sure kafka can be shutdown gracefully #689

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dd7612c
fix: remove dependency on systemd; make sure kafka can be shutdown gr…
mooc9988 Jan 17, 2024
d1d3033
refactor: remove kafka service
mooc9988 Jan 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,17 @@ MAINTAINER AutoMQ for Apache Kafka dev@automq.com
# Set mirrors
ARG general_mirror_url=""
RUN [ -z "${general_mirror_url}" ] || (sed -i "s/deb.debian.org/${general_mirror_url}/g" /etc/apt/sources.list && sed -i "s|security.debian.org/debian-security|${general_mirror_url}/debian-security|g" /etc/apt/sources.list)
RUN apt update && apt install -y vim systemctl
RUN apt update && apt install -y vim

# Do not ask for confirmations when running apt-get, etc.
ENV DEBIAN_FRONTEND=noninteractive \
AWS_DEFAULT_REGION=us-east-1
AWS_DEFAULT_REGION=us-east-1 \
KAFKA_JVM_PERFORMANCE_OPTS="-server -XX:+UseZGC -XX:ZCollectionInterval=5"

COPY "./kafka_*.tgz" /opt/kafka/kafka.tgz
RUN tar -xzvf "/opt/kafka/kafka.tgz" -C /opt/kafka/ --one-top-level=kafka --strip-components 1 && rm "/opt/kafka/kafka.tgz"
COPY scripts /opt/kafka/scripts
RUN chmod +x /opt/kafka/scripts/*.sh

RUN mkdir -p /data/kafka

Expand Down
21 changes: 18 additions & 3 deletions docker/docker-compose.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,17 @@ services:
controller:
container_name: "${CONTROLLER_DOCKER_NAME-controller}"
hostname: "${CONTROLLER_DOCKER_NAME-controller}"
stop_grace_period: 2m
image: automqinc/kafka:1.0.0-rc5
environment:
- KAFKA_S3_ACCESS_KEY=test
- KAFKA_S3_SECRET_KEY=test
- KAFKA_HEAP_OPTS=-Xms1g -Xmx1g -XX:MetaspaceSize=96m
command: bash /opt/kafka/scripts/start.sh up --process.roles controller --node.id 0 --controller.quorum.voters 0@controller:9093 --s3.bucket ko3 --s3.endpoint http://10.6.0.2:4566 --s3.region us-east-1
command:
- bash
- -c
- |
/opt/kafka/scripts/start.sh up --process.roles controller --node.id 0 --controller.quorum.voters 0@controller:9093 --s3.bucket ko3 --s3.endpoint http://10.6.0.2:4566 --s3.region us-east-1
networks:
- afk_net
depends_on:
Expand All @@ -67,6 +72,7 @@ services:
broker1:
container_name: "${BROKER1_DOCKER_NAME-broker1}"
hostname: "${BROKER1_DOCKER_NAME-broker1}"
stop_grace_period: 2m
image: automqinc/kafka:1.0.0-rc5
ports:
- "9094:9094"
Expand All @@ -81,7 +87,11 @@ services:
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,EXTERNAL://:9094
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://broker1:9092,EXTERNAL://localhost:9094
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
command: bash /opt/kafka/scripts/start.sh up --process.roles broker --node.id 1 --controller.quorum.voters 0@controller:9093 --s3.bucket ko3 --s3.endpoint http://10.6.0.2:4566 --s3.region us-east-1
command:
- bash
- -c
- |
/opt/kafka/scripts/start.sh up --process.roles broker --node.id 1 --controller.quorum.voters 0@controller:9093 --s3.bucket ko3 --s3.endpoint http://10.6.0.2:4566 --s3.region us-east-1
networks:
- afk_net
depends_on:
Expand All @@ -92,6 +102,7 @@ services:
broker2:
container_name: "${BROKER2_DOCKER_NAME-broker2}"
hostname: "${BROKER2_DOCKER_NAME-broker2}"
stop_grace_period: 2m
image: automqinc/kafka:1.0.0-rc5
ports:
- "9095:9095"
Expand All @@ -106,7 +117,11 @@ services:
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,EXTERNAL://:9095
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://broker2:9092,EXTERNAL://localhost:9095
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
command: bash /opt/kafka/scripts/start.sh up --process.roles broker --node.id 2 --controller.quorum.voters 0@controller:9093 --s3.bucket ko3 --s3.endpoint http://10.6.0.2:4566 --s3.region us-east-1
command:
- bash
- -c
- |
/opt/kafka/scripts/start.sh up --process.roles broker --node.id 2 --controller.quorum.voters 0@controller:9093 --s3.bucket ko3 --s3.endpoint http://10.6.0.2:4566 --s3.region us-east-1
networks:
- afk_net
depends_on:
Expand Down
38 changes: 0 additions & 38 deletions docker/scripts/kafka.service
View file
Open in desktop

This file was deleted.

17 changes: 4 additions & 13 deletions docker/scripts/start.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,9 +270,6 @@ kafka_up() {
setup_value "log.dirs" "${data_path}/kraft-${role}-logs" "${kafka_dir}/config/kraft/${role}.properties"
setup_value "s3.wal.path" "${data_path}/wal" "${kafka_dir}/config/kraft/${role}.properties"
setup_value "s3.endpoint" "${s3_endpoint}" "${kafka_dir}/config/kraft/${role}.properties"
setup_value "WorkingDirectory" "${kafka_dir}" "${start_dir}/kafka.service"
sed -i "s|Environment='KAFKA_S3_ACCESS_KEY.*$|Environment='KAFKA_S3_ACCESS_KEY=${s3_access_key}'|" "${start_dir}/kafka.service"
sed -i "s|Environment='KAFKA_S3_SECRET_KEY.*$|Environment='KAFKA_S3_SECRET_KEY=${s3_secret_key}'|" "${start_dir}/kafka.service"
# turn on auto_balancer
turn_on_auto_balancer "${role}" "${kafka_dir}/config/kraft/${role}.properties"
done
Expand All @@ -286,8 +283,6 @@ kafka_up() {
else
die "kafka_start_up: unknown process role ${process_role}"
fi
sed -i "s|Environment='KAFKA_HEAP_OPTS.*$|Environment='KAFKA_HEAP_OPTS=${kafka_heap_opts}'|" "${start_dir}/kafka.service"
setup_value "ExecStart" "${kafka_dir}/bin/kafka-server-start.sh ${kafka_dir}/config/kraft/${process_role}.properties" "${start_dir}/kafka.service"

# add this node's info to run.info
setup_value "node.id" "${node_id}" "${run_info_file}"
Expand All @@ -302,17 +297,13 @@ kafka_up() {
# override settings from env
configure_from_environment_variables "${kafka_dir}/config/kraft/${process_role}.properties"

# Disable the default console logger in favour of KafkaAppender (which provides the exact output)
echo "log4j.appender.stdout.Threshold=OFF" >> "${kafka_dir}/config/log4j.properties"

# format the data path
must_do -v "${kafka_dir}/bin/kafka-storage.sh format -g -t ${cluster_id} -c ${kafka_dir}/config/kraft/${process_role}.properties"

must_do -v cp "${start_dir}/kafka.service" "/etc/systemd/system/kafka.service"
must_do -v systemctl daemon-reload
must_do -v systemctl enable kafka
must_do -v systemctl start kafka

echo "** kafka_up: successfully brought up kafka node with role ${process_role} **"

tail -f /dev/null
exec "${kafka_dir}/bin/kafka-server-start.sh" "${kafka_dir}/config/kraft/${process_role}.properties"
}

# Parse command-line arguments
Expand Down