Skip to content

Commit

Permalink
Merge pull request #10005 from id/ci-try-to-fix-flaky-kafka
Browse files Browse the repository at this point in the history
ci: fix flaky kafka tests
  • Loading branch information
id committed Feb 20, 2023
2 parents a2762e5 + c869eff commit 67f88f2
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 8 deletions.
11 changes: 7 additions & 4 deletions .ci/docker-compose-file/docker-compose-kafka.yaml
Expand Up @@ -39,9 +39,12 @@ services:
container_name: kafka-1.emqx.net
hostname: kafka-1.emqx.net
depends_on:
- "kdc"
- "zookeeper"
- "ssl_cert_gen"
kdc:
condition: service_started
zookeeper:
condition: service_started
ssl_cert_gen:
condition: service_completed_successfully
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
Expand All @@ -52,7 +55,7 @@ services:
KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI
KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka
KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
KAFKA_CREATE_TOPICS_NG: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
Expand Down
Expand Up @@ -277,6 +277,9 @@ kafka_bridge_rest_api_helper(Config) ->
},
<<"kafka">> => #{
<<"topic">> => erlang:list_to_binary(KafkaTopic),
<<"buffer">> => #{
<<"memory_overload_protection">> => <<"false">>
},
<<"message">> => #{
<<"key">> => <<"${clientid}">>,
<<"value">> => <<"${.payload}">>
Expand Down Expand Up @@ -384,6 +387,13 @@ t_failed_creation_then_fix(Config) ->
"kafka_hosts_string" => HostsString,
"kafka_topic" => KafkaTopic,
"instance_id" => ResourceId,
"producer" => #{
"kafka" => #{
"buffer" => #{
"memory_overload_protection" => false
}
}
},
"ssl" => #{}
}),
%% creates, but fails to start producers
Expand Down Expand Up @@ -577,6 +587,9 @@ producer = {
topic = \"{{ kafka_topic }}\"
message = {key = \"${clientid}\", value = \"${.payload}\"}
partition_strategy = {{ partition_strategy }}
buffer = {
memory_overload_protection = false
}
}
}
""".
Expand Down
8 changes: 4 additions & 4 deletions scripts/ct/run.sh
Expand Up @@ -201,7 +201,7 @@ if [ "$STOP" = 'no' ]; then
# some left-over log file has to be deleted before a new docker-compose up
rm -f '.ci/docker-compose-file/redis/*.log'
# shellcheck disable=2086 # no quotes for F_OPTIONS
docker-compose $F_OPTIONS up -d --build --remove-orphans
docker compose $F_OPTIONS up -d --build --remove-orphans
fi

echo "Fixing file owners and permissions for $UID_GID"
Expand All @@ -218,7 +218,7 @@ set +e

if [ "$STOP" = 'yes' ]; then
# shellcheck disable=2086 # no quotes for F_OPTIONS
docker-compose $F_OPTIONS down --remove-orphans
docker compose $F_OPTIONS down --remove-orphans
elif [ "$ATTACH" = 'yes' ]; then
docker exec -it "$ERLANG_CONTAINER" bash
elif [ "$CONSOLE" = 'yes' ]; then
Expand All @@ -235,11 +235,11 @@ else
LOG='_build/test/logs/docker-compose.log'
echo "Dumping docker-compose log to $LOG"
# shellcheck disable=2086 # no quotes for F_OPTIONS
docker-compose $F_OPTIONS logs --no-color --timestamps > "$LOG"
docker compose $F_OPTIONS logs --no-color --timestamps > "$LOG"
fi
if [ "$KEEP_UP" != 'yes' ]; then
# shellcheck disable=2086 # no quotes for F_OPTIONS
docker-compose $F_OPTIONS down
docker compose $F_OPTIONS down
fi
exit $RESULT
fi

0 comments on commit 67f88f2

Please sign in to comment.