From 0611e80c4d9325c34f32669557776b6786ee206b Mon Sep 17 00:00:00 2001 From: Adrian Gonzalez-Martin Date: Tue, 5 Sep 2023 09:25:28 +0100 Subject: [PATCH 1/3] Change message.max.bytes in broker side to align with producer and consumer --- kafka/strimzi/templates/cluster.yaml | 1 + kafka/strimzi/values.yaml | 2 ++ scheduler/all-base.yaml | 1 + 3 files changed, 4 insertions(+) diff --git a/kafka/strimzi/templates/cluster.yaml b/kafka/strimzi/templates/cluster.yaml index 8887ae9d28..a6ae168dae 100644 --- a/kafka/strimzi/templates/cluster.yaml +++ b/kafka/strimzi/templates/cluster.yaml @@ -43,6 +43,7 @@ spec: default.replication.factor: {{ .Values.topic.defaultReplicationFactor }} min.insync.replicas: {{ .Values.topic.minISR }} inter.broker.protocol.version: {{ .Values.broker.interBrokerProtocolVersion }} + message.max.bytes: {{ .Values.broker.messageMaxBytes }} template: pod: tmpDirSizeLimit: {{ .Values.broker.tmpDirSizeLimit }} diff --git a/kafka/strimzi/values.yaml b/kafka/strimzi/values.yaml index f569d12476..f906e6ac55 100644 --- a/kafka/strimzi/values.yaml +++ b/kafka/strimzi/values.yaml @@ -35,6 +35,8 @@ broker: tmpDirSizeLimit: 100Mi pvcSize: 100Gi + + messageMaxBytes: 1000000000 resources: cpu: 200m diff --git a/scheduler/all-base.yaml b/scheduler/all-base.yaml index f267b9440e..6f13c95b9c 100644 --- a/scheduler/all-base.yaml +++ b/scheduler/all-base.yaml @@ -108,6 +108,7 @@ services: image: "${KAFKA_IMAGE_AND_TAG}" environment: - KAFKA_CFG_SOCKET_REQUEST_MAX_BYTES=1000000000 + - KAFKA_CFG_MESSAGE_MAX_BYTES=1000000000 volumes: - "kafka_data:/bitnami" depends_on: From c6c3367c6f16c84a4f2a52b81515304f56ff23ed Mon Sep 17 00:00:00 2001 From: Adrian Gonzalez-Martin Date: Tue, 5 Sep 2023 14:20:32 +0100 Subject: [PATCH 2/3] Increase mem resource for Kafka broker --- kafka/strimzi/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/strimzi/values.yaml b/kafka/strimzi/values.yaml index f906e6ac55..223213a3fd 100644 --- a/kafka/strimzi/values.yaml +++ b/kafka/strimzi/values.yaml @@ -40,7 +40,7 @@ broker: resources: cpu: 200m - memory: 1Gi + memory: 2Gi topic: autoCreate: true From 6eca91ab3cefb072ae56a9b09b6552f7539574da Mon Sep 17 00:00:00 2001 From: Adrian Gonzalez-Martin Date: Tue, 5 Sep 2023 16:35:38 +0100 Subject: [PATCH 3/3] Unify settings --- scheduler/all-base.yaml | 4 ++-- scheduler/all-internal.yaml | 1 - scheduler/env.all | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scheduler/all-base.yaml b/scheduler/all-base.yaml index 6f13c95b9c..ce9b35726a 100644 --- a/scheduler/all-base.yaml +++ b/scheduler/all-base.yaml @@ -107,8 +107,8 @@ services: kafka: image: "${KAFKA_IMAGE_AND_TAG}" environment: - - KAFKA_CFG_SOCKET_REQUEST_MAX_BYTES=1000000000 - - KAFKA_CFG_MESSAGE_MAX_BYTES=1000000000 + - KAFKA_CFG_SOCKET_REQUEST_MAX_BYTES=${KAFKA_MESSAGE_MAX_BYTES} + - KAFKA_CFG_MESSAGE_MAX_BYTES=${KAFKA_MESSAGE_MAX_BYTES} volumes: - "kafka_data:/bitnami" depends_on: diff --git a/scheduler/all-internal.yaml b/scheduler/all-internal.yaml index 826cba2200..07e9d02ecc 100644 --- a/scheduler/all-internal.yaml +++ b/scheduler/all-internal.yaml @@ -101,7 +101,6 @@ services: - KAFKA_BROKER_ID=1 - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 - ALLOW_PLAINTEXT_LISTENER=yes - - KAFKA_CFG_MESSAGE_MAX_BYTES=1000000000 ports: - "9092:9092" diff --git a/scheduler/env.all b/scheduler/env.all index cb174c9770..2aa8e84df3 100644 --- a/scheduler/env.all +++ b/scheduler/env.all @@ -58,3 +58,5 @@ HODOMETER_METRICS_LEVEL=feature HODOMETER_LOG_LEVEL=info HODOMETER_RECEIVER_RECORD_LEVEL=all HODOMETER_RECEIVER_LOG_LEVEL=info + +KAFKA_MESSAGE_MAX_BYTES=1000000000