GH-1271: Allow consumption of DLT published record

spring-kafka/src/main/java/org/springframework/kafka/listener/KafkaMessageListenerContainer.java

@@ -73,6 +73,7 @@
 import org.springframework.kafka.listener.ConsumerSeekAware.ConsumerSeekCallback;
 import org.springframework.kafka.listener.ContainerProperties.AckMode;
 import org.springframework.kafka.support.Acknowledgment;
+import org.springframework.kafka.support.KafkaHeaders;
 import org.springframework.kafka.support.LogIfLevelEnabled;
 import org.springframework.kafka.support.TopicPartitionInitialOffset;
 import org.springframework.kafka.support.TopicPartitionInitialOffset.SeekPosition;
@@ -1282,10 +1283,11 @@ private void invokeOnMessage(final ConsumerRecord<K, V> record,
 			if (record.key() instanceof DeserializationException) {
 				throw (DeserializationException) record.key();
 			}
-			if (record.value() == null && this.checkNullValueForExceptions) {
+			boolean notDltRecord = record.headers().lastHeader(KafkaHeaders.DLT_ORIGINAL_TOPIC) == null;
+			if (record.value() == null && this.checkNullValueForExceptions && notDltRecord) {
 				checkDeser(record, ErrorHandlingDeserializer2.VALUE_DESERIALIZER_EXCEPTION_HEADER);
 			}
-			if (record.key() == null && this.checkNullKeyForExceptions) {
+			if (record.key() == null && this.checkNullKeyForExceptions && notDltRecord) {
 				checkDeser(record, ErrorHandlingDeserializer2.KEY_DESERIALIZER_EXCEPTION_HEADER);
 			}
 			doInvokeOnMessage(record);

spring-kafka/src/test/java/org/springframework/kafka/listener/DLTConsumptionTests.java

@@ -0,0 +1,138 @@
+/*
+ * Copyright 2019 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.kafka.listener;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.kafka.annotation.EnableKafka;
+import org.springframework.kafka.annotation.KafkaListener;
+import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
+import org.springframework.kafka.core.ConsumerFactory;
+import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
+import org.springframework.kafka.core.DefaultKafkaProducerFactory;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.core.ProducerFactory;
+import org.springframework.kafka.support.serializer.ErrorHandlingDeserializer2;
+import org.springframework.kafka.test.EmbeddedKafkaBroker;
+import org.springframework.kafka.test.utils.KafkaTestUtils;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
+
+/**
+ * @author Gary Russell
+ * @since 2.2.11
+ *
+ */
+@SpringJUnitConfig
+@DirtiesContext
+public class DLTConsumptionTests {
+
+	private static final String TOPIC = "dltMain";
+
+	private static final String DLT = "dltMain.DLT";
+
+	@Autowired
+	private Config config;
+
+	@Test
+	void testDLTConsumed(@Autowired KafkaTemplate<String, String> template) throws InterruptedException {
+		template.send(TOPIC, "fail");
+		assertThat(this.config.latch.await(10, TimeUnit.SECONDS)).isTrue();
+	}
+
+	@Configuration
+	@EnableKafka
+	public static class Config {
+
+		final CountDownLatch latch = new CountDownLatch(1);
+
+		@Bean
+		public EmbeddedKafkaBroker embeddedKafka() {
+			return new EmbeddedKafkaBroker(1, true, 1, TOPIC, DLT);
+		}
+
+		@Bean
+		public ConcurrentKafkaListenerContainerFactory<String, String> kafkaListenerContainerFactory() {
+			return factory(cf());
+		}
+
+		private ConcurrentKafkaListenerContainerFactory<String, String> factory(ConsumerFactory<String, String> cf) {
+			ConcurrentKafkaListenerContainerFactory<String, String> factory =
+					new ConcurrentKafkaListenerContainerFactory<>();
+			factory.setConsumerFactory(cf);
+			factory.setErrorHandler(new SeekToCurrentErrorHandler(new DeadLetterPublishingRecoverer(dltTemplate()), 1));
+			return factory;
+		}
+
+		@Bean
+		public ConsumerFactory<String, String> cf() {
+			Map<String, Object> props = KafkaTestUtils.consumerProps(TOPIC + ".g1", "false", embeddedKafka());
+			props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+			props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ErrorHandlingDeserializer2.class.getName());
+			props.put(ErrorHandlingDeserializer2.VALUE_DESERIALIZER_CLASS, FailWhenPayloadIsFailDeserializer.class);
+			return new DefaultKafkaConsumerFactory<>(props);
+		}
+
+		@Bean
+		public ProducerFactory<String, String> pf() {
+			return new DefaultKafkaProducerFactory<>(KafkaTestUtils.producerProps(embeddedKafka()));
+		}
+
+		@Bean
+		public KafkaTemplate<String, String> template() {
+			return new KafkaTemplate<>(pf());
+		}
+
+		@Bean
+		public ProducerFactory<Object, Object> dltPf() {
+			Map<String, Object> props = KafkaTestUtils.producerProps(embeddedKafka());
+			props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class);
+			return new DefaultKafkaProducerFactory<>(props);
+		}
+
+		@Bean
+		public KafkaTemplate<Object, Object> dltTemplate() {
+			return new KafkaTemplate<>(dltPf());
+		}
+
+		@KafkaListener(id = TOPIC, topics = TOPIC)
+		public void listen(@SuppressWarnings("unused") String in) {
+
+		}
+
+		@KafkaListener(id = DLT, topics = DLT,
+				properties = "value.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer")
+		public void listen2(@SuppressWarnings("unused") ConsumerRecord<?, ?> in) {
+			this.latch.countDown();
+		}
+
+	}
+
+}

spring-kafka/src/test/java/org/springframework/kafka/listener/ErrorHandlingDeserializerTests.java

@@ -32,7 +32,6 @@
 import org.apache.kafka.common.header.Headers;
 import org.apache.kafka.common.header.internals.RecordHeaders;
 import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.serialization.ExtendedDeserializer;
 import org.apache.kafka.common.serialization.StringDeserializer;
 import org.apache.kafka.common.serialization.StringSerializer;
 import org.junit.jupiter.api.Test;
@@ -106,7 +105,8 @@ public void close() {
 		Object result = ehd.deserialize("topic", headers, "foo".getBytes());
 		assertThat(result).isNull();
 		Header deser = headers.lastHeader(ErrorHandlingDeserializer2.VALUE_DESERIALIZER_EXCEPTION_HEADER);
-		assertThat(new ObjectInputStream(new ByteArrayInputStream(deser.value())).readObject()).isInstanceOf(DeserializationException.class);
+		assertThat(new ObjectInputStream(new ByteArrayInputStream(deser.value())).readObject())
+				.isInstanceOf(DeserializationException.class);
 		ehd.close();
 	}
 
@@ -174,8 +174,9 @@ public ConsumerFactory<String, String> cf() {
 			props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
 			props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ErrorHandlingDeserializer2.class);
 			props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ErrorHandlingDeserializer2.class);
-			props.put(ErrorHandlingDeserializer2.KEY_DESERIALIZER_CLASS, FailSometimesDeserializer.class);
-			props.put(ErrorHandlingDeserializer2.VALUE_DESERIALIZER_CLASS, FailSometimesDeserializer.class.getName());
+			props.put(ErrorHandlingDeserializer2.KEY_DESERIALIZER_CLASS, FailWhenPayloadIsFailDeserializer.class);
+			props.put(ErrorHandlingDeserializer2.VALUE_DESERIALIZER_CLASS,
+					FailWhenPayloadIsFailDeserializer.class.getName());
 			return new DefaultKafkaConsumerFactory<>(props);
 		}
 
@@ -184,8 +185,9 @@ public ConsumerFactory<String, String> cfWithExplicitDeserializers() {
 			Map<String, Object> props = KafkaTestUtils.consumerProps(TOPIC + ".g2", "false", embeddedKafka());
 			props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
 			return new DefaultKafkaConsumerFactory<>(props,
-					new ErrorHandlingDeserializer2<String>(new FailSometimesDeserializer()).keyDeserializer(true),
-					new ErrorHandlingDeserializer2<String>(new FailSometimesDeserializer()));
+					new ErrorHandlingDeserializer2<String>(new FailWhenPayloadIsFailDeserializer())
+							.keyDeserializer(true),
+					new ErrorHandlingDeserializer2<String>(new FailWhenPayloadIsFailDeserializer()));
 		}
 
 		@Bean
@@ -208,30 +210,4 @@ public String toString() {
 
 	}
 
-	public static class FailSometimesDeserializer implements ExtendedDeserializer<String> {
-
-		@Override
-		public void configure(Map<String, ?> configs, boolean isKey) {
-		}
-
-		@Override
-		public String deserialize(String topic, byte[] data) {
-			return new String(data);
-		}
-
-		@Override
-		public void close() {
-		}
-
-		@Override
-		public String deserialize(String topic, Headers headers, byte[] data) {
-			String string = new String(data);
-			if ("fail".equals(string)) {
-				throw new RuntimeException("fail");
-			}
-			return string;
-		}
-
-	}
-
 }

spring-kafka/src/test/java/org/springframework/kafka/listener/FailWhenPayloadIsFailDeserializer.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2019 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.kafka.listener;
+
+import java.util.Map;
+
+import org.apache.kafka.common.header.Headers;
+import org.apache.kafka.common.serialization.ExtendedDeserializer;
+
+/**
+ * @author Gary Russell
+ * @since 2.2.11
+ */
+public class FailWhenPayloadIsFailDeserializer implements ExtendedDeserializer<String> {
+
+	@Override
+	public void configure(Map<String, ?> configs, boolean isKey) {
+	}
+
+	@Override
+	public String deserialize(String topic, byte[] data) {
+		return new String(data);
+	}
+
+	@Override
+	public void close() {
+	}
+
+	@Override
+	public String deserialize(String topic, Headers headers, byte[] data) {
+		String string = new String(data);
+		if ("fail".equals(string)) {
+			throw new RuntimeException("fail");
+		}
+		return string;
+	}
+
+}