From 2751510b80bba6b18e0452f47ac8487db0262791 Mon Sep 17 00:00:00 2001 From: Grant Nicholas Date: Tue, 26 May 2020 10:29:49 -0500 Subject: [PATCH] Handle exceptions while processing files better Now only commit offsets for a file if the entire file is processed without errors, this ensures we process every record and do not miss records --- .../java/com/github/mmolimar/kafka/connect/fs/FsSourceTask.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/github/mmolimar/kafka/connect/fs/FsSourceTask.java b/src/main/java/com/github/mmolimar/kafka/connect/fs/FsSourceTask.java index b63678d..61956a8 100644 --- a/src/main/java/com/github/mmolimar/kafka/connect/fs/FsSourceTask.java +++ b/src/main/java/com/github/mmolimar/kafka/connect/fs/FsSourceTask.java @@ -102,6 +102,7 @@ public List poll() { } catch (ConnectException | IOException e) { //when an exception happens reading a file, the connector continues log.error("Error reading file [{}]. Keep going...", metadata.getPath(), e); + return new ArrayList().stream(); } log.debug("Read [{}] records from file [{}].", records.size(), metadata.getPath());