refactor(filesystems): add module filepulse-commons-fs

This commit relocates some classes

Author: fhussonnois

checkstyle/suppressions.xml

@@ -41,21 +41,10 @@
  */
  -->
     <suppress checks="LineLength" files="RecordFilter.java"/>
-    <suppress checks="LineLength" files="CommonFilterConfig.java"/>
-    <suppress checks="LineLength" files="DelimitedRowFilterConfig.java"/>
-    <suppress checks="LineLength" files="GroupRowFilterConfig.java"/>
     <suppress checks="LineLength" files="GrokPatternCompiler.java"/>
-    <suppress checks="LineLength" files="ConvertFilterConfig.java"/>
-    <suppress checks="LineLength" files="RenameFilterConfig.java"/>
-    <suppress checks="LineLength" files="GrokFilterConfig.java"/>
-    <suppress checks="LineLength" files="SplitFilterConfig.java"/>
-    <suppress checks="LineLength" files="RowFileInputReaderConfig.java"/>
-    <suppress checks="LineLength" files="ConnectorConfig.java"/>
-    <suppress checks="LineLength" files="TaskConfig.java"/>
-    <suppress checks="LineLength" files="CommonConfig.java"/>
     <suppress checks="LineLength" files="DefaultFileSystemScanner.java"/>
     <suppress checks="LineLength" files="ComposeOffsetStrategy.java"/>
-    <suppress checks="LineLength" files="DefaultOffsetPolicyConfig.java"/>
+    <suppress checks="LineLength" files=".*Config.java"/>
     <!-- Those classes are copy from kafka-connect api-->
     <suppress checks="LineLength" files="KafkaBasedLog.java"/>
     <suppress checks="NPathComplexity" files="KafkaBasedLog.java"/>

config/connect-file-pulse-quickstart-avro.json

@@ -2,14 +2,14 @@
   "config": {
     "connector.class": "io.streamthoughts.kafka.connect.filepulse.source.FilePulseSourceConnector",
     "fs.cleanup.policy.class": "io.streamthoughts.kafka.connect.filepulse.clean.LogCleanupPolicy",
-    "fs.listing.class": "io.streamthoughts.kafka.connect.filepulse.fs.local.LocalFSDirectoryListing",
-    "fs.scan.directory.path": "/tmp/kafka-connect/examples/",
+    "fs.listing.class" : "io.streamthoughts.kafka.connect.filepulse.fs.LocalFSDirectoryListing",
+    "fs.listing.directory.path":"/tmp/kafka-connect/examples/",
     "fs.scan.interval.ms": "10000",
     "internal.kafka.reporter.bootstrap.servers": "broker:29092",
     "internal.kafka.reporter.topic": "connect-file-pulse-status",
     "offset.strategy": "name",
     "read.max.wait.ms": "5000",
-    "task.reader.class": "io.streamthoughts.kafka.connect.filepulse.fs.local.reader.AvroFileInputReader",
+    "task.reader.class": "io.streamthoughts.kafka.connect.filepulse.fs.reader.LocalAvroFileInputReader",
     "topic": "connect-file-pulse-quickstart-avro",
     "tasks.max": 1
   },

config/connect-file-pulse-quickstart-csv.json

@@ -9,8 +9,10 @@
     "filters.ParseDelimitedRow.trimColumn": "true",
     "filters.ParseDelimitedRow.type": "io.streamthoughts.kafka.connect.filepulse.filter.DelimitedRowFilter",
     "fs.cleanup.policy.class": "io.streamthoughts.kafka.connect.filepulse.clean.LogCleanupPolicy",
-    "fs.listing.class": "io.streamthoughts.kafka.connect.filepulse.fs.local.LocalFSDirectoryListing",
-    "fs.scan.directory.path": "/tmp/kafka-connect/examples/",
+    "fs.listing.class" : "io.streamthoughts.kafka.connect.filepulse.fs.LocalFSDirectoryListing",
+    "fs.listing.directory.path":"/tmp/kafka-connect/examples/",
+    "fs.listing.filters":"io.streamthoughts.kafka.connect.filepulse.fs.filter.RegexFileListFilter",
+    "file.filter.regex.pattern":".*\\.csv$",
     "fs.scan.interval.ms": "10000",
     "internal.kafka.reporter.bootstrap.servers": "broker:29092",
     "internal.kafka.reporter.topic": "connect-file-pulse-status",

config/connect-file-pulse-quickstart-log4j.json

@@ -11,14 +11,16 @@
     "filters.ParseLog4jLog.type": "io.streamthoughts.kafka.connect.filepulse.filter.GrokFilter",
     "filters.ParseLog4jLog.ignoreFailure": "true",
     "fs.cleanup.policy.class": "io.streamthoughts.kafka.connect.filepulse.clean.LogCleanupPolicy",
-    "fs.listing.class": "io.streamthoughts.kafka.connect.filepulse.fs.local.LocalFSDirectoryListing",
-    "fs.scan.directory.path": "/var/log/kafka/",
+    "fs.listing.class" : "io.streamthoughts.kafka.connect.filepulse.fs.LocalFSDirectoryListing",
+    "fs.listing.directory.path":"/tmp/kafka-connect/examples/",
+    "fs.listing.filters":"io.streamthoughts.kafka.connect.filepulse.fs.filter.RegexFileListFilter",
+    "file.filter.regex.pattern":".*\\.log$",
     "fs.scan.interval.ms": "10000",
     "internal.kafka.reporter.bootstrap.servers": "broker:29092",
     "internal.kafka.reporter.topic": "connect-file-pulse-status",
     "offset.strategy": "name",
     "read.max.wait.ms": "5000",
-    "task.reader.class": "io.streamthoughts.kafka.connect.filepulse.fs.local.reader.RowFileInputReader",
+    "task.reader.class": "io.streamthoughts.kafka.connect.filepulse.fs.reader.LocalRowFileInputReader",
     "topic": "connect-file-pulse-quickstart-log4j",
     "tasks.max": 1
   },

connect-file-pulse-api/src/main/java/io/streamthoughts/kafka/connect/filepulse/internal/StringUtils.java

@@ -18,6 +18,8 @@
  */
 package io.streamthoughts.kafka.connect.filepulse.internal;
 
+import java.util.Objects;
+
 public class StringUtils {
 
     /**
@@ -35,4 +37,8 @@ public static boolean isFastSplit(final String regex) {
                 (ch < Character.MIN_HIGH_SURROGATE ||
                         ch > Character.MAX_LOW_SURROGATE);
     }
+
+    public static boolean isNotBlank(final String string) {
+        return !(Objects.isNull(string) || string.isBlank());
+    }
 }

connect-file-pulse-filesystems/filepulse-local-fs/src/main/java/io/streamthoughts/kafka/connect/filepulse/fs/local/reader/AbstractFileInputIterator.java renamed to connect-file-pulse-api/src/main/java/io/streamthoughts/kafka/connect/filepulse/reader/AbstractFileInputIterator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2020 StreamThoughts.
+ * Copyright 2019-2021 StreamThoughts.
  *
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements. See the NOTICE file distributed with
@@ -16,9 +16,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.streamthoughts.kafka.connect.filepulse.fs.local.reader;
+package io.streamthoughts.kafka.connect.filepulse.reader;
 
-import io.streamthoughts.kafka.connect.filepulse.reader.FileInputIterator;
 import io.streamthoughts.kafka.connect.filepulse.source.FileContext;
 import io.streamthoughts.kafka.connect.filepulse.source.FileRecord;
 
@@ -47,13 +46,14 @@ public abstract class AbstractFileInputIterator<T> implements FileInputIterator<
      */
     public AbstractFileInputIterator(final IteratorManager iteratorManager,
                                      final FileContext context) {
-        Objects.requireNonNull(iteratorManager, "iteratorManager can't be null");
-        Objects.requireNonNull(context, "context can't be null");
-        this.iteratorManager = iteratorManager;
-        this.context = context;
+        this.iteratorManager =  Objects.requireNonNull(iteratorManager, "iteratorManager can't be null");
+        this.context = Objects.requireNonNull(context, "context can't be null");
         closed = new AtomicBoolean(false);
     }
 
+    /**
+     * {@inheritDoc}
+     */
     @Override
     public FileContext context() {
         return context;

connect-file-pulse-filesystems/filepulse-local-fs/src/main/java/io/streamthoughts/kafka/connect/filepulse/fs/local/reader/AbstractFileInputReader.java renamed to connect-file-pulse-api/src/main/java/io/streamthoughts/kafka/connect/filepulse/reader/AbstractFileInputReader.java

@@ -16,21 +16,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.streamthoughts.kafka.connect.filepulse.fs.local.reader;
+package io.streamthoughts.kafka.connect.filepulse.reader;
 
 import io.streamthoughts.kafka.connect.filepulse.data.TypedStruct;
-import io.streamthoughts.kafka.connect.filepulse.reader.FileInputIterator;
-import io.streamthoughts.kafka.connect.filepulse.reader.FileInputReader;
 import io.streamthoughts.kafka.connect.filepulse.source.FileContext;
 import io.streamthoughts.kafka.connect.filepulse.source.FileRecord;
-import io.streamthoughts.kafka.connect.filepulse.source.LocalFileObjectMeta;
-import io.streamthoughts.kafka.connect.filepulse.source.FileObjectMeta;
 
-import java.io.File;
-import java.net.URI;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 public abstract class AbstractFileInputReader implements FileInputReader {
@@ -42,35 +33,11 @@ public abstract class AbstractFileInputReader implements FileInputReader {
     /**
      * Creates a new {@link AbstractFileInputReader} instance.
      */
-    AbstractFileInputReader() {
+    public AbstractFileInputReader() {
         this.isClosed = new AtomicBoolean(false);
         this.openIterators = new IteratorManager();
     }
 
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public void configure(final Map<String, ?> configs) {
-
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public FileObjectMeta readMetadata(final URI fileURI) {
-        return new LocalFileObjectMeta(new File(fileURI));
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public boolean isReadable(final URI fileURI) {
-        return Files.isReadable(Paths.get(fileURI));
-    }
-
     /**
      * {@inheritDoc}
      */

connect-file-pulse-api/src/main/java/io/streamthoughts/kafka/connect/filepulse/reader/FileInputIterator.java

@@ -28,7 +28,7 @@
  *
  * @param <T> type of value.
  */
-public interface FileInputIterator<T> extends Iterator<RecordsIterable<T>> {
+public interface FileInputIterator<T> extends Iterator<RecordsIterable<T>>, AutoCloseable {
 
     /**
      * Gets the iterator context.

connect-file-pulse-api/src/main/java/io/streamthoughts/kafka/connect/filepulse/reader/FileInputReader.java

@@ -38,7 +38,9 @@ public interface FileInputReader extends Configurable, AutoCloseable {
      * @param configs the reader configuration.
      */
     @Override
-    void configure(final Map<String, ?> configs);
+    default void configure(final Map<String, ?> configs) {
+
+    }
 
     /**
      * Gets the metadata for the source object identified by the given {@link URI}.

connect-file-pulse-filesystems/filepulse-local-fs/src/main/java/io/streamthoughts/kafka/connect/filepulse/fs/local/reader/IteratorManager.java renamed to connect-file-pulse-api/src/main/java/io/streamthoughts/kafka/connect/filepulse/reader/IteratorManager.java

@@ -16,9 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.streamthoughts.kafka.connect.filepulse.fs.local.reader;
-
-import io.streamthoughts.kafka.connect.filepulse.reader.FileInputIterator;
+package io.streamthoughts.kafka.connect.filepulse.reader;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -27,14 +25,14 @@
 /**
  * Default class to easily close all open {@link FileInputIterator} instances.
  */
-class IteratorManager {
+public class IteratorManager {
 
     private final Set<FileInputIterator<?>> openIterators;
 
     /**
      * Creates a new {@link IteratorManager} instance.
      */
-    IteratorManager() {
+    public IteratorManager() {
         this.openIterators = Collections.synchronizedSet(new HashSet<>());
     }