From 04bfe1106f32c91942ee930c63ca973f06964a10 Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 26 Feb 2025 08:54:32 +0100 Subject: [PATCH 01/19] Add `getScope` method to `ActiveWatch` interface --- src/main/java/engineering/swat/watch/ActiveWatch.java | 5 +++++ .../engineering/swat/watch/impl/jdk/JDKDirectoryWatch.java | 6 ++++++ .../java/engineering/swat/watch/impl/jdk/JDKFileWatch.java | 6 ++++++ .../swat/watch/impl/jdk/JDKRecursiveDirectoryWatch.java | 6 ++++++ 4 files changed, 23 insertions(+) diff --git a/src/main/java/engineering/swat/watch/ActiveWatch.java b/src/main/java/engineering/swat/watch/ActiveWatch.java index 4be79e54..ba8899ed 100644 --- a/src/main/java/engineering/swat/watch/ActiveWatch.java +++ b/src/main/java/engineering/swat/watch/ActiveWatch.java @@ -40,4 +40,9 @@ public interface ActiveWatch extends Closeable { * Gets the path watched by this watch. */ Path getPath(); + + /** + * Gets the scope of this watch. + */ + WatchScope getScope(); } diff --git a/src/main/java/engineering/swat/watch/impl/jdk/JDKDirectoryWatch.java b/src/main/java/engineering/swat/watch/impl/jdk/JDKDirectoryWatch.java index 4fd2041b..447c22d9 100644 --- a/src/main/java/engineering/swat/watch/impl/jdk/JDKDirectoryWatch.java +++ b/src/main/java/engineering/swat/watch/impl/jdk/JDKDirectoryWatch.java @@ -38,6 +38,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import engineering.swat.watch.WatchEvent; +import engineering.swat.watch.WatchScope; import engineering.swat.watch.impl.EventHandlingWatch; import engineering.swat.watch.impl.util.BundledSubscription; import engineering.swat.watch.impl.util.SubscriptionKey; @@ -74,6 +75,11 @@ private void handleJDKEvents(List> events) { // -- JDKBaseWatch -- + @Override + public WatchScope getScope() { + return nativeRecursive ? WatchScope.PATH_AND_ALL_DESCENDANTS : WatchScope.PATH_AND_CHILDREN; + } + @Override public synchronized void close() throws IOException { if (bundledJDKWatcher != null) { diff --git a/src/main/java/engineering/swat/watch/impl/jdk/JDKFileWatch.java b/src/main/java/engineering/swat/watch/impl/jdk/JDKFileWatch.java index 01820649..520d4a16 100644 --- a/src/main/java/engineering/swat/watch/impl/jdk/JDKFileWatch.java +++ b/src/main/java/engineering/swat/watch/impl/jdk/JDKFileWatch.java @@ -36,6 +36,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import engineering.swat.watch.WatchEvent; +import engineering.swat.watch.WatchScope; import engineering.swat.watch.impl.EventHandlingWatch; /** @@ -73,6 +74,11 @@ private static Path requireNonNull(@Nullable Path p, String message) { // -- JDKBaseWatch -- + @Override + public WatchScope getScope() { + return WatchScope.PATH_ONLY; + } + @Override public void handleEvent(WatchEvent event) { internal.handleEvent(event); diff --git a/src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatch.java b/src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatch.java index 21245e5e..e6004df6 100644 --- a/src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatch.java +++ b/src/main/java/engineering/swat/watch/impl/jdk/JDKRecursiveDirectoryWatch.java @@ -48,6 +48,7 @@ import org.apache.logging.log4j.Logger; import engineering.swat.watch.WatchEvent; +import engineering.swat.watch.WatchScope; import engineering.swat.watch.impl.EventHandlingWatch; public class JDKRecursiveDirectoryWatch extends JDKBaseWatch { @@ -298,6 +299,11 @@ private void detectedMissingEntries(Path dir, ArrayList events, Hash // -- JDKBaseWatch -- + @Override + public WatchScope getScope() { + return WatchScope.PATH_AND_ALL_DESCENDANTS; + } + @Override public void handleEvent(WatchEvent event) { processEvents(event); From bcfa0dda2dc362af5c2d4720ecc9b470a53b0bbb Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 26 Feb 2025 10:33:01 +0100 Subject: [PATCH 02/19] Add `getScope` method to `ActiveWatch` interface --- .../swat/watch/impl/EventHandlingWatchTests.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/test/java/engineering/swat/watch/impl/EventHandlingWatchTests.java b/src/test/java/engineering/swat/watch/impl/EventHandlingWatchTests.java index 0eeac52b..c1ccc3d9 100644 --- a/src/test/java/engineering/swat/watch/impl/EventHandlingWatchTests.java +++ b/src/test/java/engineering/swat/watch/impl/EventHandlingWatchTests.java @@ -34,10 +34,11 @@ import org.junit.jupiter.api.Test; import engineering.swat.watch.WatchEvent; +import engineering.swat.watch.WatchScope; class EventHandlingWatchTests { - private static EventHandlingWatch emptyWatch(Path path) { + private static EventHandlingWatch emptyFileWatch(Path path) { return new EventHandlingWatch() { @Override public void handleEvent(WatchEvent event) { @@ -49,6 +50,11 @@ public void close() throws IOException { // Nothing to close } + @Override + public WatchScope getScope() { + return WatchScope.PATH_ONLY; + } + @Override public Path getPath() { return path; @@ -60,7 +66,7 @@ public Path getPath() { void relativizeTest() { var e1 = new WatchEvent(WatchEvent.Kind.OVERFLOW, Path.of("foo"), Path.of("bar", "baz.txt")); var e2 = new WatchEvent(WatchEvent.Kind.OVERFLOW, Path.of("foo", "bar", "baz.txt")); - var e3 = emptyWatch(Path.of("foo")).relativize(e2); + var e3 = emptyFileWatch(Path.of("foo")).relativize(e2); assertEquals(e1.getRootPath(), e3.getRootPath()); assertEquals(e1.getRelativePath(), e3.getRelativePath()); } From c8ed7a89a8481e65b1a5f1b5fe0ff1d26195ab3c Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 26 Feb 2025 10:34:35 +0100 Subject: [PATCH 03/19] Add `OverflowPolicy` enum to configure how overflow events should be auto-handled --- .../swat/watch/OverflowPolicy.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/main/java/engineering/swat/watch/OverflowPolicy.java diff --git a/src/main/java/engineering/swat/watch/OverflowPolicy.java b/src/main/java/engineering/swat/watch/OverflowPolicy.java new file mode 100644 index 00000000..33612eda --- /dev/null +++ b/src/main/java/engineering/swat/watch/OverflowPolicy.java @@ -0,0 +1,39 @@ +package engineering.swat.watch; + +public enum OverflowPolicy { + + /** + * When an overflow event occurs, do nothing (i.e., the user-defined event + * handler is responsible to handle overflow events). + */ + NO_RESCANS, + + /** + * When an overflow event occurs, rescan all files in the scope of the + * watch, and issue `CREATED` and `MODIFIED` events (not `DELETED` events) + * for each file. `MODIFIED` events are issued only for non-empty files. + * + * Compared to the `INDEXING_RESCANS` policy, the `MEMORYLESS_RESCANS` + * policy is less expensive in terms of memory usage, but it results in a + * larger overaproximation of the actual `CREATED` and `MODIFIED` events + * that happened, while all `DELETED` events remain undetected. + */ + MEMORYLESS_RESCANS, + + /** + * When an overflow event occurs, rescan all files in the watch scope, + * update the internal *index*, and issue `CREATED`, `MODIFIED`, and + * `DELETED` events accordingly. The index keeps track of the last modified + * time of each file, such that: (a) `CREATED` events are issued for files + * that are added to the index; (b) `DELETED` events are issued for files + * that are removed from the index; (c) `MODIFIED` events are issued for + * files in the index whose previous last-modified-time is before their + * current last-modified-time. + * + * Compared to the `MEMORYLESS_RESCANS` policy, the `INDEXING_RESCANS` + * policy results in a smaller overapproximation of the actual `CREATED`, + * `MODIFIED`, and `DELETED` events that happened, but it's more expensive + * in terms of memory usage. + */ + INDEXING_RESCANS +} From 89e1b5f8d149258bf9df15478163aadad16ee6ae Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 26 Feb 2025 10:39:19 +0100 Subject: [PATCH 04/19] Add overflow policy to watcher constructor --- .../java/engineering/swat/watch/Watcher.java | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 67d9345f..e226f28e 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -52,26 +52,35 @@ */ public class Watcher { private final Logger logger = LogManager.getLogger(); - private final WatchScope scope; private final Path path; + private final WatchScope scope; + private final OverflowPolicy overflowPolicy; private volatile Executor executor = CompletableFuture::runAsync; private static final BiConsumer EMPTY_HANDLER = (w, e) -> {}; private volatile BiConsumer eventHandler = EMPTY_HANDLER; - - private Watcher(WatchScope scope, Path path) { - this.scope = scope; + private Watcher(Path path, WatchScope scope, OverflowPolicy overflowPolicy) { this.path = path; + this.scope = scope; + this.overflowPolicy = overflowPolicy; + } + + /** + * Equivalent to: `watch(path, scope, OverflowPolicy.MEMORYLESS_RESCANS)` + */ + public static Watcher watch(Path path, WatchScope scope) { + return watch(path, scope, OverflowPolicy.MEMORYLESS_RESCANS); } /** * Watch a path for updates, optionally also get events for its children/descendants * @param path which absolute path to monitor, can be a file or a directory, but has to be absolute * @param scope for directories you can also choose to monitor it's direct children or all it's descendants + * @param overflowPolicy policy to automatically handle overflow events * @throws IllegalArgumentException in case a path is not supported (in relation to the scope) */ - public static Watcher watch(Path path, WatchScope scope) { + public static Watcher watch(Path path, WatchScope scope, OverflowPolicy overflowPolicy) { if (!path.isAbsolute()) { throw new IllegalArgumentException("We can only watch absolute paths"); } @@ -89,9 +98,8 @@ public static Watcher watch(Path path, WatchScope scope) { break; default: throw new IllegalArgumentException("Unsupported scope: " + scope); - } - return new Watcher(scope, path); + return new Watcher(path, scope, overflowPolicy); } /** From 2ff52a76b70afc8e94f2ff9bae492485ef1604fd Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 26 Feb 2025 10:40:51 +0100 Subject: [PATCH 05/19] Add license --- .../swat/watch/OverflowPolicy.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/main/java/engineering/swat/watch/OverflowPolicy.java b/src/main/java/engineering/swat/watch/OverflowPolicy.java index 33612eda..5ce73e35 100644 --- a/src/main/java/engineering/swat/watch/OverflowPolicy.java +++ b/src/main/java/engineering/swat/watch/OverflowPolicy.java @@ -1,3 +1,29 @@ +/* + * BSD 2-Clause License + * + * Copyright (c) 2023, Swat.engineering + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ package engineering.swat.watch; public enum OverflowPolicy { From 15f8313b88ab05a9647f4edcc1f2043676f2f871 Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 26 Feb 2025 10:54:16 +0100 Subject: [PATCH 06/19] Add implementation of the `MEMORYLESS_RESCANS` overflow policy --- .../impl/overflows/MemorylessRescanner.java | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java diff --git a/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java b/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java new file mode 100644 index 00000000..112ccea1 --- /dev/null +++ b/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java @@ -0,0 +1,98 @@ +/* + * BSD 2-Clause License + * + * Copyright (c) 2023, Swat.engineering + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package engineering.swat.watch.impl.overflows; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.Executor; +import java.util.function.BiConsumer; +import java.util.stream.Stream; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import engineering.swat.watch.WatchEvent; +import engineering.swat.watch.WatchScope; +import engineering.swat.watch.impl.EventHandlingWatch; + +public class MemorylessRescanner implements BiConsumer { + private final Logger logger = LogManager.getLogger(); + private final Executor exec; + + public MemorylessRescanner(Executor exec) { + this.exec = exec; + } + + /** + * Rescan all files in the scope of `watch` and issue `CREATED` and + * `MODIFIED` events (not `DELETED` events) for each file. This method + * should typically be executed asynchronously (using `exec`). + */ + protected void rescan(EventHandlingWatch watch) { + try (var content = contentOf(watch.getPath(), watch.getScope())) { + content + .flatMap(this::generateEvents) // Paths aren't properly relativized + .map(watch::relativize) // ...so they must be relativized first (wrt the root path of `watch`) + .forEach(watch::handleEvent); + } + } + + protected Stream contentOf(Path path, WatchScope scope) { + try { + var maxDepth = scope == WatchScope.PATH_AND_ALL_DESCENDANTS ? Integer.MAX_VALUE : 1; + return Files.walk(path, maxDepth).filter(p -> p != path); + } catch (IOException e) { + logger.error("Could not walk: {} ({})", path, e); + return Stream.empty(); + } + } + + protected Stream generateEvents(Path path) { + try { + var created = new WatchEvent(WatchEvent.Kind.CREATED, path); + if (Files.size(path) == 0) { + return Stream.of(created); + } else { + var modified = new WatchEvent(WatchEvent.Kind.MODIFIED, path); + return Stream.of(created, modified); + } + } catch (IOException e) { + logger.error("Could not generate events for: {} ({})", path, e); + return Stream.empty(); + } + } + + // -- BiConsumer -- + + @Override + public void accept(EventHandlingWatch watch, WatchEvent event) { + if (event.getKind() == WatchEvent.Kind.OVERFLOW) { + exec.execute(() -> rescan(watch)); + } + } +} From bab08c0f2c0acfd2f6a4f94aa368aaed7bc5f21f Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 26 Feb 2025 11:04:40 +0100 Subject: [PATCH 07/19] Add auto-handling using the `MEMORYLESS_RESCANS` overflow policy to (non-recursive) directory watching, including a test --- .../java/engineering/swat/watch/Watcher.java | 16 ++++++++- .../swat/watch/SingleDirectoryTests.java | 36 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index e226f28e..169dfd9e 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -42,6 +42,7 @@ import engineering.swat.watch.impl.jdk.JDKDirectoryWatch; import engineering.swat.watch.impl.jdk.JDKFileWatch; import engineering.swat.watch.impl.jdk.JDKRecursiveDirectoryWatch; +import engineering.swat.watch.impl.overflows.MemorylessRescanner; /** *

Watch a path for changes.

@@ -167,9 +168,11 @@ public ActiveWatch start() throws IOException { throw new IllegalStateException("There is no onEvent handler defined"); } + var h = overflowEventHandler().andThen(eventHandler); + switch (scope) { case PATH_AND_CHILDREN: { - var result = new JDKDirectoryWatch(path, executor, eventHandler, false); + var result = new JDKDirectoryWatch(path, executor, h); result.open(); return result; } @@ -196,4 +199,15 @@ public ActiveWatch start() throws IOException { throw new IllegalStateException("Not supported yet"); } } + + private BiConsumer overflowEventHandler() { + switch (overflowPolicy) { + case NO_RESCANS: + return (w, e) -> {}; + case MEMORYLESS_RESCANS: + return new MemorylessRescanner(executor); + default: + throw new UnsupportedOperationException("No event handler has been defined yet for this overflow policy"); + } + } } diff --git a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java index 72d0c656..21382405 100644 --- a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java +++ b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java @@ -31,6 +31,8 @@ import java.io.IOException; import java.nio.file.Files; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; @@ -39,6 +41,7 @@ import org.junit.jupiter.api.Test; import engineering.swat.watch.WatchEvent.Kind; +import engineering.swat.watch.impl.EventHandlingWatch; class SingleDirectoryTests { private TestDirectory testDir; @@ -118,4 +121,37 @@ public void onDeleted(WatchEvent ev) { .untilTrue(seenCreate); } } + + @Test + void memorylessRescansTest() throws IOException { + var directory = testDir.getTestDirectory(); + Files.writeString(directory.resolve("a.txt"), "foo"); + Files.writeString(directory.resolve("b.txt"), "bar"); + + var nCreated = new AtomicInteger(); + var nModified = new AtomicInteger(); + var watchConfig = Watcher.watch(directory, WatchScope.PATH_AND_CHILDREN, OverflowPolicy.MEMORYLESS_RESCANS) + .on(e -> { + switch (e.getKind()) { + case CREATED: + nCreated.incrementAndGet(); + break; + case MODIFIED: + nModified.incrementAndGet(); + break; + default: + break; + } + }); + + try (var watch = watchConfig.start()) { + var overflow = new WatchEvent(WatchEvent.Kind.OVERFLOW, directory); + ((EventHandlingWatch) watch).handleEvent(overflow); + + await("Overflow should generate create events") + .until(nCreated::get, Predicate.isEqual(6)); // 3 directories + 3 files + await("Overflow should generate modified events") + .until(nModified::get, Predicate.isEqual(5)); // 3 directories + 2 files (c.txt is still empty) + } + } } From a28ac9429603c6b9b735111688686645409ccc80 Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 26 Feb 2025 11:34:30 +0100 Subject: [PATCH 08/19] Add function to ignore overflow events in the user-defined event handler (depending on the overflow policy) --- .../java/engineering/swat/watch/Watcher.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 169dfd9e..045b4206 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -168,7 +168,7 @@ public ActiveWatch start() throws IOException { throw new IllegalStateException("There is no onEvent handler defined"); } - var h = overflowEventHandler().andThen(eventHandler); + var h = overflowEventHandler().andThen(userDefinedEventHandler()); switch (scope) { case PATH_AND_CHILDREN: { @@ -210,4 +210,18 @@ private BiConsumer overflowEventHandler() { throw new UnsupportedOperationException("No event handler has been defined yet for this overflow policy"); } } + + private BiConsumer userDefinedEventHandler() { + // If overflow events are auto-handled because of the overflow policy, + // then they should be ignored by the user-defined event handler + if (overflowPolicy != OverflowPolicy.NO_RESCANS) { + return (w, e) -> { + if (e.getKind() != WatchEvent.Kind.OVERFLOW) { + eventHandler.accept(w, e); + } + }; + } + + return eventHandler; + } } From 8d59be4a77ab6beda141ac20889ff17c3c446197 Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 26 Feb 2025 12:22:15 +0100 Subject: [PATCH 09/19] Fix comment --- .../swat/watch/impl/overflows/MemorylessRescanner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java b/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java index 112ccea1..ff69050b 100644 --- a/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java +++ b/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java @@ -56,7 +56,7 @@ public MemorylessRescanner(Executor exec) { protected void rescan(EventHandlingWatch watch) { try (var content = contentOf(watch.getPath(), watch.getScope())) { content - .flatMap(this::generateEvents) // Paths aren't properly relativized + .flatMap(this::generateEvents) // Paths aren't properly relativized yet... .map(watch::relativize) // ...so they must be relativized first (wrt the root path of `watch`) .forEach(watch::handleEvent); } From df81ecb23bdaeec6fb7f95e16dad99bfd4de0e40 Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 26 Feb 2025 13:54:23 +0100 Subject: [PATCH 10/19] Improve test --- .../swat/watch/SingleDirectoryTests.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java index 21382405..3eaf05c9 100644 --- a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java +++ b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java @@ -123,13 +123,14 @@ public void onDeleted(WatchEvent ev) { } @Test - void memorylessRescansTest() throws IOException { + void memorylessRescanOnOverflow() throws IOException, InterruptedException { var directory = testDir.getTestDirectory(); Files.writeString(directory.resolve("a.txt"), "foo"); Files.writeString(directory.resolve("b.txt"), "bar"); var nCreated = new AtomicInteger(); var nModified = new AtomicInteger(); + var nOverflow = new AtomicInteger(); var watchConfig = Watcher.watch(directory, WatchScope.PATH_AND_CHILDREN, OverflowPolicy.MEMORYLESS_RESCANS) .on(e -> { switch (e.getKind()) { @@ -139,6 +140,9 @@ void memorylessRescansTest() throws IOException { case MODIFIED: nModified.incrementAndGet(); break; + case OVERFLOW: + nOverflow.incrementAndGet(); + break; default: break; } @@ -147,11 +151,14 @@ void memorylessRescansTest() throws IOException { try (var watch = watchConfig.start()) { var overflow = new WatchEvent(WatchEvent.Kind.OVERFLOW, directory); ((EventHandlingWatch) watch).handleEvent(overflow); + Thread.sleep(TestHelper.SHORT_WAIT.toMillis()); - await("Overflow should generate create events") + await("Overflow should trigger created events") .until(nCreated::get, Predicate.isEqual(6)); // 3 directories + 3 files - await("Overflow should generate modified events") + await("Overflow should trigger modified events") .until(nModified::get, Predicate.isEqual(5)); // 3 directories + 2 files (c.txt is still empty) + await("Overflow shouldn't be visible to user-defined event handler") + .until(nOverflow::get, Predicate.isEqual(0)); } } } From 5d6e32f193aeac18ad06cf382288ef8c450abb17 Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 26 Feb 2025 15:58:38 +0100 Subject: [PATCH 11/19] Move configuration of overflow policy to separate method --- .../java/engineering/swat/watch/Watcher.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 045b4206..5ed323cf 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -55,33 +55,24 @@ public class Watcher { private final Logger logger = LogManager.getLogger(); private final Path path; private final WatchScope scope; - private final OverflowPolicy overflowPolicy; + private volatile OverflowPolicy overflowPolicy = OverflowPolicy.MEMORYLESS_RESCANS; private volatile Executor executor = CompletableFuture::runAsync; private static final BiConsumer EMPTY_HANDLER = (w, e) -> {}; private volatile BiConsumer eventHandler = EMPTY_HANDLER; - private Watcher(Path path, WatchScope scope, OverflowPolicy overflowPolicy) { + private Watcher(Path path, WatchScope scope) { this.path = path; this.scope = scope; - this.overflowPolicy = overflowPolicy; - } - - /** - * Equivalent to: `watch(path, scope, OverflowPolicy.MEMORYLESS_RESCANS)` - */ - public static Watcher watch(Path path, WatchScope scope) { - return watch(path, scope, OverflowPolicy.MEMORYLESS_RESCANS); } /** * Watch a path for updates, optionally also get events for its children/descendants * @param path which absolute path to monitor, can be a file or a directory, but has to be absolute * @param scope for directories you can also choose to monitor it's direct children or all it's descendants - * @param overflowPolicy policy to automatically handle overflow events * @throws IllegalArgumentException in case a path is not supported (in relation to the scope) */ - public static Watcher watch(Path path, WatchScope scope, OverflowPolicy overflowPolicy) { + public static Watcher watch(Path path, WatchScope scope) { if (!path.isAbsolute()) { throw new IllegalArgumentException("We can only watch absolute paths"); } @@ -100,7 +91,7 @@ public static Watcher watch(Path path, WatchScope scope, OverflowPolicy overflow default: throw new IllegalArgumentException("Unsupported scope: " + scope); } - return new Watcher(path, scope, overflowPolicy); + return new Watcher(path, scope); } /** @@ -157,6 +148,19 @@ public Watcher withExecutor(Executor callbackHandler) { return this; } + /** + * Optionally configure the overflow policy of this watcher to automatically + * handle overflow events. If not defined before this watcher is started, + * the {@link engineering.swat.watch.OverflowPolicy#MEMORYLESS_RESCANS} + * policy will be used. + * @param overflowPolicy The overflow policy to use + * @return This watcher for optional method chaining + */ + public Watcher withOverflowPolicy(OverflowPolicy overflowPolicy) { + this.overflowPolicy = overflowPolicy; + return this; + } + /** * Start watch the path for events. * @return a subscription for the watch, when closed, new events will stop being registered to the worker pool. From ac2e2123f4996442a4ebf544986724b813940f0d Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 26 Feb 2025 16:33:22 +0100 Subject: [PATCH 12/19] Remove auto-gobbling of overflow events for all overflow policies --- .../java/engineering/swat/watch/Watcher.java | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 5ed323cf..16d4abc9 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -172,7 +172,7 @@ public ActiveWatch start() throws IOException { throw new IllegalStateException("There is no onEvent handler defined"); } - var h = overflowEventHandler().andThen(userDefinedEventHandler()); + var h = applyOverflowPolicy(); switch (scope) { case PATH_AND_CHILDREN: { @@ -204,28 +204,14 @@ public ActiveWatch start() throws IOException { } } - private BiConsumer overflowEventHandler() { + private BiConsumer applyOverflowPolicy() { switch (overflowPolicy) { case NO_RESCANS: - return (w, e) -> {}; + return eventHandler; case MEMORYLESS_RESCANS: - return new MemorylessRescanner(executor); + return new MemorylessRescanner(executor).andThen(eventHandler); default: throw new UnsupportedOperationException("No event handler has been defined yet for this overflow policy"); } } - - private BiConsumer userDefinedEventHandler() { - // If overflow events are auto-handled because of the overflow policy, - // then they should be ignored by the user-defined event handler - if (overflowPolicy != OverflowPolicy.NO_RESCANS) { - return (w, e) -> { - if (e.getKind() != WatchEvent.Kind.OVERFLOW) { - eventHandler.accept(w, e); - } - }; - } - - return eventHandler; - } } From ec79d613d1bd622ec7bc20ab52b53004caaa45a9 Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 26 Feb 2025 17:08:26 +0100 Subject: [PATCH 13/19] Update test --- src/test/java/engineering/swat/watch/SingleDirectoryTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java index 3eaf05c9..9d93e6ef 100644 --- a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java +++ b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java @@ -131,7 +131,8 @@ void memorylessRescanOnOverflow() throws IOException, InterruptedException { var nCreated = new AtomicInteger(); var nModified = new AtomicInteger(); var nOverflow = new AtomicInteger(); - var watchConfig = Watcher.watch(directory, WatchScope.PATH_AND_CHILDREN, OverflowPolicy.MEMORYLESS_RESCANS) + var watchConfig = Watcher.watch(directory, WatchScope.PATH_AND_CHILDREN) + .withOverflowPolicy(OverflowPolicy.MEMORYLESS_RESCANS) .on(e -> { switch (e.getKind()) { case CREATED: From 96f4f5be2260287b2f9650bf07d9e5b4f56a884d Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Fri, 28 Feb 2025 07:35:24 +0100 Subject: [PATCH 14/19] Use `walkFileTree` instead of `walk` to avoid redundant retrieval of the `BasicFileAttributes` --- .../impl/overflows/MemorylessRescanner.java | 93 ++++++++++++++----- 1 file changed, 69 insertions(+), 24 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java b/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java index ff69050b..d1bd36b1 100644 --- a/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java +++ b/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java @@ -27,11 +27,17 @@ package engineering.swat.watch.impl.overflows; import java.io.IOException; +import java.nio.file.FileVisitOption; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; import java.util.concurrent.Executor; import java.util.function.BiConsumer; -import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -54,36 +60,75 @@ public MemorylessRescanner(Executor exec) { * should typically be executed asynchronously (using `exec`). */ protected void rescan(EventHandlingWatch watch) { - try (var content = contentOf(watch.getPath(), watch.getScope())) { - content - .flatMap(this::generateEvents) // Paths aren't properly relativized yet... - .map(watch::relativize) // ...so they must be relativized first (wrt the root path of `watch`) - .forEach(watch::handleEvent); - } - } + var start = watch.getPath(); + var options = EnumSet.noneOf(FileVisitOption.class); + var maxDepth = watch.getScope() == WatchScope.PATH_AND_ALL_DESCENDANTS ? Integer.MAX_VALUE : 1; + var visitor = new FileVisitor(watch); - protected Stream contentOf(Path path, WatchScope scope) { try { - var maxDepth = scope == WatchScope.PATH_AND_ALL_DESCENDANTS ? Integer.MAX_VALUE : 1; - return Files.walk(path, maxDepth).filter(p -> p != path); + Files.walkFileTree(start, options, maxDepth, visitor); } catch (IOException e) { - logger.error("Could not walk: {} ({})", path, e); - return Stream.empty(); + logger.error("Could not walk: {} ({})", start, e); + } + + for (var e : visitor.getEvents()) { + watch.handleEvent(e); } } - protected Stream generateEvents(Path path) { - try { - var created = new WatchEvent(WatchEvent.Kind.CREATED, path); - if (Files.size(path) == 0) { - return Stream.of(created); - } else { - var modified = new WatchEvent(WatchEvent.Kind.MODIFIED, path); - return Stream.of(created, modified); + private class FileVisitor extends SimpleFileVisitor { + private final EventHandlingWatch watch; + private final List events; + + public FileVisitor(EventHandlingWatch watch) { + this.watch = watch; + this.events = new ArrayList<>(); + } + + public List getEvents() { + return events; + } + + protected void addEvents(Path path, BasicFileAttributes attrs) { + events.add(newEvent(WatchEvent.Kind.CREATED, path)); + if (attrs.size() > 0) { + events.add(newEvent(WatchEvent.Kind.MODIFIED, path)); } - } catch (IOException e) { - logger.error("Could not generate events for: {} ({})", path, e); - return Stream.empty(); + } + + protected WatchEvent newEvent(WatchEvent.Kind kind, Path fullPath) { + var event = new WatchEvent(kind, fullPath); + return watch.relativize(event); + } + + // -- SimpleFileVisitor -- + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (!watch.getPath().equals(dir)) { + addEvents(dir, attrs); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + addEvents(file, attrs); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + logger.error("Could not generate events for file: {} ({})", file, exc); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (exc != null) { + logger.error("Could not successfully walk: {} ({})", dir, exc); + } + return FileVisitResult.CONTINUE; } } From 67ca923bd63cb05f00e7070b50964b65ec4a8c6a Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Fri, 28 Feb 2025 07:38:33 +0100 Subject: [PATCH 15/19] Update test --- .../java/engineering/swat/watch/SingleDirectoryTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java index 9d93e6ef..06f02828 100644 --- a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java +++ b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java @@ -157,9 +157,9 @@ void memorylessRescanOnOverflow() throws IOException, InterruptedException { await("Overflow should trigger created events") .until(nCreated::get, Predicate.isEqual(6)); // 3 directories + 3 files await("Overflow should trigger modified events") - .until(nModified::get, Predicate.isEqual(5)); // 3 directories + 2 files (c.txt is still empty) - await("Overflow shouldn't be visible to user-defined event handler") - .until(nOverflow::get, Predicate.isEqual(0)); + .until(nModified::get, Predicate.isEqual(2)); // 2 files (c.txt is still empty) + await("Overflow should be visible to user-defined event handler") + .until(nOverflow::get, Predicate.isEqual(1)); } } } From 97448742dbd7cb6ab4ed41232ff14b5b268a1c5c Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Fri, 28 Feb 2025 08:03:50 +0100 Subject: [PATCH 16/19] Fix test --- .../swat/watch/impl/overflows/MemorylessRescanner.java | 2 +- src/test/java/engineering/swat/watch/SingleDirectoryTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java b/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java index d1bd36b1..a0fbccdc 100644 --- a/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java +++ b/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java @@ -91,7 +91,7 @@ public List getEvents() { protected void addEvents(Path path, BasicFileAttributes attrs) { events.add(newEvent(WatchEvent.Kind.CREATED, path)); - if (attrs.size() > 0) { + if (attrs.isDirectory() || attrs.size() > 0) { events.add(newEvent(WatchEvent.Kind.MODIFIED, path)); } } diff --git a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java index 06f02828..dad8dcae 100644 --- a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java +++ b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java @@ -157,7 +157,7 @@ void memorylessRescanOnOverflow() throws IOException, InterruptedException { await("Overflow should trigger created events") .until(nCreated::get, Predicate.isEqual(6)); // 3 directories + 3 files await("Overflow should trigger modified events") - .until(nModified::get, Predicate.isEqual(2)); // 2 files (c.txt is still empty) + .until(nModified::get, Predicate.isEqual(5)); // 3 directories + 2 files (c.txt is still empty) await("Overflow should be visible to user-defined event handler") .until(nOverflow::get, Predicate.isEqual(1)); } From 4beb950c92247a16e145d5904d78bb6dc57bfed7 Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Fri, 28 Feb 2025 09:34:45 +0100 Subject: [PATCH 17/19] Fix issue that `MODIFIED` events were issued for directories when auto-handling overflows --- .../swat/watch/impl/overflows/MemorylessRescanner.java | 6 +++--- .../java/engineering/swat/watch/SingleDirectoryTests.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java b/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java index a0fbccdc..8cdebd12 100644 --- a/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java +++ b/src/main/java/engineering/swat/watch/impl/overflows/MemorylessRescanner.java @@ -89,14 +89,14 @@ public List getEvents() { return events; } - protected void addEvents(Path path, BasicFileAttributes attrs) { + private void addEvents(Path path, BasicFileAttributes attrs) { events.add(newEvent(WatchEvent.Kind.CREATED, path)); - if (attrs.isDirectory() || attrs.size() > 0) { + if (attrs.isRegularFile() && attrs.size() > 0) { events.add(newEvent(WatchEvent.Kind.MODIFIED, path)); } } - protected WatchEvent newEvent(WatchEvent.Kind kind, Path fullPath) { + private WatchEvent newEvent(WatchEvent.Kind kind, Path fullPath) { var event = new WatchEvent(kind, fullPath); return watch.relativize(event); } diff --git a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java index dad8dcae..06f02828 100644 --- a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java +++ b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java @@ -157,7 +157,7 @@ void memorylessRescanOnOverflow() throws IOException, InterruptedException { await("Overflow should trigger created events") .until(nCreated::get, Predicate.isEqual(6)); // 3 directories + 3 files await("Overflow should trigger modified events") - .until(nModified::get, Predicate.isEqual(5)); // 3 directories + 2 files (c.txt is still empty) + .until(nModified::get, Predicate.isEqual(2)); // 2 files (c.txt is still empty) await("Overflow should be visible to user-defined event handler") .until(nOverflow::get, Predicate.isEqual(1)); } From f7a7a34ca41b138c8e2d69a707123917db71b89b Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Wed, 5 Mar 2025 12:48:22 +0100 Subject: [PATCH 18/19] Rename overflow auto-handling approach constants --- .../engineering/swat/watch/OnOverflow.java | 107 ++++++++++++++++++ .../swat/watch/OverflowPolicy.java | 65 ----------- .../java/engineering/swat/watch/Watcher.java | 29 ++--- .../swat/watch/SingleDirectoryTests.java | 2 +- 4 files changed, 124 insertions(+), 79 deletions(-) create mode 100644 src/main/java/engineering/swat/watch/OnOverflow.java delete mode 100644 src/main/java/engineering/swat/watch/OverflowPolicy.java diff --git a/src/main/java/engineering/swat/watch/OnOverflow.java b/src/main/java/engineering/swat/watch/OnOverflow.java new file mode 100644 index 00000000..da3dab03 --- /dev/null +++ b/src/main/java/engineering/swat/watch/OnOverflow.java @@ -0,0 +1,107 @@ +/* + * BSD 2-Clause License + * + * Copyright (c) 2023, Swat.engineering + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package engineering.swat.watch; + +/** + * Constants to indicate for which regular files/directories in the scope of the + * watch an approximation of synthetic events (of kinds + * {@link WatchEvent.Kind#CREATED}, {@link WatchEvent.Kind#MODIFIED}, and/or + * {@link WatchEvent.Kind#DELETED}) should be issued when an overflow event + * happens. These synthetic events, as well as the overflow event itself, are + * subsequently passed to the user-defined event handler of the watch. + * Typically, the user-defined event handler can ignore the original overflow + * event (i.e., handling the synthetic events is sufficient to address the + * overflow issue), but it doesn't have to (e.g., it may carry out additional + * overflow bookkeeping). + */ +public enum OnOverflow { + + /** + * Synthetic events are issued for no regular files/directories in + * the scope of the watch. Thus, the user-defined event handler is fully + * responsible to handle overflow events. + */ + NONE, + + /** + *

+ * Synthetic events of kinds {@link WatchEvent.Kind#CREATED} and + * {@link WatchEvent.Kind#MODIFIED}, but not + * {@link WatchEvent.Kind#DELETED}, are issued for all regular + * files/directories in the scope of the watch. Specifically, when an + * overflow event happens: + * + *

    + *
  • CREATED events are issued for all regular files/directories + * (overapproximation). + *
  • MODIFIED events are issued for all non-empty, regular files + * (overapproximation) but for no directories (underapproximation). + *
  • DELETED events are issued for no regular files/directories + * (underapproximation). + *
+ * + *

+ * This approach is relatively cheap in terms of memory usage (cf. + * {@link #DIRTY}), but it results in a large over/underapproximation of the + * actual events (cf. DIRTY). + */ + ALL, + + + /** + *

+ * Synthetic events of kinds {@link WatchEvent.Kind#CREATED}, + * {@link WatchEvent.Kind#MODIFIED}, and {@link WatchEvent.Kind#DELETED} are + * issued for dirty regular files/directories in the scope of the watch, as + * determined using last-modified-times. Specifically, when an + * overflow event happens: + * + *

    + *
  • CREATED events are issued for all regular files/directories when the + * previous last-modified-time is unknown, but the current + * last-modified-time is known (i.e., the file started existing). + *
  • MODIFIED events are issued for all regular files/directories when the + * previous last-modified-time is before the current last-modified-time. + *
  • DELETED events are issued for all regular files/directories when the + * previous last-modified-time is known, but the current + * last-modified-time is unknown (i.e., the file stopped existing). + *
+ * + *

+ * To keep track of last-modified-times, an internal index is + * populated with last-modified-times of all regular files/directories in + * the scope of the watch when the watch is started. Each time when any + * event happens, the index is updated accordingly, so when an overflow + * event happens, last-modified-times can be compared as described above. + * + *

+ * This approach results in a small overapproximation (cf. {@link #ALL}), + * but it is relatively expensive in terms of memory usage (cf. ALL), as the + * watch needs to keep track of last-modified-times. + */ + DIRTY +} diff --git a/src/main/java/engineering/swat/watch/OverflowPolicy.java b/src/main/java/engineering/swat/watch/OverflowPolicy.java deleted file mode 100644 index 5ce73e35..00000000 --- a/src/main/java/engineering/swat/watch/OverflowPolicy.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * BSD 2-Clause License - * - * Copyright (c) 2023, Swat.engineering - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package engineering.swat.watch; - -public enum OverflowPolicy { - - /** - * When an overflow event occurs, do nothing (i.e., the user-defined event - * handler is responsible to handle overflow events). - */ - NO_RESCANS, - - /** - * When an overflow event occurs, rescan all files in the scope of the - * watch, and issue `CREATED` and `MODIFIED` events (not `DELETED` events) - * for each file. `MODIFIED` events are issued only for non-empty files. - * - * Compared to the `INDEXING_RESCANS` policy, the `MEMORYLESS_RESCANS` - * policy is less expensive in terms of memory usage, but it results in a - * larger overaproximation of the actual `CREATED` and `MODIFIED` events - * that happened, while all `DELETED` events remain undetected. - */ - MEMORYLESS_RESCANS, - - /** - * When an overflow event occurs, rescan all files in the watch scope, - * update the internal *index*, and issue `CREATED`, `MODIFIED`, and - * `DELETED` events accordingly. The index keeps track of the last modified - * time of each file, such that: (a) `CREATED` events are issued for files - * that are added to the index; (b) `DELETED` events are issued for files - * that are removed from the index; (c) `MODIFIED` events are issued for - * files in the index whose previous last-modified-time is before their - * current last-modified-time. - * - * Compared to the `MEMORYLESS_RESCANS` policy, the `INDEXING_RESCANS` - * policy results in a smaller overapproximation of the actual `CREATED`, - * `MODIFIED`, and `DELETED` events that happened, but it's more expensive - * in terms of memory usage. - */ - INDEXING_RESCANS -} diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 16d4abc9..6621bb2f 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -55,7 +55,7 @@ public class Watcher { private final Logger logger = LogManager.getLogger(); private final Path path; private final WatchScope scope; - private volatile OverflowPolicy overflowPolicy = OverflowPolicy.MEMORYLESS_RESCANS; + private volatile OnOverflow approximateOnOverflow = OnOverflow.ALL; private volatile Executor executor = CompletableFuture::runAsync; private static final BiConsumer EMPTY_HANDLER = (w, e) -> {}; @@ -149,15 +149,18 @@ public Watcher withExecutor(Executor callbackHandler) { } /** - * Optionally configure the overflow policy of this watcher to automatically - * handle overflow events. If not defined before this watcher is started, - * the {@link engineering.swat.watch.OverflowPolicy#MEMORYLESS_RESCANS} - * policy will be used. - * @param overflowPolicy The overflow policy to use + * Optionally configure which regular files/directories in the scope of the + * watch an approximation of synthetic events (of kinds + * {@link WatchEvent.Kind#CREATED}, {@link WatchEvent.Kind#MODIFIED}, and/or + * {@link WatchEvent.Kind#DELETED}) should be issued when an overflow event + * happens. If not defined before this watcher is started, the + * {@link engineering.swat.watch.OnOverflow#ALL} approach will be used. + * @param whichFiles Constant to indicate for which regular + * files/directories to approximate * @return This watcher for optional method chaining */ - public Watcher withOverflowPolicy(OverflowPolicy overflowPolicy) { - this.overflowPolicy = overflowPolicy; + public Watcher approximate(OnOverflow whichFiles) { + this.approximateOnOverflow = whichFiles; return this; } @@ -172,7 +175,7 @@ public ActiveWatch start() throws IOException { throw new IllegalStateException("There is no onEvent handler defined"); } - var h = applyOverflowPolicy(); + var h = applyApproximateOnOverflow(); switch (scope) { case PATH_AND_CHILDREN: { @@ -204,11 +207,11 @@ public ActiveWatch start() throws IOException { } } - private BiConsumer applyOverflowPolicy() { - switch (overflowPolicy) { - case NO_RESCANS: + private BiConsumer applyApproximateOnOverflow() { + switch (approximateOnOverflow) { + case NONE: return eventHandler; - case MEMORYLESS_RESCANS: + case ALL: return new MemorylessRescanner(executor).andThen(eventHandler); default: throw new UnsupportedOperationException("No event handler has been defined yet for this overflow policy"); diff --git a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java index 06f02828..eedf2e92 100644 --- a/src/test/java/engineering/swat/watch/SingleDirectoryTests.java +++ b/src/test/java/engineering/swat/watch/SingleDirectoryTests.java @@ -132,7 +132,7 @@ void memorylessRescanOnOverflow() throws IOException, InterruptedException { var nModified = new AtomicInteger(); var nOverflow = new AtomicInteger(); var watchConfig = Watcher.watch(directory, WatchScope.PATH_AND_CHILDREN) - .withOverflowPolicy(OverflowPolicy.MEMORYLESS_RESCANS) + .approximate(OnOverflow.ALL) .on(e -> { switch (e.getKind()) { case CREATED: From 9e71ad7ea40a13f3e29942ed69df2d06cc1b9671 Mon Sep 17 00:00:00 2001 From: Sung-Shik Jongmans Date: Fri, 7 Mar 2025 10:52:38 +0100 Subject: [PATCH 19/19] Switch order of overflow auto-handler and user-defined event handler --- src/main/java/engineering/swat/watch/Watcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/engineering/swat/watch/Watcher.java b/src/main/java/engineering/swat/watch/Watcher.java index 6621bb2f..d639fa19 100644 --- a/src/main/java/engineering/swat/watch/Watcher.java +++ b/src/main/java/engineering/swat/watch/Watcher.java @@ -212,7 +212,7 @@ private BiConsumer applyApproximateOnOverflow() case NONE: return eventHandler; case ALL: - return new MemorylessRescanner(executor).andThen(eventHandler); + return eventHandler.andThen(new MemorylessRescanner(executor)); default: throw new UnsupportedOperationException("No event handler has been defined yet for this overflow policy"); }