From d33ed88d59f6ebbf67e110e65adacabc41b2dfda Mon Sep 17 00:00:00 2001 From: MishaDemianenko Date: Tue, 13 Dec 2016 17:41:34 +0100 Subject: [PATCH] Monitor store files deletions by external applications Use file system monitoring possibilities to provide notification in case of store files deletion by third parties. --- .../ParallelBatchImporterTest.java | 1 + .../java/org/neo4j/tooling/ImportTool.java | 1 + .../io/fs/DefaultFileSystemAbstraction.java | 11 + .../io/fs/DelegateFileSystemAbstraction.java | 8 + .../neo4j/io/fs/FileSystemAbstraction.java | 11 + .../fs/watcher/DefaultFileSystemWatcher.java | 146 ++++++++ .../org/neo4j/io/fs/watcher/FileWatcher.java | 76 ++++ .../io/fs/watcher/SilentFileWatcher.java | 65 ++++ .../watcher/event/FileWatchEventListener.java | 42 +++ .../event/FileWatchEventListenerAdapter.java | 35 ++ .../io/fs/watcher/resource/WatchedFile.java | 42 +++ .../fs/watcher/resource/WatchedResource.java | 38 ++ .../fs/AdversarialFileSystemAbstraction.java | 9 + .../watcher/AdversarialFileWatcher.java | 83 +++++ .../DelegatingFileSystemAbstraction.java | 7 + .../EphemeralFileSystemAbstraction.java | 7 + .../SelectiveFileSystemAbstraction.java | 7 + .../graphdb/mockfs/SelectiveFileWatcher.java | 91 +++++ .../fs/DefaultFileSystemAbstractionTest.java | 21 +- .../fs/DelegateFileSystemAbstractionTest.java | 15 + .../SelectiveFileSystemAbstractionTest.java | 32 ++ .../watcher/DefaultFileSystemWatcherTest.java | 328 ++++++++++++++++++ .../graphdb/factory/GraphDatabaseFactory.java | 9 +- .../kernel/impl/factory/PlatformModule.java | 31 +- .../neo4j/kernel/impl/util/JobScheduler.java | 5 + .../DefaultFileDeletionEventListener.java | 45 +++ .../watcher/FileWatcherLifecycleAdapter.java | 93 +++++ .../WatcherLifecycleAdapterFactory.java | 40 +++ .../batchimport/ParallelBatchImporter.java | 3 +- .../util/FileWatcherLifecycleAdapterTest.java | 93 +++++ .../DefaultFileDeletionEventListenerTest.java | 45 +++ .../WatcherLifecycleAdapterFactoryTest.java | 53 +++ ...ersarialPageCacheGraphDatabaseFactory.java | 3 +- .../neo4j/test/TestGraphDatabaseFactory.java | 172 ++++++--- .../org/neo4j/store/watch/FileWatchIT.java | 142 ++++++++ .../MultipleIndexPopulationStressIT.java | 1 + .../test/java/upgrade/StoreUpgraderTest.java | 6 +- .../TestEnterpriseGraphDatabaseFactory.java | 74 +--- .../neo4j/upgrade/StoreMigratorFrom20IT.java | 1 + 39 files changed, 1767 insertions(+), 125 deletions(-) create mode 100644 community/io/src/main/java/org/neo4j/io/fs/watcher/DefaultFileSystemWatcher.java create mode 100644 community/io/src/main/java/org/neo4j/io/fs/watcher/FileWatcher.java create mode 100644 community/io/src/main/java/org/neo4j/io/fs/watcher/SilentFileWatcher.java create mode 100644 community/io/src/main/java/org/neo4j/io/fs/watcher/event/FileWatchEventListener.java create mode 100644 community/io/src/main/java/org/neo4j/io/fs/watcher/event/FileWatchEventListenerAdapter.java create mode 100644 community/io/src/main/java/org/neo4j/io/fs/watcher/resource/WatchedFile.java create mode 100644 community/io/src/main/java/org/neo4j/io/fs/watcher/resource/WatchedResource.java create mode 100644 community/io/src/test/java/org/neo4j/adversaries/watcher/AdversarialFileWatcher.java create mode 100644 community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileWatcher.java create mode 100644 community/io/src/test/java/org/neo4j/io/fs/watcher/DefaultFileSystemWatcherTest.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/util/watcher/DefaultFileDeletionEventListener.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/util/watcher/FileWatcherLifecycleAdapter.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/util/watcher/WatcherLifecycleAdapterFactory.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/util/FileWatcherLifecycleAdapterTest.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/util/watcher/DefaultFileDeletionEventListenerTest.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/util/watcher/WatcherLifecycleAdapterFactoryTest.java create mode 100644 community/neo4j/src/test/java/org/neo4j/store/watch/FileWatchIT.java diff --git a/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java b/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java index b76fae6f9f78a..0ef7b18f1da6a 100644 --- a/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java @@ -54,6 +54,7 @@ import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.fs.watcher.FileWatcher; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.NullLogService; import org.neo4j.kernel.impl.store.format.RecordFormats; diff --git a/community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java b/community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java index 67473559efd5c..cf4a94263f158 100644 --- a/community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java +++ b/community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java @@ -450,6 +450,7 @@ public static void doImport( PrintStream out, PrintStream err, File storeDir, Fi LogService logService = life.add( StoreLogService.inLogsDirectory( fs, logsDir ) ); life.start(); + //TODO: add file watcher here? BatchImporter importer = new ParallelBatchImporter( storeDir, fs, configuration, diff --git a/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java index 6c6d3745c5e19..896fb93c2cc7b 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/DefaultFileSystemAbstraction.java @@ -34,12 +34,16 @@ import java.nio.channels.FileChannel; import java.nio.charset.Charset; import java.nio.file.CopyOption; +import java.nio.file.FileSystems; import java.nio.file.Files; +import java.nio.file.WatchService; import java.util.HashMap; import java.util.Map; import java.util.function.Function; import org.neo4j.io.IOUtils; +import org.neo4j.io.fs.watcher.DefaultFileSystemWatcher; +import org.neo4j.io.fs.watcher.FileWatcher; import static java.lang.String.format; @@ -50,6 +54,13 @@ public class DefaultFileSystemAbstraction implements FileSystemAbstraction { static final String UNABLE_TO_CREATE_DIRECTORY_FORMAT = "Unable to create directory path [%s] for Neo4j store."; + @Override + public FileWatcher fileWatcher() throws IOException + { + WatchService watchService = FileSystems.getDefault().newWatchService(); + return new DefaultFileSystemWatcher( watchService ); + } + @Override public StoreFileChannel open( File fileName, String mode ) throws IOException { diff --git a/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java index dcaa6bd03a6b5..d86a13cdd33e6 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/DelegateFileSystemAbstraction.java @@ -44,6 +44,8 @@ import java.util.stream.Stream; import org.neo4j.io.IOUtils; +import org.neo4j.io.fs.watcher.DefaultFileSystemWatcher; +import org.neo4j.io.fs.watcher.FileWatcher; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; @@ -61,6 +63,12 @@ public DelegateFileSystemAbstraction( FileSystem fs ) this.fs = fs; } + @Override + public FileWatcher fileWatcher() throws IOException + { + return new DefaultFileSystemWatcher( fs.newWatchService() ); + } + @Override public StoreChannel open( File fileName, String mode ) throws IOException { diff --git a/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java b/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java index 503db3bc2532e..12181f144d88c 100644 --- a/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java +++ b/community/io/src/main/java/org/neo4j/io/fs/FileSystemAbstraction.java @@ -32,8 +32,19 @@ import java.util.function.Function; import java.util.zip.ZipOutputStream; +import org.neo4j.io.fs.watcher.FileWatcher; + public interface FileSystemAbstraction extends Closeable { + + /** + * Create file watcher that provides possibilities to monitor directories on underlying file system + * abstraction + * @return specific file system abstract watcher + * @throws IOException in case exception occur during file watcher creation + */ + FileWatcher fileWatcher() throws IOException; + StoreChannel open( File fileName, String mode ) throws IOException; OutputStream openAsOutputStream( File fileName, boolean append ) throws IOException; diff --git a/community/io/src/main/java/org/neo4j/io/fs/watcher/DefaultFileSystemWatcher.java b/community/io/src/main/java/org/neo4j/io/fs/watcher/DefaultFileSystemWatcher.java new file mode 100644 index 0000000000000..d0de5e5976049 --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/fs/watcher/DefaultFileSystemWatcher.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.fs.watcher; + +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.StandardWatchEventKinds; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.neo4j.io.fs.watcher.event.FileWatchEventListener; +import org.neo4j.io.fs.watcher.resource.WatchedFile; +import org.neo4j.io.fs.watcher.resource.WatchedResource; + +/** + * File watcher that monitors registered directories state using possibilities provided by {@link WatchService}. + * + * Safe to be used from multiple threads + */ +public class DefaultFileSystemWatcher implements FileWatcher +{ + private final WatchService watchService; + private final List listeners = new CopyOnWriteArrayList<>(); + private volatile boolean watch; + + public DefaultFileSystemWatcher( WatchService watchService ) + { + this.watchService = watchService; + } + + @Override + public WatchedResource watch( File file ) throws IOException + { + if ( !file.isDirectory() ) + { + throw new IllegalArgumentException( "Only directories can be registered to be monitored." ); + } + WatchKey watchKey = file.toPath() + .register( watchService, StandardWatchEventKinds.ENTRY_DELETE, StandardWatchEventKinds.ENTRY_MODIFY ); + return new WatchedFile( watchKey ); + } + + @Override + public void startWatching() throws InterruptedException + { + watch = true; + while ( watch ) + { + WatchKey key = watchService.take(); + if ( key != null ) + { + List> watchEvents = key.pollEvents(); + for ( WatchEvent watchEvent : watchEvents ) + { + WatchEvent.Kind kind = watchEvent.kind(); + if ( StandardWatchEventKinds.ENTRY_MODIFY == kind ) + { + notifyAboutModification( watchEvent ); + } + if ( StandardWatchEventKinds.ENTRY_DELETE == kind ) + { + notifyAboutDeletion( watchEvent ); + } + } + key.reset(); + } + } + } + + @Override + public void stopWatching() + { + watch = false; + } + + @Override + public void addFileWatchEventListener( FileWatchEventListener listener ) + { + listeners.add( listener ); + } + + @Override + public void removeFileWatchEventListener( FileWatchEventListener listener ) + { + listeners.remove( listener ); + } + + @Override + public void close() throws IOException + { + stopWatching(); + watchService.close(); + } + + private void notifyAboutModification( WatchEvent watchEvent ) + { + String context = getContext( watchEvent ); + if ( StringUtils.isNotEmpty( context ) ) + { + for ( FileWatchEventListener listener : listeners ) + { + listener.fileModified( context ); + } + } + } + + private void notifyAboutDeletion( WatchEvent watchEvent ) + { + String context = getContext( watchEvent ); + if ( StringUtils.isNotEmpty( context ) ) + { + for ( FileWatchEventListener listener : listeners ) + { + listener.fileDeleted( context ); + } + } + } + + private String getContext( WatchEvent watchEvent ) + { + return Objects.toString( watchEvent.context(), StringUtils.EMPTY ); + } +} diff --git a/community/io/src/main/java/org/neo4j/io/fs/watcher/FileWatcher.java b/community/io/src/main/java/org/neo4j/io/fs/watcher/FileWatcher.java new file mode 100644 index 0000000000000..7ebfb2b2eb2fe --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/fs/watcher/FileWatcher.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.fs.watcher; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.nio.file.WatchService; + +import org.neo4j.io.fs.watcher.event.FileWatchEventListener; +import org.neo4j.io.fs.watcher.resource.WatchedResource; + +/** + * Watcher that allows receive notification about files modifications/removal for particular underlying file system. + * + * To be able to get notification users need to register resource they are interested in using + * {@link #watch(File)} method call and add by adding {@link FileWatchEventListener listner} to be able to receive + * status updates. + * + * @see WatchService + */ +public interface FileWatcher extends Closeable +{ + + FileWatcher SILENT_WATCHER = new SilentFileWatcher(); + + /** + * Register provided directory in list of resources that we would like to watch and receive status modification + * updates + * @param file directory to be monitored for updates + * @return + * @throws IOException + */ + WatchedResource watch( File file ) throws IOException; + + /** + * Register listener to receive updates about registered resources. + * @param listener listener to register + */ + void addFileWatchEventListener( FileWatchEventListener listener ); + + /** + * Remove listener from a list of updates receivers. + * @param listener listener to remove + */ + void removeFileWatchEventListener( FileWatchEventListener listener ); + + /** + * Stop monitoring of registered directories + */ + void stopWatching(); + + /** + * Start monitoring of registered directories + * @throws InterruptedException when interrupted while waiting for update notification to come + * + */ + void startWatching() throws InterruptedException; +} diff --git a/community/io/src/main/java/org/neo4j/io/fs/watcher/SilentFileWatcher.java b/community/io/src/main/java/org/neo4j/io/fs/watcher/SilentFileWatcher.java new file mode 100644 index 0000000000000..965153d9e1218 --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/fs/watcher/SilentFileWatcher.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.fs.watcher; + +import java.io.File; +import java.io.IOException; + +import org.neo4j.io.fs.watcher.event.FileWatchEventListener; +import org.neo4j.io.fs.watcher.resource.WatchedResource; + +/** + * Silent file watcher implementation that do not perform any monitoring and can't observe any directories status or + * content update. + */ +public class SilentFileWatcher implements FileWatcher +{ + + @Override + public WatchedResource watch( File file ) throws IOException + { + return WatchedResource.EMPTY; + } + + @Override + public void addFileWatchEventListener( FileWatchEventListener listener ) + { + } + + @Override + public void removeFileWatchEventListener( FileWatchEventListener listener ) + { + } + + @Override + public void stopWatching() + { + } + + @Override + public void startWatching() throws InterruptedException + { + } + + @Override + public void close() throws IOException + { + } +} diff --git a/community/io/src/main/java/org/neo4j/io/fs/watcher/event/FileWatchEventListener.java b/community/io/src/main/java/org/neo4j/io/fs/watcher/event/FileWatchEventListener.java new file mode 100644 index 0000000000000..1f85951aa8451 --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/fs/watcher/event/FileWatchEventListener.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.fs.watcher.event; + +import java.util.EventListener; + +import org.neo4j.io.fs.watcher.FileWatcher; + +/** + * {@link FileWatcher} listener that allow receive state change updates for registered resources. + */ +public interface FileWatchEventListener extends EventListener +{ + /** + * Notification about deletion of file with provided name + * @param fileName deleted file name + */ + void fileDeleted( String fileName ); + + /** + *Notification about update of file with provided name + * @param fileName updated file name + */ + void fileModified( String fileName ); +} diff --git a/community/io/src/main/java/org/neo4j/io/fs/watcher/event/FileWatchEventListenerAdapter.java b/community/io/src/main/java/org/neo4j/io/fs/watcher/event/FileWatchEventListenerAdapter.java new file mode 100644 index 0000000000000..674f0b6fba14b --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/fs/watcher/event/FileWatchEventListenerAdapter.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.fs.watcher.event; + +public class FileWatchEventListenerAdapter implements FileWatchEventListener +{ + @Override + public void fileDeleted( String fileName ) + { + // empty + } + + @Override + public void fileModified( String fileName ) + { + //empty + } +} diff --git a/community/io/src/main/java/org/neo4j/io/fs/watcher/resource/WatchedFile.java b/community/io/src/main/java/org/neo4j/io/fs/watcher/resource/WatchedFile.java new file mode 100644 index 0000000000000..9351a3081513c --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/fs/watcher/resource/WatchedFile.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.fs.watcher.resource; + +import java.io.IOException; +import java.nio.file.WatchKey; + +/** + * Default {@link WatchedResource} implementation. + */ +public class WatchedFile implements WatchedResource +{ + private final WatchKey watchKey; + + public WatchedFile( WatchKey watchKey ) + { + this.watchKey = watchKey; + } + + @Override + public void close() throws IOException + { + watchKey.cancel(); + } +} diff --git a/community/io/src/main/java/org/neo4j/io/fs/watcher/resource/WatchedResource.java b/community/io/src/main/java/org/neo4j/io/fs/watcher/resource/WatchedResource.java new file mode 100644 index 0000000000000..fd1efb180ba99 --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/fs/watcher/resource/WatchedResource.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.fs.watcher.resource; + +import java.io.Closeable; + +import org.neo4j.io.fs.watcher.FileWatcher; + +/** + * Resource watched by {@link FileWatcher}. + * + * Should be closed if further monitoring is not required. + */ +public interface WatchedResource extends Closeable +{ + + WatchedResource EMPTY = () -> + { + }; + +} diff --git a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java index c33231e72f249..3b15beef84932 100644 --- a/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/adversaries/fs/AdversarialFileSystemAbstraction.java @@ -39,10 +39,12 @@ import java.util.function.Function; import org.neo4j.adversaries.Adversary; +import org.neo4j.adversaries.watcher.AdversarialFileWatcher; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.fs.StoreFileChannel; +import org.neo4j.io.fs.watcher.FileWatcher; /** * Used by the robustness suite to check for partial failures. @@ -64,6 +66,13 @@ public AdversarialFileSystemAbstraction( Adversary adversary, FileSystemAbstract this.delegate = delegate; } + @Override + public FileWatcher fileWatcher() throws IOException + { + adversary.injectFailure( UnsupportedOperationException.class, IOException.class ); + return new AdversarialFileWatcher( delegate.fileWatcher(), adversary ); + } + public StoreChannel open( File fileName, String mode ) throws IOException { adversary.injectFailure( FileNotFoundException.class, IOException.class, SecurityException.class ); diff --git a/community/io/src/test/java/org/neo4j/adversaries/watcher/AdversarialFileWatcher.java b/community/io/src/test/java/org/neo4j/adversaries/watcher/AdversarialFileWatcher.java new file mode 100644 index 0000000000000..10f5c91df72e4 --- /dev/null +++ b/community/io/src/test/java/org/neo4j/adversaries/watcher/AdversarialFileWatcher.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.adversaries.watcher; + +import java.io.File; +import java.io.IOException; + +import org.neo4j.adversaries.Adversary; +import org.neo4j.io.fs.watcher.FileWatcher; +import org.neo4j.io.fs.watcher.event.FileWatchEventListener; +import org.neo4j.io.fs.watcher.resource.WatchedResource; + +/** + * File watcher that injects additional failures using provided {@link Adversary} + * and delegate all actual watching role to provided {@link FileWatcher} + */ +public class AdversarialFileWatcher implements FileWatcher +{ + private final FileWatcher fileWatcher; + private final Adversary adversary; + + public AdversarialFileWatcher( FileWatcher fileWatcher, Adversary adversary ) + { + this.fileWatcher = fileWatcher; + this.adversary = adversary; + } + + @Override + public void close() throws IOException + { + adversary.injectFailure( IOException.class ); + fileWatcher.close(); + } + + @Override + public WatchedResource watch( File file ) throws IOException + { + adversary.injectFailure( IOException.class ); + return fileWatcher.watch( file ); + } + + @Override + public void addFileWatchEventListener( FileWatchEventListener listener ) + { + fileWatcher.addFileWatchEventListener( listener ); + } + + @Override + public void removeFileWatchEventListener( FileWatchEventListener listener ) + { + fileWatcher.removeFileWatchEventListener( listener ); + } + + @Override + public void stopWatching() + { + fileWatcher.stopWatching(); + } + + @Override + public void startWatching() throws InterruptedException + { + adversary.injectFailure( InterruptedException.class ); + fileWatcher.startWatching(); + } +} diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java index d685d26fc6465..6d11d3bf04994 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/DelegatingFileSystemAbstraction.java @@ -32,6 +32,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; +import org.neo4j.io.fs.watcher.FileWatcher; public class DelegatingFileSystemAbstraction implements FileSystemAbstraction { @@ -42,6 +43,12 @@ public DelegatingFileSystemAbstraction( FileSystemAbstraction delegate ) this.delegate = delegate; } + @Override + public FileWatcher fileWatcher() throws IOException + { + return delegate.fileWatcher(); + } + @Override public StoreChannel open( File fileName, String mode ) throws IOException { diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java index f4c689a88ea72..8645485ea9d3d 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/EphemeralFileSystemAbstraction.java @@ -69,6 +69,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.fs.StoreFileChannel; +import org.neo4j.io.fs.watcher.FileWatcher; import org.neo4j.test.impl.ChannelInputStream; import org.neo4j.test.impl.ChannelOutputStream; @@ -250,6 +251,12 @@ private void addRecursively( ZipOutputStream output, File input ) throws IOExcep } } + @Override + public FileWatcher fileWatcher() throws IOException + { + return FileWatcher.SILENT_WATCHER; + } + @Override public synchronized StoreChannel open( File fileName, String mode ) throws IOException { diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java index 949566b265be4..ba220122cb7da 100644 --- a/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileSystemAbstraction.java @@ -33,6 +33,7 @@ import org.neo4j.io.IOUtils; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; +import org.neo4j.io.fs.watcher.FileWatcher; /** * Allows you to select different file system behaviour for one file and a different file system behaviour for @@ -54,6 +55,12 @@ public SelectiveFileSystemAbstraction( File specialFile, this.defaultFileSystem = defaultFileSystem; } + @Override + public FileWatcher fileWatcher() throws IOException + { + return new SelectiveFileWatcher( specialFile, defaultFileSystem.fileWatcher(), specialFileSystem.fileWatcher() ); + } + @Override public StoreChannel open( File fileName, String mode ) throws IOException { diff --git a/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileWatcher.java b/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileWatcher.java new file mode 100644 index 0000000000000..15c5ea5b791e6 --- /dev/null +++ b/community/io/src/test/java/org/neo4j/graphdb/mockfs/SelectiveFileWatcher.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.graphdb.mockfs; + +import java.io.File; +import java.io.IOException; + +import org.neo4j.io.fs.watcher.FileWatcher; +import org.neo4j.io.fs.watcher.event.FileWatchEventListener; +import org.neo4j.io.fs.watcher.resource.WatchedResource; + +/** + * File watcher that will perform watching activities using specific file watcher in case if + * requested resource will match to provided {@link File specificFile}. + */ +public class SelectiveFileWatcher implements FileWatcher +{ + private File specialFile; + private final FileWatcher defaultFileWatcher; + private final FileWatcher specificFileWatcher; + + SelectiveFileWatcher( File specialFile, FileWatcher defaultFileWatcher, FileWatcher specificFileWatcher ) + { + this.specialFile = specialFile; + this.defaultFileWatcher = defaultFileWatcher; + this.specificFileWatcher = specificFileWatcher; + } + + @Override + public WatchedResource watch( File file ) throws IOException + { + return chooseFileWatcher( file ).watch( file ); + } + + @Override + public void addFileWatchEventListener( FileWatchEventListener listener ) + { + defaultFileWatcher.addFileWatchEventListener( listener ); + specificFileWatcher.addFileWatchEventListener( listener ); + } + + @Override + public void removeFileWatchEventListener( FileWatchEventListener listener ) + { + defaultFileWatcher.removeFileWatchEventListener( listener ); + specificFileWatcher.removeFileWatchEventListener( listener ); + } + + @Override + public void stopWatching() + { + defaultFileWatcher.stopWatching(); + specificFileWatcher.stopWatching(); + } + + @Override + public void startWatching() throws InterruptedException + { + defaultFileWatcher.startWatching(); + specificFileWatcher.startWatching(); + } + + @Override + public void close() throws IOException + { + defaultFileWatcher.close(); + specificFileWatcher.close(); + } + + private FileWatcher chooseFileWatcher( File file ) + { + return file.equals( specialFile ) ? specificFileWatcher : defaultFileWatcher; + } +} diff --git a/community/io/src/test/java/org/neo4j/io/fs/DefaultFileSystemAbstractionTest.java b/community/io/src/test/java/org/neo4j/io/fs/DefaultFileSystemAbstractionTest.java index 45f04f54a8a8a..61577abc6c992 100644 --- a/community/io/src/test/java/org/neo4j/io/fs/DefaultFileSystemAbstractionTest.java +++ b/community/io/src/test/java/org/neo4j/io/fs/DefaultFileSystemAbstractionTest.java @@ -21,6 +21,7 @@ import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import java.io.File; @@ -28,9 +29,12 @@ import java.util.UUID; import org.neo4j.graphdb.mockfs.CloseTrackingFileSystem; +import org.neo4j.io.fs.watcher.FileWatcher; +import org.neo4j.test.rule.TestDirectory; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -38,13 +42,16 @@ public class DefaultFileSystemAbstractionTest { + @Rule + public TestDirectory testDirectory = TestDirectory.testDirectory(); + private DefaultFileSystemAbstraction defaultFileSystemAbstraction; private File path; @Before public void before() throws Exception { - path = new File( "target/" + UUID.randomUUID() ); + path = testDirectory.file( "testFile" ); defaultFileSystemAbstraction = new DefaultFileSystemAbstraction(); } @@ -54,6 +61,15 @@ public void tearDown() throws IOException defaultFileSystemAbstraction.close(); } + @Test + public void fileWatcherCreation() throws IOException + { + try (FileWatcher fileWatcher = defaultFileSystemAbstraction.fileWatcher()) + { + assertNotNull( fileWatcher.watch( testDirectory.directory( "testDirectory" ) ) ); + } + } + @Test public void shouldCreatePath() throws Exception { @@ -97,7 +113,7 @@ public void shouldCreatePathThatPointsToFile() throws Exception @Test public void shouldFailGracefullyWhenPathCannotBeCreated() throws Exception { - path = new File( "target/" + UUID.randomUUID() ) + path = new File( testDirectory.directory(), String.valueOf( UUID.randomUUID() ) ) { @Override public boolean mkdirs() @@ -136,5 +152,4 @@ public void closeThirdPartyFileSystemsOnClose() throws IOException assertTrue( closeTrackingFileSystem.isClosed() ); } - } diff --git a/community/io/src/test/java/org/neo4j/io/fs/DelegateFileSystemAbstractionTest.java b/community/io/src/test/java/org/neo4j/io/fs/DelegateFileSystemAbstractionTest.java index 238e608db9cef..bf09557c6cb46 100644 --- a/community/io/src/test/java/org/neo4j/io/fs/DelegateFileSystemAbstractionTest.java +++ b/community/io/src/test/java/org/neo4j/io/fs/DelegateFileSystemAbstractionTest.java @@ -34,7 +34,10 @@ import org.neo4j.graphdb.mockfs.CloseTrackingFileSystem; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; public class DelegateFileSystemAbstractionTest { @@ -55,6 +58,18 @@ public void closeAllResourcesOnClose() throws Exception assertTrue( closeTrackingFileSystem.isClosed() ); } + @Test + public void delegatedFileSystemWatcher() throws IOException + { + FileSystem fileSystem = mock(FileSystem.class); + try ( DelegateFileSystemAbstraction abstraction = new DelegateFileSystemAbstraction( fileSystem ) ) + { + assertNotNull( abstraction.fileWatcher() ); + } + + verify( fileSystem ).newWatchService(); + } + private class TrackableFileSystem extends FileSystem { diff --git a/community/io/src/test/java/org/neo4j/io/fs/SelectiveFileSystemAbstractionTest.java b/community/io/src/test/java/org/neo4j/io/fs/SelectiveFileSystemAbstractionTest.java index 3b9de58eca499..5dff4e18445f7 100644 --- a/community/io/src/test/java/org/neo4j/io/fs/SelectiveFileSystemAbstractionTest.java +++ b/community/io/src/test/java/org/neo4j/io/fs/SelectiveFileSystemAbstractionTest.java @@ -22,12 +22,17 @@ import org.junit.Test; import java.io.File; +import java.io.IOException; import org.neo4j.graphdb.mockfs.SelectiveFileSystemAbstraction; +import org.neo4j.io.fs.watcher.FileWatcher; +import org.neo4j.io.fs.watcher.resource.WatchedResource; +import static org.junit.Assert.assertSame; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; public class SelectiveFileSystemAbstractionTest { @@ -75,4 +80,31 @@ public void shouldUseDefaultFileSystemForOtherFiles() throws Exception verifyNoMoreInteractions( normal ); } } + + @Test + public void provideSelectiveWatcher() throws IOException + { + File specialFile = new File("special"); + File otherFile = new File("other"); + + FileSystemAbstraction normal = mock( FileSystemAbstraction.class ); + FileSystemAbstraction special = mock( FileSystemAbstraction.class ); + + FileWatcher specialWatcher = mock( FileWatcher.class ); + FileWatcher normalWatcher = mock( FileWatcher.class ); + WatchedResource specialResource = mock( WatchedResource.class ); + WatchedResource normalResource = mock( WatchedResource.class ); + + when( special.fileWatcher() ).thenReturn( specialWatcher ); + when( normal.fileWatcher() ).thenReturn( normalWatcher ); + when( specialWatcher.watch( specialFile ) ).thenReturn( specialResource ); + when( normalWatcher.watch( otherFile ) ).thenReturn( normalResource ); + + try ( SelectiveFileSystemAbstraction fs = new SelectiveFileSystemAbstraction( specialFile, special, normal ) ) + { + FileWatcher fileWatcher = fs.fileWatcher(); + assertSame( specialResource, fileWatcher.watch( specialFile ) ); + assertSame( normalResource, fileWatcher.watch( otherFile ) ); + } + } } diff --git a/community/io/src/test/java/org/neo4j/io/fs/watcher/DefaultFileSystemWatcherTest.java b/community/io/src/test/java/org/neo4j/io/fs/watcher/DefaultFileSystemWatcherTest.java new file mode 100644 index 0000000000000..fa2d3306e27bc --- /dev/null +++ b/community/io/src/test/java/org/neo4j/io/fs/watcher/DefaultFileSystemWatcherTest.java @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.fs.watcher; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.internal.stubbing.answers.ThrowsExceptionClass; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.nio.file.Watchable; +import java.util.ArrayList; +import java.util.List; + +import org.neo4j.io.fs.watcher.event.FileWatchEventListenerAdapter; +import org.neo4j.io.fs.watcher.resource.WatchedResource; +import org.neo4j.test.rule.TestDirectory; + +import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; +import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; +import static java.util.Arrays.asList; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasItem; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DefaultFileSystemWatcherTest +{ + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public TestDirectory testDirectory = TestDirectory.testDirectory(); + private WatchService watchServiceMock = mock( WatchService.class ); + + @Test + public void fileWatchRegistrationIsIllegal() throws Exception + { + DefaultFileSystemWatcher watcher = createWatcher(); + + expectedException.expect( IllegalArgumentException.class ); + expectedException.expectMessage( "Only directories can be registered to be monitored." ); + + watcher.watch( new File("notADirectory") ); + } + + @Test + public void registerMultipleDirectoriesForMonitoring() throws Exception + { + try ( DefaultFileSystemWatcher watcher = new DefaultFileSystemWatcher( + FileSystems.getDefault().newWatchService() ) ) + { + File directory1 = testDirectory.directory( "test1" ); + File directory2 = testDirectory.directory( "test2" ); + WatchedResource watchedResource1 = watcher.watch( directory1 ); + WatchedResource watchedResource2 = watcher.watch( directory2 ); + assertNotSame( watchedResource1, watchedResource2 ); + } + } + + @Test + public void notifyListenersOnDeletion() throws InterruptedException + { + TestFileSystemWatcher watcher = createWatcher(); + AssertableFileEventListener listener1 = new AssertableFileEventListener(); + AssertableFileEventListener listener2 = new AssertableFileEventListener(); + + watcher.addFileWatchEventListener( listener1 ); + watcher.addFileWatchEventListener( listener2 ); + + TestWatchEvent watchEvent = new TestWatchEvent<>( ENTRY_DELETE, Paths.get( "file1" ) ); + TestWatchEvent watchEvent2 = new TestWatchEvent<>( ENTRY_DELETE, Paths.get( "file2" ) ); + TestWatchKey watchKey = new TestWatchKey( asList(watchEvent, watchEvent2) ); + + prepareWatcher( watchKey ); + + watch( watcher ); + + listener1.assertDeleted( "file1" ); + listener1.assertDeleted( "file2" ); + listener2.assertDeleted( "file1" ); + listener2.assertDeleted( "file2" ); + } + + @Test + public void notifyListenersOnModification() throws InterruptedException + { + TestFileSystemWatcher watcher = createWatcher(); + AssertableFileEventListener listener1 = new AssertableFileEventListener(); + AssertableFileEventListener listener2 = new AssertableFileEventListener(); + + watcher.addFileWatchEventListener( listener1 ); + watcher.addFileWatchEventListener( listener2 ); + + TestWatchEvent watchEvent = new TestWatchEvent<>( ENTRY_MODIFY, Paths.get( "a" ) ); + TestWatchEvent watchEvent2 = new TestWatchEvent<>( ENTRY_MODIFY, Paths.get( "b" ) ); + TestWatchEvent watchEvent3 = new TestWatchEvent<>( ENTRY_MODIFY, Paths.get( "c" ) ); + TestWatchKey watchKey = new TestWatchKey( asList(watchEvent, watchEvent2, watchEvent3) ); + + prepareWatcher( watchKey ); + + watch( watcher ); + + listener1.assertModified( "a" ); + listener1.assertModified( "b" ); + listener1.assertModified( "c" ); + listener2.assertModified( "a" ); + listener2.assertModified( "b" ); + listener2.assertModified( "c" ); + } + + @Test + public void stopWatchingAndCloseEverythingOnClosed() throws IOException + { + TestFileSystemWatcher watcher = createWatcher(); + watcher.close(); + + verify( watchServiceMock ).close(); + assertTrue( watcher.isClosed() ); + } + + @Test + public void skipEmptyEvent() throws InterruptedException + { + TestFileSystemWatcher watcher = createWatcher(); + + AssertableFileEventListener listener = new AssertableFileEventListener(); + watcher.addFileWatchEventListener( listener ); + + TestWatchEvent event = new TestWatchEvent( ENTRY_MODIFY, null ); + TestWatchKey watchKey = new TestWatchKey( asList( event ) ); + + prepareWatcher( watchKey ); + + watch( watcher ); + + listener.assertNoEvents(); + } + + private void prepareWatcher( TestWatchKey watchKey ) throws InterruptedException + { + when( watchServiceMock.take() ).thenReturn( watchKey ) + .thenAnswer( new ThrowsExceptionClass( InterruptedException.class ) ); + } + + private void watch( TestFileSystemWatcher watcher ) + { + try + { + watcher.startWatching(); + } + catch ( InterruptedException ignored ) + { + // expected + } + } + + private TestFileSystemWatcher createWatcher() + { + return new TestFileSystemWatcher( watchServiceMock ); + } + + private static class TestFileSystemWatcher extends DefaultFileSystemWatcher + { + + private boolean closed; + + TestFileSystemWatcher( WatchService watchService ) + { + super( watchService ); + } + + @Override + public void close() throws IOException + { + super.close(); + closed = true; + } + + public boolean isClosed() + { + return closed; + } + } + + private static class TestWatchKey implements WatchKey + { + private List> events; + private boolean canceled; + + TestWatchKey( List> events ) + { + this.events = events; + } + + @Override + public boolean isValid() + { + return false; + } + + @Override + public List> pollEvents() + { + return events; + } + + @Override + public boolean reset() + { + return false; + } + + @Override + public void cancel() + { + canceled = true; + } + + @Override + public Watchable watchable() + { + return null; + } + + public boolean isCanceled() + { + return canceled; + } + } + + private static class TestWatchEvent implements WatchEvent + { + + private Kind eventKind; + private T fileName; + + TestWatchEvent( Kind eventKind, T fileName ) + { + this.eventKind = eventKind; + this.fileName = fileName; + } + + @Override + public Kind kind() + { + return eventKind; + } + + @Override + public int count() + { + return 0; + } + + @Override + public T context() + { + return fileName; + } + } + + private static class AssertableFileEventListener extends FileWatchEventListenerAdapter { + + private final List deletedFileNames = new ArrayList<>(); + private final List modifiedFileNames = new ArrayList<>(); + + @Override + public void fileDeleted( String fileName ) + { + super.fileDeleted( fileName ); + deletedFileNames.add( fileName ); + } + + @Override + public void fileModified( String fileName ) + { + super.fileModified( fileName ); + modifiedFileNames.add( fileName ); + } + + void assertNoEvents() + { + assertThat( "Should not have any deletion events", deletedFileNames, empty() ); + assertThat( "Should not have any modification events", modifiedFileNames, empty() ); + } + + void assertDeleted( String fileName ) + { + assertThat( "Was expected to find notification about deletion.", deletedFileNames, hasItem( fileName ) ); + } + + void assertModified( String fileName ) + { + assertThat( "Was expected to find notification about modification.", modifiedFileNames, + hasItem( fileName ) ); + } + } + +} diff --git a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseFactory.java b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseFactory.java index 75040dfd05988..ec5e2ce6ff552 100644 --- a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseFactory.java +++ b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseFactory.java @@ -98,7 +98,7 @@ public GraphDatabaseService newDatabase( Map config ) @Override public GraphDatabaseService newDatabase( @Nonnull Config config ) { - return GraphDatabaseFactory.this.newDatabase( storeDir, + return GraphDatabaseFactory.this.newEmbeddedDatabase( storeDir, config.with( stringMap( "unsupported.dbms.ephemeral", "false" ) ), state.databaseDependencies() ); } @@ -120,6 +120,12 @@ protected GraphDatabaseService newDatabase( File storeDir, Map se return newDatabase( storeDir, Config.embeddedDefaults( settings ), dependencies ); } + protected GraphDatabaseService newEmbeddedDatabase( File storeDir, Config config, + GraphDatabaseFacadeFactory.Dependencies dependencies ) + { + return GraphDatabaseFactory.this.newDatabase( storeDir, config, dependencies ); + } + protected GraphDatabaseService newDatabase( File storeDir, Config config, GraphDatabaseFacadeFactory.Dependencies dependencies ) { @@ -149,4 +155,5 @@ public String getEdition() { return Edition.community.toString(); } + } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java index 92344ebd48f01..615aede27e02d 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java @@ -21,7 +21,6 @@ import java.io.File; import java.io.IOException; -import java.time.Clock; import java.util.List; import java.util.Map; @@ -31,6 +30,7 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileSystemLifecycleAdapter; +import org.neo4j.io.fs.watcher.FileWatcher; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.AvailabilityGuard; import org.neo4j.kernel.configuration.Config; @@ -49,6 +49,8 @@ import org.neo4j.kernel.impl.transaction.state.DataSourceManager; import org.neo4j.kernel.impl.util.JobScheduler; import org.neo4j.kernel.impl.util.Neo4jJobScheduler; +import org.neo4j.kernel.impl.util.watcher.DefaultFileDeletionEventListener; +import org.neo4j.kernel.impl.util.watcher.WatcherLifecycleAdapterFactory; import org.neo4j.kernel.info.DiagnosticsManager; import org.neo4j.kernel.info.JvmChecker; import org.neo4j.kernel.info.JvmMetadataRepository; @@ -170,6 +172,10 @@ public PlatformModule( File providedStoreDir, Config config, DatabaseInfo databa dependencies.satisfyDependency( firstImplementor( CheckPointerMonitor.class, tracers.checkPointTracer, CheckPointerMonitor.NULL ) ); + FileWatcher fileWatcher = createFileWatcher(); + dependencies.satisfyDependencies( fileWatcher ); + life.add( WatcherLifecycleAdapterFactory.createLifecycleAdapter( jobScheduler, fileWatcher ) ); + pageCache = dependencies.satisfyDependency( createPageCache( fileSystem, config, logging, tracers ) ); life.add( new PageCacheLifecycle( pageCache ) ); @@ -196,6 +202,24 @@ public PlatformModule( File providedStoreDir, Config config, DatabaseInfo databa publishPlatformInfo( dependencies.resolveDependency( UsageData.class ) ); } + protected FileWatcher createFileWatcher() + { + try + { + FileWatcher watcher = fileSystem.fileWatcher(); + watcher.addFileWatchEventListener( new DefaultFileDeletionEventListener( logging ) ); + watcher.watch( storeDir ); + return watcher; + } + catch ( Exception e ) + { + Log log = logging.getInternalLog( getClass() ); + log.warn( "Can not create file watcher for current file system. File monitoring capabilities for store " + + "files will be disabled.", e ); + return FileWatcher.SILENT_WATCHER; + } + } + protected SystemNanoClock createClock() { return Clocks.nanoClock(); @@ -272,9 +296,7 @@ protected Neo4jJobScheduler createJobScheduler() return new Neo4jJobScheduler(); } - protected PageCache createPageCache( FileSystemAbstraction fileSystem, - Config config, - LogService logging, + protected PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, LogService logging, Tracers tracers ) { Log pageCacheLog = logging.getInternalLog( PageCache.class ); @@ -311,4 +333,5 @@ private Iterable> getSettingsClasses( Iterable> settingsClasse return totalSettingsClasses; } + } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/util/JobScheduler.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/util/JobScheduler.java index 771bab6c107ea..82b63fc1688ee 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/util/JobScheduler.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/util/JobScheduler.java @@ -171,6 +171,11 @@ class Groups * Native security. */ public static Group nativeSecurity = new Group( "NativeSecurity", POOLED ); + + /** + * File watch service group + */ + public static Group fileWatch = new Group( "FileWatcher", NEW_THREAD ); } interface JobHandle diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/util/watcher/DefaultFileDeletionEventListener.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/util/watcher/DefaultFileDeletionEventListener.java new file mode 100644 index 0000000000000..8a84435ee8b89 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/util/watcher/DefaultFileDeletionEventListener.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.util.watcher; + +import org.neo4j.io.fs.watcher.event.FileWatchEventListenerAdapter; +import org.neo4j.kernel.impl.logging.LogService; +import org.neo4j.logging.Log; + +/** + * Listener that will print notification about deleted filename into internal log. + */ +public class DefaultFileDeletionEventListener extends FileWatchEventListenerAdapter +{ + + private final Log internalLog; + + public DefaultFileDeletionEventListener( LogService logService ) + { + this.internalLog = logService.getInternalLog( getClass() ); + } + + @Override + public void fileDeleted( String fileName ) + { + internalLog.info( "Store file '" + fileName + "' was deleted while database was online." ); + } + +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/util/watcher/FileWatcherLifecycleAdapter.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/util/watcher/FileWatcherLifecycleAdapter.java new file mode 100644 index 0000000000000..84a41882cb751 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/util/watcher/FileWatcherLifecycleAdapter.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.util.watcher; + +import java.util.concurrent.ThreadFactory; + +import org.neo4j.io.fs.watcher.FileWatcher; +import org.neo4j.kernel.impl.util.JobScheduler; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; + +/** + * Adapter that integrates file watching possibilities with platform lifecycle. + * Monitoring will be started when corresponding life will be started and stopped as soon as life will be stopped. + * + * In case of restart new monitoring thread will be started. + */ +public class FileWatcherLifecycleAdapter extends LifecycleAdapter +{ + private final JobScheduler jobScheduler; + private final FileWatcher fileWatcher; + private final FileSystemEventWatcher eventWatcher; + private ThreadFactory fileWatchers; + private Thread watcher; + + public FileWatcherLifecycleAdapter( JobScheduler jobScheduler, FileWatcher fileWatcher ) + { + this.jobScheduler = jobScheduler; + this.fileWatcher = fileWatcher; + this.eventWatcher = new FileSystemEventWatcher(); + } + + @Override + public void init() throws Throwable + { + fileWatchers = jobScheduler.threadFactory( JobScheduler.Groups.fileWatch ); + } + + @Override + public void start() throws Throwable + { + watcher = fileWatchers.newThread( eventWatcher ); + watcher.start(); + } + + @Override + public void stop() throws Throwable + { + eventWatcher.stopWatching(); + if ( watcher != null ) + { + watcher.interrupt(); + watcher.join(); + watcher = null; + } + } + + private class FileSystemEventWatcher implements Runnable + { + @Override + public void run() + { + try + { + fileWatcher.startWatching(); + } + catch ( InterruptedException ignored ) + { + } + } + + public void stopWatching() + { + fileWatcher.stopWatching(); + } + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/util/watcher/WatcherLifecycleAdapterFactory.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/util/watcher/WatcherLifecycleAdapterFactory.java new file mode 100644 index 0000000000000..946becbb8d333 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/util/watcher/WatcherLifecycleAdapterFactory.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.util.watcher; + +import org.neo4j.io.fs.watcher.FileWatcher; +import org.neo4j.kernel.impl.util.JobScheduler; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; + +/** + * Factory used for construction of proper adaptor for available {@link FileWatcher}. + * In case if silent matcher is used dummy adapter will be used, otherwise will use default wrapper that will bind + * monitoring cycles to corresponding lifecycle phases. + */ +public class WatcherLifecycleAdapterFactory +{ + + public static LifecycleAdapter createLifecycleAdapter( JobScheduler jobScheduler, FileWatcher fileWatcher ) + { + return FileWatcher.SILENT_WATCHER.equals( fileWatcher ) ? new LifecycleAdapter() + : new FileWatcherLifecycleAdapter( jobScheduler, + fileWatcher ); + } +} diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java index cc4914ee98e89..d63f134674fb3 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java @@ -260,8 +260,7 @@ private void importRelationships( NodeRelationshipCache nodeRelationshipCache, Set minorityRelationshipTypeSet = asSet( minorityRelationshipTypes ); PerTypeRelationshipSplitter perTypeIterator = new PerTypeRelationshipSplitter( relationships.iterator(), - allRelationshipTypes, - type -> minorityRelationshipTypeSet.contains( type ), + allRelationshipTypes, minorityRelationshipTypeSet::contains, neoStore.getRelationshipTypeRepository(), inputCache ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/util/FileWatcherLifecycleAdapterTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/util/FileWatcherLifecycleAdapterTest.java new file mode 100644 index 0000000000000..e61272b3762ec --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/util/FileWatcherLifecycleAdapterTest.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.util; + + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.concurrent.CountDownLatch; + +import org.neo4j.io.fs.watcher.FileWatcher; +import org.neo4j.io.fs.watcher.SilentFileWatcher; +import org.neo4j.kernel.impl.util.watcher.FileWatcherLifecycleAdapter; + +import static org.mockito.Mockito.verify; + +public class FileWatcherLifecycleAdapterTest +{ + + private static Neo4jJobScheduler jobScheduler; + private FileWatcher fileWatcher = Mockito.mock( FileWatcher.class ); + + @BeforeClass + public static void setUp() + { + jobScheduler = new Neo4jJobScheduler(); + } + + @AfterClass + public static void tearDown() + { + jobScheduler.shutdown(); + } + + @Test + public void startMonitoringWhenLifecycleStarting() throws Throwable + { + CountDownLatch latch = new CountDownLatch( 1 ); + FileWatcher watcher = new TestFileWatcher( latch ); + FileWatcherLifecycleAdapter watcherAdapter = new FileWatcherLifecycleAdapter( jobScheduler, watcher ); + watcherAdapter.init(); + watcherAdapter.start(); + + latch.await(); + } + + @Test + public void stopMonitoringWhenLifecycleStops() throws Throwable + { + FileWatcherLifecycleAdapter watcherAdapter = new FileWatcherLifecycleAdapter( jobScheduler, fileWatcher ); + watcherAdapter.init(); + watcherAdapter.start(); + watcherAdapter.stop(); + + verify( fileWatcher ).stopWatching(); + } + + private static class TestFileWatcher extends SilentFileWatcher + { + + private CountDownLatch latch; + + TestFileWatcher( CountDownLatch latch ) + { + this.latch = latch; + } + + @Override + public void startWatching() throws InterruptedException + { + latch.countDown(); + } + } +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/util/watcher/DefaultFileDeletionEventListenerTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/util/watcher/DefaultFileDeletionEventListenerTest.java new file mode 100644 index 0000000000000..5918ca5ff7e6d --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/util/watcher/DefaultFileDeletionEventListenerTest.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.util.watcher; + +import org.junit.Test; + +import org.neo4j.kernel.impl.logging.SimpleLogService; +import org.neo4j.logging.AssertableLogProvider; +import org.neo4j.logging.NullLogProvider; + +public class DefaultFileDeletionEventListenerTest +{ + + @Test + public void notificationInLogAboutFileDeletion() throws Exception + { + AssertableLogProvider internalLogProvider = new AssertableLogProvider( false ); + SimpleLogService logService = new SimpleLogService( NullLogProvider.getInstance(), internalLogProvider ); + DefaultFileDeletionEventListener listener = new DefaultFileDeletionEventListener( logService ); + listener.fileDeleted( "testFile" ); + listener.fileDeleted( "anotherFile" ); + + internalLogProvider.assertContainsMessageContaining( "Store file 'testFile' was " + + "deleted while database was online." ); + internalLogProvider.assertContainsMessageContaining( "Store file 'anotherFile' was " + + "deleted while database was online." ); + } +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/util/watcher/WatcherLifecycleAdapterFactoryTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/util/watcher/WatcherLifecycleAdapterFactoryTest.java new file mode 100644 index 0000000000000..6162d5007461c --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/util/watcher/WatcherLifecycleAdapterFactoryTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.util.watcher; + +import org.hamcrest.Matchers; +import org.junit.Test; + +import org.neo4j.io.fs.watcher.FileWatcher; +import org.neo4j.kernel.impl.util.JobScheduler; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; + +public class WatcherLifecycleAdapterFactoryTest +{ + + private JobScheduler scheduler = mock( JobScheduler.class ); + + @Test + public void createDummyAdapterForSilentWatcher() + { + LifecycleAdapter adapter = + WatcherLifecycleAdapterFactory.createLifecycleAdapter( scheduler, FileWatcher.SILENT_WATCHER ); + assertEquals( adapter.getClass(), LifecycleAdapter.class ); + } + + @Test + public void createDefaultWatcherAdapter() + { + LifecycleAdapter adapter = + WatcherLifecycleAdapterFactory.createLifecycleAdapter( scheduler, mock( FileWatcher.class ) ); + assertThat( adapter, Matchers.instanceOf( FileWatcherLifecycleAdapter.class ) ); + } +} diff --git a/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java b/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java index f16fd5ef635b9..1ec92f82405d3 100644 --- a/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java +++ b/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java @@ -51,7 +51,8 @@ public static GraphDatabaseFactory create( FileSystemAbstraction fs, Adversary a return new TestGraphDatabaseFactory() { @Override - protected GraphDatabaseService newDatabase( File dir, Config config, Dependencies dependencies ) + protected GraphDatabaseService newEmbeddedDatabase( File dir, Config config, Dependencies + dependencies ) { return new GraphDatabaseFacadeFactory( DatabaseInfo.COMMUNITY, CommunityEditionModule::new ) { diff --git a/community/kernel/src/test/java/org/neo4j/test/TestGraphDatabaseFactory.java b/community/kernel/src/test/java/org/neo4j/test/TestGraphDatabaseFactory.java index 88ee2d4f92ae9..799959f5dfbb8 100644 --- a/community/kernel/src/test/java/org/neo4j/test/TestGraphDatabaseFactory.java +++ b/community/kernel/src/test/java/org/neo4j/test/TestGraphDatabaseFactory.java @@ -22,6 +22,7 @@ import java.io.File; import java.util.Collections; import java.util.Map; +import java.util.function.Function; import javax.annotation.Nonnull; import org.neo4j.graphdb.GraphDatabaseService; @@ -29,6 +30,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseBuilder; import org.neo4j.graphdb.factory.GraphDatabaseFactory; import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.graphdb.mockfs.UncloseableDelegatingFileSystemAbstraction; import org.neo4j.graphdb.security.URLAccessRule; import org.neo4j.io.fs.FileSystemAbstraction; @@ -38,6 +40,7 @@ import org.neo4j.kernel.extension.KernelExtensionFactory; import org.neo4j.kernel.impl.factory.CommunityEditionModule; import org.neo4j.kernel.impl.factory.DatabaseInfo; +import org.neo4j.kernel.impl.factory.EditionModule; import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; import org.neo4j.kernel.impl.factory.PlatformModule; @@ -47,7 +50,11 @@ import org.neo4j.logging.LogProvider; import org.neo4j.logging.NullLogProvider; +import static org.neo4j.helpers.collection.MapUtil.stringMap; import static org.neo4j.kernel.configuration.Connector.ConnectorType.BOLT; +import static org.neo4j.kernel.configuration.Settings.TRUE; +import static org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory.Configuration.ephemeral; + /** * Test factory for graph databases. @@ -185,12 +192,20 @@ private TestGraphDatabaseBuilder createImpermanentGraphDatabaseBuilder( return new TestGraphDatabaseBuilder( creator ); } + @Override + protected GraphDatabaseService newEmbeddedDatabase( File storeDir, Config config, + GraphDatabaseFacadeFactory.Dependencies dependencies ) + { + return new TestGraphDatabaseFacadeFactory( getCurrentState() ).newFacade( storeDir, config, + GraphDatabaseDependencies.newDependencies( dependencies ) ); + } + protected GraphDatabaseBuilder.DatabaseCreator createImpermanentDatabaseCreator( final File storeDir, final TestGraphDatabaseFactoryState state ) { + return new GraphDatabaseBuilder.DatabaseCreator() { - @Override public GraphDatabaseService newDatabase( Map config ) { return newDatabase( Config.embeddedDefaults( config ) ); @@ -199,61 +214,116 @@ public GraphDatabaseService newDatabase( Map config ) @Override public GraphDatabaseService newDatabase( @Nonnull Config config ) { - return new GraphDatabaseFacadeFactory( DatabaseInfo.COMMUNITY, CommunityEditionModule::new ) + return new TestGraphDatabaseFacadeFactory( state, true ).newFacade( storeDir, config, + GraphDatabaseDependencies.newDependencies( state.databaseDependencies() ) ); + } + }; + } + + static class TestGraphDatabaseFacadeFactory extends GraphDatabaseFacadeFactory + { + private final TestGraphDatabaseFactoryState state; + private final boolean impermanent; + + TestGraphDatabaseFacadeFactory( TestGraphDatabaseFactoryState state, boolean impermanent ) + { + this( state, impermanent, DatabaseInfo.COMMUNITY, CommunityEditionModule::new ); + } + + TestGraphDatabaseFacadeFactory( TestGraphDatabaseFactoryState state, boolean impermanent, + DatabaseInfo databaseInfo, Function editionFactory ) + { + super(databaseInfo, editionFactory); + this.state = state; + this.impermanent = impermanent; + } + + + TestGraphDatabaseFacadeFactory( TestGraphDatabaseFactoryState state ) + { + this(state, false); + } + + @Override + protected PlatformModule createPlatform( File storeDir, Config config, + Dependencies dependencies, GraphDatabaseFacade graphDatabaseFacade ) + { + return impermanent ? + new ImpermanentTestDatabasePlatformModule( storeDir, config.with( stringMap( ephemeral.name(), TRUE ) ), + dependencies, graphDatabaseFacade, this.databaseInfo ) : + new TestDatabasePlatformModule( storeDir, config, dependencies, graphDatabaseFacade, this + .databaseInfo ); + } + + private class TestDatabasePlatformModule extends PlatformModule + { + + TestDatabasePlatformModule( File storeDir, Config config, Dependencies dependencies, + GraphDatabaseFacade graphDatabaseFacade, DatabaseInfo databaseInfo ) + { + super( storeDir, config, databaseInfo, dependencies, + graphDatabaseFacade ); + } + + + + @Override + protected FileSystemAbstraction createFileSystemAbstraction() + { + FileSystemAbstraction fs = state.getFileSystem(); + if ( fs != null ) + { + return fs; + } + else + { + return createNewFileSystem(); + } + } + + protected FileSystemAbstraction createNewFileSystem() + { + return super.createFileSystemAbstraction(); + } + + protected LogService createLogService( LogProvider logProvider ) + { + final LogProvider internalLogProvider = state.getInternalLogProvider(); + if ( internalLogProvider == null ) + { + return super.createLogService( logProvider ); + } + + final LogProvider userLogProvider = state.databaseDependencies().userLogProvider(); + return new AbstractLogService() { @Override - protected PlatformModule createPlatform( File storeDir, Config config, - Dependencies dependencies, GraphDatabaseFacade graphDatabaseFacade ) + public LogProvider getUserLogProvider() { - return new ImpermanentGraphDatabase.ImpermanentPlatformModule( storeDir, config, databaseInfo, - dependencies, graphDatabaseFacade ) - { - @Override - protected FileSystemAbstraction createFileSystemAbstraction() - { - FileSystemAbstraction fs = state.getFileSystem(); - if ( fs != null ) - { - return fs; - } - else - { - return super.createFileSystemAbstraction(); - } - } - - @Override - protected LogService createLogService( LogProvider logProvider ) - { - final LogProvider internalLogProvider = state.getInternalLogProvider(); - if ( internalLogProvider == null ) - { - return super.createLogService( logProvider ); - } - - final LogProvider userLogProvider = state.databaseDependencies().userLogProvider(); - return new AbstractLogService() - { - @Override - public LogProvider getUserLogProvider() - { - return userLogProvider; - } - - @Override - public LogProvider getInternalLogProvider() - { - return internalLogProvider; - } - }; - } - - }; + return userLogProvider; } - }.newFacade( storeDir, config, - GraphDatabaseDependencies.newDependencies( state.databaseDependencies() ) ); + @Override + public LogProvider getInternalLogProvider() + { + return internalLogProvider; + } + }; } - }; + } + + private class ImpermanentTestDatabasePlatformModule extends TestDatabasePlatformModule { + + ImpermanentTestDatabasePlatformModule( File storeDir, Config config, + Dependencies dependencies, GraphDatabaseFacade graphDatabaseFacade, DatabaseInfo databaseInfo ) + { + super( storeDir, config, dependencies, graphDatabaseFacade, databaseInfo ); + } + + protected FileSystemAbstraction createNewFileSystem() + { + return new EphemeralFileSystemAbstraction(); + } + } } } diff --git a/community/neo4j/src/test/java/org/neo4j/store/watch/FileWatchIT.java b/community/neo4j/src/test/java/org/neo4j/store/watch/FileWatchIT.java new file mode 100644 index 0000000000000..1f5e549bd9705 --- /dev/null +++ b/community/neo4j/src/test/java/org/neo4j/store/watch/FileWatchIT.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.store.watch; + +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.CountDownLatch; + +import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Transaction; +import org.neo4j.io.fs.DefaultFileSystemAbstraction; +import org.neo4j.io.fs.FileUtils; +import org.neo4j.io.fs.watcher.FileWatcher; +import org.neo4j.io.fs.watcher.event.FileWatchEventListener; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.internal.GraphDatabaseAPI; +import org.neo4j.logging.AssertableLogProvider; +import org.neo4j.test.TestGraphDatabaseFactory; +import org.neo4j.test.rule.CleanupRule; +import org.neo4j.test.rule.TestDirectory; + +import static org.junit.Assert.assertTrue; + +public class FileWatchIT +{ + + @Rule + public TestDirectory testDirectory = TestDirectory.testDirectory(); + @Rule + public CleanupRule cleanupRule = new CleanupRule(); + + @Test + public void notifyAboutStoreFileDeletion() throws Exception + { + File storeDir = testDirectory.graphDbDir(); + AssertableLogProvider logProvider = new AssertableLogProvider( true ); + GraphDatabaseService database = + new TestGraphDatabaseFactory().setInternalLogProvider( logProvider ).newEmbeddedDatabase( storeDir ); + cleanupRule.add( database ); + + CountDownLatch modificationLatch = new CountDownLatch( 1 ); + CountDownLatch deletionLatch = new CountDownLatch( 1 ); + GraphDatabaseAPI databaseAPI = (GraphDatabaseAPI) database; + DependencyResolver dependencyResolver = databaseAPI.getDependencyResolver(); + FileWatcher fileWatcher = dependencyResolver.resolveDependency( FileWatcher.class ); + fileWatcher.addFileWatchEventListener( new TestLatchEventListener( deletionLatch, modificationLatch ) ); + + createTestNode( database ); + modificationLatch.await(); + + deleteMetadataStore( storeDir ); + deletionLatch.await(); + + logProvider.assertContainsMessageContaining( + "Store file '" + MetaDataStore.DEFAULT_NAME + "' was deleted while database was online." ); + } + + @Test + public void notifyWhenFileWatchingFailToStart() + { + AssertableLogProvider logProvider = new AssertableLogProvider( true ); + GraphDatabaseService database = new TestGraphDatabaseFactory() + .setInternalLogProvider( logProvider ) + .setFileSystem( new NonWatchableFileSystemAbstraction() ) + .newEmbeddedDatabase( testDirectory.graphDbDir() ); + cleanupRule.add( database ); + + logProvider.assertContainsMessageContaining( "Can not create file watcher for current file system. " + + "File monitoring capabilities for store files will be disabled." ); + } + + + private void deleteMetadataStore( File storeDir ) + { + File metadataStore = new File( storeDir, MetaDataStore.DEFAULT_NAME ); + FileUtils.deleteFile( metadataStore ); + } + + private void createTestNode( GraphDatabaseService database ) + { + try ( Transaction transaction = database.beginTx() ) + { + database.createNode(); + transaction.success(); + } + } + + private static class NonWatchableFileSystemAbstraction extends DefaultFileSystemAbstraction + { + @Override + public FileWatcher fileWatcher() throws IOException + { + throw new IOException( "You can't watch me!"); + } + } + + private static class TestLatchEventListener implements FileWatchEventListener + { + private final CountDownLatch deletionLatch; + private final CountDownLatch modificationLatch; + + TestLatchEventListener( CountDownLatch deletionLatch, CountDownLatch modificationLatch ) + { + this.deletionLatch = deletionLatch; + this.modificationLatch = modificationLatch; + } + + @Override + public void fileDeleted( String fileName ) + { + assertTrue( fileName.endsWith( MetaDataStore.DEFAULT_NAME ) ); + deletionLatch.countDown(); + } + + @Override + public void fileModified( String fileName ) + { + modificationLatch.countDown(); + } + } +} diff --git a/community/neo4j/src/test/java/schema/MultipleIndexPopulationStressIT.java b/community/neo4j/src/test/java/schema/MultipleIndexPopulationStressIT.java index aa2b34e3bf6c0..cd7d13eab1b63 100644 --- a/community/neo4j/src/test/java/schema/MultipleIndexPopulationStressIT.java +++ b/community/neo4j/src/test/java/schema/MultipleIndexPopulationStressIT.java @@ -44,6 +44,7 @@ import org.neo4j.helpers.TimeUtil; import org.neo4j.helpers.collection.Iterables; import org.neo4j.helpers.collection.PrefetchingIterator; +import org.neo4j.io.fs.watcher.FileWatcher; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.impl.api.index.BatchingMultipleIndexPopulator; diff --git a/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java b/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java index 308de52aaa146..6eace9627e36a 100644 --- a/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java +++ b/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java @@ -242,7 +242,7 @@ public void shouldUpgradeAnOldFormatStore() throws IOException, ConsistencyCheck } @Test - public void shouldHaltUpgradeIfUpgradeConfigurationVetoesTheProcess() + public void shouldHaltUpgradeIfUpgradeConfigurationVetoesTheProcess() throws IOException { PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); Config deniedMigrationConfig = Config.embeddedDefaults( MapUtil.stringMap( GraphDatabaseSettings @@ -492,16 +492,18 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT } private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, Config config, PageCache pageCache ) + throws IOException { return newUpgrader( upgradableDatabase, pageCache, config ); } - private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCache pageCache ) + private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCache pageCache ) throws IOException { return newUpgrader( upgradableDatabase, pageCache, allowMigrateConfig ); } private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCache pageCache, Config config ) + throws IOException { check = new StoreVersionCheck( pageCache ); SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); diff --git a/enterprise/kernel/src/test/java/org/neo4j/test/TestEnterpriseGraphDatabaseFactory.java b/enterprise/kernel/src/test/java/org/neo4j/test/TestEnterpriseGraphDatabaseFactory.java index bc34e9eee6c69..ad51e8820beb9 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/test/TestEnterpriseGraphDatabaseFactory.java +++ b/enterprise/kernel/src/test/java/org/neo4j/test/TestEnterpriseGraphDatabaseFactory.java @@ -26,18 +26,11 @@ import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.factory.GraphDatabaseBuilder; import org.neo4j.graphdb.factory.GraphDatabaseFactoryState; -import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.GraphDatabaseDependencies; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.enterprise.EnterpriseEditionModule; import org.neo4j.kernel.impl.factory.DatabaseInfo; import org.neo4j.kernel.impl.factory.Edition; -import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; -import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; -import org.neo4j.kernel.impl.factory.PlatformModule; -import org.neo4j.kernel.impl.logging.AbstractLogService; -import org.neo4j.kernel.impl.logging.LogService; -import org.neo4j.logging.LogProvider; /** * Factory for test graph database. @@ -46,9 +39,10 @@ public class TestEnterpriseGraphDatabaseFactory extends TestGraphDatabaseFactory { @Override protected GraphDatabaseBuilder.DatabaseCreator createDatabaseCreator( File storeDir, - GraphDatabaseFactoryState state ) + GraphDatabaseFactoryState state ) { - return config -> { + return config -> + { config.put( "unsupported.dbms.ephemeral", "false" ); return new EnterpriseGraphDatabase( storeDir, config, state.databaseDependencies() ); }; @@ -69,63 +63,21 @@ public GraphDatabaseService newDatabase( Map config ) @Override public GraphDatabaseService newDatabase( Config config ) { - return new GraphDatabaseFacadeFactory( DatabaseInfo.ENTERPRISE, EnterpriseEditionModule::new ) - { - @Override - protected PlatformModule createPlatform( File storeDir, Config config, - Dependencies dependencies, GraphDatabaseFacade graphDatabaseFacade ) - { - return new ImpermanentGraphDatabase.ImpermanentPlatformModule( storeDir, config, databaseInfo, - dependencies, graphDatabaseFacade ) - { - @Override - protected FileSystemAbstraction createFileSystemAbstraction() - { - FileSystemAbstraction fs = state.getFileSystem(); - if ( fs != null ) - { - return fs; - } - else - { - return super.createFileSystemAbstraction(); - } - } - - @Override - protected LogService createLogService( LogProvider logProvider ) - { - final LogProvider internalLogProvider = state.getInternalLogProvider(); - if ( internalLogProvider == null ) - { - return super.createLogService( logProvider ); - } - - final LogProvider userLogProvider = state.databaseDependencies().userLogProvider(); - return new AbstractLogService() - { - @Override - public LogProvider getUserLogProvider() - { - return userLogProvider; - } - - @Override - public LogProvider getInternalLogProvider() - { - return internalLogProvider; - } - }; - } - - }; - } - }.newFacade( storeDir, config, + return new TestEnterpriseGraphDatabaseFacadeFactory( state, true ).newFacade( storeDir, config, GraphDatabaseDependencies.newDependencies( state.databaseDependencies() ) ); } }; } + static class TestEnterpriseGraphDatabaseFacadeFactory extends TestGraphDatabaseFacadeFactory + { + + TestEnterpriseGraphDatabaseFacadeFactory( TestGraphDatabaseFactoryState state, boolean impermanent ) + { + super( state, impermanent, DatabaseInfo.ENTERPRISE, EnterpriseEditionModule::new ); + } + } + @Override public String getEdition() { diff --git a/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom20IT.java b/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom20IT.java index 55ff19d94719c..ab8256b1b7c61 100644 --- a/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom20IT.java +++ b/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom20IT.java @@ -43,6 +43,7 @@ import org.neo4j.graphdb.factory.EnterpriseGraphDatabaseFactory; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.fs.watcher.FileWatcher; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory; import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProvider;