From f46cbe76d20ec362902213d511c392818f4b5352 Mon Sep 17 00:00:00 2001 From: MishaDemianenko Date: Tue, 27 Mar 2018 20:31:35 +0200 Subject: [PATCH] Add possibility to archive whole fusion index when GraphDatabaseSettings.archive_failed_index is configured --- .../schema/fusion/FusionIndexPopulator.java | 6 +- .../schema/fusion/FusionIndexProvider.java | 74 ++++++++++++++++--- .../fusion/FusionIndexPopulatorTest.java | 4 +- .../fusion/FusionIndexProviderTest.java | 2 +- .../schema/LuceneIndexProviderFactory.java | 3 +- ...iveLuceneFusionIndexProviderFactory10.java | 3 +- ...iveLuceneFusionIndexProviderFactory20.java | 3 +- .../index/IndexFailureOnStartupTest.java | 18 ++++- 8 files changed, 89 insertions(+), 24 deletions(-) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulator.java index f62a51d7f1d39..ee2f692940660 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulator.java @@ -38,18 +38,20 @@ class FusionIndexPopulator extends FusionIndexBase implements In { private final long indexId; private final DropAction dropAction; + private final boolean archiveFailedIndex; - FusionIndexPopulator( IndexPopulator[] populators, Selector selector, long indexId, DropAction dropAction ) + FusionIndexPopulator( IndexPopulator[] populators, Selector selector, long indexId, DropAction dropAction, boolean archiveFailedIndex ) { super( populators, selector ); this.indexId = indexId; this.dropAction = dropAction; + this.archiveFailedIndex = archiveFailedIndex; } @Override public void create() throws IOException { - dropAction.drop( indexId ); + dropAction.drop( indexId, archiveFailedIndex ); forAll( IndexPopulator::create, instances ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexProvider.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexProvider.java index 2605489253b37..dc609d8e05d22 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexProvider.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexProvider.java @@ -19,14 +19,26 @@ */ package org.neo4j.kernel.impl.index.schema.fusion; +import java.io.File; import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.helpers.collection.MapUtil; import org.neo4j.internal.kernel.api.IndexCapability; import org.neo4j.internal.kernel.api.IndexOrder; import org.neo4j.internal.kernel.api.IndexQuery; import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.io.fs.FileHandle; import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.fs.FileUtils; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.api.index.IndexAccessor; import org.neo4j.kernel.api.index.IndexDirectoryStructure; @@ -40,6 +52,7 @@ import org.neo4j.values.storable.Value; import org.neo4j.values.storable.ValueGroup; +import static java.util.stream.Collectors.toList; import static org.neo4j.internal.kernel.api.InternalIndexState.FAILED; import static org.neo4j.internal.kernel.api.InternalIndexState.POPULATING; import static org.neo4j.kernel.impl.index.schema.fusion.FusionIndexBase.INSTANCE_COUNT; @@ -74,6 +87,7 @@ default T select( T[] instances, Value... values ) IndexReader select( IndexReader[] instances, IndexQuery... predicates ); } + private final boolean archiveFailedIndex; private final IndexProvider[] providers = new IndexProvider[INSTANCE_COUNT]; private final Selector selector; private final DropAction dropAction; @@ -89,11 +103,13 @@ public FusionIndexProvider( Descriptor descriptor, int priority, IndexDirectoryStructure.Factory directoryStructure, - FileSystemAbstraction fs ) + FileSystemAbstraction fs, + boolean archiveFailedIndex ) { super( descriptor, priority, directoryStructure ); fillProvidersArray( stringProvider, numberProvider, spatialProvider, temporalProvider, luceneProvider ); selector.validateSatisfied( providers ); + this.archiveFailedIndex = archiveFailedIndex; this.selector = selector; this.dropAction = new FileSystemDropAction( fs, directoryStructure() ); } @@ -112,7 +128,7 @@ private void fillProvidersArray( IndexProvider stringProvider, IndexProvider num public IndexPopulator getPopulator( long indexId, SchemaIndexDescriptor descriptor, IndexSamplingConfig samplingConfig ) { return new FusionIndexPopulator( instancesAs( providers, IndexPopulator.class, - provider -> provider.getPopulator( indexId, descriptor, samplingConfig ) ), selector, indexId, dropAction ); + provider -> provider.getPopulator( indexId, descriptor, samplingConfig ) ), selector, indexId, dropAction, archiveFailedIndex ); } @Override @@ -197,26 +213,33 @@ public IndexOrder[] orderCapability( ValueGroup... valueGroups ) @Override public StoreMigrationParticipant storeMigrationParticipant( FileSystemAbstraction fs, PageCache pageCache ) { - // TODO implementation of this depends on decisions around defaults and migration. Coming soon. return StoreMigrationParticipant.NOT_PARTICIPATING; } - /** - * As an interface because this is actually dependent on whether or not an index lives on a {@link FileSystemAbstraction} - * or a page cache. At the time of writing this there's only the possibility to put these on the file system, - * but there will be a possibility to put these in the page cache file management instead and having this abstracted - * will help when making that switch/decision. - */ @FunctionalInterface interface DropAction { /** * Deletes the index directory and everything in it, as last part of dropping an index. + * Can be configured to create archive with content of index directories for future analysis. * * @param indexId the index id, for which directory to drop. + * @param archiveExistentIndex create archive with content of dropped directories * @throws IOException on I/O error. + * @see GraphDatabaseSettings#archive_failed_index */ - void drop( long indexId ) throws IOException; + void drop( long indexId, boolean archiveExistentIndex ) throws IOException; + + /** + * Deletes the index directory and everything in it, as last part of dropping an index. + * + * @param indexId the index id, for which directory to drop. + * @throws IOException on I/O error. + */ + default void drop( long indexId ) throws IOException + { + drop( indexId, false ); + } } private static class FileSystemDropAction implements DropAction @@ -231,9 +254,36 @@ private static class FileSystemDropAction implements DropAction } @Override - public void drop( long indexId ) throws IOException + public void drop( long indexId, boolean archiveExistentIndex ) throws IOException + { + File rootIndexDirectory = directoryStructure.directoryForIndex( indexId ); + if ( archiveExistentIndex && !FileUtils.isEmptyDirectory( rootIndexDirectory ) ) + { + Map env = MapUtil.stringMap( "create", "true" ); + Path rootPath = rootIndexDirectory.toPath(); + URI archiveAbsoluteURI = URI.create( "jar:file:" + archiveFile( rootIndexDirectory ).toPath() ); + + try ( FileSystem zipFs = FileSystems.newFileSystem( archiveAbsoluteURI, env ) ) + { + List fileHandles = fs.streamFilesRecursive( rootIndexDirectory ).collect( toList() ); + for ( FileHandle fileHandle : fileHandles ) + { + Path sourcePath = fileHandle.getFile().toPath(); + Path zipFsPath = zipFs.getPath( rootPath.relativize( sourcePath ).toString() ); + if ( zipFsPath.getParent() != null ) + { + Files.createDirectories( zipFsPath.getParent() ); + } + Files.copy( sourcePath, zipFsPath ); + } + } + } + fs.deleteRecursively( rootIndexDirectory ); + } + + private File archiveFile( File folder ) { - fs.deleteRecursively( directoryStructure.directoryForIndex( indexId ) ); + return new File( folder.getParent(), "archive-" + folder.getName() + "-" + System.currentTimeMillis() + ".zip" ); } } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulatorTest.java index 270eee3db51cd..4a2190609a964 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulatorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexPopulatorTest.java @@ -119,7 +119,7 @@ private void initiateMocks() throw new RuntimeException(); } } - fusionIndexPopulator = new FusionIndexPopulator( populators, fusionVersion.selector(), indexId, dropAction ); + fusionIndexPopulator = new FusionIndexPopulator( populators, fusionVersion.selector(), indexId, dropAction, false ); } private void resetMocks() @@ -150,7 +150,7 @@ public void createRemoveAnyLeftOversThatWasThereInIndexDirectoryBeforePopulation { fusionIndexPopulator.create(); - verify( dropAction ).drop( indexId ); + verify( dropAction ).drop( indexId, false ); } @Test diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexProviderTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexProviderTest.java index 09d4cd5ca6429..46b0438b12344 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexProviderTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexProviderTest.java @@ -318,7 +318,7 @@ private void setupMocks() providers[SPATIAL], providers[TEMPORAL], providers[LUCENE], - fusionVersion.selector(), DESCRIPTOR, 10, NONE, mock( FileSystemAbstraction.class ) ); + fusionVersion.selector(), DESCRIPTOR, 10, NONE, mock( FileSystemAbstraction.class ), true ); } private IndexProvider mockProvider( Class providerClass, String name ) diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneIndexProviderFactory.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneIndexProviderFactory.java index 428a84a16b906..dba6e14b3d0f2 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneIndexProviderFactory.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneIndexProviderFactory.java @@ -96,6 +96,7 @@ public static FusionIndexProvider newInstance( PageCache pageCache, File storeDi RecoveryCleanupWorkCollector recoveryCleanupWorkCollector ) { boolean readOnly = IndexProviderFactoryUtil.isReadOnly( config, operationalMode ); + boolean archiveFailedIndex = config.get( GraphDatabaseSettings.archive_failed_index ); IndexDirectoryStructure.Factory baseDirStructure = directoriesByProviderKey( storeDir ); IndexDirectoryStructure.Factory childDirectoryStructure = directoriesBySubProvider( baseDirStructure.forProvider( PROVIDER_DESCRIPTOR ) ); @@ -112,7 +113,7 @@ public static FusionIndexProvider newInstance( PageCache pageCache, File storeDi priority = 100; } return new FusionIndexProvider( EMPTY, EMPTY, spatial, temporal, lucene, new FusionSelector00(), - PROVIDER_DESCRIPTOR, priority, directoriesByProvider( storeDir ), fs ); + PROVIDER_DESCRIPTOR, priority, directoriesByProvider( storeDir ), fs, archiveFailedIndex ); } } diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/NativeLuceneFusionIndexProviderFactory10.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/NativeLuceneFusionIndexProviderFactory10.java index e11367eb3cc18..258842c52d27c 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/NativeLuceneFusionIndexProviderFactory10.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/NativeLuceneFusionIndexProviderFactory10.java @@ -77,6 +77,7 @@ public static FusionIndexProvider create( PageCache pageCache, File storeDir, Fi { IndexDirectoryStructure.Factory childDirectoryStructure = subProviderDirectoryStructure( storeDir ); boolean readOnly = IndexProviderFactoryUtil.isReadOnly( config, operationalMode ); + boolean archiveFailedIndex = config.get( GraphDatabaseSettings.archive_failed_index ); NumberIndexProvider number = IndexProviderFactoryUtil.numberProvider( pageCache, fs, childDirectoryStructure, monitor, recoveryCleanupWorkCollector, readOnly ); @@ -93,7 +94,7 @@ public static FusionIndexProvider create( PageCache pageCache, File storeDir, Fi priority = 100; } return new FusionIndexProvider( EMPTY, number, spatial, temporal, lucene, new FusionSelector10(), - DESCRIPTOR, priority, directoriesByProvider( storeDir ), fs ); + DESCRIPTOR, priority, directoriesByProvider( storeDir ), fs, archiveFailedIndex ); } private static IndexDirectoryStructure.Factory subProviderDirectoryStructure( File storeDir ) diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/NativeLuceneFusionIndexProviderFactory20.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/NativeLuceneFusionIndexProviderFactory20.java index e5b21bcb8aad3..62582fd352b75 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/NativeLuceneFusionIndexProviderFactory20.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/NativeLuceneFusionIndexProviderFactory20.java @@ -77,6 +77,7 @@ public static FusionIndexProvider create( PageCache pageCache, File storeDir, Fi { IndexDirectoryStructure.Factory childDirectoryStructure = subProviderDirectoryStructure( storeDir ); boolean readOnly = IndexProviderFactoryUtil.isReadOnly( config, operationalMode ); + boolean archiveFailedIndex = config.get( GraphDatabaseSettings.archive_failed_index ); StringIndexProvider string = IndexProviderFactoryUtil.stringProvider( pageCache, fs, childDirectoryStructure, monitor, recoveryCleanupWorkCollector, readOnly ); @@ -95,7 +96,7 @@ public static FusionIndexProvider create( PageCache pageCache, File storeDir, Fi priority = 100; } return new FusionIndexProvider( string, number, spatial, temporal, lucene, new FusionSelector20(), - DESCRIPTOR, priority, directoriesByProvider( storeDir ), fs ); + DESCRIPTOR, priority, directoriesByProvider( storeDir ), fs, archiveFailedIndex ); } public static IndexDirectoryStructure.Factory subProviderDirectoryStructure( File storeDir ) diff --git a/community/neo4j/src/test/java/org/neo4j/index/IndexFailureOnStartupTest.java b/community/neo4j/src/test/java/org/neo4j/index/IndexFailureOnStartupTest.java index 62545bb78b7d8..3c02c6979ceec 100644 --- a/community/neo4j/src/test/java/org/neo4j/index/IndexFailureOnStartupTest.java +++ b/community/neo4j/src/test/java/org/neo4j/index/IndexFailureOnStartupTest.java @@ -35,6 +35,7 @@ import org.neo4j.graphdb.schema.Schema; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.api.index.IndexDirectoryStructure; import org.neo4j.test.rule.DatabaseRule; import org.neo4j.test.rule.EmbeddedDatabaseRule; @@ -127,9 +128,8 @@ private File archiveFile() throws IOException { try ( FileSystemAbstraction fs = new DefaultFileSystemAbstraction() ) { - File indexDir = soleIndexDir( db.getStoreDir() ); - File[] files = indexDir.getParentFile() - .listFiles( pathname -> pathname.isFile() && pathname.getName().startsWith( "archive-" ) ); + File indexDir = indexRootDirectory( db.getStoreDir() ); + File[] files = indexDir.listFiles( pathname -> pathname.isFile() && pathname.getName().startsWith( "archive-" ) ); if ( files == null || files.length == 0 ) { return null; @@ -197,8 +197,18 @@ public void run( FileSystemAbstraction fs, File base ) } } + private static File indexRootDirectory( File base ) + { + return providerDirectoryStructure( base ).rootDirectory(); + } + private static File soleIndexDir( File base ) { - return subProviderDirectoryStructure( base ).forProvider( PROVIDER_DESCRIPTOR ).directoryForIndex( 1 ); + return providerDirectoryStructure( base ).directoryForIndex( 1 ); + } + + private static IndexDirectoryStructure providerDirectoryStructure( File base ) + { + return subProviderDirectoryStructure( base ).forProvider( PROVIDER_DESCRIPTOR ); } }