diff --git a/community/lucene-index/src/main/java/org/neo4j/index/impl/lucene/legacy/LuceneBatchInserterIndex.java b/community/lucene-index/src/main/java/org/neo4j/index/impl/lucene/legacy/LuceneBatchInserterIndex.java index 38e18fe196b30..6735be2d993a0 100644 --- a/community/lucene-index/src/main/java/org/neo4j/index/impl/lucene/legacy/LuceneBatchInserterIndex.java +++ b/community/lucene-index/src/main/java/org/neo4j/index/impl/lucene/legacy/LuceneBatchInserterIndex.java @@ -21,10 +21,12 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.SearcherFactory; import org.apache.lucene.search.SearcherManager; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; @@ -38,29 +40,25 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import java.util.concurrent.locks.LockSupport; import org.neo4j.collection.primitive.PrimitiveLongCollections; import org.neo4j.graphdb.ResourceIterator; import org.neo4j.graphdb.index.IndexHits; import org.neo4j.helpers.collection.LruCache; import org.neo4j.index.lucene.ValueContext; +import org.neo4j.io.IOUtils; import org.neo4j.kernel.api.LegacyIndexHits; import org.neo4j.kernel.api.impl.index.DocValuesCollector; -import org.neo4j.kernel.api.impl.index.IndexWriterFactories; -import org.neo4j.kernel.api.impl.index.ObsoleteLuceneIndexWriter; import org.neo4j.kernel.impl.index.IndexEntityType; import org.neo4j.kernel.impl.util.IoPrimitiveUtils; import org.neo4j.unsafe.batchinsert.BatchInserterIndex; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - class LuceneBatchInserterIndex implements BatchInserterIndex { private final IndexIdentifier identifier; private final IndexType type; - private ObsoleteLuceneIndexWriter writer; + private IndexWriter writer; private SearcherManager searcherManager; private final boolean createdNow; private Map>> cache; @@ -250,17 +248,19 @@ private void removeFromCache( long entityId, String key, Object value ) } } - private ObsoleteLuceneIndexWriter instantiateWriter( File directory ) + private IndexWriter instantiateWriter( File folder ) { + Directory dir = null; try { + dir = LuceneDataSource.getDirectory( folder, identifier ); IndexWriterConfig writerConfig = new IndexWriterConfig( type.analyzer ); writerConfig.setRAMBufferSizeMB( determineGoodBufferSize( writerConfig.getRAMBufferSizeMB() ) ); - Directory luceneDir = LuceneDataSource.getDirectory( directory, identifier ); - return IndexWriterFactories.batchInsert( writerConfig ).create( luceneDir ); + return new IndexWriter( dir, writerConfig ); } catch ( IOException e ) { + IOUtils.closeAllSilently( dir ); throw new RuntimeException( e ); } } @@ -272,11 +272,11 @@ private double determineGoodBufferSize( double atLeast ) return Math.min( result, 700 ); } - private static SearcherManager instantiateSearcherManager( ObsoleteLuceneIndexWriter writer ) + private static SearcherManager instantiateSearcherManager( IndexWriter writer ) { try { - return writer.createSearcherManager(); + return new SearcherManager( writer, true, new SearcherFactory() ); } catch ( IOException e ) { @@ -288,7 +288,10 @@ private void closeSearcher() { try { - this.searcherManager.close(); + if ( searcherManager != null ) + { + this.searcherManager.close(); + } } catch ( IOException e ) { @@ -306,7 +309,6 @@ private void closeWriter() { if ( this.writer != null ) { - this.writer.optimize(); this.writer.close(); } } @@ -483,10 +485,7 @@ public void flush() { try { - while ( !searcherManager.maybeRefresh() ) - { - LockSupport.parkNanos( MILLISECONDS.toNanos( 100 ) ); - } + searcherManager.maybeRefreshBlocking(); } catch ( IOException e ) { diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java index 693e87d159edd..4b728493e8ad2 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java @@ -77,6 +77,11 @@ public void open() throws IOException open = true; } + boolean isOpen() + { + return open; + } + public boolean exists() throws IOException { List folders = indexStorage.listFolders(); diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/IndexWriterFactories.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/IndexWriterConfigs.java similarity index 73% rename from community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/IndexWriterFactories.java rename to community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/IndexWriterConfigs.java index 3aa3447532b9c..9a4d93bfa0814 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/IndexWriterFactories.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/IndexWriterConfigs.java @@ -26,34 +26,22 @@ import org.neo4j.index.impl.lucene.legacy.MultipleBackupDeletionPolicy; import org.neo4j.unsafe.impl.internal.dragons.FeatureToggles; -public final class IndexWriterFactories +public final class IndexWriterConfigs { - private static final int MAX_BUFFERED_DOCS = - FeatureToggles.getInteger( IndexWriterFactories.class, "max_buffered_docs", 100000 ); + FeatureToggles.getInteger( IndexWriterConfigs.class, "max_buffered_docs", 100000 ); private static final int MERGE_POLICY_MERGE_FACTOR = - FeatureToggles.getInteger( IndexWriterFactories.class, "merge.factor", 2 ); + FeatureToggles.getInteger( IndexWriterConfigs.class, "merge.factor", 2 ); private static final double MERGE_POLICY_NO_CFS_RATIO = - FeatureToggles.getDouble( IndexWriterFactories.class, "nocfs.ratio", 1.0 ); + FeatureToggles.getDouble( IndexWriterConfigs.class, "nocfs.ratio", 1.0 ); private static final double MERGE_POLICY_MIN_MERGE_MB = - FeatureToggles.getDouble( IndexWriterFactories.class, "min.merge", 0.1 ); + FeatureToggles.getDouble( IndexWriterConfigs.class, "min.merge", 0.1 ); - private IndexWriterFactories() + private IndexWriterConfigs() { throw new AssertionError( "Not for instantiation!" ); } - - public static IndexWriterFactory standard() - { - return directory -> new ObsoleteLuceneIndexWriter( directory, standardConfig() ); - } - - public static IndexWriterFactory batchInsert( final IndexWriterConfig config ) - { - return directory -> new ObsoleteLuceneIndexWriter( directory, config ); - } - public static IndexWriterConfig standardConfig() { IndexWriterConfig writerConfig = new IndexWriterConfig( LuceneDataSource.KEYWORD_ANALYZER ); diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/IndexWriterFactory.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/IndexWriterFactory.java deleted file mode 100644 index a6959a5cdc23b..0000000000000 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/IndexWriterFactory.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.api.impl.index; - -import org.apache.lucene.store.Directory; - -import java.io.IOException; - -@FunctionalInterface -public interface IndexWriterFactory -{ - W create( Directory directory ) throws IOException; -} diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/ObsoleteLuceneIndexWriter.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/ObsoleteLuceneIndexWriter.java deleted file mode 100644 index deb1da00a5b5c..0000000000000 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/ObsoleteLuceneIndexWriter.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.api.impl.index; - -import org.apache.lucene.document.Document; -import org.apache.lucene.index.IndexDeletionPolicy; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.SearcherFactory; -import org.apache.lucene.search.SearcherManager; -import org.apache.lucene.store.Directory; - -import java.io.Closeable; -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.locks.ReentrantLock; - -import static java.util.Collections.singletonMap; - -/** - * A thin wrapper around {@link org.apache.lucene.index.IndexWriter} that exposes only some part of it's - * functionality that it really needed. - */ -@Deprecated -public class ObsoleteLuceneIndexWriter implements Closeable, LuceneIndexWriter -{ - // Lucene cannot allocate a full MAX_INT of documents, the deviation differs from JVM to JVM, but according to - // their source in future versions, the deviation can never be bigger than 128. - private static final long MAX_DOC_LIMIT = Integer.MAX_VALUE - 128; - - private static final String KEY_STATUS = "status"; - private static final String ONLINE = "online"; - private static final Map ONLINE_COMMIT_USER_DATA = singletonMap( KEY_STATUS, ONLINE ); - - protected final IndexWriter writer; - - private final ReentrantLock commitCloseLock = new ReentrantLock(); - - /** - * Package private *only* for subclasses and testing. - */ - ObsoleteLuceneIndexWriter( Directory dir, IndexWriterConfig conf ) throws IOException - { - this.writer = new IndexWriter( dir, conf ); - } - - @Override - public void addDocument( Document document ) throws IOException - { - writer.addDocument( document ); - } - - @Override - public void updateDocument( Term term, Document document ) throws IOException - { - writer.updateDocument( term, document ); - } - - @Override - public void deleteDocuments( Term term ) throws IOException - { - writer.deleteDocuments( term ); - } - - @Override - public void deleteDocuments( Query query ) throws IOException - { - writer.deleteDocuments( query ); - } - - public void optimize() throws IOException - { - writer.forceMerge( 1, true ); - } - - public SearcherManager createSearcherManager() throws IOException - { - return new SearcherManager( writer, true, new SearcherFactory() ); - } - - public void commit() throws IOException - { - commitCloseLock.lock(); - try - { - writer.commit(); - } - finally - { - commitCloseLock.unlock(); - } - } - - @Override - public void close() throws IOException - { - commitCloseLock.lock(); - try - { - writer.close(); - } - finally - { - commitCloseLock.unlock(); - } - } - - IndexDeletionPolicy getIndexDeletionPolicy() - { - return writer.getConfig().getIndexDeletionPolicy(); - } - - long maxDocLimit() - { - return MAX_DOC_LIMIT; - } -} diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/partition/IndexPartition.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/partition/IndexPartition.java index 9e3d1e8ec49a1..46beb618f4332 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/partition/IndexPartition.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/partition/IndexPartition.java @@ -30,7 +30,7 @@ import org.neo4j.graphdb.ResourceIterator; import org.neo4j.io.IOUtils; -import org.neo4j.kernel.api.impl.index.IndexWriterFactories; +import org.neo4j.kernel.api.impl.index.IndexWriterConfigs; import org.neo4j.kernel.api.impl.index.backup.LuceneIndexSnapshotFileIterator; public class IndexPartition implements Closeable @@ -44,7 +44,7 @@ public IndexPartition( File indexDirectory, Directory directory ) throws IOExcep { this.indexDirectory = indexDirectory; this.directory = directory; - this.indexWriter = new IndexWriter( directory, IndexWriterFactories.standardConfig() ); + this.indexWriter = new IndexWriter( directory, IndexWriterConfigs.standardConfig() ); this.searcherManager = new SearcherManager( indexWriter, true, new SearcherFactory() ); } diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/ObsoleteLuceneIndexWriterIntegrationTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndexIntegrationTest.java similarity index 71% rename from community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/ObsoleteLuceneIndexWriterIntegrationTest.java rename to community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndexIntegrationTest.java index 38998950b0551..53640fe192d61 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/ObsoleteLuceneIndexWriterIntegrationTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndexIntegrationTest.java @@ -24,6 +24,7 @@ import org.apache.lucene.document.LongField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IOContext; @@ -46,13 +47,17 @@ import java.util.concurrent.Future; import java.util.zip.ZipOutputStream; +import org.neo4j.io.fs.DefaultFileSystemAbstraction; +import org.neo4j.kernel.api.impl.index.partition.IndexPartition; import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory; +import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage; import org.neo4j.test.RepeatRule; import org.neo4j.test.TargetDirectory; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -public class ObsoleteLuceneIndexWriterIntegrationTest +public class AbstractLuceneIndexIntegrationTest { private static final int THREAD_NUMBER = 5; @Rule @@ -60,13 +65,17 @@ public class ObsoleteLuceneIndexWriterIntegrationTest @Rule public RepeatRule repeatRule = new RepeatRule(); + private final CountDownLatch closeRaceSignal = new CountDownLatch( 1 ); + private SyncNotifierDirectoryFactory directoryFactory; + private TestLuceneIndex luceneIndex; private ExecutorService workers; @Before - public void setUp() + public void setUp() throws IOException { - directoryFactory = new SyncNotifierDirectoryFactory(); + directoryFactory = new SyncNotifierDirectoryFactory( closeRaceSignal ); + luceneIndex = createTestLuceneIndex( directoryFactory, testDir.directory() ); workers = Executors.newFixedThreadPool( THREAD_NUMBER ); } @@ -78,50 +87,58 @@ public void tearDown() } @Test( timeout = 10000 ) - @RepeatRule.Repeat( times = 4 ) + @RepeatRule.Repeat( times = 5 ) public void testSaveCallCommitAndCloseFromMultipleThreads() throws Exception { - CountDownLatch closeRaceSignal = new CountDownLatch( 1 ); - Directory writerDirectory = directoryFactory.open( testDir.directory(), closeRaceSignal ); - ObsoleteLuceneIndexWriter indexWriter = IndexWriterFactories.standard().create( writerDirectory ); - - generateIndexData( indexWriter ); - List closeFutures = submitCloseTasks( indexWriter, closeRaceSignal ); - + generateInitialData(); + List> closeFutures = submitCloseTasks( closeRaceSignal ); - for ( Future closeFuture : closeFutures ) + for ( Future closeFuture : closeFutures ) { closeFuture.get(); } - assertFalse( indexWriter.writer.isOpen() ); + + assertFalse( luceneIndex.isOpen() ); } - private List submitCloseTasks( ObsoleteLuceneIndexWriter indexWriter, CountDownLatch closeRaceSignal ) + private static TestLuceneIndex createTestLuceneIndex( DirectoryFactory dirFactory, File folder ) throws IOException { - List closeFutures = new ArrayList<>( THREAD_NUMBER ); - closeFutures.add( workers.submit( createMainCloseTask( indexWriter ) ) ); + DefaultFileSystemAbstraction fs = new DefaultFileSystemAbstraction(); + PartitionedIndexStorage indexStorage = new PartitionedIndexStorage( dirFactory, fs, folder, "test" ); + TestLuceneIndex index = new TestLuceneIndex( indexStorage ); + index.create(); + index.open(); + return index; + } + + private List> submitCloseTasks( CountDownLatch closeRaceSignal ) + { + List> closeFutures = new ArrayList<>( THREAD_NUMBER ); + closeFutures.add( workers.submit( createMainCloseTask() ) ); for ( int i = 0; i < THREAD_NUMBER - 1; i++ ) { - closeFutures.add( workers.submit( createConcurrentCloseTask( indexWriter, closeRaceSignal ) ) ); + closeFutures.add( workers.submit( createConcurrentCloseTask( closeRaceSignal ) ) ); } return closeFutures; } - private void generateIndexData( ObsoleteLuceneIndexWriter indexWriter ) throws IOException + private void generateInitialData() throws IOException { + IndexWriter indexWriter = firstPartitionWriter(); for ( int i = 0; i < 10; i++ ) { indexWriter.addDocument( createTestDocument() ); } } - private Runnable createConcurrentCloseTask( ObsoleteLuceneIndexWriter writer, CountDownLatch closeRaceSignal ) + private Runnable createConcurrentCloseTask( CountDownLatch closeRaceSignal ) { return () -> { try { closeRaceSignal.await(); - writer.close(); + Thread.yield(); + luceneIndex.close(); } catch ( Exception e ) { @@ -130,12 +147,12 @@ private Runnable createConcurrentCloseTask( ObsoleteLuceneIndexWriter writer, Co }; } - private Runnable createMainCloseTask( ObsoleteLuceneIndexWriter writer ) + private Runnable createMainCloseTask() { return () -> { try { - writer.close(); + luceneIndex.close(); } catch ( Exception e ) { @@ -152,8 +169,31 @@ private Document createTestDocument() return document; } + private IndexWriter firstPartitionWriter() + { + List partitions = luceneIndex.getPartitions(); + assertEquals( 1, partitions.size() ); + IndexPartition partition = partitions.get( 0 ); + return partition.getIndexWriter(); + } + + private static class TestLuceneIndex extends AbstractLuceneIndex + { + TestLuceneIndex( PartitionedIndexStorage indexStorage ) + { + super( indexStorage ); + } + } + private static class SyncNotifierDirectoryFactory implements DirectoryFactory { + final CountDownLatch signal; + + SyncNotifierDirectoryFactory( CountDownLatch signal ) + { + this.signal = signal; + } + public Directory open( File dir, CountDownLatch signal ) throws IOException { Directory directory = open( dir ); @@ -164,19 +204,18 @@ public Directory open( File dir, CountDownLatch signal ) throws IOException public Directory open( File dir ) throws IOException { dir.mkdirs(); - return FSDirectory.open( dir.toPath() ); + FSDirectory fsDir = FSDirectory.open( dir.toPath() ); + return new SyncNotifierDirectory( fsDir, signal ); } @Override public void close() { - } @Override public void dumpToZip( ZipOutputStream zip, byte[] scratchPad ) throws IOException { - } private class SyncNotifierDirectory extends Directory @@ -231,6 +270,7 @@ public void sync( Collection names ) throws IOException throw new RuntimeException( e ); } } + delegate.sync( names ); } diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/backup/LuceneSnapshotterTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/backup/LuceneSnapshotterTest.java index 49fc6aa4c62b6..3b773fa3ae7d5 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/backup/LuceneSnapshotterTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/backup/LuceneSnapshotterTest.java @@ -38,7 +38,7 @@ import org.neo4j.graphdb.ResourceIterator; import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.IOUtils; -import org.neo4j.kernel.api.impl.index.IndexWriterFactories; +import org.neo4j.kernel.api.impl.index.IndexWriterConfigs; import org.neo4j.test.TargetDirectory; import static java.util.stream.Collectors.toSet; @@ -59,7 +59,7 @@ public void initializeLuceneResources() throws IOException { indexDir = testDir.directory(); dir = new RAMDirectory(); - writer = new IndexWriter( dir, IndexWriterFactories.standardConfig() ); + writer = new IndexWriter( dir, IndexWriterConfigs.standardConfig() ); } @After