From e09d44dd3af58ab862f22371d9052fd14b92ad35 Mon Sep 17 00:00:00 2001 From: Mikhaylo Demianenko Date: Fri, 22 Jul 2016 17:35:04 +0200 Subject: [PATCH] Comments, extraction, modifiers comments addressed. --- .../kernel/impl/spi/SimpleKernelContext.java | 2 +- .../api/impl/index/LuceneIndexAccessor.java | 24 ++- .../api/impl/index/LuceneLabelScanStore.java | 6 + .../impl/lucene/CloseTrackingIndexReader.java | 177 ++++++++++++++++++ .../lucene/ReadOnlyIndexReferenceTest.java | 159 +--------------- .../lucene/WritableIndexReferenceTest.java | 159 +--------------- .../impl/index/LuceneLabelScanStoreTest.java | 1 + .../StoreUpgraderInterruptionTestIT.java | 6 +- .../test/java/upgrade/StoreUpgraderTest.java | 1 + 9 files changed, 206 insertions(+), 329 deletions(-) create mode 100644 community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/CloseTrackingIndexReader.java diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/spi/SimpleKernelContext.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/spi/SimpleKernelContext.java index dc386de4dd957..a3fc9959ec5e9 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/spi/SimpleKernelContext.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/spi/SimpleKernelContext.java @@ -31,7 +31,7 @@ public class SimpleKernelContext implements KernelContext { private final FileSystemAbstraction fileSystem; private final File storeDir; - UsageDataKeys.OperationalMode operationalMode; + private final UsageDataKeys.OperationalMode operationalMode; public SimpleKernelContext( FileSystemAbstraction fileSystem, File storeDir, UsageDataKeys.OperationalMode operationalMode ) { diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java index 801e3b2982c57..2e12a80d1dee0 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneIndexAccessor.java @@ -132,16 +132,8 @@ public void close() throws IOException this.dir = dirFactory.open( indexFolder ); if ( readOnly ) { - try - { - this.writer = null; - searcherManager = - new LuceneReferenceManager.Wrap<>( new SearcherManager( dir, new SearcherFactory() ) ); - } - catch ( IndexNotFoundException e ) - { - throw new IllegalStateException( "Index creation is not supported in read only mode.", e ); - } + this.writer = null; + searcherManager = createReadOnlySearchManager(); } else { @@ -150,6 +142,18 @@ public void close() throws IOException } } + private LuceneReferenceManager.Wrap createReadOnlySearchManager() throws IOException + { + try + { + return new LuceneReferenceManager.Wrap<>( new SearcherManager( dir, new SearcherFactory() ) ); + } + catch ( IndexNotFoundException e ) + { + throw new IllegalStateException( "Index creation is not supported in read only mode.", e ); + } + } + // test only LuceneIndexAccessor( LuceneDocumentStructure documentStructure, boolean readOnly, ReservingLuceneIndexWriter writer, LuceneReferenceManager searcherManager, diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStore.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStore.java index 270e57d242efc..786850b9ca6c1 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStore.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStore.java @@ -263,6 +263,12 @@ public void init() throws IOException { searcherManager = new SearcherManager( directory, new SearcherFactory() ); } + catch ( IndexNotFoundException inf ) + { + monitor.noIndex(); + throw new IOException( "Label scan store not found while database was started in read only mode. " + + "To trigger a rebuild please restart database in writable mode.", inf ); + } catch ( IOException e ) { monitor.corruptIndex( e ); diff --git a/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/CloseTrackingIndexReader.java b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/CloseTrackingIndexReader.java new file mode 100644 index 0000000000000..9ec1c7cdd713b --- /dev/null +++ b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/CloseTrackingIndexReader.java @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.index.impl.lucene; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.TermFreqVector; +import org.apache.lucene.index.TermPositions; +import org.apache.lucene.index.TermVectorMapper; + +import java.io.IOException; +import java.util.Map; + +class CloseTrackingIndexReader extends IndexReader +{ + private boolean closed = false; + + @Override + public TermFreqVector[] getTermFreqVectors( int docNumber ) throws IOException + { + return new TermFreqVector[0]; + } + + @Override + public TermFreqVector getTermFreqVector( int docNumber, String field ) throws IOException + { + return null; + } + + @Override + public void getTermFreqVector( int docNumber, String field, TermVectorMapper mapper ) throws IOException + { + + } + + @Override + public void getTermFreqVector( int docNumber, TermVectorMapper mapper ) throws IOException + { + + } + + @Override + public int numDocs() + { + return 0; + } + + @Override + public int maxDoc() + { + return 0; + } + + @Override + public Document document( int n, FieldSelector fieldSelector ) throws CorruptIndexException, IOException + { + return null; + } + + @Override + public boolean isDeleted( int n ) + { + return false; + } + + @Override + public boolean hasDeletions() + { + return false; + } + + @Override + public byte[] norms( String field ) throws IOException + { + return new byte[0]; + } + + @Override + public void norms( String field, byte[] bytes, int offset ) throws IOException + { + + } + + @Override + protected void doSetNorm( int doc, String field, byte value ) throws CorruptIndexException, IOException + { + + } + + @Override + public TermEnum terms() throws IOException + { + return null; + } + + @Override + public TermEnum terms( Term t ) throws IOException + { + return null; + } + + @Override + public int docFreq( Term t ) throws IOException + { + return 0; + } + + @Override + public TermDocs termDocs() throws IOException + { + return null; + } + + @Override + public TermPositions termPositions() throws IOException + { + return null; + } + + @Override + protected void doDelete( int docNum ) throws CorruptIndexException, IOException + { + + } + + @Override + protected void doUndeleteAll() throws CorruptIndexException, IOException + { + + } + + @Override + protected void doCommit( Map commitUserData ) throws IOException + { + + } + + @Override + protected void doClose() throws IOException + { + closed = true; + } + + @Override + public FieldInfos getFieldInfos() + { + return null; + } + + public boolean isClosed() + { + return closed; + } +} diff --git a/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/ReadOnlyIndexReferenceTest.java b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/ReadOnlyIndexReferenceTest.java index 9fa2bfc14dd8e..9d184928d9a94 100644 --- a/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/ReadOnlyIndexReferenceTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/ReadOnlyIndexReferenceTest.java @@ -19,17 +19,6 @@ */ package org.neo4j.index.impl.lucene; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; -import org.apache.lucene.index.TermEnum; -import org.apache.lucene.index.TermFreqVector; -import org.apache.lucene.index.TermPositions; -import org.apache.lucene.index.TermVectorMapper; import org.apache.lucene.search.IndexSearcher; import org.junit.Before; import org.junit.Rule; @@ -37,7 +26,6 @@ import org.junit.rules.ExpectedException; import java.io.IOException; -import java.util.Map; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -52,7 +40,7 @@ public class ReadOnlyIndexReferenceTest public ExpectedException expectedException = ExpectedException.none(); private IndexIdentifier identifier = mock( IndexIdentifier.class ); private IndexSearcher searcher = mock( IndexSearcher.class ); - private TestIndexReader reader = new TestIndexReader(); + private CloseTrackingIndexReader reader = new CloseTrackingIndexReader(); private ReadOnlyIndexReference indexReference = new ReadOnlyIndexReference( identifier, searcher ); @Before @@ -161,147 +149,4 @@ public void closeNotReferencedIndex() { assertTrue( indexReference.close() ); } - - private class TestIndexReader extends IndexReader - { - private boolean closed = false; - - @Override - public TermFreqVector[] getTermFreqVectors( int docNumber ) throws IOException - { - return new TermFreqVector[0]; - } - - @Override - public TermFreqVector getTermFreqVector( int docNumber, String field ) throws IOException - { - return null; - } - - @Override - public void getTermFreqVector( int docNumber, String field, TermVectorMapper mapper ) throws IOException - { - - } - - @Override - public void getTermFreqVector( int docNumber, TermVectorMapper mapper ) throws IOException - { - - } - - @Override - public int numDocs() - { - return 0; - } - - @Override - public int maxDoc() - { - return 0; - } - - @Override - public Document document( int n, FieldSelector fieldSelector ) throws CorruptIndexException, IOException - { - return null; - } - - @Override - public boolean isDeleted( int n ) - { - return false; - } - - @Override - public boolean hasDeletions() - { - return false; - } - - @Override - public byte[] norms( String field ) throws IOException - { - return new byte[0]; - } - - @Override - public void norms( String field, byte[] bytes, int offset ) throws IOException - { - - } - - @Override - protected void doSetNorm( int doc, String field, byte value ) throws CorruptIndexException, IOException - { - - } - - @Override - public TermEnum terms() throws IOException - { - return null; - } - - @Override - public TermEnum terms( Term t ) throws IOException - { - return null; - } - - @Override - public int docFreq( Term t ) throws IOException - { - return 0; - } - - @Override - public TermDocs termDocs() throws IOException - { - return null; - } - - @Override - public TermPositions termPositions() throws IOException - { - return null; - } - - @Override - protected void doDelete( int docNum ) throws CorruptIndexException, IOException - { - - } - - @Override - protected void doUndeleteAll() throws CorruptIndexException, IOException - { - - } - - @Override - protected void doCommit( Map commitUserData ) throws IOException - { - - } - - @Override - protected void doClose() throws IOException - { - closed = true; - } - - @Override - public FieldInfos getFieldInfos() - { - return null; - } - - public boolean isClosed() - { - return closed; - } - } - -} \ No newline at end of file +} diff --git a/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/WritableIndexReferenceTest.java b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/WritableIndexReferenceTest.java index 6167ec49ca5c1..19fd6813004d9 100644 --- a/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/WritableIndexReferenceTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/WritableIndexReferenceTest.java @@ -20,27 +20,13 @@ package org.neo4j.index.impl.lucene; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldSelector; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; -import org.apache.lucene.index.TermEnum; -import org.apache.lucene.index.TermFreqVector; -import org.apache.lucene.index.TermPositions; -import org.apache.lucene.index.TermVectorMapper; import org.apache.lucene.search.IndexSearcher; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import java.io.IOException; -import java.util.Map; - import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; @@ -55,7 +41,7 @@ public class WritableIndexReferenceTest private IndexIdentifier identifier = mock( IndexIdentifier.class ); private IndexSearcher searcher = mock( IndexSearcher.class ); private IndexWriter indexWriter = mock( IndexWriter.class ); - private WritableIndexReferenceTest.TestIndexReader reader = new WritableIndexReferenceTest.TestIndexReader(); + private CloseTrackingIndexReader reader = new CloseTrackingIndexReader(); private WritableIndexReference indexReference = new WritableIndexReference( identifier, searcher, indexWriter); @Before @@ -89,145 +75,4 @@ public void disposeWritableIndex() throws Exception assertTrue( "Writer should be closed.", indexReference.isWriterClosed() ); } - private class TestIndexReader extends IndexReader - { - private boolean closed = false; - - @Override - public TermFreqVector[] getTermFreqVectors( int docNumber ) throws IOException - { - return new TermFreqVector[0]; - } - - @Override - public TermFreqVector getTermFreqVector( int docNumber, String field ) throws IOException - { - return null; - } - - @Override - public void getTermFreqVector( int docNumber, String field, TermVectorMapper mapper ) throws IOException - { - - } - - @Override - public void getTermFreqVector( int docNumber, TermVectorMapper mapper ) throws IOException - { - - } - - @Override - public int numDocs() - { - return 0; - } - - @Override - public int maxDoc() - { - return 0; - } - - @Override - public Document document( int n, FieldSelector fieldSelector ) throws CorruptIndexException, IOException - { - return null; - } - - @Override - public boolean isDeleted( int n ) - { - return false; - } - - @Override - public boolean hasDeletions() - { - return false; - } - - @Override - public byte[] norms( String field ) throws IOException - { - return new byte[0]; - } - - @Override - public void norms( String field, byte[] bytes, int offset ) throws IOException - { - - } - - @Override - protected void doSetNorm( int doc, String field, byte value ) throws CorruptIndexException, IOException - { - - } - - @Override - public TermEnum terms() throws IOException - { - return null; - } - - @Override - public TermEnum terms( Term t ) throws IOException - { - return null; - } - - @Override - public int docFreq( Term t ) throws IOException - { - return 0; - } - - @Override - public TermDocs termDocs() throws IOException - { - return null; - } - - @Override - public TermPositions termPositions() throws IOException - { - return null; - } - - @Override - protected void doDelete( int docNum ) throws CorruptIndexException, IOException - { - - } - - @Override - protected void doUndeleteAll() throws CorruptIndexException, IOException - { - - } - - @Override - protected void doCommit( Map commitUserData ) throws IOException - { - - } - - @Override - protected void doClose() throws IOException - { - closed = true; - } - - @Override - public FieldInfos getFieldInfos() - { - return null; - } - - public boolean isClosed() - { - return closed; - } - } -} \ No newline at end of file +} diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreTest.java index 5801931da471c..450633293c3c2 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStoreTest.java @@ -132,6 +132,7 @@ public void failToStartIfLabelScanStoreIndexDoesNotExistInReadOnlyMode() { expectedException.expectCause( Matchers.instanceOf( IOException.class ) ); startLabelScanStore( MapUtil.stringMap(GraphDatabaseSettings.read_only.name(), "true") ); + assertTrue( monitor.noIndexCalled ); } @Test diff --git a/community/neo4j/src/test/java/upgrade/StoreUpgraderInterruptionTestIT.java b/community/neo4j/src/test/java/upgrade/StoreUpgraderInterruptionTestIT.java index 905d251588ee2..aab2ea9c7ea45 100644 --- a/community/neo4j/src/test/java/upgrade/StoreUpgraderInterruptionTestIT.java +++ b/community/neo4j/src/test/java/upgrade/StoreUpgraderInterruptionTestIT.java @@ -19,7 +19,6 @@ */ package upgrade; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -132,12 +131,12 @@ public void migrate( File sourceStoreDir, File targetStoreDir, SchemaIndexProvid assertTrue( checkNeoStoreHasLatestVersion( check, workingDirectory ) ); assertTrue( allStoreFilesHaveNoTrailer( fs, workingDirectory ) ); + // Since consistency checker is in read only mode we need to start/stop db to generate label scan store. startStopDatabase( workingDirectory ); assertConsistentStore( workingDirectory ); } @Test - @Ignore public void shouldSucceedWithUpgradeAfterPreviousAttemptDiedDuringMovingFiles() throws IOException, ConsistencyCheckIncompleteException { @@ -177,8 +176,6 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT assertTrue( checkNeoStoreHasLatestVersion( check, workingDirectory ) ); assertTrue( allStoreFilesHaveNoTrailer( fs, workingDirectory ) ); - assertConsistentStore( workingDirectory ); - progressMonitor = new SilentMigrationProgressMonitor(); StoreMigrator migrator = new StoreMigrator( progressMonitor, fs, pageCache, config, logService ); newUpgrader( migrator ).migrateIfNeeded( workingDirectory, upgradableDatabase, schemaIndexProvider ); @@ -188,6 +185,7 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT pageCache.close(); + // Since consistency checker is in read only mode we need to start/stop db to generate label scan store. startStopDatabase( workingDirectory ); assertConsistentStore( workingDirectory ); } diff --git a/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java b/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java index fc103c2b4bcb1..2e395f21ed67d 100644 --- a/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java +++ b/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java @@ -145,6 +145,7 @@ public void shouldUpgradeAnOldFormatStore() throws IOException, ConsistencyCheck // We leave logical logs in place since the new version can read the old assertFalse( containsAnyStoreFiles( fileSystem, isolatedMigrationDirectoryOf( dbDirectory ) ) ); + // Since consistency checker is in read only mode we need to start/stop db to generate label scan store. startStopDatabase(); assertConsistentStore( dbDirectory ); }