Skip to content

Commit

Permalink
Cleanups, additional tests for read only cases and pieces of read onl…
Browse files Browse the repository at this point in the history
…y infrastructure.
  • Loading branch information
MishaDemianenko committed Jul 29, 2016
1 parent 2770c54 commit a26a40b
Show file tree
Hide file tree
Showing 15 changed files with 534 additions and 37 deletions.
Expand Up @@ -37,6 +37,7 @@
import org.neo4j.helpers.Exceptions;
import org.neo4j.helpers.collection.Iterators;
import org.neo4j.io.IOUtils;
import org.neo4j.kernel.api.impl.index.backup.WritableIndexSnapshotFileIterator;
import org.neo4j.kernel.api.impl.index.partition.AbstractIndexPartition;
import org.neo4j.kernel.api.impl.index.partition.IndexPartitionFactory;
import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher;
Expand Down Expand Up @@ -238,7 +239,7 @@ public LuceneAllDocumentsReader allDocumentsReader()
*
* @return iterator over all index files.
* @throws IOException
* @see org.neo4j.kernel.api.impl.index.backup.LuceneIndexSnapshotFileIterator
* @see WritableIndexSnapshotFileIterator
*/
public ResourceIterator<File> snapshot() throws IOException
{
Expand Down
Expand Up @@ -25,6 +25,7 @@
import java.util.List;

import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.kernel.api.impl.index.backup.WritableIndexSnapshotFileIterator;
import org.neo4j.kernel.api.impl.index.partition.AbstractIndexPartition;

/**
Expand Down Expand Up @@ -106,7 +107,7 @@ public interface LuceneIndex extends Closeable
*
* @return iterator over all index files.
* @throws IOException
* @see org.neo4j.kernel.api.impl.index.backup.LuceneIndexSnapshotFileIterator
* @see WritableIndexSnapshotFileIterator
*/
ResourceIterator<File> snapshot() throws IOException;

Expand Down
Expand Up @@ -56,7 +56,7 @@ public static ResourceIterator<File> forIndex( File indexFolder, IndexWriter ind
{
SnapshotDeletionPolicy policy = (SnapshotDeletionPolicy) deletionPolicy;
return hasCommits( indexWriter )
? new LuceneIndexSnapshotFileIterator( indexFolder, policy )
? new WritableIndexSnapshotFileIterator( indexFolder, policy )
: emptyIterator();
}
else
Expand All @@ -77,6 +77,10 @@ public static ResourceIterator<File> forIndex( File indexFolder, IndexWriter ind
*/
public static ResourceIterator<File> forIndex( File indexFolder, Directory directory ) throws IOException
{
if ( !hasCommits( directory ) )
{
return emptyIterator();
}
Collection<IndexCommit> indexCommits = DirectoryReader.listCommits( directory );
IndexCommit indexCommit = Iterables.last( indexCommits );
return new ReadOnlyIndexSnapshotFileIterator( indexFolder, indexCommit );
Expand All @@ -85,6 +89,11 @@ public static ResourceIterator<File> forIndex( File indexFolder, Directory direc
private static boolean hasCommits( IndexWriter indexWriter ) throws IOException
{
Directory directory = indexWriter.getDirectory();
return hasCommits( directory );
}

private static boolean hasCommits( Directory directory ) throws IOException
{
return DirectoryReader.indexExists( directory ) && SegmentInfos.readLatestCommit( directory ) != null;
}
}
Expand Up @@ -20,10 +20,10 @@
package org.neo4j.kernel.api.impl.index.backup;

/**
* Exception that is thrown by {@link LuceneIndexSnapshotFileIterator} in case if exception
* Exception that is thrown by {@link WritableIndexSnapshotFileIterator} in case if exception
* occurred during index snapshot release
*
* @see LuceneIndexSnapshotFileIterator
* @see WritableIndexSnapshotFileIterator
* @see org.apache.lucene.index.SnapshotDeletionPolicy
* @see org.apache.lucene.index.IndexCommit
*/
Expand Down
Expand Up @@ -20,10 +20,10 @@
package org.neo4j.kernel.api.impl.index.backup;

/**
* Exception that is throw by {@link LuceneIndexSnapshotFileIterator} in case if there is an attempt to create a
* Exception that is throw by {@link WritableIndexSnapshotFileIterator} in case if there is an attempt to create a
* snapshot on a index with index policy that does not support snapshots.
*
* @see LuceneIndexSnapshotFileIterator
* @see WritableIndexSnapshotFileIterator
* @see org.apache.lucene.index.SnapshotDeletionPolicy
*/
class UnsupportedIndexDeletionPolicy extends RuntimeException
Expand Down
Expand Up @@ -31,11 +31,11 @@
* Internally uses {@link SnapshotDeletionPolicy#snapshot()} to create an {@link IndexCommit} that represents
* consistent state of the index for a particular point in time.
*/
public class LuceneIndexSnapshotFileIterator extends ReadOnlyIndexSnapshotFileIterator
public class WritableIndexSnapshotFileIterator extends ReadOnlyIndexSnapshotFileIterator
{
private final SnapshotDeletionPolicy snapshotDeletionPolicy;

LuceneIndexSnapshotFileIterator( File indexDirectory, SnapshotDeletionPolicy snapshotDeletionPolicy )
WritableIndexSnapshotFileIterator( File indexDirectory, SnapshotDeletionPolicy snapshotDeletionPolicy )
throws IOException
{
super( indexDirectory, snapshotDeletionPolicy.snapshot() );
Expand Down
Expand Up @@ -70,7 +70,7 @@ public PartitionSearcher acquireSearcher() throws IOException
@Override
public void maybeRefreshBlocking() throws IOException
{

// nothing to refresh in read only partition
}

@Override
Expand Down
Expand Up @@ -24,7 +24,6 @@
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
Expand All @@ -36,62 +35,73 @@
import java.util.stream.Stream;

import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.helpers.collection.Iterators;
import org.neo4j.io.IOUtils;
import org.neo4j.kernel.api.impl.index.IndexWriterConfigs;
import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
import org.neo4j.test.TargetDirectory;

import static java.util.stream.Collectors.toSet;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;

public class LuceneIndexSnapshotFileIteratorTest
public class ReadOnlyIndexSnapshotFileIteratorTest
{
@Rule
public final TargetDirectory.TestDirectory testDir = TargetDirectory.testDirForTest( getClass() );

private File indexDir;
private Directory dir;
private IndexWriter writer;
protected File indexDir;
protected Directory dir;

@Before
public void initializeLuceneResources() throws IOException
public void setUp() throws IOException
{
indexDir = testDir.directory();
dir = new RAMDirectory();
writer = new IndexWriter( dir, IndexWriterConfigs.standard() );
dir = DirectoryFactory.PERSISTENT.open( indexDir );
}

@After
public void closeLuceneResources() throws IOException
public void tearDown() throws IOException
{
IOUtils.closeAll( writer, dir );
IOUtils.closeAll( dir );
}

@Test
public void shouldReturnRealSnapshotIfIndexAllowsIt() throws IOException
{
insertRandomDocuments( writer );
prepareIndex();

Set<String> files = listDir( dir );
assertFalse( files.isEmpty() );

try ( ResourceIterator<File> snapshot = LuceneIndexSnapshots.forIndex( indexDir, writer ) )
try ( ResourceIterator<File> snapshot = makeSnapshot() )
{
Set<String> snapshotFiles = Iterators.asList( snapshot ).stream().map( File::getName ).collect( toSet() );
Set<String> snapshotFiles = snapshot.stream().map( File::getName ).collect( toSet() );
assertEquals( files, snapshotFiles );
}
}

@Test
public void shouldReturnEmptyIteratorWhenNoCommitsHaveBeenMade() throws IOException
{
try ( ResourceIterator<File> snapshot = LuceneIndexSnapshots.forIndex( indexDir, writer ) )
try ( ResourceIterator<File> snapshot = makeSnapshot() )
{
assertFalse( snapshot.hasNext() );
}
}

private void prepareIndex() throws IOException
{
try ( IndexWriter writer = new IndexWriter( dir, IndexWriterConfigs.standard() ) )
{
insertRandomDocuments( writer );
}
}

protected ResourceIterator<File> makeSnapshot() throws IOException
{
return LuceneIndexSnapshots.forIndex( indexDir, dir );
}

private static void insertRandomDocuments( IndexWriter writer ) throws IOException
{
Document doc = new Document();
Expand All @@ -104,6 +114,9 @@ private static void insertRandomDocuments( IndexWriter writer ) throws IOExcepti
private static Set<String> listDir( Directory dir ) throws IOException
{
String[] files = dir.listAll();
return Stream.of( files ).collect( toSet() );
return Stream.of( files )
.filter( file -> !IndexWriter.WRITE_LOCK_NAME.equals( file ) )
.collect( toSet() );
}

}
@@ -0,0 +1,38 @@
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.api.impl.index.backup;

import org.apache.lucene.index.IndexWriter;

import java.io.File;
import java.io.IOException;

import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.kernel.api.impl.index.IndexWriterConfigs;

public class WritableIndexSnapshotFileIteratorTest extends ReadOnlyIndexSnapshotFileIteratorTest
{

@Override
protected ResourceIterator<File> makeSnapshot() throws IOException
{
return LuceneIndexSnapshots.forIndex( indexDir, new IndexWriter( dir, IndexWriterConfigs.standard() ) );
}
}
@@ -0,0 +1,100 @@
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.api.impl.index.partition;

import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;

import java.io.File;
import java.io.IOException;

import org.neo4j.kernel.api.impl.index.IndexWriterConfigs;
import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
import org.neo4j.test.TargetDirectory;

import static org.junit.Assert.assertEquals;

public class IndexPartitionFactoryTest
{

@Rule
public final TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() );
@Rule
public ExpectedException expectedException = ExpectedException.none();

private Directory directory;

@Before
public void setUp() throws IOException
{
directory = DirectoryFactory.PERSISTENT.open( testDirectory.directory() );
}

@Test
public void createReadOnlyPartition() throws Exception
{
prepareIndex();
try ( AbstractIndexPartition indexPartition =
new ReadOnlyIndexPartitionFactory().createPartition( testDirectory.directory(), directory ) )
{
expectedException.expect( UnsupportedOperationException.class );

indexPartition.getIndexWriter();
}
}

@Test
public void createWritablePartition() throws Exception
{
try ( AbstractIndexPartition indexPartition =
new WritableIndexPartitionFactory( IndexWriterConfigs::standard )
.createPartition( testDirectory.directory(), directory ) )
{

try ( IndexWriter indexWriter = indexPartition.getIndexWriter() )
{
indexWriter.addDocument( new Document() );
indexWriter.commit();
indexPartition.maybeRefreshBlocking();
try ( PartitionSearcher searcher = indexPartition.acquireSearcher() )
{
assertEquals( "We should be able to see newly added document ",
1, searcher.getIndexSearcher().getIndexReader().numDocs() );
}
}
}
}

private void prepareIndex() throws IOException
{
File location = testDirectory.directory();
try ( AbstractIndexPartition ignored =
new WritableIndexPartitionFactory( IndexWriterConfigs::standard )
.createPartition( location, DirectoryFactory.PERSISTENT.open( location ) ) )
{
// empty
}
}
}

0 comments on commit a26a40b

Please sign in to comment.