Skip to content

Commit

Permalink
Improved GBPTree file handling
Browse files Browse the repository at this point in the history
Introduce interface GBPTreeFileUtil that gathers file operations
used inside of components built on top of GBPTree, label index
and schema index.

Implementation decide how those file operations should be carried out.
Specifically if it should be done using normal file system or file system
provided by page cache.
  • Loading branch information
burqen committed Sep 8, 2017
1 parent 618db39 commit 18d4e37
Show file tree
Hide file tree
Showing 9 changed files with 444 additions and 53 deletions.
Expand Up @@ -24,73 +24,46 @@
import java.nio.file.NoSuchFileException;

import org.neo4j.index.internal.gbptree.GBPTree;
import org.neo4j.io.fs.FileHandle;
import org.neo4j.io.pagecache.PageCache;

import static org.neo4j.function.Predicates.alwaysTrue;

/**
* Utilities for common operations around a {@link GBPTree}.
*/
public class GBPTreeUtil
public interface GBPTreeFileUtil
{
/**
* Deletes store file backing a {@link GBPTree}.
* Undefined behaviour if storeFile is a directory.
*
* @param pageCache {@link PageCache} which manages the file.
* @param storeFile the {@link File} to delete.
* @throws NoSuchFileException if the {@code storeFile} doesn't exist according to the {@code pageCache}.
* @throws IOException on failure to delete existing {@code storeFile}.
*/
public static void delete( PageCache pageCache, File storeFile ) throws IOException
{
FileHandle fileHandle = storeFileHandle( pageCache, storeFile );
fileHandle.delete();
}
void deleteFile( File storeFile ) throws IOException;

/**
* Deletes store file backing a {@link GBPTree}, if it exists according to the {@code pageCache}.
* Undefined behaviour if storeFile is a directory.
*
* @param pageCache {@link PageCache} which manages the file.
* @param storeFile the {@link File} to delete.
* @throws IOException on failure to delete existing {@code storeFile}.
*/
public static void deleteIfPresent( PageCache pageCache, File storeFile ) throws IOException
{
try
{
delete( pageCache, storeFile );
}
catch ( NoSuchFileException e )
{
// File does not exist, we don't need to delete
}
}
void deleteFileIfPresent( File storeFile ) throws IOException;

/**
* Checks whether or not {@code storeFile} exists according to {@code pageCache}.
* Undefined behaviour if storeFile is a directory.
*
* @param pageCache {@link PageCache} which manages the file.
* @param storeFile the {@link File} to check for existence.
* @return {@code true} if {@code storeFile} exists according to {@code pageCache}, otherwise {@code false}.
*/
public static boolean storeFileExists( PageCache pageCache, File storeFile )
{
try
{
return pageCache.getCachedFileSystem().streamFilesRecursive( storeFile ).anyMatch( alwaysTrue() );
}
catch ( IOException e )
{
return false;
}
}
boolean storeFileExists( File storeFile );

private static FileHandle storeFileHandle( PageCache pageCache, File storeFile ) throws IOException
{
return pageCache.getCachedFileSystem()
.streamFilesRecursive( storeFile )
.findFirst()
.orElseThrow( () -> new NoSuchFileException( storeFile.getPath() ) );
}
/**
* Creates the directory named by this abstract pathname, including any
* necessary but nonexistent parent directories.
*
* @param dir the directory path to create
* @throws IOException on failure to create directories.
*/
void mkdirs( File dir ) throws IOException;
}
@@ -0,0 +1,87 @@
/*
* Copyright (c) 2002-2017 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.impl.index.labelscan;

import java.io.File;
import java.io.IOException;
import java.nio.file.NoSuchFileException;

import org.neo4j.io.fs.FileHandle;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.impl.index.GBPTreeFileUtil;

import static org.neo4j.function.Predicates.alwaysTrue;

/**
* Use {@link PageCache} to handle file operations on GBPTree file.
*/
public class GBPTreePageCacheFileUtil implements GBPTreeFileUtil
{
private final PageCache pageCache;

public GBPTreePageCacheFileUtil( PageCache pageCache )
{
this.pageCache = pageCache;
}

public void deleteFile( File storeFile ) throws IOException
{
FileHandle fileHandle = storeFileHandle( pageCache, storeFile );
fileHandle.delete();
}

public void deleteFileIfPresent( File storeFile ) throws IOException
{
try
{
deleteFile( storeFile );
}
catch ( NoSuchFileException e )
{
// File does not exist, we don't need to delete
}
}

public boolean storeFileExists( File storeFile )
{
try
{
return pageCache.getCachedFileSystem().streamFilesRecursive( storeFile ).anyMatch( alwaysTrue() );
}
catch ( IOException e )
{
return false;
}
}

@Override
public void mkdirs( File dir ) throws IOException
{
pageCache.getCachedFileSystem().mkdirs( dir );
}

private static FileHandle storeFileHandle( PageCache pageCache, File storeFile ) throws IOException
{
return pageCache.getCachedFileSystem()
.streamFilesRecursive( storeFile )
.findFirst()
.orElseThrow( () -> new NoSuchFileException( storeFile.getPath() ) );
}
}
Expand Up @@ -43,7 +43,7 @@
import org.neo4j.kernel.api.labelscan.LabelScanStore;
import org.neo4j.kernel.api.labelscan.LabelScanWriter;
import org.neo4j.kernel.impl.api.scan.FullStoreChangeStream;
import org.neo4j.kernel.impl.index.GBPTreeUtil;
import org.neo4j.kernel.impl.index.GBPTreeFileUtil;
import org.neo4j.kernel.impl.store.UnderlyingStorageException;
import org.neo4j.kernel.monitoring.Monitors;
import org.neo4j.storageengine.api.schema.LabelScanReader;
Expand Down Expand Up @@ -130,6 +130,11 @@ public class NativeLabelScanStore implements LabelScanStore
*/
private final int pageSize;

/**
* Used for all file operations on the gbpTree file.
*/
private final GBPTreeFileUtil gbpTreeUtil;

/**
* The index which backs this label scan store. Instantiated in {@link #init()} and considered
* started after call to {@link #start()}.
Expand Down Expand Up @@ -186,6 +191,7 @@ public NativeLabelScanStore( PageCache pageCache, File storeDir, FullStoreChange
this.monitors = monitors;
this.monitor = monitors.newMonitor( Monitor.class );
this.recoveryCleanupWorkCollector = recoveryCleanupWorkCollector;
this.gbpTreeUtil = new GBPTreePageCacheFileUtil( pageCache );
}

/**
Expand Down Expand Up @@ -342,7 +348,7 @@ public void init() throws IOException
@Override
public boolean hasStore() throws IOException
{
return GBPTreeUtil.storeFileExists( pageCache, storeFile );
return gbpTreeUtil.storeFileExists( storeFile );
}

@Override
Expand Down Expand Up @@ -402,7 +408,7 @@ private void dropStrict() throws IOException
index.close();
index = null;
}
GBPTreeUtil.delete( pageCache, storeFile );
gbpTreeUtil.deleteFile( storeFile );
}

/**
Expand Down
@@ -0,0 +1,68 @@
/*
* Copyright (c) 2002-2017 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.impl.index.schema;

import java.io.File;
import java.io.IOException;
import java.nio.file.NoSuchFileException;

import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.kernel.impl.index.GBPTreeFileUtil;

public class GBPTreeFileSystemFileUtil implements GBPTreeFileUtil
{
private final FileSystemAbstraction fs;

public GBPTreeFileSystemFileUtil( FileSystemAbstraction fs )
{
this.fs = fs;
}

@Override
public void deleteFile( File storeFile ) throws IOException
{
fs.deleteFileOrThrow( storeFile );
}

@Override
public void deleteFileIfPresent( File storeFile ) throws IOException
{
try
{
deleteFile( storeFile );
}
catch ( NoSuchFileException e )
{
// File does not exist, we don't need to delete
}
}

@Override
public boolean storeFileExists( File storeFile )
{
return fs.fileExists( storeFile );
}

@Override
public void mkdirs( File dir ) throws IOException
{
fs.mkdirs( dir );
}
}
Expand Up @@ -30,6 +30,7 @@
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.io.pagecache.PageCursor;
import org.neo4j.kernel.impl.index.GBPTreeFileUtil;

import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_READER;
import static org.neo4j.index.internal.gbptree.GBPTree.NO_MONITOR;
Expand All @@ -39,16 +40,16 @@ class NativeSchemaNumberIndex<KEY extends SchemaNumberKey, VALUE extends SchemaN
final PageCache pageCache;
final File storeFile;
final Layout<KEY,VALUE> layout;
final FileSystemAbstraction fs;
final GBPTreeFileUtil gbpTreeFileUtil;

GBPTree<KEY,VALUE> tree;

NativeSchemaNumberIndex( PageCache pageCache, FileSystemAbstraction fs, File storeFile, Layout<KEY,VALUE> layout )
{
this.pageCache = pageCache;
this.fs = fs;
this.storeFile = storeFile;
this.layout = layout;
this.gbpTreeFileUtil = new GBPTreeFileSystemFileUtil( fs );
}

void instantiateTree( RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, Consumer<PageCursor> headerWriter )
Expand All @@ -63,7 +64,7 @@ private void ensureDirectoryExist() throws IOException
{
// This will create the directory on the "normal" file system.
// When native index is put on blockdevice, page cache file system should be used instead.
fs.mkdirs( storeFile.getParentFile() );
gbpTreeFileUtil.mkdirs( storeFile.getParentFile() );
}

void closeTree() throws IOException
Expand Down
Expand Up @@ -35,7 +35,6 @@
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.PropertyAccessor;
import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
import org.neo4j.kernel.impl.index.GBPTreeUtil;
import org.neo4j.storageengine.api.schema.IndexReader;

import static org.neo4j.helpers.collection.Iterators.asResourceIterator;
Expand All @@ -59,7 +58,7 @@ public class NativeSchemaNumberIndexAccessor<KEY extends SchemaNumberKey, VALUE
public void drop() throws IOException
{
closeTree();
GBPTreeUtil.delete( pageCache, storeFile );
gbpTreeFileUtil.deleteFile( storeFile );
}

@Override
Expand Down
Expand Up @@ -41,7 +41,6 @@
import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.index.PropertyAccessor;
import org.neo4j.kernel.impl.index.GBPTreeUtil;

import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_WRITER;

Expand Down Expand Up @@ -78,7 +77,7 @@ public abstract class NativeSchemaNumberIndexPopulator<KEY extends SchemaNumberK
@Override
public synchronized void create() throws IOException
{
GBPTreeUtil.deleteIfPresent( pageCache, storeFile );
gbpTreeFileUtil.deleteFileIfPresent( storeFile );
instantiateTree( RecoveryCleanupWorkCollector.IMMEDIATE, new NativeSchemaIndexHeaderWriter( BYTE_POPULATING ) );
instantiateWriter();
workSync = new WorkSync<>( new IndexUpdateApply<>( treeKey, treeValue, singleTreeWriter, conflictDetectingValueMerger ) );
Expand All @@ -97,7 +96,7 @@ public synchronized void drop() throws IOException
{
closeWriter();
closeTree();
GBPTreeUtil.deleteIfPresent( pageCache, storeFile );
gbpTreeFileUtil.deleteFileIfPresent( storeFile );
}
finally
{
Expand Down

0 comments on commit 18d4e37

Please sign in to comment.