Skip to content

Commit

Permalink
LSS logging happens in the monitor
Browse files Browse the repository at this point in the history
  • Loading branch information
tinwelint committed Jan 22, 2017
1 parent f5fb65c commit 8794e1c
Show file tree
Hide file tree
Showing 8 changed files with 154 additions and 131 deletions.
Expand Up @@ -37,49 +37,32 @@ public interface LabelScanStore extends Lifecycle
interface Monitor interface Monitor
{ {
Monitor EMPTY = new Monitor() Monitor EMPTY = new Monitor()
{ { // empty
@Override
public void init()
{
}

@Override
public void noIndex()
{
}

@Override
public void lockedIndex( Exception e )
{
}

@Override
public void notValidIndex()
{
}

@Override
public void rebuilding()
{
}

@Override
public void rebuilt( long roughNodeCount )
{
}
}; };


void init(); default void init()
{ // empty
}


void noIndex(); default void noIndex()
{ // empty
}


void lockedIndex( Exception e ); default void lockedIndex( Exception e )
{ // empty
}


void notValidIndex(); default void notValidIndex()
{ // empty
}


void rebuilding(); default void rebuilding()
{ // empty
}


void rebuilt( long roughNodeCount ); default void rebuilt( long roughNodeCount )
{ // empty
}
} }


/** /**
Expand Down
@@ -0,0 +1,84 @@
/*
* Copyright (c) 2002-2017 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.api.labelscan;

import org.neo4j.kernel.api.labelscan.LabelScanStore.Monitor;
import org.neo4j.logging.Log;

/**
* Logs about important events about {@link LabelScanStore} {@link Monitor}.
*/
public class LoggingMonitor implements Monitor
{
private final Log log;
private final Monitor delegate;

public LoggingMonitor( Log log )
{
this( log, Monitor.EMPTY );
}

public LoggingMonitor( Log log, Monitor delegate )
{
this.log = log;
this.delegate = delegate;
}

@Override
public void init()
{
delegate.init();
}

@Override
public void noIndex()
{
log.info( "No scan store found, this might just be first use. Preparing to rebuild." );
delegate.noIndex();
}

@Override
public void lockedIndex( Exception e )
{
log.error( "Scan store is locked by another process or database", e );
delegate.lockedIndex( e );
}

@Override
public void notValidIndex()
{
log.warn( "Scan store could not be read. Preparing to rebuild." );
delegate.notValidIndex();
}

@Override
public void rebuilding()
{
log.info( "Rebuilding scan store, this may take a while" );
delegate.rebuilding();
}

@Override
public void rebuilt( long roughNodeCount )
{
log.info( "Scan store rebuilt (roughly " + roughNodeCount + " nodes)" );
delegate.rebuilt( roughNodeCount );
}
}
Expand Up @@ -23,13 +23,17 @@


import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.api.labelscan.LoggingMonitor;
import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.api.labelscan.LabelScanStore;
import org.neo4j.kernel.api.labelscan.LabelScanStore.Monitor;
import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.extension.KernelExtensionFactory; import org.neo4j.kernel.extension.KernelExtensionFactory;
import org.neo4j.kernel.impl.api.index.IndexStoreView; import org.neo4j.kernel.impl.api.index.IndexStoreView;
import org.neo4j.kernel.impl.index.labelscan.NativeLabelScanStore; import org.neo4j.kernel.impl.index.labelscan.NativeLabelScanStore;
import org.neo4j.kernel.impl.logging.LogService;
import org.neo4j.kernel.impl.spi.KernelContext; import org.neo4j.kernel.impl.spi.KernelContext;
import org.neo4j.kernel.lifecycle.Lifecycle; import org.neo4j.kernel.lifecycle.Lifecycle;
import org.neo4j.logging.Log;


public class NativeLabelScanStoreExtension extends public class NativeLabelScanStoreExtension extends
KernelExtensionFactory<NativeLabelScanStoreExtension.Dependencies> KernelExtensionFactory<NativeLabelScanStoreExtension.Dependencies>
Expand All @@ -45,6 +49,8 @@ public interface Dependencies
PageCache pageCache(); PageCache pageCache();


Supplier<IndexStoreView> indexStoreView(); Supplier<IndexStoreView> indexStoreView();

LogService getLogService();
} }


public NativeLabelScanStoreExtension() public NativeLabelScanStoreExtension()
Expand All @@ -62,6 +68,8 @@ public NativeLabelScanStoreExtension( int priority, LabelScanStore.Monitor monit
@Override @Override
public Lifecycle newInstance( KernelContext context, Dependencies dependencies ) throws Throwable public Lifecycle newInstance( KernelContext context, Dependencies dependencies ) throws Throwable
{ {
Log log = dependencies.getLogService().getInternalLog( NativeLabelScanStore.class );
Monitor monitor = new LoggingMonitor( log, this.monitor );
NativeLabelScanStore labelScanStore = new NativeLabelScanStore( NativeLabelScanStore labelScanStore = new NativeLabelScanStore(
dependencies.pageCache(), dependencies.pageCache(),
context.storeDir(), context.storeDir(),
Expand Down
Expand Up @@ -24,7 +24,7 @@
import java.io.UncheckedIOException; import java.io.UncheckedIOException;
import java.nio.file.NoSuchFileException; import java.nio.file.NoSuchFileException;
import java.util.function.IntFunction; import java.util.function.IntFunction;
import java.util.stream.Stream; import java.util.stream.Collectors;


import org.neo4j.cursor.RawCursor; import org.neo4j.cursor.RawCursor;
import org.neo4j.graphdb.ResourceIterator; import org.neo4j.graphdb.ResourceIterator;
Expand All @@ -43,6 +43,7 @@
import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.impl.store.UnderlyingStorageException;
import org.neo4j.storageengine.api.schema.LabelScanReader; import org.neo4j.storageengine.api.schema.LabelScanReader;


import static org.neo4j.helpers.collection.Iterables.single;
import static org.neo4j.helpers.collection.Iterators.asResourceIterator; import static org.neo4j.helpers.collection.Iterators.asResourceIterator;
import static org.neo4j.helpers.collection.Iterators.iterator; import static org.neo4j.helpers.collection.Iterators.iterator;
import static org.neo4j.kernel.impl.store.MetaDataStore.DEFAULT_NAME; import static org.neo4j.kernel.impl.store.MetaDataStore.DEFAULT_NAME;
Expand Down Expand Up @@ -73,8 +74,15 @@ public class NativeLabelScanStore implements LabelScanStore
{ {
static final String FILE_NAME = DEFAULT_NAME + ".labelscanstore.db"; static final String FILE_NAME = DEFAULT_NAME + ".labelscanstore.db";


/**
* Whether or not this label scan store is read-only.
*/
private final boolean readOnly; private final boolean readOnly;
private final NativeLabelScanStoreMonitor monitor;
/**
* Monitoring internal events.
*/
private final Monitor monitor;


/** /**
* {@link PageCache} to {@link PageCache#map(File, int, java.nio.file.OpenOption...)} * {@link PageCache} to {@link PageCache#map(File, int, java.nio.file.OpenOption...)}
Expand Down Expand Up @@ -117,6 +125,10 @@ public class NativeLabelScanStore implements LabelScanStore
*/ */
private boolean recoveryStarted; private boolean recoveryStarted;


/**
* Set during {@link #init()} if {@link #start()} will need to rebuild the whole label scan store from
* {@link FullStoreChangeStream}.
*/
private boolean needsRebuild; private boolean needsRebuild;


private final NativeLabelScanWriter singleWriter; private final NativeLabelScanWriter singleWriter;
Expand All @@ -139,7 +151,7 @@ public NativeLabelScanStore( PageCache pageCache, File storeDir,
this.storeFile = new File( storeDir, FILE_NAME ); this.storeFile = new File( storeDir, FILE_NAME );
this.singleWriter = new NativeLabelScanWriter( 1_000 ); this.singleWriter = new NativeLabelScanWriter( 1_000 );
this.readOnly = readOnly; this.readOnly = readOnly;
this.monitor = new NativeLabelScanStoreMonitor( monitor ); this.monitor = monitor;
} }


/** /**
Expand Down Expand Up @@ -209,12 +221,6 @@ public void force( IOLimiter limiter ) throws UnderlyingStorageException
} }
} }


/**
* Unsupported by this implementation.
*
* @return nothing since {@link UnsupportedOperationException} will be thrown.
* @throws UnsupportedOperationException since not supported by this implementation.
*/
@Override @Override
public AllEntriesLabelScanReader allNodeLabelRanges() public AllEntriesLabelScanReader allNodeLabelRanges()
{ {
Expand Down Expand Up @@ -278,16 +284,20 @@ public void init() throws IOException


try try
{ {
create( monitor );
needsRebuild = !storeExists; needsRebuild = !storeExists;
if ( !storeExists )
{
monitor.noIndex();
}

instantiateTree();
} }
catch ( MetadataMismatchException e ) catch ( MetadataMismatchException e )
{ {
// GBPTree is corrupt. Try to rebuild. // GBPTree is corrupt. Try to rebuild.
// todo log
monitor.notValidIndex(); monitor.notValidIndex();
drop(); drop();
create( GBPTree.NO_MONITOR ); instantiateTree();
needsRebuild = true; needsRebuild = true;
} }
} }
Expand All @@ -296,28 +306,28 @@ private boolean storeExists() throws IOException
{ {
try try
{ {
Stream<FileHandle> stream = pageCache.streamFilesRecursive( storeFile ); storeFileHandle();
long count = stream.count(); return true;
if ( count > 1 )
{
throw new IllegalStateException( "Multiple " + storeFile + " existed" );
}
return count == 1;
} }
catch ( NoSuchFileException e ) catch ( NoSuchFileException e )
{ {
return false; return false;
} }
} }


private void create( GBPTree.Monitor monitor ) throws IOException private FileHandle storeFileHandle() throws IOException
{
return single( pageCache.streamFilesRecursive( storeFile ).collect( Collectors.toList() ) );
}

private void instantiateTree() throws IOException
{ {
index = new GBPTree<>( pageCache, storeFile, new LabelScanLayout(), pageSize, monitor ); index = new GBPTree<>( pageCache, storeFile, new LabelScanLayout(), pageSize, GBPTree.NO_MONITOR );
} }


private void drop() throws IOException private void drop() throws IOException
{ {
storeFile.delete(); storeFileHandle().delete();
} }


/** /**
Expand All @@ -336,10 +346,8 @@ public void start() throws IOException
throw new IOException( "Tried to start label scan store " + storeFile + throw new IOException( "Tried to start label scan store " + storeFile +
" as read-only and the index needs rebuild. This makes the label scan store unusable" ); " as read-only and the index needs rebuild. This makes the label scan store unusable" );
} }
// todo log
monitor.rebuilding(); monitor.rebuilding();
long numberOfNodes = LabelScanStoreProvider.rebuild( this, fullStoreChangeStream ); long numberOfNodes = LabelScanStoreProvider.rebuild( this, fullStoreChangeStream );
// todo log
monitor.rebuilt( numberOfNodes ); monitor.rebuilt( numberOfNodes );
needsRebuild = false; needsRebuild = false;
} }
Expand Down Expand Up @@ -372,57 +380,4 @@ public void shutdown() throws IOException
{ {
index.close(); index.close();
} }

// todo make this guy log the stuffs
private class NativeLabelScanStoreMonitor implements Monitor, GBPTree.Monitor
{
private final Monitor delegate;

NativeLabelScanStoreMonitor( Monitor delegate )
{
this.delegate = delegate;
}

@Override
public void init()
{
delegate.init();
}

@Override
public void noIndex()
{
delegate.noIndex();
}

@Override
public void lockedIndex( Exception e )
{
delegate.lockedIndex( e );
}

@Override
public void notValidIndex()
{
delegate.notValidIndex();
}

@Override
public void rebuilding()
{
delegate.rebuilding();
}

@Override
public void rebuilt( long roughNodeCount )
{
delegate.rebuilt( roughNodeCount );
}

@Override
public void noStoreFile()
{
noIndex();
}
}
} }

0 comments on commit 8794e1c

Please sign in to comment.