Skip to content

Commit

Permalink
Extract separate component from RecordStorageEngine to handle id buff…
Browse files Browse the repository at this point in the history
…ering, reuse, maintenance etc.

Update feature toggles base class in RecordStorageEngine (move to RecordStorageEngine instead of NeoStoreDataSource)
  • Loading branch information
MishaDemianenko committed Jul 12, 2016
1 parent 197d45b commit 0263234
Show file tree
Hide file tree
Showing 8 changed files with 247 additions and 74 deletions.
Expand Up @@ -24,7 +24,6 @@
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;

import org.neo4j.concurrent.WorkSync;
Expand All @@ -33,7 +32,6 @@
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.pagecache.IOLimiter;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.NeoStoreDataSource;
import org.neo4j.kernel.api.TokenNameLookup;
import org.neo4j.kernel.api.exceptions.TransactionFailureException;
import org.neo4j.kernel.api.exceptions.schema.ConstraintValidationKernelException;
Expand Down Expand Up @@ -71,10 +69,12 @@
import org.neo4j.kernel.impl.index.IndexConfigStore;
import org.neo4j.kernel.impl.locking.LockGroup;
import org.neo4j.kernel.impl.locking.LockService;
import org.neo4j.kernel.impl.storageengine.impl.recordstorage.id.BufferedRecordStorageIdController;
import org.neo4j.kernel.impl.storageengine.impl.recordstorage.id.DefaultRecordStorageIdController;
import org.neo4j.kernel.impl.storageengine.impl.recordstorage.id.RecordStorageIdController;
import org.neo4j.kernel.impl.store.NeoStores;
import org.neo4j.kernel.impl.store.SchemaStorage;
import org.neo4j.kernel.impl.store.StoreFactory;
import org.neo4j.kernel.impl.store.id.BufferingIdGeneratorFactory;
import org.neo4j.kernel.impl.store.id.IdGeneratorFactory;
import org.neo4j.kernel.impl.store.id.IdReuseEligibility;
import org.neo4j.kernel.impl.store.id.configuration.IdTypeConfigurationProvider;
Expand Down Expand Up @@ -123,9 +123,9 @@
public class RecordStorageEngine implements StorageEngine, Lifecycle
{
private static final boolean takePropertyReadLocks = FeatureToggles.flag(
NeoStoreDataSource.class, "propertyReadLocks", false );
RecordStorageEngine.class, "propertyReadLocks", false );
private static final boolean safeIdBuffering = FeatureToggles.flag(
NeoStoreDataSource.class, "safeIdBuffering", true );
RecordStorageEngine.class, "safeIdBuffering", true );

private final StoreReadLayer storeLayer;
private final IndexingService indexingService;
Expand Down Expand Up @@ -154,16 +154,14 @@ public class RecordStorageEngine implements StorageEngine, Lifecycle
private final LegacyIndexProviderLookup legacyIndexProviderLookup;
private final PropertyPhysicalToLogicalConverter indexUpdatesConverter;
private final Supplier<StorageStatement> storeStatementSupplier;

private BufferingIdGeneratorFactory bufferingIdGeneratorFactory;
private final RecordStorageIdController recordStorageIdController;

// Immutable state for creating/applying commands
private final Loaders loaders;
private final RelationshipCreator relationshipCreator;
private final RelationshipDeleter relationshipDeleter;
private final PropertyCreator propertyCreator;
private final PropertyDeleter propertyDeleter;
private BufferedIdMaintenanceController idMaintenanceController;

public RecordStorageEngine(
File storeDir,
Expand Down Expand Up @@ -203,18 +201,9 @@ public RecordStorageEngine(
this.constraintSemantics = constraintSemantics;
this.legacyIndexTransactionOrdering = legacyIndexTransactionOrdering;

if ( safeIdBuffering )
{
// This buffering id generator factory will have properly buffering id generators injected into
// the stores. The buffering depends on knowledge about active transactions,
// so we'll initialize it below when all those components have been instantiated.
bufferingIdGeneratorFactory = new BufferingIdGeneratorFactory(
idGeneratorFactory, transactionsSnapshotSupplier, eligibleForReuse, idTypeConfigurationProvider );
idMaintenanceController = new BufferedIdMaintenanceController( bufferingIdGeneratorFactory );

idGeneratorFactory = bufferingIdGeneratorFactory;
}
StoreFactory factory = new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fs, logProvider );
this.recordStorageIdController = createStorageIdController( idGeneratorFactory, eligibleForReuse,
idTypeConfigurationProvider, transactionsSnapshotSupplier );
StoreFactory factory = new StoreFactory( storeDir, config, recordStorageIdController.getIdGeneratorFactory(), pageCache, fs, logProvider );
neoStores = factory.openAllNeoStores( true );

try
Expand Down Expand Up @@ -267,6 +256,17 @@ public RecordStorageEngine(
}
}

private RecordStorageIdController createStorageIdController( IdGeneratorFactory idGeneratorFactory,
IdReuseEligibility eligibleForReuse,
IdTypeConfigurationProvider idTypeConfigurationProvider,
Supplier<KernelTransactionsSnapshot> transactionsSnapshotSupplier )
{
return safeIdBuffering ?
new BufferedRecordStorageIdController( idGeneratorFactory, transactionsSnapshotSupplier,
eligibleForReuse, idTypeConfigurationProvider, scheduler ) :
new DefaultRecordStorageIdController( idGeneratorFactory );
}

private Supplier<StorageStatement> storeStatementSupplier( NeoStores neoStores )
{
Supplier<IndexReaderFactory> indexReaderFactory = () -> new IndexReaderFactory.Caching( indexingService );
Expand Down Expand Up @@ -402,10 +402,7 @@ public void satisfyDependencies( DependencySatisfier satisfier )
// providing TransactionIdStore, LogVersionRepository
satisfier.satisfyDependency( neoStores.getMetaDataStore() );
satisfier.satisfyDependency( indexStoreView );
if (idMaintenanceController != null)
{
satisfier.satisfyDependency( idMaintenanceController );
}
satisfier.satisfyDependency( recordStorageIdController );
}

@Override
Expand All @@ -431,10 +428,7 @@ public void start() throws Throwable
loadSchemaCache();
indexingService.start();
labelScanStore.start();
if ( safeIdBuffering )
{
idMaintenanceController.start();
}
recordStorageIdController.start();
}

@Override
Expand All @@ -447,21 +441,15 @@ public void loadSchemaCache()
@Override
public void clearBufferedIds()
{
if ( bufferingIdGeneratorFactory != null )
{
bufferingIdGeneratorFactory.clear();
}
recordStorageIdController.clear();
}

@Override
public void stop() throws Throwable
{
labelScanStore.stop();
indexingService.stop();
if ( safeIdBuffering )
{
idMaintenanceController.stop();
}
recordStorageIdController.stop();
}

@Override
Expand Down Expand Up @@ -519,30 +507,4 @@ public NeoStores testAccessNeoStores()
{
return neoStores;
}

public class BufferedIdMaintenanceController
{
private final BufferingIdGeneratorFactory bufferingIdGeneratorFactory;
private JobScheduler.JobHandle jobHandle;

BufferedIdMaintenanceController( BufferingIdGeneratorFactory bufferingIdGeneratorFactory )
{
this.bufferingIdGeneratorFactory = bufferingIdGeneratorFactory;
}

public void start() throws Throwable
{
jobHandle = scheduler.scheduleRecurring( JobScheduler.Groups.storageMaintenance, this::maintenance, 1, TimeUnit.SECONDS );
}

public void stop() throws Throwable
{
jobHandle.cancel( false );
}

public void maintenance()
{
bufferingIdGeneratorFactory.maintenance();
}
}
}
@@ -0,0 +1,112 @@
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.impl.storageengine.impl.recordstorage.id;


import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;

import org.neo4j.kernel.impl.api.KernelTransactionsSnapshot;
import org.neo4j.kernel.impl.store.id.BufferingIdGeneratorFactory;
import org.neo4j.kernel.impl.store.id.IdGeneratorFactory;
import org.neo4j.kernel.impl.store.id.IdReuseEligibility;
import org.neo4j.kernel.impl.store.id.configuration.IdTypeConfigurationProvider;
import org.neo4j.kernel.impl.util.JobScheduler;
import org.neo4j.kernel.lifecycle.LifecycleAdapter;

/**
* Storage id controller that provide buffering possibilities to be able so safely free and reuse ids.
* Allows perform clear and maintenance operations over currently buffered set of ids.
* @see BufferingIdGeneratorFactory
*/
public class BufferedRecordStorageIdController extends LifecycleAdapter implements RecordStorageIdController
{

private final BufferingIdGeneratorFactory bufferingIdGeneratorFactory;
private final BufferedIdMaintenanceController idMaintenanceController;

public BufferedRecordStorageIdController( IdGeneratorFactory idGeneratorFactory,
Supplier<KernelTransactionsSnapshot> transactionsSnapshotSupplier, IdReuseEligibility eligibleForReuse,
IdTypeConfigurationProvider idTypeConfigurationProvider, JobScheduler scheduler )
{
bufferingIdGeneratorFactory = new BufferingIdGeneratorFactory(
idGeneratorFactory, transactionsSnapshotSupplier, eligibleForReuse, idTypeConfigurationProvider );
idMaintenanceController = new BufferedIdMaintenanceController( bufferingIdGeneratorFactory, scheduler );
}

public IdGeneratorFactory getIdGeneratorFactory()
{
return bufferingIdGeneratorFactory;
}

@Override
public void start() throws Throwable
{
idMaintenanceController.start();
}

@Override
public void stop() throws Throwable
{
idMaintenanceController.stop();
}

@Override
public void clear()
{
bufferingIdGeneratorFactory.clear();
}

@Override
public void maintenance()
{
idMaintenanceController.maintenance();
}

private static class BufferedIdMaintenanceController
{
private final BufferingIdGeneratorFactory bufferingIdGeneratorFactory;
private final JobScheduler scheduler;
private JobScheduler.JobHandle jobHandle;

BufferedIdMaintenanceController( BufferingIdGeneratorFactory bufferingIdGeneratorFactory,
JobScheduler scheduler )
{
this.bufferingIdGeneratorFactory = bufferingIdGeneratorFactory;
this.scheduler = scheduler;
}

public void start() throws Throwable
{
jobHandle = scheduler.scheduleRecurring( JobScheduler.Groups.storageMaintenance, this::maintenance, 1,
TimeUnit.SECONDS );
}

public void stop() throws Throwable
{
jobHandle.cancel( false );
}

public void maintenance()
{
bufferingIdGeneratorFactory.maintenance();
}
}
}
@@ -0,0 +1,54 @@
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.impl.storageengine.impl.recordstorage.id;


import org.neo4j.kernel.impl.store.id.IdGeneratorFactory;
import org.neo4j.kernel.lifecycle.LifecycleAdapter;

/**
* Default implementation of {@link RecordStorageIdController}.
* Do not add any additional possibilities or functionality. Wraps provided {@link IdGeneratorFactory}.
*/
public class DefaultRecordStorageIdController extends LifecycleAdapter implements RecordStorageIdController
{

private IdGeneratorFactory idGeneratorFactory;

public DefaultRecordStorageIdController( IdGeneratorFactory idGeneratorFactory )
{
this.idGeneratorFactory = idGeneratorFactory;
}

public IdGeneratorFactory getIdGeneratorFactory()
{
return idGeneratorFactory;
}

@Override
public void clear()
{
}

@Override
public void maintenance()
{
}
}
@@ -0,0 +1,47 @@
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.impl.storageengine.impl.recordstorage.id;


import org.neo4j.kernel.impl.store.id.IdGeneratorFactory;
import org.neo4j.kernel.lifecycle.Lifecycle;

/**
* Represent abstraction that responsible for any id related operations on a storage engine level: buffering,
* maintenance, clearing, resetting, generation.
*/
public interface RecordStorageIdController extends Lifecycle
{
/**
* Retrieve id generation factory for current storage engine
* @return id generation factory
*/
IdGeneratorFactory getIdGeneratorFactory();

/**
* Clear underlying id generation infrastructure (clear buffer of ids to reuse, reset buffers, etc.)
*/
void clear();

/**
* Perform ids related maintenance.
*/
void maintenance();
}

0 comments on commit 0263234

Please sign in to comment.