From 514e7cb685a06175e57410ec70322a77643bd0c6 Mon Sep 17 00:00:00 2001 From: MishaDemianenko Date: Wed, 6 Dec 2017 18:33:27 +0100 Subject: [PATCH] Data snapshot query execution Introduce possibility to execute query based on data snapshot. Any data modification will mark corresponding pages with a version that will be equal to transaction id in which that particular change was introduced. Also allow count store to track transactions in which changes where introduced. To be able to guarantee that query result is based on a data that was present on query execution start and was not modified while it was running, the engine will verify that all paged that were accessed during query execution have the version that is less or equal that was last closed on a moment when query started. Support for snapshot query execution in CC and HA Snapshot query execution is enabled by 'unsupported.dbms.query.snapshot' setting and number of retries that engine will do while trying to get stable snapshot is controlled by 'unsupported.dbms.query.snapshot.retries'. --- .../consistency/ConsistencyCheckService.java | 5 +- .../checking/GraphStoreFixture.java | 3 +- ...etectAllRelationshipInconsistenciesIT.java | 5 +- community/cypher/cypher/pom.xml | 5 + .../CommunityCypherEngineProvider.java | 23 ++ .../internal/javacompat/EagerResult.java | 194 ++++++++++ .../cypher/internal/javacompat}/MapRow.java | 2 +- .../javacompat/SnapshotExecutionEngine.java | 113 ++++++ .../javacompat/UnstableSnapshotException.java | 31 ++ .../javacompat/EagerResultITTest.java | 345 ++++++++++++++++++ .../SnapshotExecutionEngineTest.java | 143 ++++++++ .../TransactionBoundQueryContextTest.scala | 3 +- .../index/internal/gbptree/TreePrinter.java | 5 +- .../gbptree/GBPTreePartialCreateFuzzIT.java | 6 +- .../io/pagecache/impl/muninn/CursorPool.java | 13 +- .../impl/muninn/MuninnPageCache.java | 9 +- .../impl/muninn/MuninnPageCursor.java | 5 +- .../impl/muninn/MuninnPagedFile.java | 45 ++- .../impl/muninn/MuninnReadPageCursor.java | 20 +- .../impl/muninn/MuninnWritePageCursor.java | 7 +- .../muninn/StandalonePageCacheFactory.java | 13 +- .../cursor/context/EmptyVersionContext.java | 73 ++++ .../context/EmptyVersionContextSupplier.java | 46 +++ .../cursor/context/VersionContext.java | 72 ++++ .../context/VersionContextSupplier.java | 42 +++ .../io/pagecache/PageCacheTestSupport.java | 23 +- .../impl/muninn/MuninnPageCacheFixture.java | 5 +- .../impl/muninn/MuninnPageCacheTest.java | 296 ++++++++++++++- .../RandomPageCacheTestHarness.java | 3 +- .../pagecache/stress/PageCacheStressTest.java | 3 +- .../recording/RecordingPageCursorTracer.java | 2 +- .../org/neo4j/test/rule/PageCacheRule.java | 3 +- .../factory/GraphDatabaseSettings.java | 11 + .../org/neo4j/kernel/NeoStoreDataSource.java | 44 ++- .../kernel/impl/api/KernelStatement.java | 16 +- .../api/KernelTransactionImplementation.java | 14 +- .../kernel/impl/api/KernelTransactions.java | 8 +- .../kernel/impl/api/TransactionToApply.java | 22 +- .../context/TransactionVersionContext.java | 90 +++++ .../TransactionVersionContextSupplier.java | 48 +++ .../kernel/impl/factory/DataSourceModule.java | 3 +- .../kernel/impl/factory/PlatformModule.java | 21 +- ...onfigurableStandalonePageCacheFactory.java | 13 +- .../ConfiguringPageCacheFactory.java | 9 +- .../recordstorage/RecordStorageEngine.java | 7 +- .../neo4j/kernel/impl/store/NeoStores.java | 7 +- .../neo4j/kernel/impl/store/StoreAccess.java | 3 +- .../neo4j/kernel/impl/store/StoreFactory.java | 17 +- .../impl/store/counts/CountsTracker.java | 78 ++-- .../store/counts/ReadOnlyCountsTracker.java | 3 +- .../store/kvstore/AbstractKeyValueStore.java | 9 +- .../impl/store/kvstore/ActiveState.java | 10 +- .../store/kvstore/ConcurrentMapState.java | 102 ++++-- .../kernel/impl/store/kvstore/DeadState.java | 28 +- .../impl/store/kvstore/PrototypeState.java | 4 +- .../impl/store/kvstore/RotationState.java | 3 +- .../kernel/impl/store/kvstore/State.java | 11 +- .../DirectRecordStoreMigrator.java | 12 +- .../participant/CountsMigrator.java | 5 +- .../NativeLabelScanStoreMigrator.java | 3 +- .../participant/StoreMigrator.java | 6 +- .../internal/BatchInserterImpl.java | 6 +- .../batchimport/store/BatchingNeoStores.java | 10 +- .../java/org/neo4j/kernel/RecoveryIT.java | 22 +- .../kernel/api/KernelTransactionFactory.java | 3 +- .../api/KernelTransactionTerminationTest.java | 4 +- .../impl/api/KernelTransactionTestBase.java | 3 +- .../impl/api/KernelTransactionsTest.java | 11 +- .../api/LockingStatementOperationsTest.java | 3 +- ...ropertyPhysicalToLogicalConverterTest.java | 4 +- .../StoreNodeRelationshipCursorTest.java | 6 +- .../api/store/StorePropertyCursorTest.java | 4 +- .../StoreSingleRelationshipCursorTest.java | 3 +- .../kernel/impl/core/ManyPropertyKeysIT.java | 3 +- .../ConfiguringPageCacheFactoryTest.java | 12 +- .../impl/store/FreeIdsAfterRecoveryTest.java | 3 +- ...dGeneratorRebuildFailureEmulationTest.java | 11 +- .../kernel/impl/store/MetaDataStoreTest.java | 3 +- .../kernel/impl/store/NeoStoresTest.java | 30 +- .../kernel/impl/store/NodeStoreTest.java | 15 +- .../store/RecordStoreConsistentReadTest.java | 3 +- .../store/RelationshipGroupStoreTest.java | 3 +- .../kernel/impl/store/SchemaStoreTest.java | 3 +- .../kernel/impl/store/StoreFactoryTest.java | 3 +- .../kernel/impl/store/TestArrayStore.java | 3 +- .../kernel/impl/store/TestDynamicStore.java | 3 +- .../impl/store/TestGraphProperties.java | 3 +- .../store/TestGrowingFileMemoryMapping.java | 3 +- .../impl/store/TestIdGeneratorRebuilding.java | 4 +- .../impl/store/counts/CountsComputerTest.java | 5 +- .../impl/store/counts/CountsRotationTest.java | 3 +- .../impl/store/counts/CountsTrackerTest.java | 69 +++- .../kvstore/AbstractKeyValueStoreTest.java | 6 +- .../store/kvstore/ConcurrentMapStateTest.java | 124 ++++++- .../participant/StoreMigratorIT.java | 7 +- .../state/ApplyRecoveredTransactionsTest.java | 3 +- .../transaction/state/NodeCommandTest.java | 4 +- .../state/NodeLabelsFieldTest.java | 3 +- .../state/RelationshipGroupGetterTest.java | 3 +- ...ersarialPageCacheGraphDatabaseFactory.java | 7 +- .../test/rule/ConfigurablePageCacheRule.java | 4 +- .../test/rule/NeoStoreDataSourceRule.java | 3 +- .../org/neo4j/test/rule/NeoStoresRule.java | 5 +- .../test/rule/RecordStorageEngineRule.java | 3 +- .../test/java/db/DatabaseShutdownTest.java | 8 +- .../src/test/java/db/QueryRestartIT.java | 288 +++++++++++++++ .../storemigration/StoreUpgraderTest.java | 3 +- .../ExecutionResultSerializerTest.java | 3 +- .../GraphExtractionWriterTest.java | 1 + .../RestRepresentationWriterTest.java | 10 +- .../rest/transactional/RowWriterTest.java | 1 + .../catchup/tx/BatchingTxApplier.java | 8 +- .../catchup/tx/TransactionApplier.java | 5 +- .../core/state/CoreBootstrapper.java | 3 +- .../machines/CoreStateMachinesModule.java | 15 +- .../token/ReplicatedTokenStateMachine.java | 8 +- .../tx/ReplicatedTransactionStateMachine.java | 8 +- .../EnterpriseReadReplicaEditionModule.java | 3 +- .../catchup/tx/BatchingTxApplierTest.java | 3 +- .../id/RebuildReplicatedIdGeneratorsTest.java | 3 +- .../ReplicatedTokenStateMachineTest.java | 13 +- ...tProcessStateMachineCollaborationTest.java | 3 +- ...ReplicatedTransactionStateMachineTest.java | 8 +- .../impl/muninn/VersionContextTrackingIT.java | 214 +++++++++++ .../storecopy/BatchingResponseHandler.java | 9 +- .../storecopy/ExternallyManagedPageCache.java | 5 +- ...TransactionCommittingResponseUnpacker.java | 19 +- .../neo4j/com/storecopy/ResponsePackerIT.java | 3 +- ...sactionCommittingResponseUnpackerTest.java | 3 + .../EnterpriseCypherEngineProvider.java | 23 ++ .../src/test/java/cypher/MapRow.java | 102 ------ .../feature/parser/ParsingTestSupport.scala | 2 +- .../neo4j/ha/upgrade/MasterClientTest.java | 2 + .../kernel/ha/OnDiskLastTxIdGetterTest.java | 3 +- ...llerTriggersPageTransactionTrackingIT.java | 224 ++++++++++++ .../participant/StoreMigratorTest.java | 3 +- .../org/neo4j/tools/dump/DumpCountsStore.java | 6 +- .../java/org/neo4j/tools/dump/DumpStore.java | 3 +- .../org/neo4j/tools/dump/DumpStoreChain.java | 3 +- .../neo4j/tools/rawstorereader/RsdrMain.java | 3 +- 140 files changed, 3161 insertions(+), 463 deletions(-) create mode 100644 community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EagerResult.java rename community/{server/src/test/java/org/neo4j/server/rest/transactional => cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat}/MapRow.java (98%) create mode 100644 community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngine.java create mode 100644 community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/UnstableSnapshotException.java create mode 100644 community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/EagerResultITTest.java create mode 100644 community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngineTest.java create mode 100644 community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContext.java create mode 100644 community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContextSupplier.java create mode 100644 community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContext.java create mode 100644 community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContextSupplier.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContext.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContextSupplier.java create mode 100644 community/neo4j/src/test/java/db/QueryRestartIT.java create mode 100644 enterprise/causal-clustering/src/test/java/org/neo4j/io/pagecache/impl/muninn/VersionContextTrackingIT.java delete mode 100644 enterprise/cypher/spec-suite-tools/src/test/java/cypher/MapRow.java create mode 100644 enterprise/ha/src/test/java/org/neo4j/kernel/ha/UpdatePullerTriggersPageTransactionTrackingIT.java diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java index b1f442b3371c..563af0841538 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java @@ -43,6 +43,7 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.direct.DirectStoreAccess; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.configuration.Config; @@ -150,7 +151,7 @@ public Result runFullConsistencyCheck( File storeDir, Config config, ProgressMon Log log = logProvider.getLog( getClass() ); ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( fileSystem, config, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, - logProvider.getLog( PageCache.class ) ); + logProvider.getLog( PageCache.class ), EmptyVersionContextSupplier.INSTANCE ); PageCache pageCache = pageCacheFactory.getOrCreatePageCache(); try @@ -209,7 +210,7 @@ public Result runFullConsistencyCheck( final File storeDir, Config config, Progr config.augment( GraphDatabaseSettings.read_only, TRUE ); StoreFactory factory = new StoreFactory( storeDir, config, - new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, logProvider ); + new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, logProvider, EmptyVersionContextSupplier.INSTANCE ); ConsistencySummaryStatistics summary; final File reportFile = chooseReportPath( reportDir ); diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java index 3590bdc26cf0..7eb4b385f273 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java @@ -42,6 +42,7 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.ReadOperations; import org.neo4j.kernel.api.direct.DirectStoreAccess; import org.neo4j.kernel.api.exceptions.TransactionFailureException; @@ -166,7 +167,7 @@ public DirectStoreAccess directStoreAccess() Config config = Config.defaults(); DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fileSystem ); StoreFactory storeFactory = new StoreFactory( - directory, config, idGeneratorFactory, pageCache, fileSystem, logProvider ); + directory, config, idGeneratorFactory, pageCache, fileSystem, logProvider, EmptyVersionContextSupplier.INSTANCE ); neoStore = storeFactory.openAllNeoStores(); StoreAccess nativeStores; if ( keepStatistics ) diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java index bfd934bd43e4..f01bfffee4b5 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java @@ -32,11 +32,11 @@ import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Relationship; import org.neo4j.graphdb.Transaction; -import org.neo4j.graphdb.factory.GraphDatabaseBuilder; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.progress.ProgressMonitorFactory; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.direct.DirectStoreAccess; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.configuration.Config; @@ -150,7 +150,8 @@ private StoreFactory newStoreFactory( PageCache pageCache ) { FileSystemAbstraction fileSystem = fileSystemRule.get(); return new StoreFactory( directory.directory(), getTuningConfiguration(), - new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, NullLogProvider.getInstance() ); + new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, NullLogProvider.getInstance(), + EmptyVersionContextSupplier.INSTANCE ); } private Config getTuningConfiguration() diff --git a/community/cypher/cypher/pom.xml b/community/cypher/cypher/pom.xml index 9c61f53bace2..9ec415492c97 100644 --- a/community/cypher/cypher/pom.xml +++ b/community/cypher/cypher/pom.xml @@ -385,6 +385,11 @@ commons-codec + + commons-codec + commons-codec + + diff --git a/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/CommunityCypherEngineProvider.java b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/CommunityCypherEngineProvider.java index 402c838130b4..822dd28a8113 100644 --- a/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/CommunityCypherEngineProvider.java +++ b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/CommunityCypherEngineProvider.java @@ -22,8 +22,10 @@ import org.neo4j.cypher.internal.CommunityCompatibilityFactory; import org.neo4j.cypher.javacompat.internal.GraphDatabaseCypherService; import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Service; import org.neo4j.kernel.api.KernelAPI; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.query.QueryEngineProvider; import org.neo4j.kernel.impl.query.QueryExecutionEngine; @@ -56,10 +58,31 @@ protected QueryExecutionEngine createEngine( Dependencies deps, GraphDatabaseAPI LogService logService = resolver.resolveDependency( LogService.class ); KernelAPI kernelAPI = resolver.resolveDependency( KernelAPI.class ); Monitors monitors = resolver.resolveDependency( Monitors.class ); + Config config = resolver.resolveDependency( Config.class ); LogProvider logProvider = logService.getInternalLogProvider(); CommunityCompatibilityFactory compatibilityFactory = new CommunityCompatibilityFactory( queryService, kernelAPI, monitors, logProvider ); deps.satisfyDependencies( compatibilityFactory ); + return createEngine( queryService, config, logProvider, compatibilityFactory ); + } + + private QueryExecutionEngine createEngine( GraphDatabaseCypherService queryService, Config config, + LogProvider logProvider, CommunityCompatibilityFactory compatibilityFactory ) + { + return config.get( GraphDatabaseSettings.snapshot_query ) ? + snapshotEngine( queryService, config, logProvider, compatibilityFactory ) : + standardEngine( queryService, logProvider, compatibilityFactory ); + } + + private SnapshotExecutionEngine snapshotEngine( GraphDatabaseCypherService queryService, Config config, + LogProvider logProvider, CommunityCompatibilityFactory compatibilityFactory ) + { + return new SnapshotExecutionEngine( queryService, config, logProvider, compatibilityFactory ); + } + + private ExecutionEngine standardEngine( GraphDatabaseCypherService queryService, LogProvider logProvider, + CommunityCompatibilityFactory compatibilityFactory ) + { return new ExecutionEngine( queryService, logProvider, compatibilityFactory ); } } diff --git a/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EagerResult.java b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EagerResult.java new file mode 100644 index 000000000000..61fa65953da1 --- /dev/null +++ b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EagerResult.java @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.javacompat; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.neo4j.graphdb.ExecutionPlanDescription; +import org.neo4j.graphdb.Notification; +import org.neo4j.graphdb.QueryExecutionType; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.graphdb.ResourceIterator; +import org.neo4j.graphdb.Result; + +import static java.lang.System.lineSeparator; + +/** + * Result produced as result of eager query execution for cases when {@link SnapshotExecutionEngine} is used. + */ +class EagerResult implements Result +{ + private static final String ITEM_SEPARATOR = ", "; + private final Result originalResult; + private final List> queryResult = new ArrayList<>(); + private int cursor; + + EagerResult( Result result ) + { + this.originalResult = result; + } + + public void consume() + { + while ( originalResult.hasNext() ) + { + queryResult.add( originalResult.next() ); + } + } + + @Override + public QueryExecutionType getQueryExecutionType() + { + return originalResult.getQueryExecutionType(); + } + + @Override + public List columns() + { + return originalResult.columns(); + } + + @Override + public ResourceIterator columnAs( String name ) + { + return new EagerResultResourceIterator<>( name ); + } + + @Override + public boolean hasNext() + { + return cursor < queryResult.size(); + } + + @Override + public Map next() + { + return queryResult.get( cursor++ ); + } + + @Override + public void close() + { + // nothing to close. Original result is already closed at this point + } + + @Override + public QueryStatistics getQueryStatistics() + { + return originalResult.getQueryStatistics(); + } + + @Override + public ExecutionPlanDescription getExecutionPlanDescription() + { + return originalResult.getExecutionPlanDescription(); + } + + @Override + public String resultAsString() + { + List columns = originalResult.columns(); + StringBuilder builder = new StringBuilder(); + builder.append( String.join( ITEM_SEPARATOR, columns ) ); + if ( !queryResult.isEmpty() ) + { + builder.append( lineSeparator() ); + int numberOfColumns = columns.size(); + for ( Map row : queryResult ) + { + writeRow( columns, builder, numberOfColumns, row ); + builder.append( lineSeparator() ); + } + } + return builder.toString(); + } + + @Override + public void writeAsStringTo( PrintWriter writer ) + { + writer.print( resultAsString() ); + } + + @Override + public void remove() + { + throw new UnsupportedOperationException( "Not supported" ); + } + + @Override + public Iterable getNotifications() + { + return originalResult.getNotifications(); + } + + @Override + public void accept( ResultVisitor visitor ) + throws VisitationException + { + for ( Map map : queryResult ) + { + visitor.visit( new MapRow( map ) ); + } + } + + private void writeRow( List columns, StringBuilder builder, int numberOfColumns, Map row ) + { + for ( int i = 0; i < numberOfColumns; i++ ) + { + builder.append( row.get( columns.get( i ) ) ); + if ( i != numberOfColumns - 1 ) + { + builder.append( ITEM_SEPARATOR ); + } + } + } + + private class EagerResultResourceIterator implements ResourceIterator + { + private final String column; + int cursor; + + EagerResultResourceIterator( String column ) + { + this.column = column; + } + + @Override + public boolean hasNext() + { + return cursor < queryResult.size(); + } + + @Override + public T next() + { + return (T) queryResult.get( cursor++ ).get( column ); + } + + @Override + public void close() + { + // Nothing to close. + } + } +} diff --git a/community/server/src/test/java/org/neo4j/server/rest/transactional/MapRow.java b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/MapRow.java similarity index 98% rename from community/server/src/test/java/org/neo4j/server/rest/transactional/MapRow.java rename to community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/MapRow.java index 8772b41671f3..90d0c3ab3860 100644 --- a/community/server/src/test/java/org/neo4j/server/rest/transactional/MapRow.java +++ b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/MapRow.java @@ -17,7 +17,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.server.rest.transactional; +package org.neo4j.cypher.internal.javacompat; import java.util.Map; import java.util.NoSuchElementException; diff --git a/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngine.java b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngine.java new file mode 100644 index 000000000000..ce7d723d1547 --- /dev/null +++ b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngine.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.javacompat; + +import java.util.Map; + +import org.neo4j.cypher.internal.CompatibilityFactory; +import org.neo4j.graphdb.Result; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.kernel.GraphDatabaseQueryService; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.api.KernelStatement; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.logging.LogProvider; + +/** + * {@link ExecutionEngine} engine that will try to run cypher query with guarantee that query will never see any data + * that coming from transaction that are newer then transaction that was the last closed on a moment when + * {@link VersionContext} was initialised. Observed behaviour is the same as executing query on top data snapshot for + * that version. + */ +public class SnapshotExecutionEngine extends ExecutionEngine +{ + private final int maxQueryExecutionAttempts; + + SnapshotExecutionEngine( GraphDatabaseQueryService queryService, Config config, LogProvider logProvider, + CompatibilityFactory compatibilityFactory ) + { + super( queryService, logProvider, compatibilityFactory ); + this.maxQueryExecutionAttempts = config.get( GraphDatabaseSettings.snapshot_query_retries ); + } + + @Override + public Result executeQuery( String query, Map parameters, TransactionalContext context ) + throws QueryExecutionKernelException + { + return executeWithRetries( query, parameters, context, super::executeQuery ); + } + + @Override + public Result profileQuery( String query, Map parameters, TransactionalContext context ) + throws QueryExecutionKernelException + { + return executeWithRetries( query, parameters, context, super::profileQuery ); + } + + protected Result executeWithRetries( String query, Map parameters, TransactionalContext context, + QueryExecutor executor ) throws QueryExecutionKernelException + { + VersionContext versionContext = getCursorContext( context ); + EagerResult eagerResult; + int attempt = 0; + boolean dirtySnapshot; + do + { + if ( attempt == maxQueryExecutionAttempts ) + { + return throwQueryExecutionException( + "Unable to get clean data snapshot for query '%s' after %d attempts.", query, attempt ); + } + attempt++; + versionContext.initRead(); + Result result = executor.execute( query, parameters, context ); + eagerResult = new EagerResult( result ); + eagerResult.consume(); + dirtySnapshot = versionContext.isDirty(); + if ( dirtySnapshot && result.getQueryStatistics().containsUpdates() ) + { + return throwQueryExecutionException( + "Unable to get clean data snapshot for query '%s' that perform updates.", query, attempt ); + } + } + while ( dirtySnapshot ); + return eagerResult; + } + + private Result throwQueryExecutionException( String message, Object... parameters ) throws + QueryExecutionKernelException + { + throw new QueryExecutionKernelException( new UnstableSnapshotException( message, parameters ) ); + } + + private static VersionContext getCursorContext( TransactionalContext context ) + { + return ((KernelStatement) context.statement()).getVersionContext(); + } + + @FunctionalInterface + protected interface QueryExecutor + { + Result execute( String query, Map parameters, TransactionalContext context ) throws QueryExecutionKernelException; + } + +} diff --git a/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/UnstableSnapshotException.java b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/UnstableSnapshotException.java new file mode 100644 index 000000000000..df8ea7bda019 --- /dev/null +++ b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/UnstableSnapshotException.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.javacompat; + +import org.neo4j.kernel.api.exceptions.KernelException; +import org.neo4j.kernel.api.exceptions.Status; + +class UnstableSnapshotException extends KernelException +{ + UnstableSnapshotException( String message, Object... parameters ) + { + super( Status.Transaction.Outdated, message, parameters ); + } +} diff --git a/community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/EagerResultITTest.java b/community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/EagerResultITTest.java new file mode 100644 index 000000000000..ede2b39a3db4 --- /dev/null +++ b/community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/EagerResultITTest.java @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.javacompat; + +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.LongSupplier; + +import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.QueryExecutionType; +import org.neo4j.graphdb.ResourceIterator; +import org.neo4j.graphdb.Result; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.factory.GraphDatabaseBuilder; +import org.neo4j.graphdb.factory.GraphDatabaseFactoryState; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.helpers.collection.Iterables; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; +import org.neo4j.kernel.GraphDatabaseDependencies; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.configuration.Settings; +import org.neo4j.kernel.impl.context.TransactionVersionContext; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; +import org.neo4j.kernel.impl.factory.CommunityEditionModule; +import org.neo4j.kernel.impl.factory.DatabaseInfo; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; +import org.neo4j.kernel.impl.factory.PlatformModule; +import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; +import org.neo4j.kernel.internal.GraphDatabaseAPI; +import org.neo4j.test.TestGraphDatabaseFactory; +import org.neo4j.test.rule.TestDirectory; + +import static org.hamcrest.Matchers.hasSize; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +public class EagerResultITTest +{ + @Rule + public final TestDirectory testDirectory = TestDirectory.testDirectory(); + private GraphDatabaseService database; + private TestTransactionVersionContextSupplier testContextSupplier; + private File storeDir; + private TestVersionContext testCursorContext; + + @Before + public void setUp() + { + storeDir = testDirectory.directory(); + testContextSupplier = new TestTransactionVersionContextSupplier(); + database = startRestartableDatabase(); + prepareData(); + TransactionIdStore transactionIdStore = getTransactionIdStore(); + testCursorContext = new TestVersionContext( transactionIdStore::getLastClosedTransactionId ); + testContextSupplier.setCursorContext( testCursorContext ); + } + + @After + public void tearDown() + { + if ( database != null ) + { + database.shutdown(); + } + } + + @Test + public void eagerResultContainsAllData() + { + Result result = database.execute( "MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + int rows = 0; + while ( result.hasNext() ) + { + result.next(); + rows++; + } + assertEquals( 2, rows ); + } + + @Test + public void eagerResultContainsExecutionType() + { + Result result = database.execute( "MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + assertEquals( QueryExecutionType.query( QueryExecutionType.QueryType.READ_ONLY ), result.getQueryExecutionType() ); + } + + @Test + public void eagerResultContainsColumns() + { + Result result = database.execute( "MATCH (n) RETURN n.c as a, count(n) as b" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + assertEquals( Arrays.asList("a", "b"), result.columns() ); + } + + @Test + public void useColumnAsOnEagerResult() + { + Result result = database.execute( "MATCH (n) RETURN n.c as c, n.b as b" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + ResourceIterator cValues = result.columnAs( "c" ); + int rows = 0; + while ( cValues.hasNext() ) + { + cValues.next(); + rows++; + } + assertEquals( 2, rows ); + } + + @Test + public void eagerResultHaveQueryStatistic() + { + Result result = database.execute( "MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + assertFalse( result.getQueryStatistics().containsUpdates() ); + } + + @Test + public void eagerResultHaveExecutionPlan() + { + Result result = database.execute( "profile MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + assertEquals( 2, result.getExecutionPlanDescription().getProfilerStatistics().getRows() ); + } + + @Test + public void eagerResultHaveNotifications() + { + Result result = database.execute( " CYPHER planner=rule MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + assertEquals( 1, Iterables.count( result.getNotifications() ) ); + } + + @Test + public void eagerResultToString() + { + Result result = database.execute( "MATCH (n) RETURN n.c, n.d" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + String resultString = result.resultAsString(); + assertTrue( resultString.contains( "n.c, n.d" ) ); + assertTrue( resultString.contains( "d, a" ) ); + assertTrue( resultString.contains( "y, k" ) ); + } + + @Test + public void eagerResultWriteAsStringToStream() + { + Result result = database.execute( "MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + assertEquals( result.resultAsString(), printToStream( result ) ); + } + + @Test + public void eagerResultVisit() throws Exception + { + Result result = database.execute( "MATCH (n) RETURN n.c" ); + List values = new ArrayList<>(); + result.accept( (Result.ResultVisitor) row -> + { + values.add( row.getString( "n.c" ) ); + return false; + } ); + assertThat( values, hasSize( 2 ) ); + assertThat( values, Matchers.containsInAnyOrder( "d", "y" ) ); + } + + private String printToStream( Result result ) + { + StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter( stringWriter ); + result.writeAsStringTo( printWriter ); + printWriter.flush(); + return stringWriter.toString(); + } + + private void prepareData() + { + Label label = Label.label( "label" ); + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode( label ); + node.setProperty( "c", "d" ); + node.setProperty( "d", "a" ); + transaction.success(); + } + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode( label ); + node.setProperty( "c", "y" ); + node.setProperty( "d", "k" ); + transaction.success(); + } + } + + private GraphDatabaseService startRestartableDatabase() + { + return new CustomGraphDatabaseFactory( new CustomFacadeFactory() ) + .newEmbeddedDatabaseBuilder( storeDir ) + .setConfig( GraphDatabaseSettings.snapshot_query, Settings.TRUE ) + .newGraphDatabase(); + } + + private TransactionIdStore getTransactionIdStore() + { + DependencyResolver dependencyResolver = ((GraphDatabaseAPI) database).getDependencyResolver(); + return dependencyResolver.resolveDependency( TransactionIdStore.class ); + } + + private class CustomGraphDatabaseFactory extends TestGraphDatabaseFactory + { + + private GraphDatabaseFacadeFactory customFacadeFactory; + + CustomGraphDatabaseFactory( GraphDatabaseFacadeFactory customFacadeFactory ) + { + this.customFacadeFactory = customFacadeFactory; + } + + @Override + protected GraphDatabaseBuilder.DatabaseCreator createDatabaseCreator( File storeDir, + GraphDatabaseFactoryState state ) + { + return new GraphDatabaseBuilder.DatabaseCreator() + { + @Override + public GraphDatabaseService newDatabase( Map config ) + { + return newDatabase( Config.defaults( config ) ); + } + + @Override + public GraphDatabaseService newDatabase( Config config ) + { + return customFacadeFactory.newFacade( storeDir, config, + GraphDatabaseDependencies.newDependencies( state.databaseDependencies() ) ); + } + }; + } + } + + private class CustomFacadeFactory extends GraphDatabaseFacadeFactory + { + + CustomFacadeFactory() + { + super( DatabaseInfo.COMMUNITY, CommunityEditionModule::new ); + } + + @Override + protected PlatformModule createPlatform( File storeDir, Config config, Dependencies dependencies, + GraphDatabaseFacade graphDatabaseFacade ) + { + return new PlatformModule( storeDir, config, databaseInfo, dependencies, graphDatabaseFacade ) + { + @Override + protected VersionContextSupplier createCursorContextSupplier( Config config ) + { + return testContextSupplier != null ? testContextSupplier : super.createCursorContextSupplier(config); + } + }; + } + } + + private class TestVersionContext extends TransactionVersionContext + { + + private boolean useCorrectLastCommittedTxId; + private int additionalAttempts; + + TestVersionContext( LongSupplier transactionIdSupplier ) + { + super( transactionIdSupplier ); + } + + @Override + public long lastClosedTransactionId() + { + return useCorrectLastCommittedTxId ? TransactionIdStore.BASE_TX_ID : super.lastClosedTransactionId(); + } + + @Override + public void markAsDirty() + { + super.markAsDirty(); + useCorrectLastCommittedTxId = true; + } + + @Override + public boolean isDirty() + { + additionalAttempts++; + return super.isDirty(); + } + + int getAdditionalAttempts() + { + return additionalAttempts; + } + } + + private class TestTransactionVersionContextSupplier extends TransactionVersionContextSupplier + { + void setCursorContext( VersionContext versionContext ) + { + this.cursorContext.set( versionContext ); + } + } +} diff --git a/community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngineTest.java b/community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngineTest.java new file mode 100644 index 000000000000..21a6338e3f27 --- /dev/null +++ b/community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngineTest.java @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.javacompat; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.util.Collections; +import java.util.Map; + +import org.neo4j.cypher.internal.CompatibilityFactory; +import org.neo4j.cypher.javacompat.internal.GraphDatabaseCypherService; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.graphdb.Result; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.kernel.GraphDatabaseQueryService; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.api.KernelStatement; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.test.rule.DatabaseRule; +import org.neo4j.test.rule.ImpermanentDatabaseRule; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyMap; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SnapshotExecutionEngineTest +{ + @Rule + public final DatabaseRule database = new ImpermanentDatabaseRule(); + + private CompatibilityFactory compatibilityFactory; + private TestSnapshotExecutionEngine executionEngine; + private VersionContext versionContext; + private SnapshotExecutionEngine.QueryExecutor executor; + private TransactionalContext transactionalContext; + private final Config config = Config.defaults(); + + @Before + public void setUp() throws Exception + { + GraphDatabaseQueryService cypherService = new GraphDatabaseCypherService( this.database.getGraphDatabaseAPI() ); + + compatibilityFactory = mock( CompatibilityFactory.class ); + transactionalContext = mock( TransactionalContext.class ); + KernelStatement kernelStatement = mock( KernelStatement.class ); + executor = mock( SnapshotExecutionEngine.QueryExecutor.class ); + versionContext = mock( VersionContext.class ); + + executionEngine = createExecutionEngine(cypherService); + when( kernelStatement.getVersionContext() ).thenReturn( versionContext ); + when( transactionalContext.statement() ).thenReturn( kernelStatement ); + Result result = mock( Result.class ); + when( result.getQueryStatistics() ).thenReturn( mock( QueryStatistics.class ) ); + when( executor.execute( any(), anyMap(), any() ) ).thenReturn( result ); + } + + @Test + public void executeQueryWithoutRetries() throws QueryExecutionKernelException + { + executionEngine.executeWithRetries( "query", Collections.emptyMap(), transactionalContext, executor ); + + verify( executor, times( 1 ) ).execute( any(), anyMap(), any() ); + verify( versionContext, times( 1 ) ).initRead(); + } + + @Test + public void executeQueryAfterSeveralRetries() throws QueryExecutionKernelException + { + when( versionContext.isDirty() ).thenReturn( true, true, false ); + + executionEngine.executeWithRetries( "query", Collections.emptyMap(), transactionalContext, executor ); + + verify( executor, times( 3 ) ).execute( any(), anyMap(), any() ); + verify( versionContext, times( 3 ) ).initRead(); + } + + @Test + public void failQueryAfterMaxRetriesReached() throws QueryExecutionKernelException + { + when( versionContext.isDirty() ).thenReturn( true ); + + try + { + executionEngine.executeWithRetries( "query", Collections.emptyMap(), transactionalContext, executor ); + } + catch ( QueryExecutionKernelException e ) + { + assertEquals( "Unable to get clean data snapshot for query 'query' after 5 attempts.", e.getMessage() ); + } + + verify( executor, times( 5 ) ).execute( any(), anyMap(), any() ); + verify( versionContext, times( 5 ) ).initRead(); + } + + private class TestSnapshotExecutionEngine extends SnapshotExecutionEngine + { + + TestSnapshotExecutionEngine( GraphDatabaseQueryService queryService, Config config, LogProvider logProvider, + CompatibilityFactory compatibilityFactory ) + { + super( queryService, config, logProvider, compatibilityFactory ); + } + + @Override + public Result executeWithRetries( String query, Map parameters, TransactionalContext context, + QueryExecutor executor ) throws QueryExecutionKernelException + { + return super.executeWithRetries( query, parameters, context, executor ); + } + } + + private TestSnapshotExecutionEngine createExecutionEngine( GraphDatabaseQueryService cypherService ) + { + return new TestSnapshotExecutionEngine( cypherService, config, NullLogProvider.getInstance(), + compatibilityFactory ); + } +} diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/v3_3/TransactionBoundQueryContextTest.scala b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/v3_3/TransactionBoundQueryContextTest.scala index ee4089b1fbf9..0efc4a9af1ec 100644 --- a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/v3_3/TransactionBoundQueryContextTest.scala +++ b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/v3_3/TransactionBoundQueryContextTest.scala @@ -33,6 +33,7 @@ import org.neo4j.graphdb._ import org.neo4j.graphdb.config.Setting import org.neo4j.graphdb.factory.GraphDatabaseSettings import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContext import org.neo4j.kernel.api._ import org.neo4j.kernel.api.security.SecurityContext.AUTH_DISABLED import org.neo4j.kernel.api.security.{AnonymousContext, SecurityContext} @@ -67,7 +68,7 @@ class TransactionBoundQueryContextTest extends CypherFunSuite { val operations = mock[StatementOperationParts](RETURNS_DEEP_STUBS) statement = new KernelStatement(kernelTransaction, null, storeStatement, new Procedures(), new CanWrite(), LockTracer.NONE, operations) - statement.initialize(null, PageCursorTracerSupplier.NULL.get()) + statement.initialize(null, PageCursorTracerSupplier.NULL.get(), EmptyVersionContext.INSTANCE) statement.acquire() } diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreePrinter.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreePrinter.java index ee39463823b6..d6e6c001c60c 100644 --- a/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreePrinter.java +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/TreePrinter.java @@ -33,9 +33,9 @@ import org.neo4j.io.pagecache.impl.SingleFilePageSwapperFactory; import org.neo4j.io.pagecache.impl.muninn.MuninnPageCache; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import static java.lang.String.format; - import static org.neo4j.graphdb.config.Configuration.EMPTY; import static org.neo4j.index.internal.gbptree.ConsistencyChecker.assertOnTreeNode; import static org.neo4j.index.internal.gbptree.GenerationSafePointerPair.pointer; @@ -75,7 +75,8 @@ public static void printHeader( FileSystemAbstraction fs, File file, PrintStream { SingleFilePageSwapperFactory swapper = new SingleFilePageSwapperFactory(); swapper.open( fs, EMPTY ); - try ( PageCache pageCache = new MuninnPageCache( swapper, 100, (int) kibiBytes( 8 ), NULL, PageCursorTracerSupplier.NULL ) ) + try ( PageCache pageCache = new MuninnPageCache( swapper, 100, (int) kibiBytes( 8 ), NULL, + PageCursorTracerSupplier.NULL, EmptyVersionContextSupplier.INSTANCE ) ) { printHeader( pageCache, file, out ); } diff --git a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreePartialCreateFuzzIT.java b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreePartialCreateFuzzIT.java index 36c0e4d31b66..4f5ea9bac25e 100644 --- a/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreePartialCreateFuzzIT.java +++ b/community/index/src/test/java/org/neo4j/index/internal/gbptree/GBPTreePartialCreateFuzzIT.java @@ -33,13 +33,13 @@ import org.neo4j.io.pagecache.impl.muninn.MuninnPageCache; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.test.rule.PageCacheAndDependenciesRule; import org.neo4j.test.rule.fs.DefaultFileSystemRule; -import static org.junit.Assert.assertNotEquals; - import static java.lang.ProcessBuilder.Redirect.INHERIT; import static java.util.Arrays.asList; +import static org.junit.Assert.assertNotEquals; import static org.neo4j.graphdb.config.Configuration.EMPTY; import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_READER; import static org.neo4j.io.ByteUnit.kibiBytes; @@ -105,7 +105,7 @@ public static void main( String[] args ) throws IOException SingleFilePageSwapperFactory swapper = new SingleFilePageSwapperFactory(); swapper.open( fs, EMPTY ); try ( PageCache pageCache = new MuninnPageCache( swapper, 10, (int) kibiBytes( 8 ), - PageCacheTracer.NULL, PageCursorTracerSupplier.NULL ) ) + PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, EmptyVersionContextSupplier.INSTANCE ) ) { fs.deleteFile( file ); new GBPTreeBuilder<>( pageCache, file, new SimpleLongLayout() ).build().close(); diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/CursorPool.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/CursorPool.java index 0c6bb8e208d9..fe1eba4f76c2 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/CursorPool.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/CursorPool.java @@ -22,6 +22,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; final class CursorPool extends ThreadLocal { @@ -29,6 +30,7 @@ final class CursorPool extends ThreadLocal private final long victimPage; private final PageCursorTracerSupplier pageCursorTracerSupplier; private PageCacheTracer pageCacheTracer; + private final VersionContextSupplier versionContextSupplier; /** * Cursor pool construction @@ -36,13 +38,16 @@ final class CursorPool extends ThreadLocal * @param pageCursorTracerSupplier supplier of thread local (transaction local) page cursor tracers that will * provide thread local page cache statistics * @param pageCacheTracer global page cache tracer + * @param versionContextSupplier version context supplier */ - CursorPool( MuninnPagedFile pagedFile, PageCursorTracerSupplier pageCursorTracerSupplier, PageCacheTracer pageCacheTracer ) + CursorPool( MuninnPagedFile pagedFile, PageCursorTracerSupplier pageCursorTracerSupplier, + PageCacheTracer pageCacheTracer, VersionContextSupplier versionContextSupplier ) { this.pagedFile = pagedFile; this.victimPage = pagedFile.pageCache.victimPage; this.pageCursorTracerSupplier = pageCursorTracerSupplier; this.pageCacheTracer = pageCacheTracer; + this.versionContextSupplier = versionContextSupplier; } @Override @@ -69,7 +74,8 @@ MuninnReadPageCursor takeReadCursor( long pageId, int pf_flags ) private MuninnReadPageCursor createReadCursor( CursorSets cursorSets ) { - MuninnReadPageCursor cursor = new MuninnReadPageCursor( cursorSets, victimPage, getPageCursorTracer() ); + MuninnReadPageCursor cursor = new MuninnReadPageCursor( cursorSets, victimPage, getPageCursorTracer(), versionContextSupplier + .getVersionContext() ); cursor.initialiseFile( pagedFile ); return cursor; } @@ -92,7 +98,8 @@ MuninnWritePageCursor takeWriteCursor( long pageId, int pf_flags ) private MuninnWritePageCursor createWriteCursor( CursorSets cursorSets ) { - MuninnWritePageCursor cursor = new MuninnWritePageCursor( cursorSets, victimPage, getPageCursorTracer() ); + MuninnWritePageCursor cursor = new MuninnWritePageCursor( cursorSets, victimPage, getPageCursorTracer(), versionContextSupplier + .getVersionContext() ); cursor.initialiseFile( pagedFile ); return cursor; } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java index 79202163630d..8f039e0cd7cd 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java @@ -46,6 +46,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.PageFaultEvent; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.unsafe.impl.internal.dragons.MemoryManager; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; @@ -152,6 +153,7 @@ public class MuninnPageCache implements PageCache private final int keepFree; private final PageCacheTracer pageCacheTracer; private final PageCursorTracerSupplier pageCursorTracerSupplier; + private final VersionContextSupplier versionContextSupplier; final PageList pages; // All PageCursors are initialised with their pointers pointing to the victim page. This way, we don't have to throw // exceptions on bounds checking failures; we can instead return the victim page pointer, and permit the page @@ -198,13 +200,16 @@ public class MuninnPageCache implements PageCache * @param pageCacheTracer global page cache tracer * @param pageCursorTracerSupplier supplier of thread local (transaction local) page cursor tracer that will provide * thread local page cache statistics + * @param versionContextSupplier supplier of thread local (transaction local) version context that will provide + * access to thread local version context */ public MuninnPageCache( PageSwapperFactory swapperFactory, int maxPages, int cachePageSize, PageCacheTracer pageCacheTracer, - PageCursorTracerSupplier pageCursorTracerSupplier ) + PageCursorTracerSupplier pageCursorTracerSupplier, + VersionContextSupplier versionContextSupplier ) { verifyHacks(); verifyCachePageSizeIsPowerOfTwo( cachePageSize ); @@ -216,6 +221,7 @@ public MuninnPageCache( this.keepFree = Math.min( pagesToKeepFree, maxPages / 2 ); this.pageCacheTracer = pageCacheTracer; this.pageCursorTracerSupplier = pageCursorTracerSupplier; + this.versionContextSupplier = versionContextSupplier; this.printExceptionsOnClose = true; long alignment = swapperFactory.getRequiredBufferAlignment(); @@ -338,6 +344,7 @@ else if ( !ignoredOpenOptions.contains( option ) ) swapperFactory, pageCacheTracer, pageCursorTracerSupplier, + versionContextSupplier, createIfNotExists, truncateExisting ); pagedFile.incrementRefCount(); diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCursor.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCursor.java index 03b92e3a9804..47da06877f88 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCursor.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCursor.java @@ -29,6 +29,7 @@ import org.neo4j.io.pagecache.tracing.PageFaultEvent; import org.neo4j.io.pagecache.tracing.PinEvent; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; import static org.neo4j.io.pagecache.PagedFile.PF_EAGER_FLUSH; @@ -65,6 +66,7 @@ abstract class MuninnPageCursor extends PageCursor private long pointer; private int pageSize; private int filePageSize; + protected VersionContext versionContext; private int offset; private boolean outOfBounds; private boolean isLinkedCursor; @@ -73,11 +75,12 @@ abstract class MuninnPageCursor extends PageCursor // offending code. private Object cursorException; - MuninnPageCursor( long victimPage, PageCursorTracer tracer ) + MuninnPageCursor( long victimPage, PageCursorTracer tracer, VersionContext versionContext ) { this.victimPage = victimPage; this.pointer = victimPage; this.tracer = tracer; + this.versionContext = versionContext; } final void initialiseFile( MuninnPagedFile pagedFile ) diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPagedFile.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPagedFile.java index bc13c465b9d0..d24443929dbb 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPagedFile.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPagedFile.java @@ -40,6 +40,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.PageFaultEvent; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; final class MuninnPagedFile extends PageList implements PagedFile, Flushable @@ -78,6 +79,10 @@ final class MuninnPagedFile extends PageList implements PagedFile, Flushable // Used to trace the causes of any exceptions from getLastPageId. private volatile Exception closeStackTrace; + // max modifier transaction id among evicted pages for this file + private static final long evictedTransactionIdOffset = UnsafeUtil.getFieldOffset( MuninnPagedFile.class, "highestEvictedTransactionId" ); + private volatile long highestEvictedTransactionId; + /** * The header state includes both the reference count of the PagedFile – 15 bits – and the ID of the last page in * the file – 48 bits, plus an empty file marker bit. Because our pages are usually 2^13 bytes, this means that we @@ -104,24 +109,20 @@ final class MuninnPagedFile extends PageList implements PagedFile, Flushable * @param pageCacheTracer global page cache tracer * @param pageCursorTracerSupplier supplier of thread local (transaction local) page cursor tracer that will provide * thread local page cache statistics + * @param versionContextSupplier supplier of thread local (transaction local) version context that will provide + * access to thread local version context * @param createIfNotExists should create file if it does not exists * @param truncateExisting should truncate file if it exists * @throws IOException If the {@link PageSwapper} could not be created. */ - MuninnPagedFile( - File file, - MuninnPageCache pageCache, - int filePageSize, - PageSwapperFactory swapperFactory, - PageCacheTracer pageCacheTracer, - PageCursorTracerSupplier pageCursorTracerSupplier, - boolean createIfNotExists, - boolean truncateExisting ) throws IOException + MuninnPagedFile( File file, MuninnPageCache pageCache, int filePageSize, PageSwapperFactory swapperFactory, + PageCacheTracer pageCacheTracer, PageCursorTracerSupplier pageCursorTracerSupplier, + VersionContextSupplier versionContextSupplier, boolean createIfNotExists, boolean truncateExisting ) throws IOException { super( pageCache.pages ); this.pageCache = pageCache; this.filePageSize = filePageSize; - this.cursorPool = new CursorPool( this, pageCursorTracerSupplier, pageCacheTracer ); + this.cursorPool = new CursorPool( this, pageCursorTracerSupplier, pageCacheTracer, versionContextSupplier ); this.pageCacheTracer = pageCacheTracer; this.pageFaultLatches = new LatchMap(); @@ -655,6 +656,25 @@ void evictPage( long filePageId ) UnsafeUtil.putIntVolatile( chunk, chunkOffset, UNMAPPED_TTE ); } + void setHighestEvictedTransactionId( long modifiedTransactionId ) + { + long highestModifier; + do + { + highestModifier = this.highestEvictedTransactionId; + if ( highestModifier >= modifiedTransactionId ) + { + return; + } + } + while ( !UnsafeUtil.compareAndSwapLong( this, evictedTransactionIdOffset, highestModifier, modifiedTransactionId ) ); + } + + long getHighestEvictedTransactionId() + { + return highestEvictedTransactionId; + } + /** * Expand the translation table such that it can include at least the given chunkId. * @param maxChunkId The new translation table must be big enough to include at least this chunkId. @@ -701,4 +721,9 @@ static long computeChunkOffset( long filePageId ) int index = (int) (filePageId & translationTableChunkSizeMask); return UnsafeUtil.arrayOffset( index, translationTableChunkArrayBase, translationTableChunkArrayScale ); } + + public void setLastModifiedTxId( long l ) + { + + } } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnReadPageCursor.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnReadPageCursor.java index f2750dfe5c69..d4a1140670b2 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnReadPageCursor.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnReadPageCursor.java @@ -23,6 +23,7 @@ import org.neo4j.io.pagecache.PageSwapper; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; final class MuninnReadPageCursor extends MuninnPageCursor { @@ -30,9 +31,10 @@ final class MuninnReadPageCursor extends MuninnPageCursor private long lockStamp; MuninnReadPageCursor nextCursor; - MuninnReadPageCursor( CursorPool.CursorSets cursorSets, long victimPage, PageCursorTracer pageCursorTracer ) + MuninnReadPageCursor( CursorPool.CursorSets cursorSets, long victimPage, PageCursorTracer pageCursorTracer, + VersionContext versionContext ) { - super( victimPage, pageCursorTracer ); + super( victimPage, pageCursorTracer, versionContext ); this.cursorSets = cursorSets; } @@ -58,10 +60,24 @@ public boolean next() throws IOException } pin( nextPageId, false ); currentPageId = nextPageId; + verifyContext(); nextPageId++; return true; } + private void verifyContext() + { + if ( versionContext.lastClosedTransactionId() == Long.MAX_VALUE ) + { + return; + } +// if ( page.getLastModifiedTxId() > versionContext.lastClosedTransactionId() || +// pagedFile.getHighestEvictedTransactionId() > versionContext.lastClosedTransactionId() ) +// { +// versionContext.markAsDirty(); +// } + } + @Override protected boolean tryLockPage( long pageRef ) { diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnWritePageCursor.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnWritePageCursor.java index bd383409a080..084f6a7937b0 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnWritePageCursor.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnWritePageCursor.java @@ -24,15 +24,17 @@ import org.neo4j.io.pagecache.PageSwapper; import org.neo4j.io.pagecache.PagedFile; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; final class MuninnWritePageCursor extends MuninnPageCursor { private final CursorPool.CursorSets cursorSets; MuninnWritePageCursor nextCursor; - MuninnWritePageCursor( CursorPool.CursorSets cursorSets, long victimPage, PageCursorTracer pageCursorTracer ) + MuninnWritePageCursor( CursorPool.CursorSets cursorSets, long victimPage, PageCursorTracer pageCursorTracer, + VersionContext versionContext ) { - super( victimPage, pageCursorTracer ); + super( victimPage, pageCursorTracer, versionContext ); this.cursorSets = cursorSets; } @@ -124,6 +126,7 @@ protected void pinCursorToPage( long pageRef, long filePageId, PageSwapper swapp // be closed and the page lock will be released. assertPagedFileStillMappedAndGetIdOfLastPage(); pagedFile.incrementUsage( pageRef ); + pagedFile.setLastModifiedTxId( versionContext.committingTransactionId() ); } @Override diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/StandalonePageCacheFactory.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/StandalonePageCacheFactory.java index 6a055c229fe2..737af3c487c0 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/StandalonePageCacheFactory.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/StandalonePageCacheFactory.java @@ -27,6 +27,8 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; /* * This class is an helper to allow to construct properly a page cache in the few places we need it without all @@ -43,16 +45,18 @@ private StandalonePageCacheFactory() public static PageCache createPageCache( FileSystemAbstraction fileSystem ) { - return createPageCache( fileSystem, null, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE ); + return createPageCache( fileSystem, null, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE, + EmptyVersionContextSupplier.INSTANCE ); } public static PageCache createPageCache( FileSystemAbstraction fileSystem, Integer pageSize ) { - return createPageCache( fileSystem, pageSize, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE ); + return createPageCache( fileSystem, pageSize, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE, + EmptyVersionContextSupplier.INSTANCE ); } public static PageCache createPageCache( FileSystemAbstraction fileSystem, Integer pageSize, - PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier ) + PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier, VersionContextSupplier versionContextSupplier ) { SingleFilePageSwapperFactory factory = new SingleFilePageSwapperFactory(); factory.open( fileSystem, Configuration.EMPTY ); @@ -60,6 +64,7 @@ public static PageCache createPageCache( FileSystemAbstraction fileSystem, Integ int cachePageSize = pageSize != null ? pageSize : factory.getCachePageSizeHint(); long pageCacheMemory = ByteUnit.mebiBytes( 8 ); long pageCount = pageCacheMemory / cachePageSize; - return new MuninnPageCache( factory, (int) pageCount, cachePageSize, tracer, cursorTracerSupplier ); + return new MuninnPageCache( factory, (int) pageCount, cachePageSize, tracer, cursorTracerSupplier, + versionContextSupplier ); } } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContext.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContext.java new file mode 100644 index 000000000000..f327c68f07b2 --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContext.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.tracing.cursor.context; + +/** + * {@link VersionContext} that does not perform any kind of version tracking for cases when its not required. + * @see VersionContext + */ +public class EmptyVersionContext implements VersionContext +{ + public static final VersionContext INSTANCE = new EmptyVersionContext(); + + private EmptyVersionContext() + { + } + + @Override + public void initRead() + { + } + + @Override + public void initWrite( long committingTransactionId ) + { + + } + + @Override + public long committingTransactionId() + { + return 0; + } + + @Override + public long lastClosedTransactionId() + { + return Long.MAX_VALUE; + } + + @Override + public void markAsDirty() + { + + } + + @Override + public boolean isDirty() + { + return false; + } + + @Override + public void clear() + { + } +} diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContextSupplier.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContextSupplier.java new file mode 100644 index 000000000000..b03be30110e2 --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContextSupplier.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.tracing.cursor.context; + +import java.util.function.LongSupplier; + +/** + * {@link VersionContextSupplier} version that supply same {@link EmptyVersionContext} each time for cases + * where version context mechanics is not required + */ +public class EmptyVersionContextSupplier implements VersionContextSupplier +{ + public static final VersionContextSupplier INSTANCE = new EmptyVersionContextSupplier(); + + private EmptyVersionContextSupplier() + { + } + + @Override + public void init( LongSupplier lastClosedTransactionIdSupplier ) + { + } + + @Override + public VersionContext getVersionContext() + { + return EmptyVersionContext.INSTANCE; + } +} diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContext.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContext.java new file mode 100644 index 000000000000..442105aafad4 --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContext.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.tracing.cursor.context; + +/** + * Context that contains state of ongoing versioned data read or write. + * + * Read context performing data read with a version that is equal to last closed transaction. + * Write context data modification with have version that is equal to transaction that is currently committing. + * + * As soon reader that associated with a context will observe data version that it should not see, context will be + * marked as dirty. + */ +public interface VersionContext +{ + /** + * Initialise read context with latest closed transaction id as it current version. + */ + void initRead(); + + /** + * Initialise write context with committingTxId as modification version. + * @param committingTxId currently committing transaction id + */ + void initWrite( long committingTxId ); + + /** + * Context currently committing transaction id + * @return committing transaction id + */ + long committingTransactionId(); + + /** + * Last closed transaction id that read context was initialised with + * @return last closed transaction id + */ + long lastClosedTransactionId(); + + /** + * Mark current context as dirty + */ + void markAsDirty(); + + /** + * Check whenever current context is dirty + * @return true if context is dirty, false otherwise + */ + boolean isDirty(); + + /** + * Clear transaction ids that read/write context was initialised with + */ + void clear(); + +} diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContextSupplier.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContextSupplier.java new file mode 100644 index 000000000000..e655dc75b67b --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContextSupplier.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.tracing.cursor.context; + +import java.util.function.LongSupplier; + +/** + * Supplier to create {@link VersionContext} used during version data read and write operations + */ +public interface VersionContextSupplier +{ + /** + * Initialise current supplier with provider of last closed transaction ids + * for future version context to be able to get version ids + * @param lastClosedTransactionIdSupplier closed transaction id supplier. + */ + void init( LongSupplier lastClosedTransactionIdSupplier ); + + /** + * Provide version context + * @return instance of version context + */ + VersionContext getVersionContext(); + +} diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java index 702147267d27..a2cf14f6baa6 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java @@ -44,6 +44,8 @@ import org.neo4j.io.pagecache.impl.SingleFilePageSwapperFactory; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.test.rule.RepeatRule; import static org.junit.Assert.assertThat; @@ -111,15 +113,30 @@ public void tearDown() throws Exception protected final T createPageCache( PageSwapperFactory swapperFactory, int maxPages, int pageSize, PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier ) { - return fixture.createPageCache( swapperFactory, maxPages, pageSize, tracer, cursorTracerSupplier ); + return createPageCache( swapperFactory, maxPages, pageSize, tracer, cursorTracerSupplier, + EmptyVersionContextSupplier.INSTANCE ); + } + + protected final T createPageCache( PageSwapperFactory swapperFactory, int maxPages, int pageSize, + PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier, VersionContextSupplier versionContextSupplier ) + { + return fixture.createPageCache( swapperFactory, maxPages, pageSize, tracer, cursorTracerSupplier, + versionContextSupplier ); } protected T createPageCache( FileSystemAbstraction fs, int maxPages, int pageSize, PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier ) + { + return createPageCache( fs, maxPages, pageSize, tracer, cursorTracerSupplier, + EmptyVersionContextSupplier.INSTANCE ); + } + + protected T createPageCache( FileSystemAbstraction fs, int maxPages, int pageSize, PageCacheTracer tracer, + PageCursorTracerSupplier cursorTracerSupplier, VersionContextSupplier versionContextSupplier ) { PageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory(); swapperFactory.open( fs, Configuration.EMPTY ); - return createPageCache( swapperFactory, maxPages, pageSize, tracer, cursorTracerSupplier ); + return createPageCache( swapperFactory, maxPages, pageSize, tracer, cursorTracerSupplier, versionContextSupplier ); } protected final T getPageCache( FileSystemAbstraction fs, int maxPages, int pageSize, PageCacheTracer tracer, @@ -319,7 +336,7 @@ protected void verifyRecordsInFile( ReadableByteChannel channel, int recordCount public abstract static class Fixture { public abstract T createPageCache( PageSwapperFactory swapperFactory, int maxPages, int pageSize, - PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier ); + PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier, VersionContextSupplier contextSupplier ); public abstract void tearDownPageCache( T pageCache ) throws IOException; diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java index 4e87d0f754c3..9a08bc60b7df 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java @@ -26,6 +26,7 @@ import org.neo4j.io.pagecache.PageSwapperFactory; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; public class MuninnPageCacheFixture extends PageCacheTestSupport.Fixture { @@ -33,9 +34,9 @@ public class MuninnPageCacheFixture extends PageCacheTestSupport.Fixture 0 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 7 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 1 ); + } + + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); +// assertEquals( 7, ((MuninnPageCursor) cursor).page.getLastModifiedTxId() ); + assertEquals( 1, cursor.getLong() ); + } + } + + @Test + public void pareModificationTrackingNoticeWriteFromAnotherThread() throws Exception + { + TestVersionContext cursorContext = new TestVersionContext( () -> 0 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 7 ); + + Future future = executor.submit( () -> + { + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 1 ); + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + } ); + future.get(); + + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); +// assertEquals( 7, ((MuninnPageCursor) cursor).page.getLastModifiedTxId() ); + assertEquals( 1, cursor.getLong() ); + } + } + + @Test + public void pageModificationTracksHighestModifierTransactionId() throws IOException + { + TestVersionContext cursorContext = new TestVersionContext( () -> 0 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 1 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 1 ); + } + cursorContext.initWrite( 12 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 2 ); + } + cursorContext.initWrite( 7 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 3 ); + } + + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); +// assertEquals( 12, ((MuninnPageCursor) cursor).page.getLastModifiedTxId() ); + assertEquals( 3, cursor.getLong() ); + } + } + + @Test + public void markCursorContextAsDirtyWhenReadingDataFromMoreRecentTransactions() throws IOException + { + TestVersionContext cursorContext = new TestVersionContext( () -> 3 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 7 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 3 ); + } + + cursorContext.initRead(); + assertFalse( cursorContext.isDirty() ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); + assertEquals( 3, cursor.getLong() ); + assertTrue( cursorContext.isDirty() ); + } + } + + @Test + public void doNotMarkCursorContextAsDirtyWhenReadingDataFromOlderTransactions() throws IOException + { + TestVersionContext cursorContext = new TestVersionContext( () -> 23 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 17 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 3 ); + } + + cursorContext.initRead(); + assertFalse( cursorContext.isDirty() ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); + assertEquals( 3, cursor.getLong() ); + assertFalse( cursorContext.isDirty() ); + } + } + + @Test + public void markContextAsDirtyWhenAnyEvictedPageHaveModificationTransactionHigherThenReader() throws IOException + { + TestVersionContext cursorContext = new TestVersionContext( () -> 5 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 3 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 3 ); + } + + cursorContext.initWrite( 13 ); + try ( PageCursor cursor = pagedFile.io( 1, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 4 ); + } + pageCache.evictPages( 1, 1, EvictionRunEvent.NULL ); + + cursorContext.initRead(); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); + assertEquals( 3, cursor.getLong() ); + assertTrue( cursorContext.isDirty() ); + } + } + + @Test + public void doNotMarkContextAsDirtyWhenAnyEvictedPageHaveModificationTransactionLowerThenReader() throws IOException + { + TestVersionContext cursorContext = new TestVersionContext( () -> 15 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 3 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 3 ); + } + + cursorContext.initWrite( 13 ); + try ( PageCursor cursor = pagedFile.io( 1, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 4 ); + } + pageCache.evictPages( 1, 1, EvictionRunEvent.NULL ); + + cursorContext.initRead(); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); + assertEquals( 3, cursor.getLong() ); + assertFalse( cursorContext.isDirty() ); + } + } + @Test public void closingTheCursorMustUnlockModifiedPage() throws Exception { @@ -362,6 +578,84 @@ private ByteBuffer readIntoBuffer( String fileName ) throws IOException channel.read( buffer ); } buffer.flip(); - return buffer; + return buffer; + } + + private static class ConfiguredVersionContextSupplier implements VersionContextSupplier + { + + private final VersionContext versionContext; + + ConfiguredVersionContextSupplier( VersionContext versionContext ) + { + this.versionContext = versionContext; + } + + @Override + public void init( LongSupplier lastClosedTransactionIdSupplier ) + { + } + + @Override + public VersionContext getVersionContext() + { + return versionContext; + } + } + + private static class TestVersionContext implements VersionContext + { + + private final IntSupplier closedTxIdSupplier; + private long committingTxId; + private long lastClosedTxId; + private boolean dirty; + + TestVersionContext( IntSupplier closedTxIdSupplier ) + { + this.closedTxIdSupplier = closedTxIdSupplier; + } + + @Override + public void initRead() + { + this.lastClosedTxId = closedTxIdSupplier.getAsInt(); + } + + @Override + public void initWrite( long committingTxId ) + { + this.committingTxId = committingTxId; + } + + @Override + public long committingTransactionId() + { + return committingTxId; + } + + @Override + public long lastClosedTransactionId() + { + return lastClosedTxId; + } + + @Override + public void markAsDirty() + { + dirty = true; + } + + @Override + public boolean isDirty() + { + return dirty; + } + + @Override + public void clear() + { + + } } } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/randomharness/RandomPageCacheTestHarness.java b/community/io/src/test/java/org/neo4j/io/pagecache/randomharness/RandomPageCacheTestHarness.java index 6e13f11de1f4..a65b9f8435c9 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/randomharness/RandomPageCacheTestHarness.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/randomharness/RandomPageCacheTestHarness.java @@ -47,6 +47,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.io.pagecache.tracing.linear.LinearHistoryPageCacheTracerTest; import org.neo4j.io.pagecache.tracing.linear.LinearTracers; @@ -385,7 +386,7 @@ private void runIteration( long timeout, TimeUnit unit ) throws Exception PageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory(); swapperFactory.open( fs, Configuration.EMPTY ); MuninnPageCache cache = new MuninnPageCache( swapperFactory, cachePageCount, cachePageSize, tracer, - cursorTracerSupplier ); + cursorTracerSupplier, EmptyVersionContextSupplier.INSTANCE ); cache.setPrintExceptionsOnClose( false ); Map fileMap = new HashMap<>( files.length ); for ( int i = 0; i < Math.min( files.length, initialMappedFiles ); i++ ) diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/stress/PageCacheStressTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/stress/PageCacheStressTest.java index aa0183a2ae9c..3c97cdd3eea3 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/stress/PageCacheStressTest.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/stress/PageCacheStressTest.java @@ -31,6 +31,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -91,7 +92,7 @@ public void run() throws Exception swapperFactory.open( fs, Configuration.EMPTY ); try ( PageCache pageCacheUnderTest = new MuninnPageCache( swapperFactory, numberOfCachePages, cachePageSize, - tracer, pageCursorTracerSupplier ) ) + tracer, pageCursorTracerSupplier, EmptyVersionContextSupplier.INSTANCE ) ) { PageCacheStresser pageCacheStresser = new PageCacheStresser( numberOfPages, numberOfThreads, workingDirectory ); diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java index 81d52ec7451f..05e782368234 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java @@ -180,7 +180,7 @@ public void init( PageCacheTracer tracer ) @Override public void reportEvents() { - Objects.nonNull( tracer ); + Objects.requireNonNull( tracer ); tracer.pins( pins ); tracer.faults( faults ); } diff --git a/community/io/src/test/java/org/neo4j/test/rule/PageCacheRule.java b/community/io/src/test/java/org/neo4j/test/rule/PageCacheRule.java index 385e1ab54865..646365039801 100644 --- a/community/io/src/test/java/org/neo4j/test/rule/PageCacheRule.java +++ b/community/io/src/test/java/org/neo4j/test/rule/PageCacheRule.java @@ -33,6 +33,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; public class PageCacheRule extends ExternalResource { @@ -172,7 +173,7 @@ public PageCache getPageCache( FileSystemAbstraction fs, PageCacheConfig overrid selectConfig( baseConfig.pageSize, overriddenConfig.pageSize, null ), selectConfig( baseConfig.tracer, overriddenConfig.tracer, PageCacheTracer.NULL ), selectConfig( baseConfig.pageCursorTracerSupplier, overriddenConfig.pageCursorTracerSupplier, - DefaultPageCursorTracerSupplier.INSTANCE )); + DefaultPageCursorTracerSupplier.INSTANCE ), EmptyVersionContextSupplier.INSTANCE); pageCachePostConstruct( overriddenConfig ); return pageCache; } diff --git a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java index a69702a6b3ea..7ba1515d92a4 100644 --- a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java +++ b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java @@ -492,6 +492,17 @@ public class GraphDatabaseSettings implements LoadableConfig public static final Setting rebuild_idgenerators_fast = setting( "unsupported.dbms.id_generator_fast_rebuild_enabled", BOOLEAN, TRUE ); + @Description( "Specifies if engine should run cypher query based on a snapshot of accessed data. " + + "Query will be restarted in case if concurrent modification of data will be detected." ) + @Internal + public static final Setting snapshot_query = setting( "unsupported.dbms.query.snapshot", BOOLEAN, FALSE ); + + @Description( "Specifies number or retries that query engine will do to execute query based on " + + "stable accessed data snapshot before giving up." ) + @Internal + public static final Setting snapshot_query_retries = buildSetting( "unsupported.dbms.query.snapshot.retries", + INTEGER, "5" ).constraint( range( 1, Integer.MAX_VALUE ) ).build(); + // Store memory settings @Description( "Target size for pages of mapped memory. If set to 0, then a reasonable default is chosen, " + "depending on the storage device used." ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java index b3fb8d2a531f..20cbad1fd74c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java @@ -37,6 +37,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.KernelAPI; import org.neo4j.kernel.api.TokenNameLookup; import org.neo4j.kernel.api.exceptions.KernelException; @@ -272,6 +273,7 @@ boolean applicable( DiagnosticsPhase phase ) private final IdController idController; private final OperationalMode operationalMode; private final RecoveryCleanupWorkCollector recoveryCleanupWorkCollector; + private final VersionContextSupplier versionContextSupplier; private final AccessCapability accessCapability; private StorageEngine storageEngine; @@ -284,24 +286,13 @@ boolean applicable( DiagnosticsPhase phase ) * core API are now slowly accumulating in the Kernel implementation. Over time, these components should be * refactored into bigger components that wrap the very granular things we depend on here. */ - public NeoStoreDataSource( - File storeDir, - Config config, - IdGeneratorFactory idGeneratorFactory, - LogService logService, - JobScheduler scheduler, - TokenNameLookup tokenNameLookup, - DependencyResolver dependencyResolver, - PropertyKeyTokenHolder propertyKeyTokens, - LabelTokenHolder labelTokens, - RelationshipTypeTokenHolder relationshipTypeTokens, - StatementLocksFactory statementLocksFactory, - SchemaWriteGuard schemaWriteGuard, - TransactionEventHandlers transactionEventHandlers, - IndexingService.Monitor indexingServiceMonitor, - FileSystemAbstraction fs, - TransactionMonitor transactionMonitor, - DatabaseHealth databaseHealth, + public NeoStoreDataSource( File storeDir, Config config, IdGeneratorFactory idGeneratorFactory, + LogService logService, JobScheduler scheduler, TokenNameLookup tokenNameLookup, + DependencyResolver dependencyResolver, PropertyKeyTokenHolder propertyKeyTokens, + LabelTokenHolder labelTokens, RelationshipTypeTokenHolder relationshipTypeTokens, + StatementLocksFactory statementLocksFactory, SchemaWriteGuard schemaWriteGuard, + TransactionEventHandlers transactionEventHandlers, IndexingService.Monitor indexingServiceMonitor, + FileSystemAbstraction fs, TransactionMonitor transactionMonitor, DatabaseHealth databaseHealth, PhysicalLogFile.Monitor physicalLogMonitor, TransactionHeaderInformationFactory transactionHeaderInformationFactory, StartupStatisticsProvider startupStatistics, @@ -320,7 +311,8 @@ public NeoStoreDataSource( StoreCopyCheckPointMutex storeCopyCheckPointMutex, RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, IdController idController, - OperationalMode operationalMode ) + OperationalMode operationalMode, + VersionContextSupplier versionContextSupplier ) { this.storeDir = storeDir; this.config = config; @@ -359,6 +351,7 @@ public NeoStoreDataSource( readOnly = config.get( Configuration.read_only ); this.idController = idController; this.operationalMode = operationalMode; + this.versionContextSupplier = versionContextSupplier; msgLog = logProvider.getLog( getClass() ); this.lockService = new ReentrantLockService(); this.explicitIndexProviderLookup = new ExplicitIndexProviderLookup() @@ -444,9 +437,12 @@ public void start() throws IOException storageEngine = buildStorageEngine( propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, explicitIndexProviderLookup, - indexConfigStore, databaseSchemaState, explicitIndexTransactionOrdering, operationalMode ); + indexConfigStore, databaseSchemaState, explicitIndexTransactionOrdering, operationalMode, versionContextSupplier ); TransactionIdStore transactionIdStore = dependencies.resolveDependency( TransactionIdStore.class ); + + versionContextSupplier.init( transactionIdStore::getLastClosedTransactionId ); + LogVersionRepository logVersionRepository = dependencies.resolveDependency( LogVersionRepository.class ); NeoStoreTransactionLogModule transactionLogModule = buildTransactionLogs( logFiles, config, logProvider, scheduler, fs, @@ -574,7 +570,8 @@ private StorageEngine buildStorageEngine( PropertyKeyTokenHolder propertyKeyTokenHolder, LabelTokenHolder labelTokens, RelationshipTypeTokenHolder relationshipTypeTokens, ExplicitIndexProviderLookup explicitIndexProviderLookup, IndexConfigStore indexConfigStore, - SchemaState schemaState, SynchronizedArrayIdOrderingQueue explicitIndexTransactionOrdering, OperationalMode operationalMode ) + SchemaState schemaState, SynchronizedArrayIdOrderingQueue explicitIndexTransactionOrdering, + OperationalMode operationalMode, VersionContextSupplier versionContextSupplier ) { RecordStorageEngine storageEngine = new RecordStorageEngine( storeDir, config, pageCache, fs, logProvider, propertyKeyTokenHolder, @@ -582,7 +579,7 @@ private StorageEngine buildStorageEngine( tokenNameLookup, lockService, schemaIndexProviderMap, indexingServiceMonitor, databaseHealth, explicitIndexProviderLookup, indexConfigStore, explicitIndexTransactionOrdering, idGeneratorFactory, idController, monitors, recoveryCleanupWorkCollector, - operationalMode ); + operationalMode, versionContextSupplier ); // We pretend that the storage engine abstract hides all details within it. Whereas that's mostly // true it's not entirely true for the time being. As long as we need this call below, which @@ -726,7 +723,8 @@ private NeoStoreKernelModule buildKernel( TransactionAppender appender, KernelTransactions kernelTransactions = life.add( new KernelTransactions( statementLocksFactory, constraintIndexCreator, statementOperationParts, schemaWriteGuard, transactionHeaderInformationFactory, transactionCommitProcess, indexConfigStore, explicitIndexProviderLookup, hooks, transactionMonitor, - availabilityGuard, tracers, storageEngine, procedures, transactionIdStore, clock, accessCapability ) ); + availabilityGuard, tracers, storageEngine, procedures, transactionIdStore, clock, accessCapability, + versionContextSupplier ) ); buildTransactionMonitor( kernelTransactions, clock, config ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelStatement.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelStatement.java index 2dd95ccce979..05fd8cf0219c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelStatement.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelStatement.java @@ -32,6 +32,8 @@ import org.neo4j.graphdb.NotInTransactionException; import org.neo4j.graphdb.TransactionTerminatedException; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; import org.neo4j.kernel.api.AssertOpen; import org.neo4j.kernel.api.DataWriteOperations; import org.neo4j.kernel.api.ExecutionStatisticsOperations; @@ -66,14 +68,14 @@ *
    *
  1. Construct {@link KernelStatement} when {@link KernelTransactionImplementation} is constructed
    2. *
  2. For every transaction...
    3. - *
  3. Call {@link #initialize(StatementLocks, PageCursorTracer)} which makes this instance + *
  4. Call {@link #initialize(StatementLocks, PageCursorTracer, VersionContext)} which makes this instance * full available and ready to use. Call when the {@link KernelTransactionImplementation} is initialized.
    5. *
  5. Alternate {@link #acquire()} / {@link #close()} when acquiring / closing a statement for the transaction... * Temporarily asymmetric number of calls to {@link #acquire()} / {@link #close()} is supported, although in * the end an equal number of calls must have been issued.
    6. *
  6. To be safe call {@link #forceClose()} at the end of a transaction to force a close of the statement, * even if there are more than one current call to {@link #acquire()}. This instance is now again ready - * to be {@link #initialize(StatementLocks, PageCursorTracer)} initialized} and used for the transaction + * to be {@link #initialize(StatementLocks, PageCursorTracer, VersionContext)} initialized} and used for the transaction * instance again, when it's initialized.
    7. *
*/ @@ -95,6 +97,7 @@ public class KernelStatement implements TxStateHolder, Statement, AssertOpen private volatile ExecutingQueryList executingQueryList; private final LockTracer systemLockTracer; private final Deque statementOpenCloseCalls; + private VersionContext versionContext = EmptyVersionContext.INSTANCE; public KernelStatement( KernelTransactionImplementation transaction, TxStateHolder txStateHolder, @@ -214,10 +217,11 @@ public void assertOpen() } } - public void initialize( StatementLocks statementLocks, PageCursorTracer pageCursorCounters ) + public void initialize( StatementLocks statementLocks, PageCursorTracer pageCursorCounters, VersionContext versionContext ) { this.statementLocks = statementLocks; this.pageCursorTracer = pageCursorCounters; + this.versionContext = versionContext; } public StatementLocks locks() @@ -305,6 +309,7 @@ private void cleanupResources() { // closing is done by KTI storeStatement.release(); + versionContext.clear(); executingQueryList = ExecutingQueryList.EMPTY; } @@ -313,6 +318,11 @@ public KernelTransactionImplementation getTransaction() return transaction; } + public VersionContext getVersionContext() + { + return versionContext; + } + void assertAllows( Function allows, String mode ) { AccessMode accessMode = transaction.securityContext().mode(); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactionImplementation.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactionImplementation.java index abe88cf31799..392b692cc77c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactionImplementation.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactionImplementation.java @@ -35,6 +35,8 @@ import org.neo4j.collection.pool.Pool; import org.neo4j.graphdb.TransactionTerminatedException; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.KeyReadTokenNameLookup; import org.neo4j.kernel.api.exceptions.ConstraintViolationTransactionFailureException; @@ -105,6 +107,7 @@ public class KernelTransactionImplementation implements KernelTransaction, TxSta private final TransactionCommitProcess commitProcess; private final TransactionMonitor transactionMonitor; private final PageCursorTracerSupplier cursorTracerSupplier; + private final VersionContextSupplier versionContextSupplier; private final StoreReadLayer storeLayer; private final Clock clock; @@ -114,6 +117,7 @@ public class KernelTransactionImplementation implements KernelTransaction, TxSta private ExplicitIndexTransactionState explicitIndexTransactionState; private TransactionWriteState writeState; private TransactionHooks.TransactionHooksState hooksState; + private VersionContext versionContext; private final KernelStatement currentStatement; private final StorageStatement storageStatement; private final List closeListeners = new ArrayList<>( 2 ); @@ -160,7 +164,8 @@ public KernelTransactionImplementation( StatementOperationParts statementOperati LockTracer lockTracer, PageCursorTracerSupplier cursorTracerSupplier, StorageEngine storageEngine, - AccessCapability accessCapability ) + AccessCapability accessCapability, + VersionContextSupplier versionContextSupplier ) { this.statementOperations = statementOperations; this.schemaWriteGuard = schemaWriteGuard; @@ -176,6 +181,7 @@ public KernelTransactionImplementation( StatementOperationParts statementOperati this.clock = clock; this.transactionTracer = transactionTracer; this.cursorTracerSupplier = cursorTracerSupplier; + this.versionContextSupplier = versionContextSupplier; this.storageStatement = storeLayer.newStatement(); this.currentStatement = new KernelStatement( this, this, storageStatement, procedures, accessCapability, lockTracer, statementOperations ); @@ -197,7 +203,6 @@ public KernelTransactionImplementation initialize( this.beforeHookInvoked = false; this.failure = false; this.success = false; - this.beforeHookInvoked = false; this.writeState = TransactionWriteState.NONE; this.startTimeMillis = clock.millis(); this.timeoutMillis = transactionTimeout; @@ -208,7 +213,8 @@ public KernelTransactionImplementation initialize( this.securityContext = frozenSecurityContext; this.transactionId = NOT_COMMITTED_TRANSACTION_ID; this.commitTime = NOT_COMMITTED_TRANSACTION_COMMIT_TIME; - this.currentStatement.initialize( statementLocks, cursorTracerSupplier.get() ); + this.versionContext = this.versionContextSupplier.getVersionContext(); + this.currentStatement.initialize( statementLocks, cursorTracerSupplier.get(), versionContext ); return this; } @@ -593,7 +599,7 @@ private long commit() throws TransactionFailureException // Commit the transaction success = true; - TransactionToApply batch = new TransactionToApply( transactionRepresentation ); + TransactionToApply batch = new TransactionToApply( transactionRepresentation, versionContext ); txId = transactionId = commitProcess.commit( batch, commitEvent, INTERNAL ); commitTime = timeCommitted; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactions.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactions.java index 731c219c8070..b0b410939c78 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactions.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactions.java @@ -31,6 +31,7 @@ import org.neo4j.function.Factory; import org.neo4j.graphdb.DatabaseShutdownException; import org.neo4j.graphdb.TransactionFailureException; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.AvailabilityGuard; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.KernelTransactionHandle; @@ -80,6 +81,7 @@ public class KernelTransactions extends LifecycleAdapter implements Supplier explicitIndexTxStateSupplier; + private final VersionContextSupplier versionContextSupplier; private final Clock clock; private final ReentrantReadWriteLock newTransactionsLock = new ReentrantReadWriteLock(); @@ -126,7 +128,8 @@ public KernelTransactions( StatementLocksFactory statementLocksFactory, StorageEngine storageEngine, Procedures procedures, TransactionIdStore transactionIdStore, - Clock clock, AccessCapability accessCapability ) + Clock clock, AccessCapability accessCapability, + VersionContextSupplier versionContextSupplier ) { this.statementLocksFactory = statementLocksFactory; this.constraintIndexCreator = constraintIndexCreator; @@ -144,6 +147,7 @@ public KernelTransactions( StatementLocksFactory statementLocksFactory, this.accessCapability = accessCapability; this.explicitIndexTxStateSupplier = () -> new CachingExplicitIndexTransactionState( new ExplicitIndexTransactionStateImpl( indexConfigStore, explicitIndexProviderLookup ) ); + this.versionContextSupplier = versionContextSupplier; this.clock = clock; blockNewTransactions(); } @@ -327,7 +331,7 @@ public KernelTransactionImplementation newInstance() constraintIndexCreator, procedures, transactionHeaderInformationFactory, transactionCommitProcess, transactionMonitor, explicitIndexTxStateSupplier, localTxPool, clock, tracers.transactionTracer, tracers.lockTracer, tracers.pageCursorTracerSupplier, - storageEngine, accessCapability ); + storageEngine, accessCapability, versionContextSupplier ); this.transactions.add( tx ); return tx; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/TransactionToApply.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/TransactionToApply.java index 81d4b4bbb909..397cb002d42b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/TransactionToApply.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/TransactionToApply.java @@ -23,6 +23,8 @@ import java.util.function.LongConsumer; import org.neo4j.helpers.collection.Visitor; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; import org.neo4j.kernel.impl.transaction.TransactionRepresentation; import org.neo4j.kernel.impl.transaction.log.Commitment; import org.neo4j.kernel.impl.transaction.log.LogPosition; @@ -66,6 +68,7 @@ public class TransactionToApply implements CommandsToApply, AutoCloseable // These fields are provided by user private final TransactionRepresentation transactionRepresentation; private long transactionId; + private final VersionContext versionContext; private TransactionToApply nextTransactionInBatch; // These fields are provided by commit process, storage engine, or recovery process @@ -78,17 +81,28 @@ public class TransactionToApply implements CommandsToApply, AutoCloseable */ public TransactionToApply( TransactionRepresentation transactionRepresentation ) { - this( transactionRepresentation, TRANSACTION_ID_NOT_SPECIFIED ); + this( transactionRepresentation, EmptyVersionContext.INSTANCE ); } /** - * Used as convenience when committing a transaction that has already gotten a transaction id assigned, - * i.e. when replicating a transaction. + * Used when committing a transaction that hasn't already gotten a transaction id assigned. */ + public TransactionToApply( TransactionRepresentation transactionRepresentation, VersionContext versionContext ) + { + this( transactionRepresentation, TRANSACTION_ID_NOT_SPECIFIED, versionContext ); + } + public TransactionToApply( TransactionRepresentation transactionRepresentation, long transactionId ) + { + this( transactionRepresentation, transactionId, EmptyVersionContext.INSTANCE ); + } + + public TransactionToApply( TransactionRepresentation transactionRepresentation, long transactionId, + VersionContext versionContext ) { this.transactionRepresentation = transactionRepresentation; this.transactionId = transactionId; + this.versionContext = versionContext; } // These methods are called by the user when building a batch @@ -130,6 +144,7 @@ public void commitment( Commitment commitment, long transactionId ) { this.commitment = commitment; this.transactionId = transactionId; + this.versionContext.initWrite( transactionId ); } public void logPosition( LogPosition position ) @@ -151,6 +166,7 @@ public void onClose( LongConsumer closedCallback ) @Override public void close() { + versionContext.clear(); if ( closedCallback != null ) { closedCallback.accept( transactionId ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContext.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContext.java new file mode 100644 index 000000000000..6a539860eded --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContext.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.context; + +import java.util.function.LongSupplier; + +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; + +import static org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_ID; + +/** + * Transactional version context that used by read transaction to read data of specific version. + * Or perform versioned data modification. + */ +public class TransactionVersionContext implements VersionContext +{ + private final LongSupplier lastClosedTxIdSupplier; + private long transactionId = BASE_TX_ID; + private long lastClosedTxId = Long.MAX_VALUE; + private boolean dirty; + + public TransactionVersionContext( LongSupplier lastClosedTxIdSupplier ) + { + this.lastClosedTxIdSupplier = lastClosedTxIdSupplier; + } + + @Override + public void initRead() + { + long txId = lastClosedTxIdSupplier.getAsLong(); + assert txId >= BASE_TX_ID; + lastClosedTxId = txId; + dirty = false; + } + + @Override + public void initWrite( long committingTxId ) + { + assert committingTxId >= BASE_TX_ID; + transactionId = committingTxId; + } + + @Override + public long committingTransactionId() + { + return transactionId; + } + + @Override + public long lastClosedTransactionId() + { + return lastClosedTxId; + } + + @Override + public void markAsDirty() + { + dirty = true; + } + + @Override + public boolean isDirty() + { + return dirty; + } + + @Override + public void clear() + { + transactionId = BASE_TX_ID; + lastClosedTxId = Long.MAX_VALUE; + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContextSupplier.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContextSupplier.java new file mode 100644 index 000000000000..f00275f2b03f --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContextSupplier.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.context; + +import java.util.function.LongSupplier; + +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; + +/** + * {@link VersionContextSupplier} that supplier thread bound version context that should be used in a context of + * transaction(Committing or reading). + */ +public class TransactionVersionContextSupplier implements VersionContextSupplier +{ + protected ThreadLocal cursorContext; + + @Override + public void init( LongSupplier lastClosedTransactionIdSupplier ) + { + this.cursorContext = ThreadLocal.withInitial( () -> new TransactionVersionContext( lastClosedTransactionIdSupplier ) ); + } + + @Override + public VersionContext getVersionContext() + { + return cursorContext == null ? EmptyVersionContext.INSTANCE : cursorContext.get(); + } + +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java index 359d4a5b2de9..df3936e21331 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java @@ -220,7 +220,8 @@ public DataSourceModule( final PlatformModule platformModule, EditionModule edit platformModule.storeCopyCheckPointMutex, platformModule.recoveryCleanupWorkCollector, editionModule.idController, - platformModule.databaseInfo.operationalMode ) ); + platformModule.databaseInfo.operationalMode, + platformModule.versionContextSupplier ) ); dataSourceManager.register( neoStoreDataSource ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java index 55321c47940d..449859ee4c56 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java @@ -30,12 +30,15 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileSystemLifecycleAdapter; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.AvailabilityGuard; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.ConnectorPortRegister; import org.neo4j.kernel.extension.KernelExtensions; import org.neo4j.kernel.extension.UnsatisfiedDependencyStrategies; import org.neo4j.kernel.impl.api.LogRotationMonitor; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.logging.StoreLogService; import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory; @@ -113,6 +116,7 @@ public class PlatformModule public final SystemNanoClock clock; public final StoreCopyCheckPointMutex storeCopyCheckPointMutex; + public final VersionContextSupplier versionContextSupplier; public final RecoveryCleanupWorkCollector recoveryCleanupWorkCollector; @@ -173,7 +177,11 @@ public PlatformModule( File providedStoreDir, Config config, DatabaseInfo databa dependencies.satisfyDependency( firstImplementor( CheckPointerMonitor.class, tracers.checkPointTracer, CheckPointerMonitor.NULL ) ); - pageCache = dependencies.satisfyDependency( createPageCache( fileSystem, config, logging, tracers ) ); + versionContextSupplier = createCursorContextSupplier( config ); + dependencies.satisfyDependency( versionContextSupplier ); + pageCache = dependencies.satisfyDependency( createPageCache( fileSystem, config, logging, tracers, + versionContextSupplier ) ); + life.add( new PageCacheLifecycle( pageCache ) ); diagnosticsManager = life.add( dependencies @@ -203,6 +211,12 @@ public PlatformModule( File providedStoreDir, Config config, DatabaseInfo databa publishPlatformInfo( dependencies.resolveDependency( UsageData.class ) ); } + protected VersionContextSupplier createCursorContextSupplier( Config config ) + { + return config.get( GraphDatabaseSettings.snapshot_query ) ? new TransactionVersionContextSupplier() + : EmptyVersionContextSupplier.INSTANCE; + } + protected AvailabilityGuard createAvailabilityGuard() { return new AvailabilityGuard( clock, logging.getInternalLog( AvailabilityGuard.class ) ); @@ -295,11 +309,12 @@ protected JobScheduler createJobScheduler() } protected PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, LogService logging, - Tracers tracers ) + Tracers tracers, VersionContextSupplier versionContextSupplier ) { Log pageCacheLog = logging.getInternalLog( PageCache.class ); ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( - fileSystem, config, tracers.pageCacheTracer, tracers.pageCursorTracerSupplier, pageCacheLog ); + fileSystem, config, tracers.pageCacheTracer, tracers.pageCursorTracerSupplier, pageCacheLog, + versionContextSupplier ); PageCache pageCache = pageCacheFactory.getOrCreatePageCache(); if ( config.get( GraphDatabaseSettings.dump_configuration ) ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfigurableStandalonePageCacheFactory.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfigurableStandalonePageCacheFactory.java index 53102fea3c89..86f4e3bf83b8 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfigurableStandalonePageCacheFactory.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfigurableStandalonePageCacheFactory.java @@ -27,6 +27,8 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.logging.FormattedLogProvider; @@ -46,12 +48,13 @@ private ConfigurableStandalonePageCacheFactory() public static PageCache createPageCache( FileSystemAbstraction fileSystem ) { return createPageCache( fileSystem, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE, - Config.defaults() ); + Config.defaults(), EmptyVersionContextSupplier.INSTANCE ); } public static PageCache createPageCache( FileSystemAbstraction fileSystem, Config config ) { - return createPageCache( fileSystem, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE, config ); + return createPageCache( fileSystem, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE, config, + EmptyVersionContextSupplier.INSTANCE ); } /** @@ -61,17 +64,19 @@ public static PageCache createPageCache( FileSystemAbstraction fileSystem, Confi * @param pageCursorTracerSupplier supplier of thread local (transaction local) page cursor tracer that will provide * thread local page cache statistics * @param config page cache configuration + * @param versionContextSupplier version context supplier * @return created page cache instance */ public static PageCache createPageCache( FileSystemAbstraction fileSystem, PageCacheTracer pageCacheTracer, - PageCursorTracerSupplier pageCursorTracerSupplier, Config config ) + PageCursorTracerSupplier pageCursorTracerSupplier, Config config, + VersionContextSupplier versionContextSupplier ) { config.augmentDefaults( GraphDatabaseSettings.pagecache_memory, "8M" ); ZoneId logTimeZone = config.get( GraphDatabaseSettings.log_timezone ).getZoneId(); FormattedLogProvider logProvider = FormattedLogProvider.withZoneId( logTimeZone ).toOutputStream( System.err ); ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( fileSystem, config, pageCacheTracer, pageCursorTracerSupplier, - logProvider.getLog( PageCache.class ) ); + logProvider.getLog( PageCache.class ), versionContextSupplier ); return pageCacheFactory.getOrCreatePageCache(); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactory.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactory.java index 50d7777d3aab..a44fd0377236 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactory.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactory.java @@ -28,6 +28,7 @@ import org.neo4j.io.pagecache.impl.muninn.MuninnPageCache; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.util.OsBeanUtil; import org.neo4j.logging.Log; @@ -46,6 +47,7 @@ public class ConfiguringPageCacheFactory private final Config config; private final PageCacheTracer pageCacheTracer; private final Log log; + private final VersionContextSupplier versionContextSupplier; private PageCache pageCache; private PageCursorTracerSupplier pageCursorTracerSupplier; @@ -57,11 +59,14 @@ public class ConfiguringPageCacheFactory * @param pageCursorTracerSupplier supplier of thread local (transaction local) page cursor tracer that will provide * thread local page cache statistics * @param log page cache factory log + * @param versionContextSupplier cursor context factory */ public ConfiguringPageCacheFactory( FileSystemAbstraction fs, Config config, PageCacheTracer pageCacheTracer, - PageCursorTracerSupplier pageCursorTracerSupplier, Log log ) + PageCursorTracerSupplier pageCursorTracerSupplier, Log log, + VersionContextSupplier versionContextSupplier ) { this.fs = fs; + this.versionContextSupplier = versionContextSupplier; this.config = config; this.pageCacheTracer = pageCacheTracer; this.log = log; @@ -85,7 +90,7 @@ protected PageCache createPageCache() return new MuninnPageCache( swapperFactory, maxPages, - cachePageSize, pageCacheTracer, pageCursorTracerSupplier ); + cachePageSize, pageCacheTracer, pageCursorTracerSupplier, versionContextSupplier ); } public int calculateMaxPages( Config config, int cachePageSize ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java index c9216ca45d5d..efbfcfcc2a81 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java @@ -33,6 +33,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.TokenNameLookup; import org.neo4j.kernel.api.exceptions.TransactionApplyKernelException; import org.neo4j.kernel.api.exceptions.TransactionFailureException; @@ -179,7 +180,8 @@ public RecordStorageEngine( IdController idController, Monitors monitors, RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, - OperationalMode operationalMode ) + OperationalMode operationalMode, + VersionContextSupplier versionContextSupplier ) { this.propertyKeyTokenHolder = propertyKeyTokenHolder; this.relationshipTypeTokenHolder = relationshipTypeTokens; @@ -193,7 +195,8 @@ public RecordStorageEngine( this.explicitIndexTransactionOrdering = explicitIndexTransactionOrdering; this.idController = idController; - StoreFactory factory = new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fs, logProvider ); + StoreFactory factory = new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fs, logProvider, + versionContextSupplier ); neoStores = factory.openAllNeoStores( true ); try diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java index 668071f93712..316d63cb7cd1 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java @@ -37,6 +37,7 @@ import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PagedFile; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.NeoStoresDiagnostics; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.CountsAccessor; @@ -104,6 +105,7 @@ public boolean test( StoreType type ) private final IdGeneratorFactory idGeneratorFactory; private final PageCache pageCache; private final LogProvider logProvider; + private final VersionContextSupplier versionContextSupplier; private final boolean createIfNotExist; private final File storeDir; private final File neoStoreFileName; @@ -121,6 +123,7 @@ public boolean test( StoreType type ) PageCache pageCache, final LogProvider logProvider, FileSystemAbstraction fileSystemAbstraction, + VersionContextSupplier versionContextSupplier, RecordFormats recordFormats, boolean createIfNotExist, StoreType[] storeTypes, @@ -132,6 +135,7 @@ public boolean test( StoreType type ) this.pageCache = pageCache; this.logProvider = logProvider; this.fileSystemAbstraction = fileSystemAbstraction; + this.versionContextSupplier = versionContextSupplier; this.recordFormats = recordFormats; this.createIfNotExist = createIfNotExist; this.openOptions = openOptions; @@ -416,7 +420,8 @@ public CountsTracker getCounts() private CountsTracker createWritableCountsTracker( File fileName ) { - return new CountsTracker( logProvider, fileSystemAbstraction, pageCache, config, fileName ); + return new CountsTracker( logProvider, fileSystemAbstraction, pageCache, config, fileName, + versionContextSupplier ); } private ReadOnlyCountsTracker createReadOnlyCountsTracker( File fileName ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreAccess.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreAccess.java index d1ff806168f3..7f3c0516c254 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreAccess.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreAccess.java @@ -23,6 +23,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.CountsAccessor; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; @@ -75,7 +76,7 @@ public StoreAccess( NeoStores store ) public StoreAccess( FileSystemAbstraction fileSystem, PageCache pageCache, File storeDir, Config config ) { this( new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fileSystem ), pageCache, - fileSystem, NullLogProvider.getInstance() ).openAllNeoStores() ); + fileSystem, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ).openAllNeoStores() ); this.closeable = true; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreFactory.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreFactory.java index 3771285e1fde..b5377d414c8a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreFactory.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreFactory.java @@ -25,6 +25,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.format.RecordFormatPropertyConfigurator; import org.neo4j.kernel.impl.store.format.RecordFormats; @@ -69,29 +70,32 @@ public class StoreFactory private final PageCache pageCache; private final RecordFormats recordFormats; private final OpenOption[] openOptions; + private final VersionContextSupplier versionContextSupplier; public StoreFactory( File storeDir, Config config, IdGeneratorFactory idGeneratorFactory, PageCache pageCache, - FileSystemAbstraction fileSystemAbstraction, LogProvider logProvider ) + FileSystemAbstraction fileSystemAbstraction, LogProvider logProvider, VersionContextSupplier versionContextSupplier ) { this( storeDir, config, idGeneratorFactory, pageCache, fileSystemAbstraction, selectForStoreOrConfig( config, storeDir, fileSystemAbstraction, pageCache, logProvider ), - logProvider ); + logProvider, versionContextSupplier ); } public StoreFactory( File storeDir, Config config, IdGeneratorFactory idGeneratorFactory, PageCache pageCache, - FileSystemAbstraction fileSystemAbstraction, RecordFormats recordFormats, LogProvider logProvider ) + FileSystemAbstraction fileSystemAbstraction, RecordFormats recordFormats, LogProvider logProvider, + VersionContextSupplier versionContextSupplier ) { this( storeDir, MetaDataStore.DEFAULT_NAME, config, idGeneratorFactory, pageCache, fileSystemAbstraction, - recordFormats, logProvider ); + recordFormats, logProvider, versionContextSupplier ); } public StoreFactory( File storeDir, String storeName, Config config, IdGeneratorFactory idGeneratorFactory, PageCache pageCache, FileSystemAbstraction fileSystemAbstraction, RecordFormats recordFormats, - LogProvider logProvider, OpenOption... openOptions ) + LogProvider logProvider, VersionContextSupplier versionContextSupplier, OpenOption... openOptions ) { this.config = config; this.idGeneratorFactory = idGeneratorFactory; this.fileSystemAbstraction = fileSystemAbstraction; + this.versionContextSupplier = versionContextSupplier; this.recordFormats = recordFormats; this.openOptions = openOptions; new RecordFormatPropertyConfigurator( recordFormats, config ).configure(); @@ -154,6 +158,7 @@ public NeoStores openNeoStores( boolean createStoreIfNotExists, StoreType... sto } } return new NeoStores( neoStoreFileName, config, idGeneratorFactory, pageCache, logProvider, - fileSystemAbstraction, recordFormats, createStoreIfNotExists, storeTypes, openOptions ); + fileSystemAbstraction, versionContextSupplier, recordFormats, createStoreIfNotExists, storeTypes, + openOptions ); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/CountsTracker.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/CountsTracker.java index 443d64b8f079..7c43b3ff275d 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/CountsTracker.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/CountsTracker.java @@ -25,6 +25,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.CountsAccessor; import org.neo4j.kernel.impl.api.CountsVisitor; @@ -85,46 +86,16 @@ public class CountsTracker extends AbstractKeyValueStore public static final String TYPE_DESCRIPTOR = "CountsStore"; public CountsTracker( final LogProvider logProvider, FileSystemAbstraction fs, PageCache pages, Config config, - File baseFile ) + File baseFile, VersionContextSupplier versionContextSupplier ) { - this( logProvider, fs, pages, config, baseFile, Clocks.nanoClock() ); + this( logProvider, fs, pages, config, baseFile, Clocks.nanoClock(), versionContextSupplier ); } public CountsTracker( final LogProvider logProvider, FileSystemAbstraction fs, PageCache pages, Config config, - File baseFile, SystemNanoClock clock ) + File baseFile, SystemNanoClock clock, VersionContextSupplier versionContextSupplier ) { - super( fs, pages, baseFile, new RotationMonitor() - { - final Log log = logProvider.getLog( CountsTracker.class ); - - @Override - public void failedToOpenStoreFile( File path, Exception error ) - { - log.error( "Failed to open counts store file: " + path, error ); - } - - @Override - public void beforeRotation( File source, File target, Headers headers ) - { - log.info( format( "About to rotate counts store at transaction %d to [%s], from [%s].", - headers.get( FileVersion.FILE_VERSION ).txId, target, source ) ); - } - - @Override - public void rotationSucceeded( File source, File target, Headers headers ) - { - log.info( format( "Successfully rotated counts store at transaction %d to [%s], from [%s].", - headers.get( FileVersion.FILE_VERSION ).txId, target, source ) ); - } - - @Override - public void rotationFailed( File source, File target, Headers headers, Exception e ) - { - log.error( format( "Failed to rotate counts store at transaction %d to [%s], from [%s].", - headers.get( FileVersion.FILE_VERSION ).txId, target, source ), e ); - } - }, new RotationTimerFactory( clock, - config.get( counts_store_rotation_timeout ).toMillis() ), 16, 16, HEADER_FIELDS ); + super( fs, pages, baseFile, new CountsTrackerRotationMonitor( logProvider ), new RotationTimerFactory( clock, + config.get( counts_store_rotation_timeout ).toMillis() ), versionContextSupplier,16, 16, HEADER_FIELDS ); } public CountsTracker setInitializer( final DataInitializer initializer ) @@ -293,6 +264,43 @@ protected void writeFormatSpecifier( WritableBuffer formatSpecifier ) formatSpecifier.put( 0, FORMAT ); } + private static class CountsTrackerRotationMonitor implements RotationMonitor + { + final Log log; + + CountsTrackerRotationMonitor( LogProvider logProvider ) + { + log = logProvider.getLog( CountsTracker.class ); + } + + @Override + public void failedToOpenStoreFile( File path, Exception error ) + { + log.error( "Failed to open counts store file: " + path, error ); + } + + @Override + public void beforeRotation( File source, File target, Headers headers ) + { + log.info( format( "About to rotate counts store at transaction %d to [%s], from [%s].", + headers.get( FileVersion.FILE_VERSION ).txId, target, source ) ); + } + + @Override + public void rotationSucceeded( File source, File target, Headers headers ) + { + log.info( format( "Successfully rotated counts store at transaction %d to [%s], from [%s].", + headers.get( FileVersion.FILE_VERSION ).txId, target, source ) ); + } + + @Override + public void rotationFailed( File source, File target, Headers headers, Exception e ) + { + log.error( format( "Failed to rotate counts store at transaction %d to [%s], from [%s].", + headers.get( FileVersion.FILE_VERSION ).txId, target, source ), e ); + } + } + private class DelegatingVisitor extends Visitor implements MetadataVisitor { private final CountsVisitor visitor; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/ReadOnlyCountsTracker.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/ReadOnlyCountsTracker.java index 5934f19c2f5a..9f0e958ab75d 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/ReadOnlyCountsTracker.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/ReadOnlyCountsTracker.java @@ -24,6 +24,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.kvstore.State; import org.neo4j.logging.LogProvider; @@ -34,7 +35,7 @@ public class ReadOnlyCountsTracker extends CountsTracker public ReadOnlyCountsTracker( LogProvider logProvider, FileSystemAbstraction fileSystem, PageCache pageCache, Config config, File baseFile ) { - super( logProvider, fileSystem, pageCache, config, baseFile ); + super( logProvider, fileSystem, pageCache, config, baseFile, EmptyVersionContextSupplier.INSTANCE ); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStore.java index de857f64eac0..40636547da3c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStore.java @@ -29,6 +29,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.impl.locking.LockWrapper; import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.lifecycle.LifecycleAdapter; @@ -49,6 +50,7 @@ public abstract class AbstractKeyValueStore extends LifecycleAdapter private final ReadWriteLock updateLock = new ReentrantReadWriteLock( /*fair=*/true ); private final Format format; final RotationStrategy rotationStrategy; + private final VersionContextSupplier versionContextSupplier; private final RotationTimerFactory rotationTimerFactory; volatile ProgressiveState state; private DataInitializer> stateInitializer; @@ -58,9 +60,11 @@ public abstract class AbstractKeyValueStore extends LifecycleAdapter private volatile boolean stopped; public AbstractKeyValueStore( FileSystemAbstraction fs, PageCache pages, File base, RotationMonitor monitor, - RotationTimerFactory timerFactory, int keySize, int valueSize, HeaderField... headerFields ) + RotationTimerFactory timerFactory, VersionContextSupplier versionContextSupplier, int keySize, + int valueSize, HeaderField... headerFields ) { this.fs = fs; + this.versionContextSupplier = versionContextSupplier; this.keySize = keySize; this.valueSize = valueSize; Rotation rotation = getClass().getAnnotation( Rotation.class ); @@ -71,7 +75,8 @@ public AbstractKeyValueStore( FileSystemAbstraction fs, PageCache pages, File ba this.format = new Format( headerFields ); this.rotationStrategy = rotation.value().create( fs, pages, format, monitor, base, rotation.parameters() ); this.rotationTimerFactory = timerFactory; - this.state = new DeadState.Stopped<>( format, getClass().getAnnotation( State.class ).value() ); + this.state = new DeadState.Stopped<>( format, getClass().getAnnotation( State.class ).value(), + versionContextSupplier ); } protected final void setEntryUpdaterInitializer( DataInitializer> stateInitializer ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ActiveState.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ActiveState.java index 0623099eb963..9c698113575b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ActiveState.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ActiveState.java @@ -24,18 +24,22 @@ import java.util.Optional; import java.util.concurrent.locks.Lock; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; + public abstract class ActiveState extends ProgressiveState { public interface Factory { - ActiveState open( ReadableState store, File file ); + ActiveState open( ReadableState store, File file, VersionContextSupplier versionContextSupplier ); } protected final ReadableState store; + protected final VersionContextSupplier versionContextSupplier; - public ActiveState( ReadableState store ) + public ActiveState( ReadableState store, VersionContextSupplier versionContextSupplier ) { this.store = store; + this.versionContextSupplier = versionContextSupplier; } @Override @@ -82,7 +86,7 @@ final EntryUpdater resetter( Lock lock, Runnable closeAction ) final ProgressiveState stop() throws IOException { close(); - return new DeadState.Stopped<>( keyFormat(), factory() ); + return new DeadState.Stopped<>( keyFormat(), factory(), versionContextSupplier ); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapState.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapState.java index 8c653c0b917c..48edc86132dd 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapState.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapState.java @@ -30,30 +30,37 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; +import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; + class ConcurrentMapState extends ActiveState { - private final ConcurrentMap changes; + private final ConcurrentMap changes; + private final VersionContextSupplier versionContextSupplier; private final File file; private final AtomicLong highestAppliedVersion; private final AtomicLong appliedChanges; private final AtomicBoolean hasTrackedChanges; private final long previousVersion; - ConcurrentMapState( ReadableState store, File file ) + ConcurrentMapState( ReadableState store, File file, VersionContextSupplier versionContextSupplier ) { - super( store ); + super( store, versionContextSupplier ); this.previousVersion = store.version(); this.file = file; + this.versionContextSupplier = versionContextSupplier; this.highestAppliedVersion = new AtomicLong( previousVersion ); this.changes = new ConcurrentHashMap<>(); this.appliedChanges = new AtomicLong(); hasTrackedChanges = new AtomicBoolean(); } - private ConcurrentMapState( Prototype prototype, ReadableState store, File file ) + private ConcurrentMapState( Prototype prototype, ReadableState store, File file, VersionContextSupplier versionContextSupplier ) { - super( store ); + super( store, versionContextSupplier ); this.previousVersion = store.version(); + this.versionContextSupplier = versionContextSupplier; this.file = file; this.hasTrackedChanges = prototype.hasTrackedChanges; this.changes = prototype.changes; @@ -76,35 +83,38 @@ public EntryUpdater updater( long version, Lock lock ) } update( highestAppliedVersion, version ); hasTrackedChanges.set( true ); - return new Updater<>( lock, store, changes, appliedChanges ); + return new Updater<>( lock, store, changes, appliedChanges, version ); } @Override public EntryUpdater unsafeUpdater( Lock lock ) { hasTrackedChanges.set( true ); - return new Updater<>( lock, store, changes, null ); + return new Updater<>( lock, store, changes, null, TransactionIdStore.BASE_TX_ID ); } private static class Updater extends EntryUpdater { private AtomicLong changeCounter; private final ReadableState store; - private final ConcurrentMap changes; + private final ConcurrentMap changes; + private final long version; - Updater( Lock lock, ReadableState store, ConcurrentMap changes, AtomicLong changeCounter ) + Updater( Lock lock, ReadableState store, ConcurrentMap changes, AtomicLong changeCounter, + long version ) { super( lock ); this.changeCounter = changeCounter; this.store = store; this.changes = changes; + this.version = version; } @Override public void apply( Key key, ValueUpdate update ) throws IOException { ensureOpenOnSameThread(); - applyUpdate( store, changes, key, update, false ); + applyUpdate( store, changes, key, update, false, version ); } @Override @@ -138,7 +148,7 @@ protected EntryUpdater resettingUpdater( Lock lock, final Runnable closeAct public void apply( Key key, ValueUpdate update ) throws IOException { ensureOpen(); - applyUpdate( store, changes, key, update, true ); + applyUpdate( store, changes, key, update, true, highestAppliedVersion.get() ); } @Override @@ -163,16 +173,16 @@ protected PrototypeState prototype( long version ) } @SuppressWarnings( "SynchronizationOnLocalVariableOrMethodParameter" ) - static void applyUpdate( ReadableState store, ConcurrentMap changes, - Key key, ValueUpdate update, boolean reset ) throws IOException + static void applyUpdate( ReadableState store, ConcurrentMap changes, + Key key, ValueUpdate update, boolean reset, long version ) throws IOException { - byte[] value = changes.get( key ); + ChangeEntry value = changes.get( key ); if ( value == null ) { final byte[] proposal = new byte[store.keyFormat().valueSize()]; synchronized ( proposal ) { - value = changes.putIfAbsent( key, proposal ); + value = changes.putIfAbsent( key, ChangeEntry.of( proposal, version ) ); if ( value == null ) { BigEndianByteArrayBuffer buffer = new BigEndianByteArrayBuffer( proposal ); @@ -191,7 +201,8 @@ static void applyUpdate( ReadableState store, ConcurrentMap extends PrototypeState { - final ConcurrentMap changes = new ConcurrentHashMap<>(); + final ConcurrentMap changes = new ConcurrentHashMap<>(); final AtomicLong highestAppliedVersion; final AtomicLong appliedChanges = new AtomicLong(); + final VersionContextSupplier versionContextSupplier; final AtomicBoolean hasTrackedChanges; private final long threshold; Prototype( ConcurrentMapState state, long version ) { super( state ); + this.versionContextSupplier = state.versionContextSupplier; threshold = version; hasTrackedChanges = new AtomicBoolean(); this.highestAppliedVersion = new AtomicLong( version ); } @Override - protected ActiveState create( ReadableState sub, File file ) + protected ActiveState create( ReadableState sub, File file, VersionContextSupplier versionContextSupplier ) { - return new ConcurrentMapState<>( this, sub, file ); + return new ConcurrentMapState<>( this, sub, file, versionContextSupplier ); } @Override @@ -245,11 +258,11 @@ protected EntryUpdater updater( long version, Lock lock ) if ( version > threshold ) { hasTrackedChanges.set( true ); - return new Updater<>( lock, store, changes, appliedChanges ); + return new Updater<>( lock, store, changes, appliedChanges, version ); } else { - return new Updater<>( lock, store, changes, null ); + return new Updater<>( lock, store, changes, null, version ); } } @@ -257,7 +270,7 @@ protected EntryUpdater updater( long version, Lock lock ) protected EntryUpdater unsafeUpdater( Lock lock ) { hasTrackedChanges.set( true ); - return new Updater<>( lock, store, changes, null ); + return new Updater<>( lock, store, changes, null, highestAppliedVersion.get() ); } @Override @@ -275,7 +288,7 @@ protected long version() @Override protected boolean lookup( Key key, ValueSink sink ) throws IOException { - return performLookup( store, changes, key, sink ); + return performLookup( store, versionContextSupplier.getVersionContext(), changes, key, sink ); } @Override @@ -340,16 +353,20 @@ protected Factory factory() @Override protected boolean lookup( Key key, ValueSink sink ) throws IOException { - return performLookup( store, changes, key, sink ); + return performLookup( store, versionContextSupplier.getVersionContext(), changes, key, sink ); } - private static boolean performLookup( ReadableState store, ConcurrentMap changes, - Key key, ValueSink sink ) throws IOException + private static boolean performLookup( ReadableState store, VersionContext versionContext, + ConcurrentMap changes, Key key, ValueSink sink ) throws IOException { - byte[] value = changes.get( key ); - if ( value != null ) + ChangeEntry change = changes.get( key ); + if ( change != null ) { - sink.value( new BigEndianByteArrayBuffer( value ) ); + if ( change.version > versionContext.lastClosedTransactionId() ) + { + versionContext.markAsDirty(); + } + sink.value( new BigEndianByteArrayBuffer( change.data ) ); return true; } return store.lookup( key, sink ); @@ -364,7 +381,7 @@ public DataProvider dataProvider() throws IOException return dataProvider( store, changes ); } - private static DataProvider dataProvider( ReadableState store, ConcurrentMap changes ) + private static DataProvider dataProvider( ReadableState store, ConcurrentMap changes ) throws IOException { if ( changes.isEmpty() ) @@ -379,16 +396,16 @@ private static DataProvider dataProvider( ReadableState store, Concur } } - private static byte[][] sortedUpdates( KeyFormat keys, ConcurrentMap changes ) + private static byte[][] sortedUpdates( KeyFormat keys, ConcurrentMap changes ) { Entry[] buffer = new Entry[changes.size()]; - Iterator> entries = changes.entrySet().iterator(); + Iterator> entries = changes.entrySet().iterator(); for ( int i = 0; i < buffer.length; i++ ) { - Map.Entry next = entries.next(); // we hold the lock, so this should succeed + Map.Entry next = entries.next(); // we hold the lock, so this should succeed byte[] key = new byte[keys.keySize()]; keys.writeKey( next.getKey(), new BigEndianByteArrayBuffer( key ) ); - buffer[i] = new Entry( key, next.getValue() ); + buffer[i] = new Entry( key, next.getValue().data ); } Arrays.sort( buffer ); assert !entries.hasNext() : "We hold the lock, so we should see 'size' entries."; @@ -447,4 +464,21 @@ public void close() throws IOException { } } + + private static class ChangeEntry + { + private byte[] data; + private long version; + + static ChangeEntry of( byte[] data, long version ) + { + return new ChangeEntry( data, version ); + } + + ChangeEntry( byte[] data, long version ) + { + this.data = data; + this.version = version; + } + } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/DeadState.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/DeadState.java index 43b41b22adc7..8d4f45c5a54e 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/DeadState.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/DeadState.java @@ -28,6 +28,7 @@ import java.util.function.Function; import org.neo4j.helpers.collection.Pair; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; abstract class DeadState extends ProgressiveState { @@ -99,18 +100,20 @@ protected KeyFormat keyFormat() private final KeyFormat keys; final ActiveState.Factory stateFactory; + final VersionContextSupplier versionContextSupplier; - private DeadState( KeyFormat keys, ActiveState.Factory stateFactory ) + private DeadState( KeyFormat keys, ActiveState.Factory stateFactory, VersionContextSupplier versionContextSupplier ) { this.keys = keys; this.stateFactory = stateFactory; + this.versionContextSupplier = versionContextSupplier; } static class Stopped extends DeadState { - Stopped( KeyFormat keys, ActiveState.Factory stateFactory ) + Stopped( KeyFormat keys, ActiveState.Factory stateFactory, VersionContextSupplier versionContextSupplier ) { - super( keys, stateFactory ); + super( keys, stateFactory, versionContextSupplier ); } @Override @@ -125,10 +128,10 @@ ProgressiveState initialize( RotationStrategy rotation ) throws IOException Pair opened = rotation.open(); if ( opened == null ) { - return new NeedsCreation<>( keyFormat(), stateFactory, rotation ); + return new NeedsCreation<>( keyFormat(), stateFactory, rotation, versionContextSupplier ); } return new Prepared<>( stateFactory.open( ReadableState.store( keyFormat(), opened.other() ), - opened.first() ) ); + opened.first(), versionContextSupplier ) ); } @Override @@ -143,16 +146,17 @@ private static class NeedsCreation extends DeadState { private final RotationStrategy rotation; - private NeedsCreation( KeyFormat keys, ActiveState.Factory stateFactory, RotationStrategy rotation ) + private NeedsCreation( KeyFormat keys, ActiveState.Factory stateFactory, RotationStrategy rotation, + VersionContextSupplier versionContextSupplier ) { - super( keys, stateFactory ); + super( keys, stateFactory, versionContextSupplier ); this.rotation = rotation; } @Override ProgressiveState stop() throws IOException { - return new Stopped<>( keyFormat(), stateFactory ); + return new Stopped<>( keyFormat(), stateFactory, versionContextSupplier ); } @Override @@ -169,14 +173,16 @@ ActiveState start( DataInitializer> initializer ) throws throw new IllegalStateException( "Store needs to be created, and no initializer is given." ); } Pair created = initialState( initializer ); - return stateFactory.open( ReadableState.store( keyFormat(), created.other() ), created.first() ); + return stateFactory.open( ReadableState.store( keyFormat(), created.other() ), created.first(), + versionContextSupplier ); } private Pair initialState( DataInitializer> initializer ) throws IOException { long version = initializer.initialVersion(); - ActiveState creation = stateFactory.open( ReadableState.empty( keyFormat(), version ), null ); + ActiveState creation = stateFactory.open( ReadableState.empty( keyFormat(), version ), null, + versionContextSupplier ); try { try ( EntryUpdater updater = creation.resetter( new ReentrantLock(), () -> @@ -246,7 +252,7 @@ private static class Prepared extends DeadState private Prepared( ActiveState state ) { - super( state.keyFormat(), state.factory() ); + super( state.keyFormat(), state.factory(), state.versionContextSupplier ); this.state = state; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/PrototypeState.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/PrototypeState.java index 1ad0c82683ff..da57e6f68d68 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/PrototypeState.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/PrototypeState.java @@ -24,6 +24,8 @@ import java.util.Optional; import java.util.concurrent.locks.Lock; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; + public abstract class PrototypeState extends WritableState { protected final ActiveState store; @@ -33,7 +35,7 @@ public PrototypeState( ActiveState store ) this.store = store; } - protected abstract ActiveState create( ReadableState sub, File file ); + protected abstract ActiveState create( ReadableState sub, File file, VersionContextSupplier versionContextSupplier ); @Override protected final Headers headers() diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/RotationState.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/RotationState.java index 016af4f092a7..bf5c73a26914 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/RotationState.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/RotationState.java @@ -90,7 +90,8 @@ ActiveState rotate( boolean force, RotationStrategy strategy, RotationTimer } Pair next = strategy .next( file(), updateHeaders( headersUpdater ), keyFormat().filter( preState.dataProvider() ) ); - return postState.create( ReadableState.store( preState.keyFormat(), next.other() ), next.first() ); + return postState.create( ReadableState.store( preState.keyFormat(), next.other() ), next.first(), + preState.versionContextSupplier ); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/State.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/State.java index f1f05bd95ea3..b95ba414a72f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/State.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/State.java @@ -26,6 +26,9 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; + @Inherited @Target( ElementType.TYPE ) @Retention( RetentionPolicy.RUNTIME ) @@ -38,17 +41,17 @@ enum Strategy implements ActiveState.Factory CONCURRENT_HASH_MAP { @Override - public ActiveState open( ReadableState store, File file ) + public ActiveState open( ReadableState store, File file, VersionContextSupplier versionContextSupplier ) { - return new ConcurrentMapState<>( store, file ); + return new ConcurrentMapState<>( store, file, versionContextSupplier ); } }, READ_ONLY_CONCURRENT_HASH_MAP { @Override - public ActiveState open( ReadableState store, File file ) + public ActiveState open( ReadableState store, File file, VersionContextSupplier versionContextSupplier ) { - return new ConcurrentMapState( store, file ) + return new ConcurrentMapState( store, file, EmptyVersionContextSupplier.INSTANCE ) { @Override protected boolean hasChanges() diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DirectRecordStoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DirectRecordStoreMigrator.java index 6223cd6c1ecd..79f7b560875f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DirectRecordStoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DirectRecordStoreMigrator.java @@ -27,6 +27,7 @@ import org.neo4j.helpers.ArrayUtil; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.RecordStore; @@ -65,11 +66,12 @@ public void migrate( File fromStoreDir, RecordFormats fromFormat, File toStoreDi try ( NeoStores fromStores = new StoreFactory( fromStoreDir, config, new DefaultIdGeneratorFactory( fs ), - pageCache, fs, fromFormat, NullLogProvider.getInstance() ).openNeoStores( true, storesToOpen ); - NeoStores toStores = new StoreFactory( toStoreDir, - withPersistedStoreHeadersAsConfigFrom( fromStores, storesToOpen ), - new DefaultIdGeneratorFactory( fs ), - pageCache, fs, toFormat, NullLogProvider.getInstance() ).openNeoStores( true, storesToOpen ) ) + pageCache, fs, fromFormat, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ) + .openNeoStores( true, storesToOpen ); + NeoStores toStores = new StoreFactory( toStoreDir, withPersistedStoreHeadersAsConfigFrom( fromStores, storesToOpen ), + new DefaultIdGeneratorFactory( fs ), pageCache, fs, toFormat, NullLogProvider.getInstance(), + EmptyVersionContextSupplier.INSTANCE ) + .openNeoStores( true, storesToOpen ) ) { for ( StoreType type : types ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/CountsMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/CountsMigrator.java index 1af0f18d32e8..7fbea955561e 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/CountsMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/CountsMigrator.java @@ -25,6 +25,7 @@ import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.CountsComputer; import org.neo4j.kernel.impl.store.MetaDataStore; @@ -158,7 +159,7 @@ private void rebuildCountsFromScratch( File storeDirToReadFrom, File migrationDi RecordFormats recordFormats = selectForVersion( expectedStoreVersion ); IdGeneratorFactory idGeneratorFactory = new ReadOnlyIdGeneratorFactory( fileSystem ); StoreFactory storeFactory = new StoreFactory( storeDirToReadFrom, config, idGeneratorFactory, pageCache, - fileSystem, recordFormats, logProvider ); + fileSystem, recordFormats, logProvider, EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = storeFactory .openNeoStores( StoreType.NODE, StoreType.RELATIONSHIP, StoreType.LABEL_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN ) ) @@ -174,7 +175,7 @@ private void rebuildCountsFromScratch( File storeDirToReadFrom, File migrationDi highRelationshipTypeId, NumberArrayFactory.auto( pageCache, migrationDir, true ), progressMonitor ); life.add( new CountsTracker( logProvider, fileSystem, pageCache, config, - storeFileBase ).setInitializer( initializer ) ); + storeFileBase, EmptyVersionContextSupplier.INSTANCE ).setInitializer( initializer ) ); } } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigrator.java index dd077215dc30..c4a824311129 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/NativeLabelScanStoreMigrator.java @@ -29,6 +29,7 @@ import org.neo4j.io.fs.FileHandle; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.labelscan.NodeLabelUpdate; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.index.IndexStoreView; @@ -140,7 +141,7 @@ private StoreFactory getStoreFactory( File storeDir, String versionToMigrateFrom RecordFormats recordFormats = selectForVersion( versionToMigrateFrom ); IdGeneratorFactory idGeneratorFactory = new ReadOnlyIdGeneratorFactory( fileSystem ); return new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fileSystem, - recordFormats, logProvider ); + recordFormats, logProvider, EmptyVersionContextSupplier.INSTANCE ); } private boolean isNativeLabelScanStoreMigrationRequired( File storeDir ) throws IOException diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java index b98bf56fc007..00ab8e6d06b3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java @@ -47,6 +47,7 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PagedFile; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.AssertOpen; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.store.StorePropertyCursor; @@ -426,7 +427,7 @@ public boolean parallelRecordReadsWhenWriting() private NeoStores instantiateLegacyStore( RecordFormats format, File storeDir ) { return new StoreFactory( storeDir, config, new ReadOnlyIdGeneratorFactory(), pageCache, fileSystem, - format, NullLogProvider.getInstance() ).openAllNeoStores( true ); + format, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ).openAllNeoStores( true ); } private void prepareBatchImportMigration( File storeDir, File migrationDir, RecordFormats oldFormat, @@ -495,7 +496,8 @@ private void createStore( File migrationDir, RecordFormats newFormat ) IdGeneratorFactory idGeneratorFactory = new ReadOnlyIdGeneratorFactory( fileSystem ); NullLogProvider logProvider = NullLogProvider.getInstance(); StoreFactory storeFactory = new StoreFactory( - migrationDir, config, idGeneratorFactory, pageCache, fileSystem, newFormat, logProvider ); + migrationDir, config, idGeneratorFactory, pageCache, fileSystem, newFormat, logProvider, + EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = storeFactory.openAllNeoStores( true ) ) { neoStores.getMetaDataStore(); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java index 0b8864a139e5..3fbe77101b0e 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java @@ -54,6 +54,7 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.exceptions.KernelException; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.exceptions.schema.AlreadyConstrainedException; @@ -236,7 +237,8 @@ public BatchInserterImpl( final File storeDir, final FileSystemAbstraction fileS life = new LifeSupport(); this.storeDir = storeDir; ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( - fileSystem, config, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, NullLog.getInstance() ); + fileSystem, config, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, NullLog.getInstance(), + EmptyVersionContextSupplier.INSTANCE ); PageCache pageCache = pageCacheFactory.getOrCreatePageCache(); life.add( new PageCacheLifecycle( pageCache ) ); @@ -254,7 +256,7 @@ public BatchInserterImpl( final File storeDir, final FileSystemAbstraction fileS RecordFormats recordFormats = RecordFormatSelector.selectForStoreOrConfig( config, storeDir, fileSystem, pageCache, internalLogProvider ); StoreFactory sf = new StoreFactory( this.storeDir, config, idGeneratorFactory, pageCache, fileSystem, - recordFormats, internalLogProvider ); + recordFormats, internalLogProvider, EmptyVersionContextSupplier.INSTANCE ); maxNodeId = recordFormats.node().getMaxId(); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java index cf6ab0df2845..e178af9abfe2 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java @@ -32,6 +32,8 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.scan.FullStoreChangeStream; @@ -165,7 +167,7 @@ public static BatchingNeoStores batchingNeoStores( FileSystemAbstraction fileSys Config neo4jConfig = getNeo4jConfig( config, dbConfig ); final PageCacheTracer tracer = new DefaultPageCacheTracer(); PageCache pageCache = createPageCache( fileSystem, neo4jConfig, logService.getInternalLogProvider(), tracer, - DefaultPageCursorTracerSupplier.INSTANCE ); + DefaultPageCursorTracerSupplier.INSTANCE, EmptyVersionContextSupplier.INSTANCE ); return new BatchingNeoStores( fileSystem, pageCache, storeDir, recordFormats, neo4jConfig, config, logService, initialIds, false, tracer::bytesWritten ); @@ -190,10 +192,10 @@ private static Config getNeo4jConfig( Configuration config, Config dbConfig ) } private static PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, LogProvider log, - PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier ) + PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier, VersionContextSupplier contextSupplier ) { return new ConfiguringPageCacheFactory( fileSystem, config, tracer, cursorTracerSupplier, - log.getLog( BatchingNeoStores.class ) ).getOrCreatePageCache(); + log.getLog( BatchingNeoStores.class ), contextSupplier ).getOrCreatePageCache(); } private boolean alreadyContainsData( NeoStores neoStores ) @@ -205,7 +207,7 @@ private StoreFactory newStoreFactory( String name, OpenOption... openOptions ) { return new StoreFactory( storeDir, name, neo4jConfig, new BatchingIdGeneratorFactory( fileSystem ), pageCache, fileSystem, recordFormats, logProvider, - openOptions ); + EmptyVersionContextSupplier.INSTANCE, openOptions ); } /** diff --git a/community/kernel/src/test/java/org/neo4j/kernel/RecoveryIT.java b/community/kernel/src/test/java/org/neo4j/kernel/RecoveryIT.java index 85f83e1b9764..fa048ec2245c 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/RecoveryIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/RecoveryIT.java @@ -34,7 +34,6 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; - import javax.annotation.Nonnull; import org.neo4j.adversaries.ClassGuardedAdversary; @@ -58,6 +57,8 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.index.IndexEntryUpdate; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.index.inmemory.InMemoryIndexProvider; @@ -94,6 +95,9 @@ import org.neo4j.test.rule.TestDirectory; import org.neo4j.test.rule.fs.DefaultFileSystemRule; +import static java.lang.Long.max; +import static java.util.Arrays.asList; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.hamcrest.Matchers.instanceOf; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -101,11 +105,6 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; - -import static java.lang.Long.max; -import static java.util.Arrays.asList; -import static java.util.concurrent.TimeUnit.SECONDS; - import static org.neo4j.graphdb.RelationshipType.withName; import static org.neo4j.helpers.ArrayUtil.array; import static org.neo4j.helpers.collection.Iterables.asList; @@ -444,15 +443,18 @@ private long lastCommittedTxId( GraphDatabaseService db ) private void assertSameStoreContents( EphemeralFileSystemAbstraction fs1, EphemeralFileSystemAbstraction fs2, File storeDir ) { NullLogProvider logProvider = NullLogProvider.getInstance(); + VersionContextSupplier contextSupplier = EmptyVersionContextSupplier.INSTANCE; try ( PageCache pageCache1 = new ConfiguringPageCacheFactory( fs1, defaults(), PageCacheTracer.NULL, - PageCursorTracerSupplier.NULL, NullLog.getInstance() ).getOrCreatePageCache(); + PageCursorTracerSupplier.NULL, NullLog.getInstance(), contextSupplier ) + .getOrCreatePageCache(); PageCache pageCache2 = new ConfiguringPageCacheFactory( fs2, defaults(), PageCacheTracer.NULL, - PageCursorTracerSupplier.NULL, NullLog.getInstance() ).getOrCreatePageCache(); + PageCursorTracerSupplier.NULL, NullLog.getInstance(), contextSupplier ) + .getOrCreatePageCache(); NeoStores store1 = new StoreFactory( storeDir, defaults(), new DefaultIdGeneratorFactory( fs1 ), - pageCache1, fs1, logProvider ).openAllNeoStores(); + pageCache1, fs1, logProvider, contextSupplier ).openAllNeoStores(); NeoStores store2 = new StoreFactory( storeDir, defaults(), new DefaultIdGeneratorFactory( fs2 ), - pageCache2, fs2, logProvider ).openAllNeoStores(); + pageCache2, fs2, logProvider, contextSupplier ).openAllNeoStores(); ) { for ( StoreType storeType : StoreType.values() ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/api/KernelTransactionFactory.java b/community/kernel/src/test/java/org/neo4j/kernel/api/KernelTransactionFactory.java index 36c035eb70da..5d04ccd2651a 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/api/KernelTransactionFactory.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/api/KernelTransactionFactory.java @@ -23,6 +23,7 @@ import org.neo4j.collection.pool.Pool; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.security.SecurityContext; import org.neo4j.kernel.impl.api.KernelTransactionImplementation; import org.neo4j.kernel.impl.api.SchemaWriteGuard; @@ -95,7 +96,7 @@ static Instances kernelTransactionWithInternals( SecurityContext securityContext NULL, LockTracer.NONE, PageCursorTracerSupplier.NULL, - storageEngine, new CanWrite() ); + storageEngine, new CanWrite(), EmptyVersionContextSupplier.INSTANCE ); StatementLocks statementLocks = new SimpleStatementLocks( new NoOpClient() ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTerminationTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTerminationTest.java index beb40ae765ba..6809c4f84f57 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTerminationTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTerminationTest.java @@ -31,6 +31,7 @@ import org.neo4j.collection.pool.Pool; import org.neo4j.graphdb.TransactionTerminatedException; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.exceptions.Status; import org.neo4j.kernel.api.exceptions.TransactionFailureException; @@ -336,7 +337,8 @@ private static class TestKernelTransaction extends KernelTransactionImplementati mock( TransactionCommitProcess.class ), monitor, () -> mock( ExplicitIndexTransactionState.class ), mock( Pool.class ), Clocks.fakeClock(), TransactionTracer.NULL, LockTracer.NONE, PageCursorTracerSupplier.NULL, - mock( StorageEngine.class, RETURNS_MOCKS ), new CanWrite() ); + mock( StorageEngine.class, RETURNS_MOCKS ), new CanWrite(), + EmptyVersionContextSupplier.INSTANCE ); this.monitor = monitor; } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTestBase.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTestBase.java index bcd363a77487..02823436462e 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTestBase.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTestBase.java @@ -27,6 +27,7 @@ import org.neo4j.collection.pool.Pool; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.KernelTransaction.Type; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.api.security.SecurityContext; @@ -143,7 +144,7 @@ public KernelTransactionImplementation newNotInitializedTransaction() return new KernelTransactionImplementation( statementOperations, schemaWriteGuard, hooks, null, null, headerInformationFactory, commitProcess, transactionMonitor, explicitIndexStateSupplier, txPool, clock, TransactionTracer.NULL, LockTracer.NONE, - PageCursorTracerSupplier.NULL, storageEngine, new CanWrite() ); + PageCursorTracerSupplier.NULL, storageEngine, new CanWrite(), EmptyVersionContextSupplier.INSTANCE ); } public class CapturingCommitProcess implements TransactionCommitProcess diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionsTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionsTest.java index a5dd2ba80dee..451f0a4ee71d 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionsTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionsTest.java @@ -35,6 +35,8 @@ import org.neo4j.graphdb.DatabaseShutdownException; import org.neo4j.graphdb.security.AuthorizationExpiredException; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.AvailabilityGuard; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.KernelTransactionHandle; @@ -560,7 +562,8 @@ private static KernelTransactions createTransactions( StorageEngine storageEngin null, statementOperations, null, DEFAULT, commitProcess, null, null, new TransactionHooks(), mock( TransactionMonitor.class ), availabilityGuard, - tracers, storageEngine, new Procedures(), transactionIdStore, clock, new CanWrite() ); + tracers, storageEngine, new Procedures(), transactionIdStore, clock, new CanWrite(), + EmptyVersionContextSupplier.INSTANCE ); } private static TestKernelTransactions createTestTransactions( StorageEngine storageEngine, @@ -572,7 +575,7 @@ private static TestKernelTransactions createTestTransactions( StorageEngine stor null, DEFAULT, commitProcess, null, null, new TransactionHooks(), mock( TransactionMonitor.class ), availabilityGuard, tracers, storageEngine, new Procedures(), transactionIdStore, clock, - new CanWrite() ); + new CanWrite(), EmptyVersionContextSupplier.INSTANCE ); } private static TransactionCommitProcess newRememberingCommitProcess( final TransactionRepresentation[] slot ) @@ -621,13 +624,13 @@ private static class TestKernelTransactions extends KernelTransactions ExplicitIndexProviderLookup explicitIndexProviderLookup, TransactionHooks hooks, TransactionMonitor transactionMonitor, AvailabilityGuard availabilityGuard, Tracers tracers, StorageEngine storageEngine, Procedures procedures, TransactionIdStore transactionIdStore, Clock clock, - AccessCapability accessCapability ) + AccessCapability accessCapability, VersionContextSupplier versionContextSupplier ) { super( statementLocksFactory, constraintIndexCreator, statementOperations, schemaWriteGuard, txHeaderFactory, transactionCommitProcess, indexConfigStore, explicitIndexProviderLookup, hooks, transactionMonitor, availabilityGuard, tracers, storageEngine, procedures, transactionIdStore, clock, - accessCapability ); + accessCapability, versionContextSupplier ); } @Override diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LockingStatementOperationsTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LockingStatementOperationsTest.java index 40844500bd91..4924f6a2b86a 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LockingStatementOperationsTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LockingStatementOperationsTest.java @@ -27,6 +27,7 @@ import org.neo4j.helpers.collection.Iterators; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContext; import org.neo4j.kernel.api.exceptions.EntityNotFoundException; import org.neo4j.kernel.api.exceptions.InvalidTransactionTypeKernelException; import org.neo4j.kernel.api.exceptions.KernelException; @@ -106,7 +107,7 @@ public LockingStatementOperationsTest() lockingOps = new LockingStatementOperations( entityReadOps, entityWriteOps, schemaReadOps, schemaWriteOps, schemaStateOps ); - state.initialize( new SimpleStatementLocks( locks ), PageCursorTracer.NULL ); + state.initialize( new SimpleStatementLocks( locks ), PageCursorTracer.NULL, EmptyVersionContext.INSTANCE ); state.acquire(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/PropertyPhysicalToLogicalConverterTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/PropertyPhysicalToLogicalConverterTest.java index 45ccfc68ceca..e3e9bed25274 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/PropertyPhysicalToLogicalConverterTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/PropertyPhysicalToLogicalConverterTest.java @@ -28,6 +28,7 @@ import java.io.File; import org.neo4j.helpers.collection.Iterables; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.PropertyStore; @@ -226,7 +227,8 @@ public void before() throws Exception File storeDir = new File( "dir" ); fs.get().mkdirs( storeDir ); StoreFactory storeFactory = new StoreFactory( storeDir, Config.defaults(), new DefaultIdGeneratorFactory( fs.get() ), - pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance(), + EmptyVersionContextSupplier.INSTANCE ); neoStores = storeFactory.openAllNeoStores( true ); store = neoStores.getPropertyStore(); converter = new PropertyPhysicalToLogicalConverter( store ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursorTest.java index f84be3df3ed5..88561df94f70 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursorTest.java @@ -34,6 +34,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory; import org.neo4j.kernel.impl.store.NeoStores; @@ -103,12 +104,13 @@ public static void setupStores() fs = new DefaultFileSystemAbstraction(); Config config = Config.defaults( pagecache_memory, "8m" ); pageCache = new ConfiguringPageCacheFactory( fs, - config, NULL, PageCursorTracerSupplier.NULL, NullLog.getInstance() ) + config, NULL, PageCursorTracerSupplier.NULL, NullLog.getInstance(), EmptyVersionContextSupplier.INSTANCE ) .getOrCreatePageCache(); DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fs ); NullLogProvider logProvider = NullLogProvider.getInstance(); StoreFactory storeFactory = - new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fs, logProvider ); + new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fs, logProvider, + EmptyVersionContextSupplier.INSTANCE ); neoStores = storeFactory.openAllNeoStores( true ); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StorePropertyCursorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StorePropertyCursorTest.java index 6adcebdd48b1..36beeae49302 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StorePropertyCursorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StorePropertyCursorTest.java @@ -38,6 +38,7 @@ import org.neo4j.cursor.Cursor; import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.DynamicArrayStore; import org.neo4j.kernel.impl.store.DynamicRecordAllocator; @@ -294,7 +295,8 @@ public static void setUp() throws IOException Config config = Config.defaults(); DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fs ); NullLogProvider logProvider = NullLogProvider.getInstance(); - neoStores = new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fs, logProvider ) + neoStores = new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fs, logProvider, + EmptyVersionContextSupplier.INSTANCE ) .openAllNeoStores( true ); propertyStore = neoStores.getPropertyStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreSingleRelationshipCursorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreSingleRelationshipCursorTest.java index e07f3c6e03ed..70a69a66a0ee 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreSingleRelationshipCursorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreSingleRelationshipCursorTest.java @@ -25,6 +25,7 @@ import org.junit.Test; import org.junit.rules.RuleChain; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.locking.LockService; import org.neo4j.kernel.impl.store.NeoStores; @@ -113,7 +114,7 @@ private StoreFactory getStoreFactory() return new StoreFactory( testDirectory.directory(), Config.defaults(), new DefaultIdGeneratorFactory( fileSystemRule.get() ), pageCacheRule.getPageCache( fileSystemRule.get() ), fileSystemRule.get(), - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); } private StoreSingleRelationshipCursor createRelationshipCursor() diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/core/ManyPropertyKeysIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/core/ManyPropertyKeysIT.java index df3baafb2398..d28f16c41121 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/core/ManyPropertyKeysIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/core/ManyPropertyKeysIT.java @@ -33,6 +33,7 @@ import org.neo4j.graphdb.Transaction; import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.KernelAPI; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.Statement; @@ -134,7 +135,7 @@ private GraphDatabaseAPI databaseWithManyPropertyKeys( int propertyKeyCount ) th StoreFactory storeFactory = new StoreFactory( storeDir, Config.defaults(), new DefaultIdGeneratorFactory( fileSystemRule.get() ), pageCache, fileSystemRule.get(), - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = storeFactory.openAllNeoStores( true ); PropertyKeyTokenStore store = neoStores.getPropertyKeyTokenStore(); for ( int i = 0; i < propertyKeyCount; i++ ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactoryTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactoryTest.java index 60f69fa6220b..297ce78caeec 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactoryTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactoryTest.java @@ -27,6 +27,7 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.logging.AssertableLogProvider; import org.neo4j.logging.Log; @@ -66,7 +67,7 @@ public void shouldFitAsManyPagesAsItCan() throws Throwable // When ConfiguringPageCacheFactory factory = new ConfiguringPageCacheFactory( fsRule.get(), config, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, - NullLog.getInstance() ); + NullLog.getInstance(), EmptyVersionContextSupplier.INSTANCE ); // Then try ( PageCache cache = factory.getOrCreatePageCache() ) @@ -88,7 +89,7 @@ public void shouldWarnWhenCreatedWithConfiguredPageCache() throws Exception // When ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( fsRule.get(), config, - PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, log ); + PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, log, EmptyVersionContextSupplier.INSTANCE ); // Then try ( PageCache pageCache = pageCacheFactory.getOrCreatePageCache() ) @@ -111,7 +112,7 @@ public void mustUseAndLogConfiguredPageSwapper() throws Exception // When ConfiguringPageCacheFactory cacheFactory = new ConfiguringPageCacheFactory( fsRule.get(), config, PageCacheTracer.NULL, - PageCursorTracerSupplier.NULL, log ); + PageCursorTracerSupplier.NULL, log, EmptyVersionContextSupplier.INSTANCE ); try ( PageCache pageCache = cacheFactory.getOrCreatePageCache() ) { // empty block @@ -133,7 +134,8 @@ public void mustThrowIfConfiguredPageSwapperCannotBeFound() throws Exception // When try ( PageCache pageCache = new ConfiguringPageCacheFactory( fsRule.get(), config, PageCacheTracer.NULL, - PageCursorTracerSupplier.NULL, NullLog.getInstance() ).getOrCreatePageCache() ) + PageCursorTracerSupplier.NULL, NullLog.getInstance(), EmptyVersionContextSupplier.INSTANCE ) + .getOrCreatePageCache() ) { //empty } @@ -151,7 +153,7 @@ public void mustIgnoreExplicitlySpecifiedCachePageSizeIfPageSwapperHintIsStrict( // When ConfiguringPageCacheFactory factory = new ConfiguringPageCacheFactory( fsRule.get(), config, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, - NullLog.getInstance() ); + NullLog.getInstance(), EmptyVersionContextSupplier.INSTANCE ); // Then try ( PageCache cache = factory.getOrCreatePageCache() ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/FreeIdsAfterRecoveryTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/FreeIdsAfterRecoveryTest.java index 1e5fefd66cac..7d48db20c0ad 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/FreeIdsAfterRecoveryTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/FreeIdsAfterRecoveryTest.java @@ -25,6 +25,7 @@ import java.io.File; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; import org.neo4j.kernel.impl.store.id.IdGeneratorImpl; @@ -55,7 +56,7 @@ public void shouldCompletelyRebuildIdGeneratorsAfterCrash() throws Exception StoreFactory storeFactory = new StoreFactory( directory.directory(), Config.defaults(), new DefaultIdGeneratorFactory( fileSystemRule.get() ), pageCacheRule.getPageCache( fileSystemRule.get() ), fileSystemRule.get(), - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); long highId; try ( NeoStores stores = storeFactory.openAllNeoStores( true ) ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/IdGeneratorRebuildFailureEmulationTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/IdGeneratorRebuildFailureEmulationTest.java index e6e265d6cdb3..1e85f21b4928 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/IdGeneratorRebuildFailureEmulationTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/IdGeneratorRebuildFailureEmulationTest.java @@ -19,10 +19,6 @@ */ package org.neo4j.kernel.impl.store; -import java.io.File; -import java.util.HashMap; -import java.util.Map; - import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -32,6 +28,10 @@ import org.junit.runners.Suite; import org.junit.runners.Suite.SuiteClasses; +import java.io.File; +import java.util.HashMap; +import java.util.Map; + import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.PropertyContainer; @@ -41,6 +41,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.impl.factory.CommunityEditionModule; @@ -123,7 +124,7 @@ public void initialize() params.put( GraphDatabaseSettings.rebuild_idgenerators_fast.name(), Settings.FALSE ); Config config = Config.defaults( params ); factory = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs ), - pageCacheRule.getPageCache( fs ), fs, NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs ), fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); } @After diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/MetaDataStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/MetaDataStoreTest.java index f31246fd9bec..8dcf1e9cbadc 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/MetaDataStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/MetaDataStoreTest.java @@ -39,6 +39,7 @@ import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PagedFile; import org.neo4j.io.pagecache.impl.DelegatingPageCursor; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.format.standard.Standard; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; @@ -115,7 +116,7 @@ private MetaDataStore newMetaDataStore() throws IOException { LogProvider logProvider = NullLogProvider.getInstance(); StoreFactory storeFactory = new StoreFactory( STORE_DIR, Config.defaults(), new DefaultIdGeneratorFactory( fs ), - pageCacheWithFakeOverflow, fs, logProvider ); + pageCacheWithFakeOverflow, fs, logProvider, EmptyVersionContextSupplier.INSTANCE ); return storeFactory.openNeoStores( true, StoreType.META_DATA ).getMetaDataStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java index f7c39ea4133e..f838e59e61ee 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java @@ -49,6 +49,7 @@ import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.exceptions.EntityNotFoundException; @@ -148,7 +149,7 @@ public void setUpNeoStores() throws Exception Config config = Config.defaults(); pageCache = pageCacheRule.getPageCache( fs.get() ); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), pageCache, - fs.get(), NullLogProvider.getInstance() ); + fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); sf.openAllNeoStores( true ).close(); } @@ -157,7 +158,7 @@ public void impossibleToGetStoreFromClosedNeoStoresContainer() { Config config = Config.defaults(); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), pageCache, - fs.get(), NullLogProvider.getInstance() ); + fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = sf.openAllNeoStores( true ); assertNotNull( neoStores.getMetaDataStore() ); @@ -174,7 +175,7 @@ public void notAllowCreateDynamicStoreWithNegativeBlockSize() { Config config = Config.defaults(); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), pageCache, - fs.get(), NullLogProvider.getInstance() ); + fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); exception.expect( IllegalArgumentException.class ); exception.expectMessage( "Block size of dynamic array store should be positive integer." ); @@ -190,7 +191,7 @@ public void impossibleToGetNotRequestedStore() { Config config = Config.defaults(); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), pageCache, - fs.get(), NullLogProvider.getInstance() ); + fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = sf.openNeoStores( true, StoreType.NODE_LABEL ); exception.expect( IllegalStateException.class ); @@ -540,7 +541,7 @@ public void setVersion() throws Exception Config config = Config.defaults(); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fileSystem ), pageCache, - fileSystem, LOG_PROVIDER ); + fileSystem, LOG_PROVIDER, EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = sf.openAllNeoStores(); assertEquals( 12, neoStores.getMetaDataStore().getCurrentLogVersion() ); @@ -597,7 +598,7 @@ public void testSetLatestConstraintTx() throws Exception // given Config config = Config.defaults(); StoreFactory sf = new StoreFactory( dir.directory(), config, new DefaultIdGeneratorFactory( fs.get() ), - pageCacheRule.getPageCache( fs.get() ), fs.get(), LOG_PROVIDER ); + pageCacheRule.getPageCache( fs.get() ), fs.get(), LOG_PROVIDER, EmptyVersionContextSupplier.INSTANCE ); // when NeoStores neoStores = sf.openAllNeoStores( true ); @@ -629,7 +630,7 @@ public void shouldInitializeTheTxIdToOne() new StoreFactory( new File( "graph.db/neostore" ), Config.defaults(), new DefaultIdGeneratorFactory( fs.get() ), pageCache, fs.get(), - LOG_PROVIDER ); + LOG_PROVIDER, EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = factory.openAllNeoStores( true ) ) { @@ -650,7 +651,7 @@ public void shouldThrowUnderlyingStorageExceptionWhenFailingToLoadStorage() File neoStoreDir = new File( "/tmp/graph.db/neostore" ).getAbsoluteFile(); StoreFactory factory = new StoreFactory( neoStoreDir, Config.defaults(), new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, - LOG_PROVIDER ); + LOG_PROVIDER, EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = factory.openAllNeoStores( true ) ) { @@ -729,7 +730,7 @@ public void shouldSetHighestTransactionIdWhenNeeded() throws Throwable fileSystem.mkdirs( storeDir ); StoreFactory factory = new StoreFactory( storeDir, Config.defaults(), new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, - LOG_PROVIDER ); + LOG_PROVIDER, EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStore = factory.openAllNeoStores( true ) ) { @@ -755,7 +756,7 @@ public void shouldNotSetHighestTransactionIdWhenNeeded() throws Throwable fileSystem.mkdirs( storeDir ); StoreFactory factory = new StoreFactory( storeDir, Config.defaults(), new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, - LOG_PROVIDER ); + LOG_PROVIDER, EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStore = factory.openAllNeoStores( true ) ) { @@ -781,7 +782,7 @@ public void shouldCloseAllTheStoreEvenIfExceptionsAreThrown() throws Exception Config defaults = Config.defaults( counts_store_rotation_timeout, "60m" ); StoreFactory factory = new StoreFactory( storeDir, defaults, new DefaultIdGeneratorFactory( fileSystem ), pageCache, - fileSystem, LOG_PROVIDER ); + fileSystem, LOG_PROVIDER, EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStore = factory.openAllNeoStores( true ); // let's hack the counts store so it fails to rotate and hence it fails to close as well... @@ -840,7 +841,7 @@ public void isPresentAfterCreatingAllStores() throws Exception fileSystem.deleteRecursively( storeDir ); DefaultIdGeneratorFactory idFactory = new DefaultIdGeneratorFactory( fileSystem ); StoreFactory factory = new StoreFactory( storeDir, Config.defaults(), idFactory, pageCache, fileSystem, - LOG_PROVIDER ); + LOG_PROVIDER, EmptyVersionContextSupplier.INSTANCE ); // when try ( NeoStores ignore = factory.openAllNeoStores( true ) ) @@ -858,7 +859,7 @@ public void isPresentFalseAfterCreatingAllButLastStoreType() throws Exception fileSystem.deleteRecursively( storeDir ); DefaultIdGeneratorFactory idFactory = new DefaultIdGeneratorFactory( fileSystem ); StoreFactory factory = new StoreFactory( storeDir, Config.defaults(), idFactory, pageCache, fileSystem, - LOG_PROVIDER ); + LOG_PROVIDER, EmptyVersionContextSupplier.INSTANCE ); StoreType[] allStoreTypes = StoreType.values(); StoreType[] allButLastStoreTypes = Arrays.copyOf( allStoreTypes, allStoreTypes.length - 1 ); @@ -911,7 +912,8 @@ private static StoreFactory newStoreFactory( File neoStoreDir, PageCache pageCac RecordFormats recordFormats = RecordFormatSelector.defaultFormat(); Config config = Config.defaults(); IdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fs ); - return new StoreFactory( neoStoreDir, config, idGeneratorFactory, pageCache, fs, recordFormats, LOG_PROVIDER ); + return new StoreFactory( neoStoreDir, config, idGeneratorFactory, pageCache, fs, recordFormats, LOG_PROVIDER, + EmptyVersionContextSupplier.INSTANCE ); } private Token createDummyIndex( int id, String key ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeStoreTest.java index 9e0bb6c90a02..ef9f09b77b01 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeStoreTest.java @@ -19,6 +19,12 @@ */ package org.neo4j.kernel.impl.store; +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.junit.After; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; + import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; @@ -29,12 +35,6 @@ import java.util.function.Supplier; import java.util.stream.LongStream; -import org.apache.commons.lang3.mutable.MutableBoolean; -import org.junit.After; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; - import org.neo4j.collection.primitive.Primitive; import org.neo4j.collection.primitive.PrimitiveLongSet; import org.neo4j.graphdb.mockfs.DelegatingFileSystemAbstraction; @@ -45,6 +45,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.allocator.ReusableRecordsAllocator; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; @@ -381,7 +382,7 @@ protected IdGenerator instantiate( FileSystemAbstraction fs, File fileName, int } } ); StoreFactory factory = new StoreFactory( storeDir, Config.defaults(), idGeneratorFactory, pageCache, fs, - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); neoStores = factory.openAllNeoStores( true ); nodeStore = neoStores.getNodeStore(); return nodeStore; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RecordStoreConsistentReadTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RecordStoreConsistentReadTest.java index ee45a47e3ffc..c0afc1bfedc5 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RecordStoreConsistentReadTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RecordStoreConsistentReadTest.java @@ -35,6 +35,7 @@ import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.allocator.ReusableRecordsAllocator; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; @@ -87,7 +88,7 @@ private NeoStores storeFixture() config().withInconsistentReads( nextReadIsInconsistent ) ); File storeDir = new File( "stores" ); StoreFactory factory = new StoreFactory( storeDir, Config.defaults(), new DefaultIdGeneratorFactory( fs ), - pageCache, fs, NullLogProvider.getInstance() ); + pageCache, fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = factory.openAllNeoStores( true ); S store = initialiseStore( neoStores ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RelationshipGroupStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RelationshipGroupStoreTest.java index e8b7e336ecfb..c34358274a5f 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RelationshipGroupStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RelationshipGroupStoreTest.java @@ -40,6 +40,7 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.MyRelTypes; import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine; @@ -170,7 +171,7 @@ private StoreFactory factory( Integer customThreshold, PageCache pageCache ) customConfig.put( GraphDatabaseSettings.dense_node_threshold.name(), "" + customThreshold ); } return new StoreFactory( directory, Config.defaults( customConfig ), new DefaultIdGeneratorFactory( fs ), pageCache, - fs, NullLogProvider.getInstance() ); + fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); } @Test diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/SchemaStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/SchemaStoreTest.java index bcefe1862d93..c9f1c4e3027a 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/SchemaStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/SchemaStoreTest.java @@ -31,6 +31,7 @@ import java.util.stream.IntStream; import org.neo4j.helpers.collection.Iterables; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.api.schema.constaints.ConstraintDescriptorFactory; import org.neo4j.kernel.api.schema.index.IndexDescriptorFactory; @@ -70,7 +71,7 @@ public void before() throws Exception config = Config.defaults(); DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fs.get() ); storeFactory = new StoreFactory( storeDir, config, idGeneratorFactory, pageCacheRule.getPageCache( fs.get() ), - fs.get(), NullLogProvider.getInstance() ); + fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); neoStores = storeFactory.openAllNeoStores( true ); store = neoStores.getSchemaStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/StoreFactoryTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/StoreFactoryTest.java index 80bf8e7904a0..f66525676fe5 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/StoreFactoryTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/StoreFactoryTest.java @@ -31,6 +31,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.impl.store.format.RecordFormats; @@ -74,7 +75,7 @@ private StoreFactory storeFactory( Config config, OpenOption... openOptions ) LogProvider logProvider = NullLogProvider.getInstance(); RecordFormats recordFormats = selectForStoreOrConfig( config, storeDir, fsRule.get(), pageCache, logProvider ); return new StoreFactory( storeDir, DEFAULT_NAME, config, idGeneratorFactory, pageCache, fsRule.get(), - recordFormats, logProvider, openOptions ); + recordFormats, logProvider, EmptyVersionContextSupplier.INSTANCE, openOptions ); } private File directory( String name ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestArrayStore.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestArrayStore.java index 65e5568613bd..050607e6372d 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestArrayStore.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestArrayStore.java @@ -35,6 +35,7 @@ import org.neo4j.helpers.collection.Pair; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; import org.neo4j.kernel.impl.store.record.DynamicRecord; @@ -68,7 +69,7 @@ public void before() throws Exception DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fs ); PageCache pageCache = pageCacheRule.getPageCache( fs ); StoreFactory factory = new StoreFactory( dir, Config.defaults(), idGeneratorFactory, pageCache, fs, - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); neoStores = factory.openAllNeoStores( true ); arrayStore = neoStores.getPropertyStore().getArrayStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestDynamicStore.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestDynamicStore.java index 401eda06fc1c..d927a2efda21 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestDynamicStore.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestDynamicStore.java @@ -34,6 +34,7 @@ import java.util.Set; import org.neo4j.helpers.collection.Iterables; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; import org.neo4j.kernel.impl.store.record.DynamicRecord; @@ -63,7 +64,7 @@ public void setUp() fs.get().mkdir( storeDir ); config = config(); storeFactory = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), - pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); } @After diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGraphProperties.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGraphProperties.java index 6316064734f0..2e06192a2aab 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGraphProperties.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGraphProperties.java @@ -33,6 +33,7 @@ import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.graphdb.mockfs.UncloseableDelegatingFileSystemAbstraction; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.core.NodeManager; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; @@ -191,7 +192,7 @@ public void firstRecordOtherThanZeroIfNotFirst() throws Exception Config config = Config.defaults(); StoreFactory storeFactory = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), - pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = storeFactory.openAllNeoStores(); long prop = neoStores.getMetaDataStore().getGraphNextProp(); assertTrue( prop != 0 ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGrowingFileMemoryMapping.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGrowingFileMemoryMapping.java index 6f8b1c3ce8a2..05325fb7553b 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGrowingFileMemoryMapping.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGrowingFileMemoryMapping.java @@ -29,6 +29,7 @@ import org.neo4j.io.ByteUnit; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.format.standard.NodeRecordFormat; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; @@ -70,7 +71,7 @@ public void shouldGrowAFileWhileContinuingToMemoryMapNewRegions() throws Excepti DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fileSystemAbstraction ); PageCache pageCache = pageCacheRule.getPageCache( fileSystemAbstraction, config ); StoreFactory storeFactory = new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, - fileSystemAbstraction, NullLogProvider.getInstance() ); + fileSystemAbstraction, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = storeFactory.openAllNeoStores( true ); NodeStore nodeStore = neoStores.getNodeStore(); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestIdGeneratorRebuilding.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestIdGeneratorRebuilding.java index 19fbfedc2082..c1b23c730280 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestIdGeneratorRebuilding.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestIdGeneratorRebuilding.java @@ -32,6 +32,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; @@ -123,7 +124,8 @@ public void verifyDynamicSizedStoresCanRebuildIdGeneratorSlowly() throws Excepti Config config = Config.defaults( GraphDatabaseSettings.rebuild_idgenerators_fast, "false" ); StoreFactory storeFactory = new StoreFactory( testDirectory.graphDbDir(), config, - new DefaultIdGeneratorFactory( fs ), pageCacheRule.getPageCache( fs ), fs, NullLogProvider.getInstance() ); + new DefaultIdGeneratorFactory( fs ), pageCacheRule.getPageCache( fs ), fs, NullLogProvider + .getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = storeFactory.openAllNeoStores( true ); DynamicStringStore store = neoStores.getPropertyStore().getStringStore(); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsComputerTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsComputerTest.java index e15fb4e170eb..68b47aa5453b 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsComputerTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsComputerTest.java @@ -37,6 +37,7 @@ import org.neo4j.graphdb.mockfs.UncloseableDelegatingFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.CountsComputer; import org.neo4j.kernel.impl.store.MetaDataStore; @@ -316,7 +317,7 @@ private void cleanupCountsForRebuilding() private CountsTracker createCountsTracker() { return new CountsTracker( LOG_PROVIDER, fs, pageCache, - CONFIG, new File( dir, COUNTS_STORE_BASE ) ); + CONFIG, new File( dir, COUNTS_STORE_BASE ), EmptyVersionContextSupplier.INSTANCE ); } private void rebuildCounts( long lastCommittedTransactionId ) throws IOException @@ -324,7 +325,7 @@ private void rebuildCounts( long lastCommittedTransactionId ) throws IOException cleanupCountsForRebuilding(); IdGeneratorFactory idGenFactory = new DefaultIdGeneratorFactory( fs ); - StoreFactory storeFactory = new StoreFactory( dir, CONFIG, idGenFactory, pageCache, fs, LOG_PROVIDER ); + StoreFactory storeFactory = new StoreFactory( dir, CONFIG, idGenFactory, pageCache, fs, LOG_PROVIDER, EmptyVersionContextSupplier.INSTANCE ); try ( Lifespan life = new Lifespan(); NeoStores neoStores = storeFactory.openAllNeoStores() ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsRotationTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsRotationTest.java index 4401fc09156f..8080fc775274 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsRotationTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsRotationTest.java @@ -50,6 +50,7 @@ import org.neo4j.helpers.collection.Pair; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.CountsAccessor; import org.neo4j.kernel.impl.api.CountsVisitor; @@ -388,7 +389,7 @@ private CountsTracker createCountsTracker( PageCache pageCache ) private CountsTracker createCountsTracker( PageCache pageCache, Config config ) { return new CountsTracker( NullLogProvider.getInstance(), fs, pageCache, config, - new File( dir.getPath(), COUNTS_STORE_BASE ) ); + new File( dir.getPath(), COUNTS_STORE_BASE ), EmptyVersionContextSupplier.INSTANCE ); } private void checkPoint( GraphDatabaseAPI db ) throws IOException diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsTrackerTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsTrackerTest.java index cbbc71a1a91c..a390f07533bd 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsTrackerTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsTrackerTest.java @@ -30,9 +30,13 @@ import org.neo4j.function.IOFunction; import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.CountsAccessor; import org.neo4j.kernel.impl.api.CountsVisitor; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; import org.neo4j.kernel.impl.store.CountsOracle; import org.neo4j.kernel.impl.store.counts.keys.CountsKey; import org.neo4j.kernel.impl.store.counts.keys.CountsKeyFactory; @@ -52,6 +56,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; @@ -190,6 +195,54 @@ public void shouldUpdateCountsOnExistingStore() throws Exception } } + @Test + public void detectInMemoryDirtyVersionRead() + { + int labelId = 1; + long lastClosedTransactionId = 11L; + long writeTransactionId = 22L; + TransactionVersionContextSupplier versionContextSupplier = new TransactionVersionContextSupplier(); + versionContextSupplier.init( () -> lastClosedTransactionId ); + VersionContext versionContext = versionContextSupplier.getVersionContext(); + + try ( Lifespan life = new Lifespan() ) + { + CountsTracker tracker = life.add( newTracker( versionContextSupplier ) ); + try ( CountsAccessor.Updater updater = tracker.apply( writeTransactionId ).get() ) + { + updater.incrementNodeCount( labelId, 1 ); + } + + versionContext.initRead(); + tracker.nodeCount( labelId, Registers.newDoubleLongRegister() ); + assertTrue( versionContext.isDirty() ); + } + } + + @Test + public void allowNonDirtyInMemoryDirtyVersionRead() + { + int labelId = 1; + long lastClosedTransactionId = 15L; + long writeTransactionId = 13L; + TransactionVersionContextSupplier versionContextSupplier = new TransactionVersionContextSupplier(); + versionContextSupplier.init( () -> lastClosedTransactionId ); + VersionContext versionContext = versionContextSupplier.getVersionContext(); + + try ( Lifespan life = new Lifespan() ) + { + CountsTracker tracker = life.add( newTracker( versionContextSupplier ) ); + try ( CountsAccessor.Updater updater = tracker.apply( writeTransactionId ).get() ) + { + updater.incrementNodeCount( labelId, 1 ); + } + + versionContext.initRead(); + tracker.nodeCount( labelId, Registers.newDoubleLongRegister() ); + assertFalse( versionContext.isDirty() ); + } + } + @Test public void shouldBeAbleToReadUpToDateValueWhileAnotherThreadIsPerformingRotation() throws Exception { @@ -219,7 +272,7 @@ public void shouldBeAbleToReadUpToDateValueWhileAnotherThreadIsPerformingRotatio final Barrier.Control barrier = new Barrier.Control(); CountsTracker tracker = life.add( new CountsTracker( resourceManager.logProvider(), resourceManager.fileSystem(), resourceManager.pageCache(), - Config.defaults(), resourceManager.testPath() ) + Config.defaults(), resourceManager.testPath(), EmptyVersionContextSupplier.INSTANCE ) { @Override protected boolean include( CountsKey countsKey, ReadableBuffer value ) @@ -360,7 +413,7 @@ public void shouldNotEndUpInBrokenStateAfterRotationFailure() throws Exception // GIVEN FakeClock clock = Clocks.fakeClock(); CallTrackingClock callTrackingClock = new CallTrackingClock( clock ); - CountsTracker tracker = resourceManager.managed( newTracker( callTrackingClock ) ); + CountsTracker tracker = resourceManager.managed( newTracker( callTrackingClock, EmptyVersionContextSupplier.INSTANCE ) ); int labelId = 1; try ( CountsAccessor.Updater tx = tracker.apply( 2 ).get() ) { @@ -410,13 +463,19 @@ public void shouldNotEndUpInBrokenStateAfterRotationFailure() throws Exception private CountsTracker newTracker() { - return newTracker( Clocks.nanoClock() ); + return newTracker( Clocks.nanoClock(), EmptyVersionContextSupplier.INSTANCE ); + } + + private CountsTracker newTracker( VersionContextSupplier versionContextSupplier ) + { + return newTracker( Clocks.nanoClock(), versionContextSupplier ); } - private CountsTracker newTracker( SystemNanoClock clock ) + private CountsTracker newTracker( SystemNanoClock clock, VersionContextSupplier versionContextSupplier ) { return new CountsTracker( resourceManager.logProvider(), resourceManager.fileSystem(), - resourceManager.pageCache(), Config.defaults(), resourceManager.testPath(), clock ) + resourceManager.pageCache(), Config.defaults(), resourceManager.testPath(), clock, + versionContextSupplier ) .setInitializer( new DataInitializer() { @Override diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStoreTest.java index 6628479908aa..25b069e8f35e 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStoreTest.java @@ -37,6 +37,7 @@ import org.neo4j.function.ThrowingConsumer; import org.neo4j.helpers.collection.Pair; import org.neo4j.io.fs.StoreChannel; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; import org.neo4j.kernel.lifecycle.Lifespan; import org.neo4j.test.rule.Resources; @@ -93,7 +94,7 @@ public String toString() public void retryLookupOnConcurrentStoreStateChange() throws IOException { Store testStore = resourceManager.managed( createTestStore( TimeUnit.DAYS.toMillis( 2 ) ) ); - ConcurrentMapState newState = new ConcurrentMapState<>( testStore.state, mock( File.class ) ); + ConcurrentMapState newState = new ConcurrentMapState<>( testStore.state, mock( File.class ), EmptyVersionContextSupplier.INSTANCE ); testStore.put( "test", "value" ); CountingErroneousReader countingErroneousReader = new CountingErroneousReader( testStore, newState ); @@ -595,7 +596,8 @@ private Store( HeaderField... headerFields ) private Store( long rotationTimeout, HeaderField... headerFields ) { super( resourceManager.fileSystem(), resourceManager.pageCache(), resourceManager.testPath(), null, - new RotationTimerFactory( Clocks.nanoClock(), rotationTimeout ), 16, 16, headerFields ); + new RotationTimerFactory( Clocks.nanoClock(), rotationTimeout ), + EmptyVersionContextSupplier.INSTANCE, 16, 16, headerFields ); this.headerFields = headerFields; setEntryUpdaterInitializer( new DataInitializer>() { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapStateTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapStateTest.java index 964a3d33e743..4b603fefa19d 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapStateTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapStateTest.java @@ -19,30 +19,48 @@ */ package org.neo4j.kernel.impl.store.kvstore; +import org.junit.Before; import org.junit.Test; import java.io.File; +import java.io.IOException; import java.util.concurrent.locks.Lock; +import org.neo4j.concurrent.Runnables; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; + import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class ConcurrentMapStateTest { - private final ReadableState store = mock( ReadableState.class ); + private final ReadableState store = mock( ReadableState.class ); private final File file = mock( File.class ); private final Lock lock = mock( Lock.class ); + @Before + public void setUp() throws Exception + { + KeyFormat keyFormat = mock( KeyFormat.class ); + when( keyFormat.valueSize() ).thenReturn( Long.BYTES ); + when( store.keyFormat() ).thenReturn( keyFormat ); + } + @Test public void shouldCreateAnUpdaterForTheNextUnseenVersionUpdate() throws Exception { // given long initialVersion = 42; when( store.version() ).thenReturn( initialVersion ); - ConcurrentMapState state = new ConcurrentMapState<>( store, file ); + ConcurrentMapState state = createMapState(); // when long updateVersion = 43; @@ -60,7 +78,7 @@ public void shouldCreateAnUpdaterForAnUnseenVersionUpdateWithAGap() throws Excep // given long initialVersion = 42; when( store.version() ).thenReturn( initialVersion ); - ConcurrentMapState state = new ConcurrentMapState<>( store, file ); + ConcurrentMapState state = createMapState(); // when long updateVersion = 45; @@ -79,7 +97,7 @@ public void shouldCreateAnUpdaterForMultipleVersionUpdatesInOrder() throws Excep // given long initialVersion = 42; when( store.version() ).thenReturn( initialVersion ); - ConcurrentMapState state = new ConcurrentMapState<>( store, file ); + ConcurrentMapState state = createMapState(); // when EntryUpdater updater; @@ -108,7 +126,7 @@ public void shouldCreateAnUpdaterForMultipleVersionUpdatesNotInOrder() throws Ex // given long initialVersion = 42; when( store.version() ).thenReturn( initialVersion ); - ConcurrentMapState state = new ConcurrentMapState<>( store, file ); + ConcurrentMapState state = createMapState(); // when EntryUpdater updater; @@ -137,7 +155,7 @@ public void shouldUseEmptyUpdaterOnVersionLowerOrEqualToTheInitialVersion() thro // given long initialVersion = 42; when( store.version() ).thenReturn( initialVersion ); - ConcurrentMapState state = new ConcurrentMapState<>( store, file ); + ConcurrentMapState state = createMapState(); // when EntryUpdater updater = state.updater( initialVersion, lock ); @@ -146,4 +164,98 @@ public void shouldUseEmptyUpdaterOnVersionLowerOrEqualToTheInitialVersion() thro assertEquals( "Empty updater should be used for version less or equal to initial", EntryUpdater.noUpdates(), updater ); } + + @Test + public void markDirtyVersionLookupOnKeyUpdate() throws IOException + { + long updaterVersionTxId = 25; + long lastClosedTxId = 20; + TransactionVersionContextSupplier versionContextSupplier = new TransactionVersionContextSupplier(); + versionContextSupplier.init( () -> lastClosedTxId ); + ConcurrentMapState mapState = createMapState( versionContextSupplier ); + VersionContext versionContext = versionContextSupplier.getVersionContext(); + try ( EntryUpdater updater = mapState.updater( updaterVersionTxId, lock ) ) + { + updater.apply( "a", new SimpleValueUpdate( 1 ) ); + updater.apply( "b", new SimpleValueUpdate( 2 ) ); + } + + assertEquals( updaterVersionTxId, mapState.version() ); + versionContext.initRead(); + mapState.lookup( "a", new EmptyValueSink() ); + assertTrue( versionContext.isDirty() ); + } + + @Test + public void markDirtyVersionLookupOnKeyReset() throws IOException + { + long updaterVersionTxId = 25; + long lastClosedTxId = 20; + when( store.version() ).thenReturn( updaterVersionTxId ); + TransactionVersionContextSupplier versionContextSupplier = new TransactionVersionContextSupplier(); + versionContextSupplier.init( () -> lastClosedTxId ); + VersionContext versionContext = versionContextSupplier.getVersionContext(); + + ConcurrentMapState mapState = createMapState( versionContextSupplier ); + + versionContext.initRead(); + mapState.resettingUpdater( lock, Runnables.EMPTY_RUNNABLE ).apply( "a", new SimpleValueUpdate( 1 ) ); + mapState.lookup( "a", new EmptyValueSink() ); + assertTrue( versionContext.isDirty() ); + } + + @Test + public void doNotMarkVersionAsDirtyOnAnotherKeyUpdate() throws IOException + { + long updaterVersionTxId = 25; + long lastClosedTxId = 20; + TransactionVersionContextSupplier versionContextSupplier = new TransactionVersionContextSupplier(); + versionContextSupplier.init( () -> lastClosedTxId ); + ConcurrentMapState mapState = createMapState( versionContextSupplier ); + VersionContext versionContext = versionContextSupplier.getVersionContext(); + try ( EntryUpdater updater = mapState.updater( updaterVersionTxId, lock ) ) + { + updater.apply( "b", new SimpleValueUpdate( 2 ) ); + } + + assertEquals( updaterVersionTxId, mapState.version() ); + versionContext.initRead(); + mapState.lookup( "a", new EmptyValueSink() ); + assertFalse( versionContext.isDirty() ); + } + + private ConcurrentMapState createMapState() + { + return createMapState( EmptyVersionContextSupplier.INSTANCE ); + } + + private ConcurrentMapState createMapState( VersionContextSupplier versionContextSupplier ) + { + return new ConcurrentMapState<>( store, file, versionContextSupplier ); + } + + private static class SimpleValueUpdate implements ValueUpdate + { + private final long value; + + SimpleValueUpdate( long value ) + { + this.value = value; + } + + @Override + public void update( WritableBuffer target ) + { + target.putLong( 0, value ); + } + } + + private static class EmptyValueSink extends ValueSink + { + @Override + protected void value( ReadableBuffer value ) + { + + } + } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java index d71f04ef44fd..ea2fe9310e1b 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java @@ -33,6 +33,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.logging.NullLogService; @@ -138,7 +139,7 @@ public void shouldBeAbleToResumeMigrationOnMoving() throws Exception // THEN starting the new store should be successful StoreFactory storeFactory = new StoreFactory( storeDirectory, CONFIG, new DefaultIdGeneratorFactory( fs ), pageCache, fs, - logService.getInternalLogProvider() ); + logService.getInternalLogProvider(), EmptyVersionContextSupplier.INSTANCE ); storeFactory.openAllNeoStores().close(); } @@ -178,7 +179,7 @@ public void shouldBeAbleToMigrateWithoutErrors() throws Exception // THEN starting the new store should be successful StoreFactory storeFactory = new StoreFactory( storeDirectory, CONFIG, new DefaultIdGeneratorFactory( fs ), pageCache, fs, - logService.getInternalLogProvider() ); + logService.getInternalLogProvider(), EmptyVersionContextSupplier.INSTANCE ); storeFactory.openAllNeoStores().close(); logProvider.assertNoLogCallContaining( "ERROR" ); } @@ -218,7 +219,7 @@ public void shouldBeAbleToResumeMigrationOnRebuildingCounts() throws Exception // THEN starting the new store should be successful StoreFactory storeFactory = new StoreFactory( storeDirectory, CONFIG, new DefaultIdGeneratorFactory( fs ), pageCache, fs, - logService.getInternalLogProvider() ); + logService.getInternalLogProvider(), EmptyVersionContextSupplier.INSTANCE ); storeFactory.openAllNeoStores().close(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/ApplyRecoveredTransactionsTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/ApplyRecoveredTransactionsTest.java index f9e5ee07bec5..b6fffa1000f2 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/ApplyRecoveredTransactionsTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/ApplyRecoveredTransactionsTest.java @@ -28,6 +28,7 @@ import java.util.Arrays; import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.TransactionToApply; import org.neo4j.kernel.impl.core.CacheAccessBackDoor; @@ -114,7 +115,7 @@ public void before() FileSystemAbstraction fs = fsr.get(); File storeDir = new File( "dir" ); StoreFactory storeFactory = new StoreFactory( storeDir, Config.defaults(), new DefaultIdGeneratorFactory( fs ), - pageCacheRule.getPageCache( fs ), fs, NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs ), fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); neoStores = storeFactory.openAllNeoStores( true ); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeCommandTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeCommandTest.java index 1c0aa7ba0dd7..8f3c4d5598d6 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeCommandTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeCommandTest.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Set; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeLabels; @@ -245,7 +246,8 @@ public void before() throws Exception fs.get().mkdirs( dir ); @SuppressWarnings( "deprecation" ) StoreFactory storeFactory = new StoreFactory( dir, Config.defaults(), new DefaultIdGeneratorFactory( fs.get() ), - pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance(), + EmptyVersionContextSupplier.INSTANCE ); neoStores = storeFactory.openAllNeoStores( true ); nodeStore = neoStores.getNodeStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeLabelsFieldTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeLabelsFieldTest.java index cadbcf26d418..83e61156ac75 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeLabelsFieldTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeLabelsFieldTest.java @@ -38,6 +38,7 @@ import org.neo4j.helpers.CloneableInPublic; import org.neo4j.helpers.collection.Iterables; import org.neo4j.helpers.collection.Pair; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.DynamicNodeLabels; import org.neo4j.kernel.impl.store.NeoStores; @@ -518,7 +519,7 @@ public void startUp() fs.get().mkdirs( storeDir ); Config config = Config.defaults( GraphDatabaseSettings.label_block_size, "60" ); StoreFactory storeFactory = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), - pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); neoStores = storeFactory.openAllNeoStores( true ); nodeStore = neoStores.getNodeStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetterTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetterTest.java index b93e3afcdb68..91955acd7070 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetterTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetterTest.java @@ -26,6 +26,7 @@ import java.io.File; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.RecordStore; @@ -66,7 +67,7 @@ public void shouldAbortLoadingGroupChainIfComeTooFar() throws Exception LogProvider logProvider = NullLogProvider.getInstance(); StoreFactory storeFactory = new StoreFactory( dir, Config.defaults(), new DefaultIdGeneratorFactory( fs.get() ), pageCache.getPageCache( fs.get() ), fs.get(), - logProvider ); + logProvider, EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores stores = storeFactory.openNeoStores( true, StoreType.RELATIONSHIP_GROUP ) ) { RecordStore store = spy( stores.getRelationshipGroupStore() ); diff --git a/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java b/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java index b558bbd8fd74..8381409b178d 100644 --- a/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java +++ b/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java @@ -20,7 +20,6 @@ package org.neo4j.test; import java.io.File; -import java.util.Map; import org.neo4j.adversaries.Adversary; import org.neo4j.adversaries.pagecache.AdversarialPageCache; @@ -28,6 +27,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseFactory; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.factory.CommunityEditionModule; import org.neo4j.kernel.impl.factory.DatabaseInfo; @@ -71,9 +71,10 @@ protected FileSystemAbstraction createFileSystemAbstraction() @Override protected PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, - LogService logging, Tracers tracers ) + LogService logging, Tracers tracers, VersionContextSupplier versionContextSupplier ) { - PageCache pageCache = super.createPageCache( fileSystem, config, logging, tracers ); + PageCache pageCache = super.createPageCache( fileSystem, config, logging, tracers, + versionContextSupplier ); return new AdversarialPageCache( pageCache, adversary ); } }; diff --git a/community/kernel/src/test/java/org/neo4j/test/rule/ConfigurablePageCacheRule.java b/community/kernel/src/test/java/org/neo4j/test/rule/ConfigurablePageCacheRule.java index 2e4da0175e5c..9d50e238184f 100644 --- a/community/kernel/src/test/java/org/neo4j/test/rule/ConfigurablePageCacheRule.java +++ b/community/kernel/src/test/java/org/neo4j/test/rule/ConfigurablePageCacheRule.java @@ -26,6 +26,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory; import org.neo4j.logging.FormattedLogProvider; @@ -53,7 +54,8 @@ private PageCache createPageCache( FileSystemAbstraction fs, PageCacheConfig pag config.augmentDefaults( GraphDatabaseSettings.pagecache_memory, "8M" ); FormattedLogProvider logProvider = FormattedLogProvider.toOutputStream( System.err ); ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( - fs, config, tracer, cursorTracerSupplier, logProvider.getLog( PageCache.class ) ) + fs, config, tracer, cursorTracerSupplier, logProvider.getLog( PageCache.class ), + EmptyVersionContextSupplier.INSTANCE ) { @Override public int calculatePageSize( Config config, PageSwapperFactory swapperFactory ) diff --git a/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoreDataSourceRule.java b/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoreDataSourceRule.java index 07fee022deec..3c18209e63b7 100644 --- a/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoreDataSourceRule.java +++ b/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoreDataSourceRule.java @@ -36,6 +36,7 @@ import org.neo4j.kernel.impl.api.explicitindex.InternalAutoIndexing; import org.neo4j.kernel.impl.api.index.IndexingService; import org.neo4j.kernel.impl.constraints.StandardConstraintSemantics; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; import org.neo4j.kernel.impl.core.DatabasePanicEventGenerator; import org.neo4j.kernel.impl.core.LabelTokenHolder; import org.neo4j.kernel.impl.core.PropertyKeyTokenHolder; @@ -139,7 +140,7 @@ logService, mock( JobScheduler.class, RETURNS_MOCKS ), mock( TokenNameLookup.cla new BufferedIdController( new BufferingIdGeneratorFactory( idGeneratorFactory, IdReuseEligibility.ALWAYS, idConfigurationProvider ), jobScheduler ), - OperationalMode.single ); + OperationalMode.single, new TransactionVersionContextSupplier() ); return dataSource; } diff --git a/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoresRule.java b/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoresRule.java index a49e5ec8ea5c..ae1c71f16399 100644 --- a/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoresRule.java +++ b/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoresRule.java @@ -28,6 +28,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory; import org.neo4j.kernel.impl.store.NeoStores; @@ -80,7 +81,7 @@ private NeoStores open( FileSystemAbstraction fs, PageCache pageCache, RecordFor File storeDir = testDirectory.makeGraphDbDir(); Config configuration = configOf( config ); StoreFactory storeFactory = new StoreFactory( storeDir, configuration, idGeneratorFactory.apply( fs ), - pageCache, fs, format, NullLogProvider.getInstance() ); + pageCache, fs, format, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); return neoStores = stores.length == 0 ? storeFactory.openAllNeoStores( true ) : storeFactory.openNeoStores( true, stores ); @@ -105,7 +106,7 @@ private static PageCache getOrCreatePageCache( Config config, FileSystemAbstract { Log log = NullLog.getInstance(); ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( fs, config, NULL, - PageCursorTracerSupplier.NULL, log ); + PageCursorTracerSupplier.NULL, log, EmptyVersionContextSupplier.INSTANCE ); return pageCacheFactory.getOrCreatePageCache(); } diff --git a/community/kernel/src/test/java/org/neo4j/test/rule/RecordStorageEngineRule.java b/community/kernel/src/test/java/org/neo4j/test/rule/RecordStorageEngineRule.java index de5cd4e12558..7e12ac45b947 100644 --- a/community/kernel/src/test/java/org/neo4j/test/rule/RecordStorageEngineRule.java +++ b/community/kernel/src/test/java/org/neo4j/test/rule/RecordStorageEngineRule.java @@ -26,6 +26,7 @@ import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.TokenNameLookup; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.configuration.Config; @@ -209,7 +210,7 @@ private class ExtendedRecordStorageEngine extends RecordStorageEngine relationshipTypeTokens, schemaState, constraintSemantics, scheduler, tokenNameLookup, lockService, new DefaultSchemaIndexProviderMap( indexProvider ), indexingServiceMonitor, databaseHealth, explicitIndexProviderLookup, indexConfigStore, explicitIndexTransactionOrdering, idGeneratorFactory, - idController, monitors, recoveryCleanupWorkCollector, operationalMode ); + idController, monitors, recoveryCleanupWorkCollector, operationalMode, EmptyVersionContextSupplier.INSTANCE ); this.transactionApplierTransformer = transactionApplierTransformer; } diff --git a/community/neo4j/src/test/java/db/DatabaseShutdownTest.java b/community/neo4j/src/test/java/db/DatabaseShutdownTest.java index 1711c73b0073..039bc8714f34 100644 --- a/community/neo4j/src/test/java/db/DatabaseShutdownTest.java +++ b/community/neo4j/src/test/java/db/DatabaseShutdownTest.java @@ -32,6 +32,7 @@ import org.neo4j.io.pagecache.DelegatingPageCache; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.factory.CommunityEditionModule; @@ -107,10 +108,11 @@ protected PlatformModule createPlatform( File storeDir, Config config, return new PlatformModule( storeDir, config, databaseInfo, dependencies, graphDatabaseFacade ) { @Override - protected PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, - LogService logging, Tracers tracers ) + protected PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, LogService logging, + Tracers tracers, VersionContextSupplier versionContextSupplier ) { - PageCache pageCache = super.createPageCache( fileSystem, config, logging, tracers ); + PageCache pageCache = super.createPageCache( fileSystem, config, logging, tracers, + versionContextSupplier ); return new DelegatingPageCache( pageCache ) { @Override diff --git a/community/neo4j/src/test/java/db/QueryRestartIT.java b/community/neo4j/src/test/java/db/QueryRestartIT.java new file mode 100644 index 000000000000..07946f286bbe --- /dev/null +++ b/community/neo4j/src/test/java/db/QueryRestartIT.java @@ -0,0 +1,288 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package db; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.util.Map; +import java.util.function.LongSupplier; + +import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.QueryExecutionException; +import org.neo4j.graphdb.Result; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.factory.GraphDatabaseBuilder; +import org.neo4j.graphdb.factory.GraphDatabaseFactoryState; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; +import org.neo4j.kernel.GraphDatabaseDependencies; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.configuration.Settings; +import org.neo4j.kernel.impl.context.TransactionVersionContext; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; +import org.neo4j.kernel.impl.factory.CommunityEditionModule; +import org.neo4j.kernel.impl.factory.DatabaseInfo; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; +import org.neo4j.kernel.impl.factory.PlatformModule; +import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; +import org.neo4j.kernel.internal.GraphDatabaseAPI; +import org.neo4j.test.TestGraphDatabaseFactory; +import org.neo4j.test.rule.TestDirectory; + +import static org.junit.Assert.assertEquals; + +public class QueryRestartIT +{ + @Rule + public final TestDirectory testDirectory = TestDirectory.testDirectory(); + private GraphDatabaseService database; + private TestTransactionVersionContextSupplier testContextSupplier; + private File storeDir; + private TestVersionContext testCursorContext; + + @Before + public void setUp() + { + storeDir = testDirectory.directory(); + testContextSupplier = new TestTransactionVersionContextSupplier(); + database = startSnapshotQueryDb(); + createData(); + + testCursorContext = testCursorContext(); + testContextSupplier.setCursorContext( testCursorContext ); + } + + @After + public void tearDown() + { + if ( database != null ) + { + database.shutdown(); + } + } + + @Test + public void executeQueryWithoutRestarts() + { + testCursorContext.setWrongLastClosedTxId( false ); + + Result result = database.execute( "MATCH (n:label) RETURN n.c" ); + while ( result.hasNext() ) + { + assertEquals( "d", result.next().get( "n.c" ) ); + } + assertEquals( 0, testCursorContext.getAdditionalAttempts() ); + } + + @Test + public void executeQueryWithSingleRetry() + { + Result result = database.execute( "MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + while ( result.hasNext() ) + { + assertEquals( "d", result.next().get( "n.c" ) ); + } + } + + @Test + public void executeCountStoreQueryWithSingleRetry() + { + Result result = database.execute( "MATCH (n:toRetry) RETURN count(n)" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + while ( result.hasNext() ) + { + assertEquals( 1L, result.next().get( "count(n)" ) ); + } + } + + @Test + public void executeLabelScanQueryWithSingleRetry() + { + Result result = database.execute( "MATCH (n:toRetry) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + while ( result.hasNext() ) + { + assertEquals( "d", result.next().get( "n.c" ) ); + } + } + + @Test + public void queryThatModifyDataAndSeeUnstableSnapshotThrowException() + { + try + { + database.execute( "MATCH (n:toRetry) CREATE () RETURN n.c" ); + } + catch ( QueryExecutionException e ) + { + assertEquals( "Unable to get clean data snapshot for query " + + "'MATCH (n:toRetry) CREATE () RETURN n.c' that perform updates.", e.getMessage() ); + } + } + + private GraphDatabaseService startSnapshotQueryDb() + { + return new CustomGraphDatabaseFactory( new CustomFacadeFactory() ) + .newEmbeddedDatabaseBuilder( storeDir ) + .setConfig( GraphDatabaseSettings.snapshot_query, Settings.TRUE ) + .newGraphDatabase(); + } + + private void createData() + { + Label label = Label.label( "toRetry" ); + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode( label ); + node.setProperty( "c", "d" ); + transaction.success(); + } + } + + private TestVersionContext testCursorContext() + { + TransactionIdStore transactionIdStore = getTransactionIdStore(); + return new TestVersionContext( transactionIdStore::getLastClosedTransactionId ); + } + + private TransactionIdStore getTransactionIdStore() + { + DependencyResolver dependencyResolver = ((GraphDatabaseAPI) database).getDependencyResolver(); + return dependencyResolver.resolveDependency( TransactionIdStore.class ); + } + + private class CustomGraphDatabaseFactory extends TestGraphDatabaseFactory + { + private GraphDatabaseFacadeFactory customFacadeFactory; + + CustomGraphDatabaseFactory( GraphDatabaseFacadeFactory customFacadeFactory ) + { + this.customFacadeFactory = customFacadeFactory; + } + + @Override + protected GraphDatabaseBuilder.DatabaseCreator createDatabaseCreator( File storeDir, + GraphDatabaseFactoryState state ) + { + return new GraphDatabaseBuilder.DatabaseCreator() + { + @Override + public GraphDatabaseService newDatabase( Map config ) + { + return newDatabase( Config.defaults( config ) ); + } + + @Override + public GraphDatabaseService newDatabase( Config config ) + { + return customFacadeFactory.newFacade( storeDir, config, + GraphDatabaseDependencies.newDependencies( state.databaseDependencies() ) ); + } + }; + } + } + + private class CustomFacadeFactory extends GraphDatabaseFacadeFactory + { + + CustomFacadeFactory() + { + super( DatabaseInfo.COMMUNITY, CommunityEditionModule::new ); + } + + @Override + protected PlatformModule createPlatform( File storeDir, Config config, Dependencies dependencies, + GraphDatabaseFacade graphDatabaseFacade ) + { + return new PlatformModule( storeDir, config, databaseInfo, dependencies, graphDatabaseFacade ) + { + @Override + protected VersionContextSupplier createCursorContextSupplier( Config config ) + { + return testContextSupplier != null ? testContextSupplier : super.createCursorContextSupplier(config); + } + }; + } + } + + private class TestVersionContext extends TransactionVersionContext + { + + private boolean wrongLastClosedTxId = true; + private int additionalAttempts; + + TestVersionContext( LongSupplier transactionIdSupplier ) + { + super( transactionIdSupplier ); + } + + @Override + public long lastClosedTransactionId() + { + return wrongLastClosedTxId ? TransactionIdStore.BASE_TX_ID : super.lastClosedTransactionId(); + } + + @Override + public void markAsDirty() + { + super.markAsDirty(); + wrongLastClosedTxId = false; + } + + void setWrongLastClosedTxId( boolean wrongLastClosedTxId ) + { + this.wrongLastClosedTxId = wrongLastClosedTxId; + } + + @Override + public boolean isDirty() + { + boolean dirty = super.isDirty(); + if ( dirty ) + { + additionalAttempts++; + } + return dirty; + } + + int getAdditionalAttempts() + { + return additionalAttempts; + } + } + + private class TestTransactionVersionContextSupplier extends TransactionVersionContextSupplier + { + void setCursorContext( VersionContext versionContext ) + { + this.cursorContext.set( versionContext ); + } + } +} diff --git a/community/neo4j/src/test/java/org/neo4j/kernel/impl/storemigration/StoreUpgraderTest.java b/community/neo4j/src/test/java/org/neo4j/kernel/impl/storemigration/StoreUpgraderTest.java index c6f73bfa3b85..eb1237bccea2 100644 --- a/community/neo4j/src/test/java/org/neo4j/kernel/impl/storemigration/StoreUpgraderTest.java +++ b/community/neo4j/src/test/java/org/neo4j/kernel/impl/storemigration/StoreUpgraderTest.java @@ -38,6 +38,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.index.inmemory.InMemoryIndexProvider; @@ -255,7 +256,7 @@ public void upgradedNeoStoreShouldHaveNewUpgradeTimeAndUpgradeId() throws Except // Then StoreFactory factory = new StoreFactory( dbDirectory, allowMigrateConfig, new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = factory.openAllNeoStores() ) { assertThat( neoStores.getMetaDataStore().getUpgradeTransaction(), diff --git a/community/server/src/test/java/org/neo4j/server/rest/transactional/ExecutionResultSerializerTest.java b/community/server/src/test/java/org/neo4j/server/rest/transactional/ExecutionResultSerializerTest.java index 258394a6d565..ff3ac1248b33 100644 --- a/community/server/src/test/java/org/neo4j/server/rest/transactional/ExecutionResultSerializerTest.java +++ b/community/server/src/test/java/org/neo4j/server/rest/transactional/ExecutionResultSerializerTest.java @@ -22,8 +22,6 @@ import org.codehaus.jackson.JsonNode; import org.junit.Test; import org.mockito.internal.stubbing.answers.ThrowsException; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -42,6 +40,7 @@ import java.util.TreeMap; import java.util.TreeSet; +import org.neo4j.cypher.internal.javacompat.MapRow; import org.neo4j.graphdb.ExecutionPlanDescription; import org.neo4j.graphdb.InputPosition; import org.neo4j.graphdb.Node; diff --git a/community/server/src/test/java/org/neo4j/server/rest/transactional/GraphExtractionWriterTest.java b/community/server/src/test/java/org/neo4j/server/rest/transactional/GraphExtractionWriterTest.java index e31e9d5d19ba..733c237fcb00 100644 --- a/community/server/src/test/java/org/neo4j/server/rest/transactional/GraphExtractionWriterTest.java +++ b/community/server/src/test/java/org/neo4j/server/rest/transactional/GraphExtractionWriterTest.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Map; +import org.neo4j.cypher.internal.javacompat.MapRow; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Relationship; import org.neo4j.kernel.api.Statement; diff --git a/community/server/src/test/java/org/neo4j/server/rest/transactional/RestRepresentationWriterTest.java b/community/server/src/test/java/org/neo4j/server/rest/transactional/RestRepresentationWriterTest.java index 73417f42cf8c..42ea0052490c 100644 --- a/community/server/src/test/java/org/neo4j/server/rest/transactional/RestRepresentationWriterTest.java +++ b/community/server/src/test/java/org/neo4j/server/rest/transactional/RestRepresentationWriterTest.java @@ -19,15 +19,17 @@ */ package org.neo4j.server.rest.transactional; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.net.URI; - import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.JsonNode; import org.hamcrest.MatcherAssert; import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; + +import org.neo4j.cypher.internal.javacompat.MapRow; import org.neo4j.server.rest.domain.JsonParseException; import static java.util.Arrays.asList; diff --git a/community/server/src/test/java/org/neo4j/server/rest/transactional/RowWriterTest.java b/community/server/src/test/java/org/neo4j/server/rest/transactional/RowWriterTest.java index 09e98b8ceaba..201eaecae48d 100644 --- a/community/server/src/test/java/org/neo4j/server/rest/transactional/RowWriterTest.java +++ b/community/server/src/test/java/org/neo4j/server/rest/transactional/RowWriterTest.java @@ -28,6 +28,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import org.neo4j.cypher.internal.javacompat.MapRow; import org.neo4j.server.rest.domain.JsonParseException; import static java.util.Arrays.asList; diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplier.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplier.java index 732a8a5d0f94..ebcc031e0440 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplier.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplier.java @@ -22,6 +22,7 @@ import java.util.function.Supplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionQueue; import org.neo4j.kernel.impl.api.TransactionToApply; @@ -46,6 +47,7 @@ public class BatchingTxApplier extends LifecycleAdapter private final PullRequestMonitor monitor; private final PageCursorTracerSupplier pageCursorTracerSupplier; + private final VersionContextSupplier versionContextSupplier; private final Log log; private TransactionQueue txQueue; @@ -56,7 +58,8 @@ public class BatchingTxApplier extends LifecycleAdapter public BatchingTxApplier( int maxBatchSize, Supplier txIdStoreSupplier, Supplier commitProcessSupplier, Monitors monitors, - PageCursorTracerSupplier pageCursorTracerSupplier, LogProvider logProvider ) + PageCursorTracerSupplier pageCursorTracerSupplier, + VersionContextSupplier versionContextSupplier, LogProvider logProvider ) { this.maxBatchSize = maxBatchSize; this.txIdStoreSupplier = txIdStoreSupplier; @@ -64,6 +67,7 @@ public BatchingTxApplier( int maxBatchSize, Supplier txIdSto this.pageCursorTracerSupplier = pageCursorTracerSupplier; this.log = logProvider.getLog( getClass() ); this.monitor = monitors.newMonitor( PullRequestMonitor.class ); + this.versionContextSupplier = versionContextSupplier; } @Override @@ -107,7 +111,7 @@ public void queue( CommittedTransactionRepresentation tx ) throws Exception return; } - txQueue.queue( new TransactionToApply( tx.getTransactionRepresentation(), receivedTxId ) ); + txQueue.queue( new TransactionToApply( tx.getTransactionRepresentation(), receivedTxId, versionContextSupplier.getVersionContext() ) ); if ( !stopped ) { diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionApplier.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionApplier.java index 81a2bf55e185..bc27306944d8 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionApplier.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionApplier.java @@ -24,6 +24,7 @@ import org.neo4j.com.TransactionStreamResponse; import org.neo4j.com.storecopy.TransactionObligationFulfiller; import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionRepresentationCommitProcess; @@ -47,17 +48,19 @@ public class TransactionApplier { private final TransactionRepresentationCommitProcess commitProcess; + private final VersionContextSupplier versionContextSupplier; public TransactionApplier( DependencyResolver resolver ) { commitProcess = new TransactionRepresentationCommitProcess( resolver.resolveDependency( TransactionAppender.class ), resolver.resolveDependency( StorageEngine.class ) ); + versionContextSupplier = resolver.resolveDependency( VersionContextSupplier.class ); } public void appendToLogAndApplyToStore( CommittedTransactionRepresentation tx ) throws TransactionFailureException { commitProcess.commit( new TransactionToApply( tx.getTransactionRepresentation(), - tx.getCommitEntry().getTxId() ), NULL, EXTERNAL ); + tx.getCommitEntry().getTxId(), versionContextSupplier.getVersionContext() ), NULL, EXTERNAL ); } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java index 5b68f1102e71..85b851ba8ac1 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java @@ -35,6 +35,7 @@ import org.neo4j.causalclustering.identity.MemberId; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.NeoStores; @@ -109,7 +110,7 @@ public class CoreBootstrapper public CoreSnapshot bootstrap( Set members ) throws IOException { StoreFactory factory = new StoreFactory( storeDir, config, - new DefaultIdGeneratorFactory( fs ), pageCache, fs, logProvider ); + new DefaultIdGeneratorFactory( fs ), pageCache, fs, logProvider, EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = factory.openAllNeoStores( true ); neoStores.close(); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/CoreStateMachinesModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/CoreStateMachinesModule.java index fd02301d5583..fd8d23c1c99f 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/CoreStateMachinesModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/CoreStateMachinesModule.java @@ -30,6 +30,7 @@ import org.neo4j.causalclustering.core.consensus.RaftMachine; import org.neo4j.causalclustering.core.replication.RaftReplicator; import org.neo4j.causalclustering.core.replication.Replicator; +import org.neo4j.causalclustering.core.state.machines.dummy.DummyMachine; import org.neo4j.causalclustering.core.state.machines.id.CommandIndexTracker; import org.neo4j.causalclustering.core.state.machines.id.IdAllocationState; import org.neo4j.causalclustering.core.state.machines.id.IdReusabilityCondition; @@ -39,7 +40,6 @@ import org.neo4j.causalclustering.core.state.machines.locks.LeaderOnlyLockManager; import org.neo4j.causalclustering.core.state.machines.locks.ReplicatedLockTokenState; import org.neo4j.causalclustering.core.state.machines.locks.ReplicatedLockTokenStateMachine; -import org.neo4j.causalclustering.core.state.machines.dummy.DummyMachine; import org.neo4j.causalclustering.core.state.machines.token.ReplicatedLabelTokenHolder; import org.neo4j.causalclustering.core.state.machines.token.ReplicatedPropertyKeyTokenHolder; import org.neo4j.causalclustering.core.state.machines.token.ReplicatedRelationshipTypeTokenHolder; @@ -53,6 +53,7 @@ import org.neo4j.causalclustering.identity.MemberId; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.CommitProcessFactory; import org.neo4j.kernel.impl.core.LabelTokenHolder; @@ -163,20 +164,24 @@ public CoreStateMachinesModule( MemberId myself, PlatformModule platformModule, ReplicatedLockTokenStateMachine replicatedLockTokenStateMachine = new ReplicatedLockTokenStateMachine( lockTokenState ); + VersionContextSupplier versionContextSupplier = platformModule.versionContextSupplier; ReplicatedTokenStateMachine labelTokenStateMachine = - new ReplicatedTokenStateMachine<>( labelTokenRegistry, new Token.Factory(), logProvider ); + new ReplicatedTokenStateMachine<>( labelTokenRegistry, new Token.Factory(), logProvider, + versionContextSupplier ); ReplicatedTokenStateMachine propertyKeyTokenStateMachine = - new ReplicatedTokenStateMachine<>( propertyKeyTokenRegistry, new Token.Factory(), logProvider ); + new ReplicatedTokenStateMachine<>( propertyKeyTokenRegistry, new Token.Factory(), logProvider, + versionContextSupplier ); ReplicatedTokenStateMachine relationshipTypeTokenStateMachine = new ReplicatedTokenStateMachine<>( relationshipTypeTokenRegistry, new RelationshipTypeToken.Factory(), - logProvider ); + logProvider, versionContextSupplier ); PageCursorTracerSupplier cursorTracerSupplier = platformModule.tracers.pageCursorTracerSupplier; ReplicatedTransactionStateMachine replicatedTxStateMachine = new ReplicatedTransactionStateMachine( commandIndexTracker, replicatedLockTokenStateMachine, - config.get( state_machine_apply_max_batch_size ), logProvider, cursorTracerSupplier ); + config.get( state_machine_apply_max_batch_size ), logProvider, cursorTracerSupplier, + versionContextSupplier ); dependencies.satisfyDependencies( replicatedTxStateMachine ); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachine.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachine.java index d788111b1a9e..4c36e20a8c3f 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachine.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachine.java @@ -25,6 +25,8 @@ import org.neo4j.causalclustering.core.state.Result; import org.neo4j.causalclustering.core.state.machines.StateMachine; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionToApply; @@ -50,15 +52,17 @@ public class ReplicatedTokenStateMachine implements StateMa private final TokenRegistry tokenRegistry; private final TokenFactory tokenFactory; + private final VersionContext versionContext; private final Log log; private long lastCommittedIndex = -1; public ReplicatedTokenStateMachine( TokenRegistry tokenRegistry, TokenFactory tokenFactory, - LogProvider logProvider ) + LogProvider logProvider, VersionContextSupplier versionContextSupplier ) { this.tokenRegistry = tokenRegistry; this.tokenFactory = tokenFactory; + this.versionContext = versionContextSupplier.getVersionContext(); this.log = logProvider.getLog( getClass() ); } @@ -108,7 +112,7 @@ private int applyToStore( Collection commands, long logIndex ) t try ( LockGroup ignored = new LockGroup() ) { - commitProcess.commit( new TransactionToApply( representation ), CommitEvent.NULL, + commitProcess.commit( new TransactionToApply( representation, versionContext ), CommitEvent.NULL, TransactionApplicationMode.EXTERNAL ); } catch ( TransactionFailureException e ) diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachine.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachine.java index b0367f154d54..3bdbb9144457 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachine.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachine.java @@ -27,6 +27,7 @@ import org.neo4j.causalclustering.core.state.machines.id.CommandIndexTracker; import org.neo4j.causalclustering.core.state.machines.locks.ReplicatedLockTokenStateMachine; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionQueue; @@ -49,6 +50,7 @@ public class ReplicatedTransactionStateMachine implements StateMachine { callback.accept( Result.of( txId ) ); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java index 39c19ea327d4..d32280a7b24d 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java @@ -211,7 +211,8 @@ public class EnterpriseReadReplicaEditionModule extends EditionModule int maxBatchSize = config.get( CausalClusteringSettings.read_replica_transaction_applier_batch_size ); BatchingTxApplier batchingTxApplier = new BatchingTxApplier( maxBatchSize, dependencies.provideDependency( TransactionIdStore.class ), writableCommitProcess, - platformModule.monitors, platformModule.tracers.pageCursorTracerSupplier, logProvider ); + platformModule.monitors, platformModule.tracers.pageCursorTracerSupplier, + platformModule.versionContextSupplier, logProvider ); TimerService timerService = new TimerService( platformModule.jobScheduler, logProvider ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplierTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplierTest.java index 82692e47f8ea..982caeb1eee2 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplierTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplierTest.java @@ -28,6 +28,7 @@ import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionToApply; @@ -56,7 +57,7 @@ public class BatchingTxApplierTest private final BatchingTxApplier txApplier = new BatchingTxApplier( maxBatchSize, () -> idStore, () -> commitProcess, new Monitors(), PageCursorTracerSupplier.NULL, - NullLogProvider.getInstance() ); + EmptyVersionContextSupplier.INSTANCE, NullLogProvider.getInstance() ); @Before public void before() throws Throwable diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java index b5c14d3b1a8a..efaed8dae286 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java @@ -28,6 +28,7 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileUtils; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.enterprise.id.EnterpriseIdTypeConfigurationProvider; import org.neo4j.kernel.impl.store.MetaDataStore; @@ -66,7 +67,7 @@ public void rebuildReplicatedIdGeneratorsOnRecovery() throws Exception StoreFactory storeFactory = new StoreFactory( testDirectory.graphDbDir(), Config.defaults(), getIdGenerationFactory( fileSystem ), pageCacheRule.getPageCache( fileSystem ), fileSystem, - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = storeFactory.openAllNeoStores( true ) ) { NodeStore nodeStore = neoStores.getNodeStore(); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachineTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachineTest.java index c6f787108ad0..aca5df9a2e85 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachineTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachineTest.java @@ -19,12 +19,13 @@ */ package org.neo4j.causalclustering.core.state.machines.token; +import org.junit.Test; + import java.util.ArrayList; import java.util.List; -import org.junit.Test; - import org.neo4j.graphdb.TransactionFailureException; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionRepresentationCommitProcess; import org.neo4j.kernel.impl.api.TransactionToApply; @@ -40,10 +41,8 @@ import org.neo4j.storageengine.api.TransactionApplicationMode; import static java.util.Collections.singletonList; - import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; - import static org.neo4j.causalclustering.core.state.machines.token.ReplicatedTokenRequestSerializer.commandBytes; import static org.neo4j.causalclustering.core.state.machines.token.TokenType.LABEL; import static org.neo4j.causalclustering.core.state.machines.tx.LogIndexTxHeaderEncoding.decodeLogIndexFromTxHeader; @@ -59,7 +58,7 @@ public void shouldCreateTokenId() throws Exception // given TokenRegistry registry = new TokenRegistry<>( "Label" ); ReplicatedTokenStateMachine stateMachine = new ReplicatedTokenStateMachine<>( registry, - new Token.Factory(), NullLogProvider.getInstance() ); + new Token.Factory(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( mock( TransactionCommitProcess.class ), -1 ); // when @@ -76,7 +75,7 @@ public void shouldAllocateTokenIdToFirstReplicateRequest() throws Exception // given TokenRegistry registry = new TokenRegistry<>( "Label" ); ReplicatedTokenStateMachine stateMachine = new ReplicatedTokenStateMachine<>( registry, - new Token.Factory(), NullLogProvider.getInstance() ); + new Token.Factory(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( mock( TransactionCommitProcess.class ), -1 ); @@ -102,7 +101,7 @@ public void shouldStoreRaftLogIndexInTransactionHeader() throws Exception StubTransactionCommitProcess commitProcess = new StubTransactionCommitProcess( null, null ); ReplicatedTokenStateMachine stateMachine = new ReplicatedTokenStateMachine<>( new TokenRegistry<>( "Token" ), new Token.Factory(), - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( commitProcess, -1 ); // when diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/CommitProcessStateMachineCollaborationTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/CommitProcessStateMachineCollaborationTest.java index 56499ccd3043..0895998ae4a4 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/CommitProcessStateMachineCollaborationTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/CommitProcessStateMachineCollaborationTest.java @@ -26,6 +26,7 @@ import org.neo4j.causalclustering.core.state.machines.locks.ReplicatedLockTokenRequest; import org.neo4j.causalclustering.core.state.machines.locks.ReplicatedLockTokenStateMachine; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionToApply; @@ -52,7 +53,7 @@ public void shouldFailTransactionIfLockSessionChanges() throws Exception ReplicatedTransactionStateMachine stateMachine = new ReplicatedTransactionStateMachine( mock( CommandIndexTracker.class ), lockState( finalLockSessionId ), 16, NullLogProvider.getInstance(), - PageCursorTracerSupplier.NULL ); + PageCursorTracerSupplier.NULL, EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( localCommitProcess, -1L ); DirectReplicator replicator = new DirectReplicator<>( stateMachine ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachineTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachineTest.java index 38590d30d39c..587ce0cc5554 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachineTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachineTest.java @@ -28,6 +28,7 @@ import org.neo4j.causalclustering.core.state.machines.locks.ReplicatedLockTokenStateMachine; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.exceptions.Status; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.TransactionCommitProcess; @@ -68,7 +69,8 @@ public void shouldCommitTransaction() throws Exception PageCursorTracer cursorTracer = mock( PageCursorTracer.class ); ReplicatedTransactionStateMachine stateMachine = new ReplicatedTransactionStateMachine( - commandIndexTracker, lockState( lockSessionId ), batchSize, logProvider, () -> cursorTracer ); + commandIndexTracker, lockState( lockSessionId ), batchSize, logProvider, () -> cursorTracer, + EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( localCommitProcess, -1L ); // when @@ -96,7 +98,7 @@ public void shouldFailFutureForTransactionCommittedUnderWrongLockSession() throw final ReplicatedTransactionStateMachine stateMachine = new ReplicatedTransactionStateMachine( commandIndexTracker, lockState( currentLockSessionId ), batchSize, logProvider, - PageCursorTracerSupplier.NULL ); + PageCursorTracerSupplier.NULL, EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( localCommitProcess, -1L ); AtomicBoolean called = new AtomicBoolean(); @@ -139,7 +141,7 @@ public void shouldAcceptTransactionCommittedWithNoLockManager() throws Exception ReplicatedTransactionStateMachine stateMachine = new ReplicatedTransactionStateMachine( commandIndexTracker, lockState( currentLockSessionId ), batchSize, logProvider, - PageCursorTracerSupplier.NULL ); + PageCursorTracerSupplier.NULL, EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( localCommitProcess, -1L ); AtomicBoolean called = new AtomicBoolean(); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/io/pagecache/impl/muninn/VersionContextTrackingIT.java b/enterprise/causal-clustering/src/test/java/org/neo4j/io/pagecache/impl/muninn/VersionContextTrackingIT.java new file mode 100644 index 000000000000..013c0b43720e --- /dev/null +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/io/pagecache/impl/muninn/VersionContextTrackingIT.java @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.impl.muninn; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import org.neo4j.causalclustering.core.CoreGraphDatabase; +import org.neo4j.causalclustering.discovery.Cluster; +import org.neo4j.causalclustering.discovery.CoreClusterMember; +import org.neo4j.causalclustering.readreplica.ReadReplicaGraphDatabase; +import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.PagedFile; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; +import org.neo4j.test.causalclustering.ClusterRule; + +import static org.junit.Assert.assertEquals; +import static org.neo4j.causalclustering.discovery.Cluster.dataMatchesEventually; +import static org.neo4j.kernel.configuration.Settings.TRUE; + +public class VersionContextTrackingIT +{ + @Rule + public final ClusterRule clusterRule = new ClusterRule( getClass() ); + private static final int NUMBER_OF_TRANSACTIONS = 3; + private Cluster cluster; + + @Before + public void setup() throws Exception + { + cluster = clusterRule.withSharedCoreParam( GraphDatabaseSettings.snapshot_query, TRUE ) + .withSharedReadReplicaParam( GraphDatabaseSettings.snapshot_query, TRUE ) + .startCluster(); + } + + @Test + public void coreMemberTransactionIdPageTracking() throws Exception + { + long baseTxId = getBaseTransactionId(); + for ( int i = 1; i < 4; i++ ) + { + generateData(); + dataMatchesEventually( anyCoreClusterMember(), cluster.coreMembers() ); + assertEquals( getExpectedLatestPageVersion( baseTxId, i ), getLatestPageVersion( getAnyCore() ) ); + } + } + + @Test + public void readReplicateTransactionIdPageTracking() throws Exception + { + long baseTxId = getBaseTransactionId(); + for ( int i = 1; i < 4; i++ ) + { + generateData(); + dataMatchesEventually( anyCoreClusterMember(), cluster.readReplicas() ); + assertEquals( getExpectedLatestPageVersion( baseTxId, i ), getLatestPageVersion( getAnyReadReplica() ) ); + } + } + + private long getExpectedLatestPageVersion( long baseTxId, int round ) + { + return baseTxId + round * NUMBER_OF_TRANSACTIONS; + } + + private long getBaseTransactionId() + { + DependencyResolver dependencyResolver = getAnyCore().getDependencyResolver(); + TransactionIdStore transactionIdStore = dependencyResolver.resolveDependency( TransactionIdStore.class ); + return transactionIdStore.getLastClosedTransactionId(); + } + + private CoreClusterMember anyCoreClusterMember() + { + return cluster.coreMembers().iterator().next(); + } + + private CoreGraphDatabase getAnyCore() + { + return anyCoreClusterMember().database(); + } + + private ReadReplicaGraphDatabase getAnyReadReplica() + { + return cluster.findAnyReadReplica().database(); + } + + private long getLatestPageVersion( GraphDatabaseFacade databaseFacade ) throws IOException + { + DependencyResolver dependencyResolver = databaseFacade.getDependencyResolver(); + PageCache pageCache = dependencyResolver.resolveDependency( PageCache.class ); + NeoStores neoStores = dependencyResolver.resolveDependency( RecordStorageEngine.class ).testAccessNeoStores(); + File storeFile = neoStores.getNodeStore().getStorageFileName(); + long maxTransactionId = Long.MIN_VALUE; + try ( PagedFile pageFile = pageCache.getExistingMapping( storeFile ).get() ) + { + long lastPageId = pageFile.getLastPageId(); + for ( int i = 0; i <= lastPageId; i++ ) + { +// try ( CursorPageAccessor pageCursor = new CursorPageAccessor( +// (MuninnPageCursor) pageFile.io( i, PagedFile.PF_SHARED_READ_LOCK ) ) ) +// { +// if ( pageCursor.next() ) +// { +// maxTransactionId = Math.max( maxTransactionId, pageCursor.lastTxModifierId() ); +// } +// } + } + } + return maxTransactionId; + } + + private void generateData() throws Exception + { + for ( int i = 0; i < NUMBER_OF_TRANSACTIONS; i++ ) + { + cluster.coreTx( ( coreGraphDatabase, transaction ) -> + { + coreGraphDatabase.createNode(); + transaction.success(); + } ); + } + } + +// private class CursorPageAccessor extends MuninnPageCursor +// { +// +// private MuninnPageCursor delegate; +// +// CursorPageAccessor( MuninnPageCursor delegate ) +// { +// super( -1, PageCursorTracer.NULL, EmptyVersionContext.INSTANCE ); +// this.delegate = delegate; +// } +// +// long lastTxModifierId() +// { +// return delegate.page.getLastModifiedTxId(); +// } +// +// @Override +// protected void unpinCurrentPage() +// { +// delegate.unpinCurrentPage(); +// } +// +// @Override +// protected void convertPageFaultLock( MuninnPage page ) +// { +// delegate.convertPageFaultLock( page ); +// } +// +// @Override +// protected void pinCursorToPage( MuninnPage page, long filePageId, PageSwapper swapper ) +// { +// delegate.pinCursorToPage( page, filePageId, swapper ); +// } +// +// @Override +// protected boolean tryLockPage( MuninnPage page ) +// { +// return delegate.tryLockPage( page ); +// } +// +// @Override +// protected void unlockPage( MuninnPage page ) +// { +// delegate.unlockPage( page ); +// } +// +// @Override +// protected void releaseCursor() +// { +// delegate.releaseCursor(); +// } +// +// @Override +// public boolean next() throws IOException +// { +// return delegate.next(); +// } +// +// @Override +// public boolean shouldRetry() throws IOException +// { +// return delegate.shouldRetry(); +// } +// } +} diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/BatchingResponseHandler.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/BatchingResponseHandler.java index 7068787665ba..82caf0d5f457 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/BatchingResponseHandler.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/BatchingResponseHandler.java @@ -26,6 +26,7 @@ import org.neo4j.com.Response.Handler; import org.neo4j.com.storecopy.ResponseUnpacker.TxHandler; import org.neo4j.helpers.collection.Visitor; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.impl.api.TransactionQueue; import org.neo4j.kernel.impl.api.TransactionToApply; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; @@ -42,14 +43,17 @@ class BatchingResponseHandler implements Response.Handler, { private final TransactionQueue queue; private final TxHandler txHandler; + private final VersionContextSupplier versionContextSupplier; private final TransactionObligationFulfiller obligationFulfiller; private final Log log; BatchingResponseHandler( int maxBatchSize, TransactionQueue.Applier applier, - TransactionObligationFulfiller obligationFulfiller, TxHandler txHandler, Log log ) + TransactionObligationFulfiller obligationFulfiller, TxHandler txHandler, + VersionContextSupplier versionContextSupplier, Log log ) { this.obligationFulfiller = obligationFulfiller; this.txHandler = txHandler; + this.versionContextSupplier = versionContextSupplier; this.queue = new TransactionQueue( maxBatchSize, applier ); this.log = log; } @@ -88,7 +92,8 @@ public boolean visit( CommittedTransactionRepresentation transaction ) throws Ex { queue.queue( new TransactionToApply( transaction.getTransactionRepresentation(), - transaction.getCommitEntry().getTxId() ) + transaction.getCommitEntry().getTxId(), + versionContextSupplier.getVersionContext() ) { @Override public void commitment( Commitment commitment, long transactionId ) diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java index 8febc3524feb..7220d8811d06 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.nio.file.OpenOption; import java.util.Optional; -import java.util.stream.Stream; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.factory.GraphDatabaseFactory; @@ -31,6 +30,7 @@ import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PagedFile; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.extension.KernelExtensionFactory; import org.neo4j.kernel.impl.enterprise.EnterpriseEditionModule; @@ -134,10 +134,9 @@ protected PlatformModule createPlatform( File storeDir, Config config, Dependenc { return new PlatformModule( storeDir, config, databaseInfo, dependencies, graphDatabaseFacade ) { - @Override protected PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, - LogService logging, Tracers tracers ) + LogService logging, Tracers tracers, VersionContextSupplier versionContextSupplier ) { return new ExternallyManagedPageCache( delegatePageCache ); } diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpacker.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpacker.java index c2028abe6878..52817e86f713 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpacker.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpacker.java @@ -23,6 +23,7 @@ import org.neo4j.com.TransactionStream; import org.neo4j.com.TransactionStreamResponse; import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.impl.api.KernelTransactions; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionRepresentationCommitProcess; @@ -200,6 +201,11 @@ public interface Dependencies LogService logService(); KernelTransactions kernelTransactions(); + + /** + * Version context supplier + */ + VersionContextSupplier versionContextSupplier(); } /** @@ -253,13 +259,16 @@ public KernelTransactions kernelTransactions() { return resolver.resolveDependency( KernelTransactions.class ); } + + @Override + public VersionContextSupplier versionContextSupplier() + { + return resolver.resolveDependency( VersionContextSupplier.class ); + } } public static final int DEFAULT_BATCH_SIZE = 100; - static final String msg = "Kernel panic detected: pulled transactions cannot be applied to a non-healthy database. " - + "In order to resolve this issue a manual restart of this instance is required."; - // Assigned in constructor private final Dependencies dependencies; private final int maxBatchSize; @@ -268,6 +277,7 @@ public KernelTransactions kernelTransactions() // Assigned in start() private TransactionObligationFulfiller obligationFulfiller; private TransactionBatchCommitter batchCommitter; + private VersionContextSupplier versionContextSupplier; private Log log; // Assigned in stop() private volatile boolean stopped; @@ -295,7 +305,7 @@ public void unpackResponse( Response response, TxHandler txHandler ) throws E } BatchingResponseHandler responseHandler = new BatchingResponseHandler( maxBatchSize, - batchCommitter, obligationFulfiller, txHandler, log ); + batchCommitter, obligationFulfiller, txHandler, versionContextSupplier, log ); try { response.accept( responseHandler ); @@ -311,6 +321,7 @@ public void start() { this.obligationFulfiller = dependencies.obligationFulfiller(); this.log = dependencies.logService().getInternalLog( BatchingResponseHandler.class ); + this.versionContextSupplier = dependencies.versionContextSupplier(); this.batchCommitter = new TransactionBatchCommitter( dependencies.kernelTransactions(), idReuseSafeZoneTime, dependencies.commitProcess(), log ); this.stopped = false; diff --git a/enterprise/com/src/test/java/org/neo4j/com/storecopy/ResponsePackerIT.java b/enterprise/com/src/test/java/org/neo4j/com/storecopy/ResponsePackerIT.java index 3ad98fd615a8..5b0970af19ab 100644 --- a/enterprise/com/src/test/java/org/neo4j/com/storecopy/ResponsePackerIT.java +++ b/enterprise/com/src/test/java/org/neo4j/com/storecopy/ResponsePackerIT.java @@ -32,6 +32,7 @@ import org.neo4j.helpers.collection.Visitor; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.NeoStores; @@ -112,7 +113,7 @@ private NeoStores createNeoStore( FileSystemAbstraction fs, PageCache pageCache DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fs ); NullLogProvider logProvider = NullLogProvider.getInstance(); StoreFactory storeFactory = - new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fs, logProvider ); + new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fs, logProvider, EmptyVersionContextSupplier.INSTANCE ); return storeFactory.openAllNeoStores( true ); } } diff --git a/enterprise/com/src/test/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpackerTest.java b/enterprise/com/src/test/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpackerTest.java index 87eba0bb404c..5a8cfc0b8b34 100644 --- a/enterprise/com/src/test/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpackerTest.java +++ b/enterprise/com/src/test/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpackerTest.java @@ -32,6 +32,7 @@ import org.neo4j.com.TransactionStream; import org.neo4j.com.TransactionStreamResponse; import org.neo4j.com.storecopy.TransactionCommittingResponseUnpacker.Dependencies; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.KernelTransactions; import org.neo4j.kernel.impl.api.TransactionCommitProcess; @@ -78,6 +79,7 @@ public void shouldUnfreezeKernelTransactionsAfterApplyIfBatchIsLarge() throws Th TransactionObligationFulfiller fulfiller = mock( TransactionObligationFulfiller.class ); when( dependencies.obligationFulfiller() ).thenReturn( fulfiller ); when( dependencies.logService() ).thenReturn( NullLogService.getInstance() ); + when( dependencies.versionContextSupplier() ).thenReturn( EmptyVersionContextSupplier.INSTANCE ); KernelTransactions kernelTransactions = mock( KernelTransactions.class ); when( dependencies.kernelTransactions() ).thenReturn( kernelTransactions ); TransactionCommitProcess commitProcess = mock( TransactionCommitProcess.class ); @@ -124,6 +126,7 @@ public void shouldCommitTransactionsInBatches() throws Exception TransactionCountingTransactionCommitProcess commitProcess = new TransactionCountingTransactionCommitProcess(); when( dependencies.commitProcess() ).thenReturn( commitProcess ); when( dependencies.logService() ).thenReturn( NullLogService.getInstance() ); + when( dependencies.versionContextSupplier() ).thenReturn( EmptyVersionContextSupplier.INSTANCE ); KernelTransactions kernelTransactions = mock( KernelTransactions.class ); when( dependencies.kernelTransactions() ).thenReturn( kernelTransactions ); TransactionCommittingResponseUnpacker unpacker = diff --git a/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EnterpriseCypherEngineProvider.java b/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EnterpriseCypherEngineProvider.java index 2ebb69ce5844..99c17fd04800 100644 --- a/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EnterpriseCypherEngineProvider.java +++ b/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EnterpriseCypherEngineProvider.java @@ -23,8 +23,10 @@ import org.neo4j.cypher.internal.EnterpriseCompatibilityFactory; import org.neo4j.cypher.javacompat.internal.GraphDatabaseCypherService; import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Service; import org.neo4j.kernel.api.KernelAPI; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.query.QueryEngineProvider; import org.neo4j.kernel.impl.query.QueryExecutionEngine; @@ -57,6 +59,7 @@ protected QueryExecutionEngine createEngine( Dependencies deps, GraphDatabaseAPI LogService logService = resolver.resolveDependency( LogService.class ); KernelAPI kernelAPI = resolver.resolveDependency( KernelAPI.class ); Monitors monitors = resolver.resolveDependency( Monitors.class ); + Config config = resolver.resolveDependency( Config.class ); LogProvider logProvider = logService.getInternalLogProvider(); CommunityCompatibilityFactory inner = new CommunityCompatibilityFactory( queryService, kernelAPI, monitors, logProvider ); @@ -64,6 +67,26 @@ protected QueryExecutionEngine createEngine( Dependencies deps, GraphDatabaseAPI EnterpriseCompatibilityFactory compatibilityFactory = new EnterpriseCompatibilityFactory( inner, queryService, kernelAPI, monitors, logProvider ); deps.satisfyDependency( compatibilityFactory ); + return createEngine( queryService, config, logProvider, compatibilityFactory ); + } + + private QueryExecutionEngine createEngine( GraphDatabaseCypherService queryService, Config config, + LogProvider logProvider, EnterpriseCompatibilityFactory compatibilityFactory ) + { + return config.get( GraphDatabaseSettings.snapshot_query ) ? + snapshotEngine( queryService, config, logProvider, compatibilityFactory ) : + standardEngine( queryService, logProvider, compatibilityFactory ); + } + + private SnapshotExecutionEngine snapshotEngine( GraphDatabaseCypherService queryService, Config config, + LogProvider logProvider, EnterpriseCompatibilityFactory compatibilityFactory ) + { + return new SnapshotExecutionEngine( queryService, config, logProvider, compatibilityFactory ); + } + + private ExecutionEngine standardEngine( GraphDatabaseCypherService queryService, LogProvider logProvider, + EnterpriseCompatibilityFactory compatibilityFactory ) + { return new ExecutionEngine( queryService, logProvider, compatibilityFactory ); } } diff --git a/enterprise/cypher/spec-suite-tools/src/test/java/cypher/MapRow.java b/enterprise/cypher/spec-suite-tools/src/test/java/cypher/MapRow.java deleted file mode 100644 index 32a97dcdeca9..000000000000 --- a/enterprise/cypher/spec-suite-tools/src/test/java/cypher/MapRow.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package cypher; - -import java.util.Map; -import java.util.NoSuchElementException; - -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Path; -import org.neo4j.graphdb.Relationship; -import org.neo4j.graphdb.Result; - -import static org.neo4j.helpers.Exceptions.withCause; - -public class MapRow implements Result.ResultRow -{ - private final Map map; - - public MapRow( Map map ) - { - this.map = map; - } - - private T get( String key, Class type ) - { - Object value = map.get( key ); - if ( value == null ) - { - if ( !map.containsKey( key ) ) - { - throw new NoSuchElementException( "No such entry: " + key ); - } - } - try - { - return type.cast( value ); - } - catch ( ClassCastException e ) - { - throw withCause( new NoSuchElementException( "Element '" + key + "' is not a " + type.getSimpleName() ), e ); - } - } - - @Override - public Node getNode( String key ) - { - return get( key, Node.class ); - } - - @Override - public Relationship getRelationship( String key ) - { - return get( key, Relationship.class ); - } - - @Override - public Object get( String key ) - { - return get( key, Object.class ); - } - - @Override - public String getString( String key ) - { - return get( key, String.class ); - } - - @Override - public Number getNumber( String key ) - { - return get( key, Number.class ); - } - - @Override - public Boolean getBoolean( String key ) - { - return get( key, Boolean.class ); - } - - @Override - public Path getPath( String key ) - { - return get( key, Path.class ); - } -} diff --git a/enterprise/cypher/spec-suite-tools/src/test/scala/cypher/feature/parser/ParsingTestSupport.scala b/enterprise/cypher/spec-suite-tools/src/test/scala/cypher/feature/parser/ParsingTestSupport.scala index d41767e5e4c9..4e5db6fdf51e 100644 --- a/enterprise/cypher/spec-suite-tools/src/test/scala/cypher/feature/parser/ParsingTestSupport.scala +++ b/enterprise/cypher/spec-suite-tools/src/test/scala/cypher/feature/parser/ParsingTestSupport.scala @@ -21,12 +21,12 @@ package cypher.feature.parser import java.util.Collections -import cypher.MapRow import cypher.feature.parser.matchers.ResultMatcher import org.junit.runner.RunWith import org.mockito.Mockito._ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer +import org.neo4j.cypher.internal.javacompat.MapRow import org.neo4j.graphdb.Result.ResultVisitor import org.neo4j.graphdb._ import org.scalatest.junit.JUnitRunner diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/upgrade/MasterClientTest.java b/enterprise/ha/src/test/java/org/neo4j/ha/upgrade/MasterClientTest.java index 183261fa8364..00abe5ecf0e5 100644 --- a/enterprise/ha/src/test/java/org/neo4j/ha/upgrade/MasterClientTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/ha/upgrade/MasterClientTest.java @@ -37,6 +37,7 @@ import org.neo4j.com.storecopy.TransactionCommittingResponseUnpacker; import org.neo4j.com.storecopy.TransactionCommittingResponseUnpacker.Dependencies; import org.neo4j.helpers.HostnamePort; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.ha.MasterClient320; import org.neo4j.kernel.ha.com.master.ConversationManager; @@ -127,6 +128,7 @@ public void clientShouldReadAndApplyTransactionLogsOnNewLockSessionRequest() thr TransactionCommitProcess commitProcess = mock( TransactionCommitProcess.class ); when( deps.commitProcess() ).thenReturn( commitProcess ); when( deps.logService() ).thenReturn( NullLogService.getInstance() ); + when( deps.versionContextSupplier() ).thenReturn( EmptyVersionContextSupplier.INSTANCE ); when( deps.kernelTransactions() ).thenReturn( mock( KernelTransactions.class ) ); ResponseUnpacker unpacker = life.add( diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/OnDiskLastTxIdGetterTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/OnDiskLastTxIdGetterTest.java index 1adfdeaf3ab0..ed23190cade6 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/OnDiskLastTxIdGetterTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/OnDiskLastTxIdGetterTest.java @@ -25,6 +25,7 @@ import java.io.File; import java.util.function.LongSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.ha.transaction.OnDiskLastTxIdGetter; import org.neo4j.kernel.impl.store.NeoStores; @@ -59,7 +60,7 @@ public void lastTransactionIdIsBaseTxIdWhileNeoStoresAreStopped() final StoreFactory storeFactory = new StoreFactory( new File( "store" ), Config.defaults(), new DefaultIdGeneratorFactory( fs.get() ), pageCacheRule.getPageCache( fs.get() ), fs.get(), - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); final NeoStores neoStores = storeFactory.openAllNeoStores( true ); neoStores.close(); diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/UpdatePullerTriggersPageTransactionTrackingIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/UpdatePullerTriggersPageTransactionTrackingIT.java new file mode 100644 index 000000000000..6167395e2560 --- /dev/null +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/UpdatePullerTriggersPageTransactionTrackingIT.java @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.ha; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.util.Map; +import java.util.function.LongSupplier; + +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.QueryExecutionException; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.factory.GraphDatabaseBuilder; +import org.neo4j.graphdb.factory.GraphDatabaseFactoryState; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.graphdb.factory.TestHighlyAvailableGraphDatabaseFactory; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.ha.factory.HighlyAvailableEditionModule; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; +import org.neo4j.kernel.impl.factory.DatabaseInfo; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; +import org.neo4j.kernel.impl.factory.PlatformModule; +import org.neo4j.kernel.impl.ha.ClusterManager; +import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; +import org.neo4j.test.ha.ClusterRule; + +import static org.junit.Assert.assertEquals; +import static org.neo4j.helpers.collection.MapUtil.stringMap; +import static org.neo4j.kernel.configuration.Settings.TRUE; + +public class UpdatePullerTriggersPageTransactionTrackingIT +{ + @Rule + public final ClusterRule clusterRule = new ClusterRule( getClass() ); + private final Label NODE_LABEL = Label.label( "mark" ); + private final TestTransactionVersionContextSupplier contextSupplier = new TestTransactionVersionContextSupplier(); + private ClusterManager.ManagedCluster cluster; + + @Before + public void setup() throws Exception + { + CustomGraphDatabaseFactory customGraphDatabaseFactory = new CustomGraphDatabaseFactory(); + cluster = clusterRule.withSharedSetting( GraphDatabaseSettings.snapshot_query, TRUE ) + .withDbFactory( customGraphDatabaseFactory ) + .startCluster(); + HighlyAvailableGraphDatabase master = cluster.getMaster(); + for ( int i = 0; i < 3; i++ ) + { + try ( Transaction tx = master.beginTx() ) + { + master.createNode( NODE_LABEL ); + tx.success(); + } + } + cluster.sync(); + } + + @Test + public void updatePullerTriggerPageTransactionTracking() + { + HighlyAvailableGraphDatabase slave = cluster.getAnySlave(); + TransactionIdStore slaveTransactionIdStore = + slave.getDependencyResolver().resolveDependency( TransactionIdStore.class ); + assertEquals( 5, slaveTransactionIdStore.getLastClosedTransactionId() ); + + ByzantineLongSupplier byzantineIdSupplier = contextSupplier.getByzantineIdSupplier(); + byzantineIdSupplier.useWrongTxId(); + try ( Transaction ignored = slave.beginTx() ) + { + slave.execute( "match (n) return n" ); + } + catch ( QueryExecutionException executionException ) + { + assertEquals( "Unable to get clean data snapshot for query 'match (n) return n' after 5 attempts.", executionException.getMessage()); + } + byzantineIdSupplier.useCorrectTxId(); + slave.execute( "match (n) return n" ).close(); + } + + private class CustomGraphDatabaseFactory extends TestHighlyAvailableGraphDatabaseFactory + { + @Override + protected GraphDatabaseBuilder.DatabaseCreator createDatabaseCreator( File storeDir, + GraphDatabaseFactoryState state ) + { + return new GraphDatabaseBuilder.DatabaseCreator() + { + @Override + public GraphDatabaseService newDatabase( Map config ) + { + return newDatabase( Config.defaults( config ) ); + } + + @Override + public GraphDatabaseService newDatabase( Config config ) + { + config.augment( stringMap( "unsupported.dbms.ephemeral", "false" ) ); + return new CustomHighlyAvailableGraphDatabase( storeDir, + config, state.databaseDependencies() ) ; + } + }; + } + } + + private class CustomHighlyAvailableGraphDatabase extends HighlyAvailableGraphDatabase + { + + CustomHighlyAvailableGraphDatabase( File storeDir, Config config, + GraphDatabaseFacadeFactory.Dependencies dependencies ) + { + super( storeDir, config, dependencies ); + } + + @Override + protected GraphDatabaseFacadeFactory newHighlyAvailableFacadeFactory() + { + return new CustomFacadeFactory(); + } + } + + private class CustomFacadeFactory extends GraphDatabaseFacadeFactory + { + CustomFacadeFactory() + { + super( DatabaseInfo.HA, HighlyAvailableEditionModule::new ); + } + + @Override + public GraphDatabaseFacade newFacade( File storeDir, Config config, Dependencies dependencies ) + { + return initFacade( storeDir, config, dependencies, new HighlyAvailableGraphDatabase( storeDir, config, dependencies ) ); + } + + @Override + protected PlatformModule createPlatform( File storeDir, Config config, Dependencies dependencies, + GraphDatabaseFacade graphDatabaseFacade ) + { + return new PlatformModule( storeDir, config, databaseInfo, dependencies, graphDatabaseFacade ) + { + @Override + protected VersionContextSupplier createCursorContextSupplier( Config config ) + { + return contextSupplier; + } + }; + } + } + + private class TestTransactionVersionContextSupplier extends TransactionVersionContextSupplier + { + + private volatile ByzantineLongSupplier byzantineLongSupplier; + + @Override + public void init( LongSupplier lastClosedTransactionIdSupplier ) + { + byzantineLongSupplier = new ByzantineLongSupplier( lastClosedTransactionIdSupplier ); + super.init( byzantineLongSupplier ); + } + + @Override + public VersionContext getVersionContext() + { + return super.getVersionContext(); + } + + ByzantineLongSupplier getByzantineIdSupplier() + { + return byzantineLongSupplier; + } + } + + private class ByzantineLongSupplier implements LongSupplier + { + + private volatile boolean wrongTxId; + private final LongSupplier originalIdSupplier; + + ByzantineLongSupplier( LongSupplier originalIdSupplier ) + { + this.originalIdSupplier = originalIdSupplier; + } + + @Override + public long getAsLong() + { + return wrongTxId ? 1 : originalIdSupplier.getAsLong(); + } + + void useWrongTxId() + { + wrongTxId = true; + } + + void useCorrectTxId() + { + wrongTxId = false; + } + } +} diff --git a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java index 04e99ab8e48e..2809f3d19928 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java +++ b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java @@ -34,6 +34,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.NullLogService; import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory; @@ -75,7 +76,7 @@ public void shouldNotDoActualStoreMigrationBetween3_0_5_and_next() throws Except try ( FileSystemAbstraction fs = new DefaultFileSystemAbstraction(); PageCache pageCache = new ConfiguringPageCacheFactory( fs, config, NULL, - PageCursorTracerSupplier.NULL, NullLog.getInstance() ) + PageCursorTracerSupplier.NULL, NullLog.getInstance(), EmptyVersionContextSupplier.INSTANCE ) .getOrCreatePageCache() ) { // For test code sanity diff --git a/tools/src/main/java/org/neo4j/tools/dump/DumpCountsStore.java b/tools/src/main/java/org/neo4j/tools/dump/DumpCountsStore.java index c8576d2b0eaf..792e2fcaf9ef 100644 --- a/tools/src/main/java/org/neo4j/tools/dump/DumpCountsStore.java +++ b/tools/src/main/java/org/neo4j/tools/dump/DumpCountsStore.java @@ -31,6 +31,7 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.ReadOperations; import org.neo4j.kernel.api.schema.index.IndexDescriptor; import org.neo4j.kernel.configuration.Config; @@ -83,8 +84,7 @@ public static void dumpCountsStore( FileSystemAbstraction fs, File path, PrintSt if ( fs.isDirectory( path ) ) { StoreFactory factory = new StoreFactory( path, Config.defaults(), new DefaultIdGeneratorFactory( fs ), - pages, fs, - logProvider ); + pages, fs, logProvider, EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = factory.openAllNeoStores(); SchemaStorage schemaStorage = new SchemaStorage( neoStores.getSchemaStore() ); @@ -273,7 +273,7 @@ private static class VisitableCountsTracker extends CountsTracker VisitableCountsTracker( LogProvider logProvider, FileSystemAbstraction fs, PageCache pages, Config config, File baseFile ) { - super( logProvider, fs, pages, config, baseFile ); + super( logProvider, fs, pages, config, baseFile, EmptyVersionContextSupplier.INSTANCE ); } @Override diff --git a/tools/src/main/java/org/neo4j/tools/dump/DumpStore.java b/tools/src/main/java/org/neo4j/tools/dump/DumpStore.java index 29d21fe8bea1..ebb1cf89b4d4 100644 --- a/tools/src/main/java/org/neo4j/tools/dump/DumpStore.java +++ b/tools/src/main/java/org/neo4j/tools/dump/DumpStore.java @@ -26,6 +26,7 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.CommonAbstractStore; import org.neo4j.kernel.impl.store.NeoStores; @@ -72,7 +73,7 @@ public static void main( String... args ) throws Exception { final DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fs ); Function createStoreFactory = file -> new StoreFactory( file.getParentFile(), - Config.defaults(), idGeneratorFactory, pageCache, fs, logProvider() ); + Config.defaults(), idGeneratorFactory, pageCache, fs, logProvider(), EmptyVersionContextSupplier.INSTANCE ); for ( String arg : args ) { diff --git a/tools/src/main/java/org/neo4j/tools/dump/DumpStoreChain.java b/tools/src/main/java/org/neo4j/tools/dump/DumpStoreChain.java index b24edc8c977a..f74c6e492048 100644 --- a/tools/src/main/java/org/neo4j/tools/dump/DumpStoreChain.java +++ b/tools/src/main/java/org/neo4j/tools/dump/DumpStoreChain.java @@ -28,6 +28,7 @@ import org.neo4j.helpers.Args; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeStore; @@ -121,7 +122,7 @@ void dump( File storeDir ) throws IOException DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fs ); Config config = Config.defaults(); StoreFactory storeFactory = new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fs, - logProvider() ); + logProvider(), EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = storeFactory.openNeoStores( getStoreTypes() ) ) { diff --git a/tools/src/main/java/org/neo4j/tools/rawstorereader/RsdrMain.java b/tools/src/main/java/org/neo4j/tools/rawstorereader/RsdrMain.java index 70aa2711bbd6..7514902e633c 100644 --- a/tools/src/main/java/org/neo4j/tools/rawstorereader/RsdrMain.java +++ b/tools/src/main/java/org/neo4j/tools/rawstorereader/RsdrMain.java @@ -35,6 +35,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.InvalidRecordException; import org.neo4j.kernel.impl.store.MetaDataStore; @@ -107,7 +108,7 @@ private static StoreFactory openStore( FileSystemAbstraction fileSystem, File st { IdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fileSystem ); NullLogProvider logProvider = NullLogProvider.getInstance(); - return new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fileSystem, logProvider ); + return new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fileSystem, logProvider, EmptyVersionContextSupplier.INSTANCE ); } private static void interact( FileSystemAbstraction fileSystem, NeoStores neoStores ) throws IOException