From 3795750fd91c8e02f2b355f374824485e4a21f13 Mon Sep 17 00:00:00 2001 From: MishaDemianenko Date: Wed, 6 Dec 2017 18:33:27 +0100 Subject: [PATCH] Data snapshot query execution Introduce possibility to execute query based on data snapshot. Any data modification will mark corresponding pages with a version that will be equal to transaction id in which that particular change was introduced. Also allow count store to track transactions in which changes where introduced. To be able to guarantee that query result is based on a data that was present on query execution start and was not modified while it was running, the engine will verify that all paged that were accessed during query execution have the version that is less or equal that was last closed on a moment when query started. Support for snapshot query execution in CC and HA Snapshot query execution is enabled by 'unsupported.dbms.query.snapshot' setting and number of retries that engine will do while trying to get stable snapshot is controlled by 'unsupported.dbms.query.snapshot.retries'. --- .../consistency/ConsistencyCheckService.java | 5 +- .../checking/GraphStoreFixture.java | 3 +- ...etectAllRelationshipInconsistenciesIT.java | 5 +- community/cypher/cypher/pom.xml | 5 + .../CommunityCypherEngineProvider.java | 25 +- .../internal/javacompat/EagerResult.java | 194 ++++++++++ .../cypher/internal/javacompat}/MapRow.java | 2 +- .../javacompat/SnapshotExecutionEngine.java | 113 ++++++ .../javacompat/UnstableSnapshotException.java | 31 ++ .../javacompat/EagerResultITTest.java | 345 ++++++++++++++++++ .../SnapshotExecutionEngineTest.java | 142 +++++++ .../TransactionBoundQueryContextTest.scala | 6 +- .../neo4j/index/internal/gbptree/GBPTree.java | 2 +- .../io/pagecache/impl/muninn/CursorPool.java | 13 +- .../io/pagecache/impl/muninn/MuninnPage.java | 34 +- .../impl/muninn/MuninnPageCache.java | 10 +- .../impl/muninn/MuninnPageCursor.java | 5 +- .../muninn/MuninnPageEvictionCallback.java | 1 + .../impl/muninn/MuninnPagedFile.java | 40 +- .../impl/muninn/MuninnReadPageCursor.java | 20 +- .../impl/muninn/MuninnWritePageCursor.java | 7 +- .../muninn/StandalonePageCacheFactory.java | 13 +- .../cursor/context/EmptyVersionContext.java | 73 ++++ .../context/EmptyVersionContextSupplier.java | 46 +++ .../cursor/context/VersionContext.java | 72 ++++ .../context/VersionContextSupplier.java | 42 +++ .../io/pagecache/PageCacheTestSupport.java | 26 +- .../impl/muninn/MuninnPageCacheFixture.java | 5 +- .../impl/muninn/MuninnPageCacheTest.java | 294 +++++++++++++++ .../RandomPageCacheTestHarness.java | 3 +- .../pagecache/stress/PageCacheStressTest.java | 3 +- .../recording/RecordingPageCursorTracer.java | 2 +- .../org/neo4j/test/rule/PageCacheRule.java | 4 +- .../factory/GraphDatabaseSettings.java | 11 + .../org/neo4j/kernel/NeoStoreDataSource.java | 62 ++-- .../kernel/impl/api/KernelStatement.java | 18 +- .../api/KernelTransactionImplementation.java | 14 +- .../kernel/impl/api/KernelTransactions.java | 8 +- .../kernel/impl/api/TransactionToApply.java | 22 +- .../context/TransactionVersionContext.java | 90 +++++ .../TransactionVersionContextSupplier.java | 48 +++ .../kernel/impl/factory/DataSourceModule.java | 3 +- .../kernel/impl/factory/PlatformModule.java | 21 +- ...onfigurableStandalonePageCacheFactory.java | 13 +- .../ConfiguringPageCacheFactory.java | 11 +- .../recordstorage/RecordStorageEngine.java | 7 +- .../neo4j/kernel/impl/store/NeoStores.java | 8 +- .../neo4j/kernel/impl/store/StoreAccess.java | 3 +- .../neo4j/kernel/impl/store/StoreFactory.java | 26 +- .../impl/store/counts/CountsTracker.java | 80 ++-- .../store/counts/ReadOnlyCountsTracker.java | 4 +- .../store/kvstore/AbstractKeyValueStore.java | 9 +- .../impl/store/kvstore/ActiveState.java | 10 +- .../store/kvstore/ConcurrentMapState.java | 102 ++++-- .../kernel/impl/store/kvstore/DeadState.java | 28 +- .../impl/store/kvstore/PrototypeState.java | 4 +- .../impl/store/kvstore/RotationState.java | 3 +- .../kernel/impl/store/kvstore/State.java | 11 +- .../DirectRecordStoreMigrator.java | 12 +- .../PropertyDeduplicator.java | 4 +- .../participant/StoreMigrator.java | 10 +- .../internal/BatchInserterImpl.java | 6 +- .../batchimport/store/BatchingNeoStores.java | 10 +- .../kernel/api/KernelTransactionFactory.java | 3 +- .../api/KernelTransactionTerminationTest.java | 4 +- .../impl/api/KernelTransactionTestBase.java | 3 +- .../impl/api/KernelTransactionsTest.java | 11 +- .../api/LockingStatementOperationsTest.java | 3 +- ...ropertyPhysicalToLogicalConverterTest.java | 4 +- .../StoreNodeRelationshipCursorTest.java | 5 +- .../api/store/StorePropertyCursorTest.java | 3 +- .../StoreSingleRelationshipCursorTest.java | 3 +- .../kernel/impl/core/ManyPropertyKeysIT.java | 3 +- .../ConfiguringPageCacheFactoryTest.java | 11 +- .../impl/store/FreeIdsAfterRecoveryTest.java | 8 +- ...dGeneratorRebuildFailureEmulationTest.java | 11 +- .../kernel/impl/store/MetaDataStoreTest.java | 4 +- .../kernel/impl/store/NeoStoresTest.java | 26 +- .../kernel/impl/store/NodeStoreTest.java | 15 +- .../store/RecordStoreConsistentReadTest.java | 3 +- .../store/RelationshipGroupStoreTest.java | 3 +- .../kernel/impl/store/SchemaStoreTest.java | 3 +- .../kernel/impl/store/StoreFactoryTest.java | 3 +- .../kernel/impl/store/TestArrayStore.java | 3 +- .../kernel/impl/store/TestDynamicStore.java | 3 +- .../impl/store/TestGraphProperties.java | 3 +- .../store/TestGrowingFileMemoryMapping.java | 3 +- .../impl/store/TestIdGeneratorRebuilding.java | 3 +- .../impl/store/counts/CountsComputerTest.java | 5 +- .../impl/store/counts/CountsRotationTest.java | 3 +- .../impl/store/counts/CountsTrackerTest.java | 68 +++- .../kvstore/AbstractKeyValueStoreTest.java | 6 +- .../store/kvstore/ConcurrentMapStateTest.java | 124 ++++++- .../participant/StoreMigratorIT.java | 5 +- .../state/ApplyRecoveredTransactionsTest.java | 3 +- .../transaction/state/NodeCommandTest.java | 4 +- .../state/NodeLabelsFieldTest.java | 3 +- .../state/RelationshipGroupGetterTest.java | 4 +- ...ersarialPageCacheGraphDatabaseFactory.java | 7 +- .../test/rule/ConfigurablePageCacheRule.java | 4 +- .../test/rule/NeoStoreDataSourceRule.java | 4 +- .../org/neo4j/test/rule/NeoStoresRule.java | 5 +- .../test/rule/RecordStorageEngineRule.java | 3 +- .../test/java/db/DatabaseShutdownTest.java | 8 +- .../src/test/java/db/QueryRestartIT.java | 288 +++++++++++++++ .../test/java/upgrade/StoreUpgraderTest.java | 3 +- .../ExecutionResultSerializerTest.java | 6 +- .../GraphExtractionWriterTest.java | 1 + .../RestRepresentationWriterTest.java | 10 +- .../rest/transactional/RowWriterTest.java | 8 +- .../catchup/tx/BatchingTxApplier.java | 8 +- .../catchup/tx/TransactionApplier.java | 5 +- .../core/state/CoreBootstrapper.java | 3 +- .../machines/CoreStateMachinesModule.java | 15 +- .../token/ReplicatedTokenStateMachine.java | 8 +- .../tx/ReplicatedTransactionStateMachine.java | 8 +- .../EnterpriseReadReplicaEditionModule.java | 3 +- .../catchup/tx/BatchingTxApplierTest.java | 3 +- .../id/RebuildReplicatedIdGeneratorsTest.java | 3 +- .../ReplicatedTokenStateMachineTest.java | 13 +- ...tProcessStateMachineCollaborationTest.java | 3 +- ...ReplicatedTransactionStateMachineTest.java | 8 +- .../impl/muninn/VersionContextTrackingIT.java | 217 +++++++++++ .../storecopy/BatchingResponseHandler.java | 9 +- .../storecopy/ExternallyManagedPageCache.java | 7 +- ...TransactionCommittingResponseUnpacker.java | 19 +- .../neo4j/com/storecopy/ResponsePackerIT.java | 3 +- ...sactionCommittingResponseUnpackerTest.java | 3 + .../EnterpriseCypherEngineProvider.java | 23 ++ .../src/test/java/cypher/MapRow.java | 102 ------ .../feature/parser/ParsingTestSupport.scala | 2 +- .../neo4j/ha/upgrade/MasterClientTest.java | 2 + .../kernel/ha/OnDiskLastTxIdGetterTest.java | 3 +- ...llerTriggersPageTransactionTrackingIT.java | 224 ++++++++++++ .../participant/StoreMigratorTest.java | 3 +- .../neo4j/upgrade/StoreMigratorFrom20IT.java | 3 +- .../org/neo4j/tools/dump/DumpCountsStore.java | 5 +- .../java/org/neo4j/tools/dump/DumpStore.java | 3 +- .../org/neo4j/tools/dump/DumpStoreChain.java | 3 +- .../neo4j/tools/rawstorereader/RsdrMain.java | 3 +- 140 files changed, 3204 insertions(+), 475 deletions(-) create mode 100644 community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EagerResult.java rename community/{server/src/test/java/org/neo4j/server/rest/transactional => cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat}/MapRow.java (98%) create mode 100644 community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngine.java create mode 100644 community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/UnstableSnapshotException.java create mode 100644 community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/EagerResultITTest.java create mode 100644 community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngineTest.java create mode 100644 community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContext.java create mode 100644 community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContextSupplier.java create mode 100644 community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContext.java create mode 100644 community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContextSupplier.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContext.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContextSupplier.java create mode 100644 community/neo4j/src/test/java/db/QueryRestartIT.java create mode 100644 enterprise/causal-clustering/src/test/java/org/neo4j/io/pagecache/impl/muninn/VersionContextTrackingIT.java delete mode 100644 enterprise/cypher/spec-suite-tools/src/test/java/cypher/MapRow.java create mode 100644 enterprise/ha/src/test/java/org/neo4j/kernel/ha/UpdatePullerTriggersPageTransactionTrackingIT.java diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java index 29bafcfc08850..513025c80f08c 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java @@ -43,6 +43,7 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.direct.DirectStoreAccess; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.api.labelscan.LabelScanStore; @@ -158,7 +159,7 @@ public Result runFullConsistencyCheck( File storeDir, Config config, ProgressMon Log log = logProvider.getLog( getClass() ); ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( fileSystem, config, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, - logProvider.getLog( PageCache.class ) ); + logProvider.getLog( PageCache.class ), EmptyVersionContextSupplier.INSTANCE ); PageCache pageCache = pageCacheFactory.getOrCreatePageCache(); try @@ -218,7 +219,7 @@ public Result runFullConsistencyCheck( final File storeDir, Config config, Progr GraphDatabaseSettings.read_only.name(), TRUE, GraphDatabaseSettings.label_index.name(), LabelIndex.AUTO.name() ) ); StoreFactory factory = new StoreFactory( storeDir, config, - new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, logProvider ); + new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, logProvider, EmptyVersionContextSupplier.INSTANCE ); ConsistencySummaryStatistics summary; final File reportFile = chooseReportPath( reportDir ); diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java index 90be248ddb0ff..0e9821516f6bf 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java @@ -41,6 +41,7 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.ReadOperations; import org.neo4j.kernel.api.direct.DirectStoreAccess; import org.neo4j.kernel.api.exceptions.TransactionFailureException; @@ -164,7 +165,7 @@ public DirectStoreAccess directStoreAccess() fileSystem = new DefaultFileSystemAbstraction(); PageCache pageCache = getPageCache( fileSystem ); LogProvider logProvider = NullLogProvider.getInstance(); - StoreFactory storeFactory = new StoreFactory( directory, pageCache, fileSystem, logProvider ); + StoreFactory storeFactory = new StoreFactory( directory, pageCache, fileSystem, logProvider, EmptyVersionContextSupplier.INSTANCE ); neoStore = storeFactory.openAllNeoStores(); StoreAccess nativeStores; if ( keepStatistics ) diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java index f4ed1974e3c63..e8e49c3df7775 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java @@ -37,6 +37,7 @@ import org.neo4j.helpers.progress.ProgressMonitorFactory; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.direct.DirectStoreAccess; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.api.labelscan.LabelScanStore; @@ -59,7 +60,6 @@ import org.neo4j.test.rule.fs.DefaultFileSystemRule; import static org.junit.Assert.assertTrue; - import static org.neo4j.graphdb.Label.label; import static org.neo4j.helpers.collection.MapUtil.stringMap; @@ -151,7 +151,8 @@ private StoreFactory newStoreFactory( PageCache pageCache ) { FileSystemAbstraction fileSystem = fileSystemRule.get(); return new StoreFactory( directory.directory(), getTuningConfiguration(), - new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, NullLogProvider.getInstance() ); + new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, NullLogProvider.getInstance(), + EmptyVersionContextSupplier.INSTANCE ); } private Config getTuningConfiguration() diff --git a/community/cypher/cypher/pom.xml b/community/cypher/cypher/pom.xml index aefbd71f3399f..33156ba35acad 100644 --- a/community/cypher/cypher/pom.xml +++ b/community/cypher/cypher/pom.xml @@ -348,6 +348,11 @@ test + + commons-codec + commons-codec + + diff --git a/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/CommunityCypherEngineProvider.java b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/CommunityCypherEngineProvider.java index 57edfd5a88892..0d47d2cfc92f2 100644 --- a/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/CommunityCypherEngineProvider.java +++ b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/CommunityCypherEngineProvider.java @@ -22,8 +22,10 @@ import org.neo4j.cypher.internal.CommunityCompatibilityFactory; import org.neo4j.cypher.javacompat.internal.GraphDatabaseCypherService; import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Service; import org.neo4j.kernel.api.KernelAPI; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.query.QueryEngineProvider; import org.neo4j.kernel.impl.query.QueryExecutionEngine; @@ -56,10 +58,31 @@ protected QueryExecutionEngine createEngine( Dependencies deps, GraphDatabaseAPI LogService logService = resolver.resolveDependency( LogService.class ); KernelAPI kernelAPI = resolver.resolveDependency( KernelAPI.class ); Monitors monitors = resolver.resolveDependency( Monitors.class ); + Config config = resolver.resolveDependency( Config.class ); LogProvider logProvider = logService.getInternalLogProvider(); CommunityCompatibilityFactory compatibilityFactory = new CommunityCompatibilityFactory( queryService, kernelAPI, monitors, logProvider ); deps.satisfyDependencies( compatibilityFactory ); - return new ExecutionEngine( queryService, logProvider, compatibilityFactory); + return createEngine( queryService, config, logProvider, compatibilityFactory ); + } + + private QueryExecutionEngine createEngine( GraphDatabaseCypherService queryService, Config config, + LogProvider logProvider, CommunityCompatibilityFactory compatibilityFactory ) + { + return config.get( GraphDatabaseSettings.snapshot_query ) ? + snapshotEngine( queryService, config, logProvider, compatibilityFactory ) : + standardEngine( queryService, logProvider, compatibilityFactory ); + } + + private SnapshotExecutionEngine snapshotEngine( GraphDatabaseCypherService queryService, Config config, + LogProvider logProvider, CommunityCompatibilityFactory compatibilityFactory ) + { + return new SnapshotExecutionEngine( queryService, config, logProvider, compatibilityFactory); + } + + private ExecutionEngine standardEngine( GraphDatabaseCypherService queryService, LogProvider logProvider, + CommunityCompatibilityFactory compatibilityFactory ) + { + return new ExecutionEngine( queryService, logProvider, compatibilityFactory ); } } diff --git a/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EagerResult.java b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EagerResult.java new file mode 100644 index 0000000000000..eb28d4b277998 --- /dev/null +++ b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EagerResult.java @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.javacompat; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.neo4j.graphdb.ExecutionPlanDescription; +import org.neo4j.graphdb.Notification; +import org.neo4j.graphdb.QueryExecutionType; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.graphdb.ResourceIterator; +import org.neo4j.graphdb.Result; + +import static java.lang.System.lineSeparator; + +/** + * Result produced as result of eager query execution for cases when {@link SnapshotExecutionEngine} is used. + */ +class EagerResult implements Result +{ + private static final String ITEM_SEPARATOR = ", "; + private final Result originalResult; + private final List> queryResult = new ArrayList<>(); + private int cursor; + + EagerResult( Result result ) + { + this.originalResult = result; + } + + public void consume() + { + while ( originalResult.hasNext() ) + { + queryResult.add( originalResult.next() ); + } + } + + @Override + public QueryExecutionType getQueryExecutionType() + { + return originalResult.getQueryExecutionType(); + } + + @Override + public List columns() + { + return originalResult.columns(); + } + + @Override + public ResourceIterator columnAs( String name ) + { + return new EagerResultResourceIterator<>( name ); + } + + @Override + public boolean hasNext() + { + return cursor < queryResult.size(); + } + + @Override + public Map next() + { + return queryResult.get( cursor++ ); + } + + @Override + public void close() + { + // nothing to close. Original result is already closed at this point + } + + @Override + public QueryStatistics getQueryStatistics() + { + return originalResult.getQueryStatistics(); + } + + @Override + public ExecutionPlanDescription getExecutionPlanDescription() + { + return originalResult.getExecutionPlanDescription(); + } + + @Override + public String resultAsString() + { + List columns = originalResult.columns(); + StringBuilder builder = new StringBuilder(); + builder.append( String.join( ITEM_SEPARATOR, columns ) ); + if ( !queryResult.isEmpty() ) + { + builder.append( lineSeparator() ); + int numberOfColumns = columns.size(); + for ( Map row : queryResult ) + { + writeRow( columns, builder, numberOfColumns, row ); + builder.append( lineSeparator() ); + } + } + return builder.toString(); + } + + @Override + public void writeAsStringTo( PrintWriter writer ) + { + writer.print( resultAsString() ); + } + + @Override + public void remove() + { + throw new UnsupportedOperationException( "Not supported" ); + } + + @Override + public Iterable getNotifications() + { + return originalResult.getNotifications(); + } + + @Override + public void accept( ResultVisitor visitor ) + throws VisitationException + { + for ( Map map : queryResult ) + { + visitor.visit( new MapRow( map ) ); + } + } + + private void writeRow( List columns, StringBuilder builder, int numberOfColumns, Map row ) + { + for ( int i = 0; i < numberOfColumns; i++ ) + { + builder.append( row.get( columns.get( i ) ) ); + if ( i != numberOfColumns - 1 ) + { + builder.append( ITEM_SEPARATOR ); + } + } + } + + private class EagerResultResourceIterator implements ResourceIterator + { + private final String column; + int cursor; + + EagerResultResourceIterator( String column ) + { + this.column = column; + } + + @Override + public boolean hasNext() + { + return cursor < queryResult.size(); + } + + @Override + public T next() + { + return (T) queryResult.get( cursor++ ).get( column ); + } + + @Override + public void close() + { + // Nothing to close. + } + } +} diff --git a/community/server/src/test/java/org/neo4j/server/rest/transactional/MapRow.java b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/MapRow.java similarity index 98% rename from community/server/src/test/java/org/neo4j/server/rest/transactional/MapRow.java rename to community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/MapRow.java index 8772b41671f3e..90d0c3ab38609 100644 --- a/community/server/src/test/java/org/neo4j/server/rest/transactional/MapRow.java +++ b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/MapRow.java @@ -17,7 +17,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.server.rest.transactional; +package org.neo4j.cypher.internal.javacompat; import java.util.Map; import java.util.NoSuchElementException; diff --git a/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngine.java b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngine.java new file mode 100644 index 0000000000000..df9ae4a1f64c1 --- /dev/null +++ b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngine.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.javacompat; + +import java.util.Map; + +import org.neo4j.cypher.internal.CompatibilityFactory; +import org.neo4j.graphdb.Result; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.kernel.GraphDatabaseQueryService; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.api.KernelStatement; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.logging.LogProvider; + +/** + * {@link ExecutionEngine} engine that will try to run cypher query with guarantee that query will never see any data + * that coming from transaction that are newer then transaction that was the last closed on a moment when + * {@link VersionContext} was initialised. Observed behaviour is the same as executing query on top data snapshot for + * that version. + */ +public class SnapshotExecutionEngine extends ExecutionEngine +{ + private final int maxQueryExecutionAttempts; + + SnapshotExecutionEngine( GraphDatabaseQueryService queryService, Config config, LogProvider logProvider, + CompatibilityFactory compatibilityFactory ) + { + super( queryService, logProvider, compatibilityFactory ); + this.maxQueryExecutionAttempts = config.get( GraphDatabaseSettings.snapshot_query_retries ); + } + + @Override + public Result executeQuery( String query, Map parameters, TransactionalContext context ) + throws QueryExecutionKernelException + { + return executeWithRetries( query, parameters, context, super::executeQuery ); + } + + @Override + public Result profileQuery( String query, Map parameters, TransactionalContext context ) + throws QueryExecutionKernelException + { + return executeWithRetries( query, parameters, context, super::profileQuery ); + } + + protected Result executeWithRetries( String query, Map parameters, TransactionalContext context, + QueryExecutor executor ) throws QueryExecutionKernelException + { + VersionContext versionContext = getCursorContext( context ); + EagerResult eagerResult; + int attempt = 0; + boolean dirtySnapshot; + do + { + if ( attempt == maxQueryExecutionAttempts ) + { + return throwQueryExecutionException( + "Unable to get clean data snapshot for query '%s' after %d attempts.", query, attempt ); + } + attempt++; + versionContext.initRead(); + Result result = executor.execute( query, parameters, context ); + eagerResult = new EagerResult( result ); + eagerResult.consume(); + dirtySnapshot = versionContext.isDirty(); + if ( dirtySnapshot && result.getQueryStatistics().containsUpdates() ) + { + return throwQueryExecutionException( + "Unable to get clean data snapshot for query '%s' that perform updates.", query, attempt ); + } + } + while ( dirtySnapshot ); + return eagerResult; + } + + private Result throwQueryExecutionException( String message, Object... parameters ) throws + QueryExecutionKernelException + { + throw new QueryExecutionKernelException( new UnstableSnapshotException( message, parameters ) ); + } + + private static VersionContext getCursorContext( TransactionalContext context ) + { + return ((KernelStatement) context.statement()).getVersionContext(); + } + + @FunctionalInterface + protected interface QueryExecutor + { + Result execute( String query, Map parameters, TransactionalContext context ) throws QueryExecutionKernelException; + } + +} diff --git a/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/UnstableSnapshotException.java b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/UnstableSnapshotException.java new file mode 100644 index 0000000000000..2a881f69328e8 --- /dev/null +++ b/community/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/UnstableSnapshotException.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.javacompat; + +import org.neo4j.kernel.api.exceptions.KernelException; +import org.neo4j.kernel.api.exceptions.Status; + +class UnstableSnapshotException extends KernelException +{ + UnstableSnapshotException( String message, Object... parameters ) + { + super( Status.Transaction.Outdated, message, parameters ); + } +} diff --git a/community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/EagerResultITTest.java b/community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/EagerResultITTest.java new file mode 100644 index 0000000000000..f91967fff43a4 --- /dev/null +++ b/community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/EagerResultITTest.java @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.javacompat; + +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.LongSupplier; + +import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.QueryExecutionType; +import org.neo4j.graphdb.ResourceIterator; +import org.neo4j.graphdb.Result; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.factory.GraphDatabaseBuilder; +import org.neo4j.graphdb.factory.GraphDatabaseFactoryState; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.helpers.collection.Iterables; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; +import org.neo4j.kernel.GraphDatabaseDependencies; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.configuration.Settings; +import org.neo4j.kernel.impl.context.TransactionVersionContext; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; +import org.neo4j.kernel.impl.factory.CommunityEditionModule; +import org.neo4j.kernel.impl.factory.DatabaseInfo; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; +import org.neo4j.kernel.impl.factory.PlatformModule; +import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; +import org.neo4j.kernel.internal.GraphDatabaseAPI; +import org.neo4j.test.TestGraphDatabaseFactory; +import org.neo4j.test.rule.TestDirectory; + +import static org.hamcrest.Matchers.hasSize; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +public class EagerResultITTest +{ + @Rule + public final TestDirectory testDirectory = TestDirectory.testDirectory(); + private GraphDatabaseService database; + private TestTransactionVersionContextSupplier testContextSupplier; + private File storeDir; + private TestVersionContext testCursorContext; + + @Before + public void setUp() + { + storeDir = testDirectory.directory(); + testContextSupplier = new TestTransactionVersionContextSupplier(); + database = startRestartableDatabase(); + prepareData(); + TransactionIdStore transactionIdStore = getTransactionIdStore(); + testCursorContext = new TestVersionContext( transactionIdStore::getLastClosedTransactionId ); + testContextSupplier.setCursorContext( testCursorContext ); + } + + @After + public void tearDown() + { + if ( database != null ) + { + database.shutdown(); + } + } + + @Test + public void eagerResultContainsAllData() + { + Result result = database.execute( "MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + int rows = 0; + while ( result.hasNext() ) + { + result.next(); + rows++; + } + assertEquals( 2, rows ); + } + + @Test + public void eagerResultContainsExecutionType() + { + Result result = database.execute( "MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + assertEquals( QueryExecutionType.query( QueryExecutionType.QueryType.READ_ONLY ), result.getQueryExecutionType() ); + } + + @Test + public void eagerResultContainsColumns() + { + Result result = database.execute( "MATCH (n) RETURN n.c as a, count(n) as b" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + assertEquals( Arrays.asList("a", "b"), result.columns() ); + } + + @Test + public void useColumnAsOnEagerResult() + { + Result result = database.execute( "MATCH (n) RETURN n.c as c, n.b as b" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + ResourceIterator cValues = result.columnAs( "c" ); + int rows = 0; + while ( cValues.hasNext() ) + { + cValues.next(); + rows++; + } + assertEquals( 2, rows ); + } + + @Test + public void eagerResultHaveQueryStatistic() + { + Result result = database.execute( "MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + assertFalse( result.getQueryStatistics().containsUpdates() ); + } + + @Test + public void eagerResultHaveExecutionPlan() + { + Result result = database.execute( "profile MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + assertEquals( 2, result.getExecutionPlanDescription().getProfilerStatistics().getRows() ); + } + + @Test + public void eagerResultHaveNotifications() + { + Result result = database.execute( " CYPHER planner=rule MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + assertEquals( 1, Iterables.count( result.getNotifications() ) ); + } + + @Test + public void eagerResultToString() + { + Result result = database.execute( "MATCH (n) RETURN n.c, n.d" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + String resultString = result.resultAsString(); + assertTrue( resultString.contains( "n.c, n.d" ) ); + assertTrue( resultString.contains( "d, a" ) ); + assertTrue( resultString.contains( "y, k" ) ); + } + + @Test + public void eagerResultWriteAsStringToStream() + { + Result result = database.execute( "MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + assertEquals( result.resultAsString(), printToStream( result ) ); + } + + @Test + public void eagerResultVisit() throws Exception + { + Result result = database.execute( "MATCH (n) RETURN n.c" ); + List values = new ArrayList<>(); + result.accept( (Result.ResultVisitor) row -> + { + values.add( row.getString( "n.c" ) ); + return false; + } ); + assertThat( values, hasSize( 2 ) ); + assertThat( values, Matchers.containsInAnyOrder( "d", "y" ) ); + } + + private String printToStream( Result result ) + { + StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter( stringWriter ); + result.writeAsStringTo( printWriter ); + printWriter.flush(); + return stringWriter.toString(); + } + + private void prepareData() + { + Label label = Label.label( "label" ); + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode( label ); + node.setProperty( "c", "d" ); + node.setProperty( "d", "a" ); + transaction.success(); + } + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode( label ); + node.setProperty( "c", "y" ); + node.setProperty( "d", "k" ); + transaction.success(); + } + } + + private GraphDatabaseService startRestartableDatabase() + { + return new CustomGraphDatabaseFactory( new CustomFacadeFactory() ) + .newEmbeddedDatabaseBuilder( storeDir ) + .setConfig( GraphDatabaseSettings.snapshot_query, Settings.TRUE ) + .newGraphDatabase(); + } + + private TransactionIdStore getTransactionIdStore() + { + DependencyResolver dependencyResolver = ((GraphDatabaseAPI) database).getDependencyResolver(); + return dependencyResolver.resolveDependency( TransactionIdStore.class ); + } + + private class CustomGraphDatabaseFactory extends TestGraphDatabaseFactory + { + + private GraphDatabaseFacadeFactory customFacadeFactory; + + CustomGraphDatabaseFactory( GraphDatabaseFacadeFactory customFacadeFactory ) + { + this.customFacadeFactory = customFacadeFactory; + } + + @Override + protected GraphDatabaseBuilder.DatabaseCreator createDatabaseCreator( File storeDir, + GraphDatabaseFactoryState state ) + { + return new GraphDatabaseBuilder.DatabaseCreator() + { + @Override + public GraphDatabaseService newDatabase( Map config ) + { + return newDatabase( Config.embeddedDefaults( config ) ); + } + + @Override + public GraphDatabaseService newDatabase( Config config ) + { + return customFacadeFactory.newFacade( storeDir, config, + GraphDatabaseDependencies.newDependencies( state.databaseDependencies() ) ); + } + }; + } + } + + private class CustomFacadeFactory extends GraphDatabaseFacadeFactory + { + + CustomFacadeFactory() + { + super( DatabaseInfo.COMMUNITY, CommunityEditionModule::new ); + } + + @Override + protected PlatformModule createPlatform( File storeDir, Config config, Dependencies dependencies, + GraphDatabaseFacade graphDatabaseFacade ) + { + return new PlatformModule( storeDir, config, databaseInfo, dependencies, graphDatabaseFacade ) + { + @Override + protected VersionContextSupplier createCursorContextSupplier( Config config ) + { + return testContextSupplier != null ? testContextSupplier : super.createCursorContextSupplier(config); + } + }; + } + } + + private class TestVersionContext extends TransactionVersionContext + { + + private boolean useCorrectLastCommittedTxId = false; + private int additionalAttempts; + + TestVersionContext( LongSupplier transactionIdSupplier ) + { + super( transactionIdSupplier ); + } + + @Override + public long lastClosedTransactionId() + { + return useCorrectLastCommittedTxId ? TransactionIdStore.BASE_TX_ID : super.lastClosedTransactionId(); + } + + @Override + public void markAsDirty() + { + super.markAsDirty(); + useCorrectLastCommittedTxId = true; + } + + @Override + public boolean isDirty() + { + additionalAttempts++; + return super.isDirty(); + } + + int getAdditionalAttempts() + { + return additionalAttempts; + } + } + + private class TestTransactionVersionContextSupplier extends TransactionVersionContextSupplier + { + void setCursorContext( VersionContext versionContext ) + { + this.cursorContext.set( versionContext ); + } + } +} diff --git a/community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngineTest.java b/community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngineTest.java new file mode 100644 index 0000000000000..005fb9f71daae --- /dev/null +++ b/community/cypher/cypher/src/test/java/org/neo4j/cypher/internal/javacompat/SnapshotExecutionEngineTest.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.cypher.internal.javacompat; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.util.Collections; +import java.util.Map; + +import org.neo4j.cypher.internal.CompatibilityFactory; +import org.neo4j.cypher.javacompat.internal.GraphDatabaseCypherService; +import org.neo4j.graphdb.Result; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.kernel.GraphDatabaseQueryService; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.api.KernelStatement; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.test.rule.DatabaseRule; +import org.neo4j.test.rule.ImpermanentDatabaseRule; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyMap; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SnapshotExecutionEngineTest +{ + @Rule + public final DatabaseRule database = new ImpermanentDatabaseRule(); + + private CompatibilityFactory compatibilityFactory; + private TestSnapshotExecutionEngine executionEngine; + private VersionContext versionContext; + private SnapshotExecutionEngine.QueryExecutor executor; + private TransactionalContext transactionalContext; + private final Config config = Config.defaults(); + + @Before + public void setUp() throws Exception + { + GraphDatabaseQueryService cypherService = new GraphDatabaseCypherService( this.database.getGraphDatabaseAPI() ); + + compatibilityFactory = mock( CompatibilityFactory.class ); + transactionalContext = mock( TransactionalContext.class ); + KernelStatement kernelStatement = mock( KernelStatement.class ); + executor = mock( SnapshotExecutionEngine.QueryExecutor.class ); + versionContext = mock( VersionContext.class ); + + executionEngine = createExecutionEngine(cypherService); + when( kernelStatement.getVersionContext() ).thenReturn( versionContext ); + when( transactionalContext.statement() ).thenReturn( kernelStatement ); + Result result = mock( Result.class ); + when( result.getQueryStatistics() ).thenReturn( mock( QueryStatistics.class ) ); + when( executor.execute( any(), anyMap(), any() ) ).thenReturn( result ); + } + + @Test + public void executeQueryWithoutRetries() throws QueryExecutionKernelException + { + executionEngine.executeWithRetries( "query", Collections.emptyMap(), transactionalContext, executor ); + + verify( executor, times( 1 ) ).execute( any(), anyMap(), any() ); + verify( versionContext, times( 1 ) ).initRead(); + } + + @Test + public void executeQueryAfterSeveralRetries() throws QueryExecutionKernelException + { + when( versionContext.isDirty() ).thenReturn( true, true, false ); + + executionEngine.executeWithRetries( "query", Collections.emptyMap(), transactionalContext, executor ); + + verify( executor, times( 3 ) ).execute( any(), anyMap(), any() ); + verify( versionContext, times( 3 ) ).initRead(); + } + + @Test + public void failQueryAfterMaxRetriesReached() throws QueryExecutionKernelException + { + when( versionContext.isDirty() ).thenReturn( true ); + + try + { + executionEngine.executeWithRetries( "query", Collections.emptyMap(), transactionalContext, executor ); + } + catch ( QueryExecutionKernelException e ) + { + assertEquals( "Unable to get clean data snapshot for query 'query' after 5 attempts.", e.getMessage() ); + } + + verify( executor, times( 5 ) ).execute( any(), anyMap(), any() ); + verify( versionContext, times( 5 ) ).initRead(); + } + + private class TestSnapshotExecutionEngine extends SnapshotExecutionEngine + { + + TestSnapshotExecutionEngine( GraphDatabaseQueryService queryService, Config config, LogProvider logProvider, + CompatibilityFactory compatibilityFactory ) + { + super( queryService, config, logProvider, compatibilityFactory ); + } + + @Override + public Result executeWithRetries( String query, Map parameters, TransactionalContext context, + QueryExecutor executor ) throws QueryExecutionKernelException + { + return super.executeWithRetries( query, parameters, context, executor ); + } + } + + private TestSnapshotExecutionEngine createExecutionEngine( GraphDatabaseQueryService cypherService ) + { + return new TestSnapshotExecutionEngine( cypherService, config, NullLogProvider.getInstance(), + compatibilityFactory ); + } +} diff --git a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/v3_2/TransactionBoundQueryContextTest.scala b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/v3_2/TransactionBoundQueryContextTest.scala index c87d4e5fed2e5..624791708f5f3 100644 --- a/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/v3_2/TransactionBoundQueryContextTest.scala +++ b/community/cypher/cypher/src/test/scala/org/neo4j/cypher/internal/spi/v3_2/TransactionBoundQueryContextTest.scala @@ -32,6 +32,7 @@ import org.neo4j.graphdb._ import org.neo4j.graphdb.config.Setting import org.neo4j.graphdb.factory.GraphDatabaseSettings import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContext import org.neo4j.kernel.api._ import org.neo4j.kernel.api.security.SecurityContext.AUTH_DISABLED import org.neo4j.kernel.api.security.{AnonymousContext, SecurityContext} @@ -63,8 +64,9 @@ class TransactionBoundQueryContextTest extends CypherFunSuite { when(kernelTransaction.securityContext()).thenReturn(AUTH_DISABLED) val storeStatement = mock[StorageStatement] val operations = mock[StatementOperationParts](RETURNS_DEEP_STUBS) - statement = new KernelStatement(kernelTransaction, null, storeStatement, new Procedures(), new CanWrite(), LockTracer.NONE) - statement.initialize(null, operations, PageCursorTracerSupplier.NULL.get()) + statement = new KernelStatement(kernelTransaction, null, storeStatement, new Procedures(), new CanWrite(), + LockTracer.NONE ) + statement.initialize(null, operations, PageCursorTracerSupplier.NULL.get(), EmptyVersionContext.INSTANCE ) statement.acquire() } diff --git a/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java b/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java index 069b206d96254..ea911a4dba9a2 100644 --- a/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java +++ b/community/index/src/main/java/org/neo4j/index/internal/gbptree/GBPTree.java @@ -397,7 +397,7 @@ public GBPTree( PageCache pageCache, File indexFile, Layout layout, i { close(); } - catch ( IOException e ) + catch ( Throwable e ) { t.addSuppressed( e ); } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/CursorPool.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/CursorPool.java index 0c6bb8e208d9f..fe1eba4f76c22 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/CursorPool.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/CursorPool.java @@ -22,6 +22,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; final class CursorPool extends ThreadLocal { @@ -29,6 +30,7 @@ final class CursorPool extends ThreadLocal private final long victimPage; private final PageCursorTracerSupplier pageCursorTracerSupplier; private PageCacheTracer pageCacheTracer; + private final VersionContextSupplier versionContextSupplier; /** * Cursor pool construction @@ -36,13 +38,16 @@ final class CursorPool extends ThreadLocal * @param pageCursorTracerSupplier supplier of thread local (transaction local) page cursor tracers that will * provide thread local page cache statistics * @param pageCacheTracer global page cache tracer + * @param versionContextSupplier version context supplier */ - CursorPool( MuninnPagedFile pagedFile, PageCursorTracerSupplier pageCursorTracerSupplier, PageCacheTracer pageCacheTracer ) + CursorPool( MuninnPagedFile pagedFile, PageCursorTracerSupplier pageCursorTracerSupplier, + PageCacheTracer pageCacheTracer, VersionContextSupplier versionContextSupplier ) { this.pagedFile = pagedFile; this.victimPage = pagedFile.pageCache.victimPage; this.pageCursorTracerSupplier = pageCursorTracerSupplier; this.pageCacheTracer = pageCacheTracer; + this.versionContextSupplier = versionContextSupplier; } @Override @@ -69,7 +74,8 @@ MuninnReadPageCursor takeReadCursor( long pageId, int pf_flags ) private MuninnReadPageCursor createReadCursor( CursorSets cursorSets ) { - MuninnReadPageCursor cursor = new MuninnReadPageCursor( cursorSets, victimPage, getPageCursorTracer() ); + MuninnReadPageCursor cursor = new MuninnReadPageCursor( cursorSets, victimPage, getPageCursorTracer(), versionContextSupplier + .getVersionContext() ); cursor.initialiseFile( pagedFile ); return cursor; } @@ -92,7 +98,8 @@ MuninnWritePageCursor takeWriteCursor( long pageId, int pf_flags ) private MuninnWritePageCursor createWriteCursor( CursorSets cursorSets ) { - MuninnWritePageCursor cursor = new MuninnWritePageCursor( cursorSets, victimPage, getPageCursorTracer() ); + MuninnWritePageCursor cursor = new MuninnWritePageCursor( cursorSets, victimPage, getPageCursorTracer(), versionContextSupplier + .getVersionContext() ); cursor.initialiseFile( pagedFile ); return cursor; } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPage.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPage.java index b5e1399b75fbb..d2fe0eae6ba3a 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPage.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPage.java @@ -35,7 +35,10 @@ final class MuninnPage extends SequenceLock implements Page { + private static final int UNBOUND_LAST_MODIFIED_TX_ID = -1; + private static final long usageStampOffset = UnsafeUtil.getFieldOffset( MuninnPage.class, "usageStamp" ); + private static final long lastModifiedTxIdOffset = UnsafeUtil.getFieldOffset( MuninnPage.class, "lastModifiedTxId" ); // The sign bit is used as a dirty flag for the page. // The other 7 bits are used as an exponent for computing the cache page size (as a power of two). @@ -47,6 +50,10 @@ final class MuninnPage extends SequenceLock implements Page private long pointer; + // max transaction id that updated this page + @SuppressWarnings( "unused" ) + private volatile long lastModifiedTxId = UNBOUND_LAST_MODIFIED_TX_ID; + // Optimistically incremented; occasionally truncated to a max of 4. // accessed through unsafe @SuppressWarnings( "unused" ) @@ -84,6 +91,32 @@ public long address() return pointer; } + public long getLastModifiedTxId() + { + return lastModifiedTxId; + } + + public long resetLastModifiedTxId() + { + long value = lastModifiedTxId; + lastModifiedTxId = UNBOUND_LAST_MODIFIED_TX_ID; + return value; + } + + public void setLastModifiedTxId( long modifierTxId ) + { + long pageLatestModifier; + do + { + pageLatestModifier = this.lastModifiedTxId; + if ( pageLatestModifier >= modifierTxId ) + { + return; + } + } + while ( !UnsafeUtil.compareAndSwapLong( this, lastModifiedTxIdOffset, pageLatestModifier, modifierTxId ) ); + } + /** * NOTE: Should be called under a page lock. */ @@ -208,7 +241,6 @@ public void evict( EvictionEvent evictionEvent ) throws IOException flush( evictionEvent.flushEventOpportunity() ); this.filePageId = PageCursor.UNBOUND_PAGE_ID; - this.swapper = null; if ( swapper != null ) { diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java index 9ab5c968b2f29..5f5b2af79e972 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCache.java @@ -48,6 +48,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.PageFaultEvent; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.unsafe.impl.internal.dragons.MemoryManager; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; @@ -196,6 +197,7 @@ public class MuninnPageCache implements PageCache // 'true' (the default) if we should print any exceptions we get when unmapping a file. private boolean printExceptionsOnClose; private PageCursorTracerSupplier pageCursorTracerSupplier; + private VersionContextSupplier versionContextSupplier; /** * Create page cache @@ -205,9 +207,11 @@ public class MuninnPageCache implements PageCache * @param pageCacheTracer global page cache tracer * @param pageCursorTracerSupplier supplier of thread local (transaction local) page cursor tracer that will provide * thread local page cache statistics + * @param versionContextSupplier supplier of thread local (transaction local) version context that will provide + * access to thread local version context */ public MuninnPageCache( PageSwapperFactory swapperFactory, int maxPages, int cachePageSize, PageCacheTracer pageCacheTracer, - PageCursorTracerSupplier pageCursorTracerSupplier ) + PageCursorTracerSupplier pageCursorTracerSupplier, VersionContextSupplier versionContextSupplier ) { verifyHacks(); verifyCachePageSizeIsPowerOfTwo( cachePageSize ); @@ -219,6 +223,7 @@ public MuninnPageCache( PageSwapperFactory swapperFactory, int maxPages, int cac this.keepFree = Math.min( pagesToKeepFree, maxPages / 2 ); this.pageCacheTracer = pageCacheTracer; this.pageCursorTracerSupplier = pageCursorTracerSupplier; + this.versionContextSupplier = versionContextSupplier; this.pages = new MuninnPage[maxPages]; this.printExceptionsOnClose = true; @@ -363,7 +368,7 @@ else if ( !ignoredOpenOptions.contains( option ) ) file, this, filePageSize, - swapperFactory, pageCacheTracer, pageCursorTracerSupplier, + swapperFactory, pageCacheTracer, pageCursorTracerSupplier, versionContextSupplier, createIfNotExists, truncateExisting ); pagedFile.incrementRefCount(); @@ -991,7 +996,6 @@ int evictPages( int pageCountToEvict, int clockArm, EvictionRunEvent evictionRun */ private boolean evictPage( MuninnPage page, EvictionEvent evictionEvent ) { - //noinspection TryWithIdenticalCatches - this warning is a false positive; bug in Intellij inspection try { page.evict( evictionEvent ); diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCursor.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCursor.java index 410628a6accfd..fd8ec59cfb344 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCursor.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCursor.java @@ -30,6 +30,7 @@ import org.neo4j.io.pagecache.tracing.PageFaultEvent; import org.neo4j.io.pagecache.tracing.PinEvent; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_WRITE_LOCK; @@ -65,6 +66,7 @@ abstract class MuninnPageCursor extends PageCursor private long pointer; private int pageSize; private int filePageSize; + protected VersionContext versionContext; private int offset; private boolean outOfBounds; private boolean isLinkedCursor; @@ -73,11 +75,12 @@ abstract class MuninnPageCursor extends PageCursor // offending code. private Object cursorException; - MuninnPageCursor( long victimPage, PageCursorTracer tracer ) + MuninnPageCursor( long victimPage, PageCursorTracer tracer, VersionContext versionContext ) { this.victimPage = victimPage; this.pointer = victimPage; this.tracer = tracer; + this.versionContext = versionContext; } final void initialiseFile( MuninnPagedFile pagedFile ) diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageEvictionCallback.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageEvictionCallback.java index b069efa575700..56b17075bb18d 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageEvictionCallback.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageEvictionCallback.java @@ -38,5 +38,6 @@ public void onEvict( long filePageId, Page page ) assert removed == page : "Removed unexpected page when cleaning up translation table for filePageId " + filePageId + ". " + "Evicted " + page + " but removed " + removed + " from the translation table."; + file.setHighestEvictedTransactionId( removed.resetLastModifiedTxId() ); } } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPagedFile.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPagedFile.java index dbcdc939e0636..0af926b77a97a 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPagedFile.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnPagedFile.java @@ -39,6 +39,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.PageFaultEvent; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.unsafe.impl.internal.dragons.UnsafeUtil; final class MuninnPagedFile implements PagedFile, Flushable @@ -74,6 +75,10 @@ final class MuninnPagedFile implements PagedFile, Flushable // Used to trace the causes of any exceptions from getLastPageId. private volatile Exception closeStackTrace; + // max modifier transaction id among evicted pages for this file + private static final long evictedTransactionIdOffset = UnsafeUtil.getFieldOffset( MuninnPagedFile.class, "highestEvictedTransactionId" ); + private volatile long highestEvictedTransactionId; + /** * The header state includes both the reference count of the PagedFile – 15 bits – and the ID of the last page in * the file – 48 bits, plus an empty file marker bit. Because our pages are usually 2^13 bytes, this means that we @@ -100,23 +105,19 @@ final class MuninnPagedFile implements PagedFile, Flushable * @param pageCacheTracer global page cache tracer * @param pageCursorTracerSupplier supplier of thread local (transaction local) page cursor tracer that will provide * thread local page cache statistics + * @param versionContextSupplier supplier of thread local (transaction local) version context that will provide + * access to thread local version context * @param createIfNotExists should create file if it does not exists * @param truncateExisting should truncate file if it exists * @throws IOException */ - MuninnPagedFile( - File file, - MuninnPageCache pageCache, - int filePageSize, - PageSwapperFactory swapperFactory, - PageCacheTracer pageCacheTracer, - PageCursorTracerSupplier pageCursorTracerSupplier, - boolean createIfNotExists, - boolean truncateExisting ) throws IOException + MuninnPagedFile( File file, MuninnPageCache pageCache, int filePageSize, PageSwapperFactory swapperFactory, + PageCacheTracer pageCacheTracer, PageCursorTracerSupplier pageCursorTracerSupplier, + VersionContextSupplier versionContextSupplier, boolean createIfNotExists, boolean truncateExisting ) throws IOException { this.pageCache = pageCache; this.filePageSize = filePageSize; - this.cursorPool = new CursorPool( this, pageCursorTracerSupplier, pageCacheTracer ); + this.cursorPool = new CursorPool( this, pageCursorTracerSupplier, pageCacheTracer, versionContextSupplier ); this.pageCacheTracer = pageCacheTracer; // The translation table is an array of arrays of references to either null, MuninnPage objects, or Latch @@ -609,6 +610,25 @@ MuninnPage evictPage( long filePageId ) return (MuninnPage) element; } + void setHighestEvictedTransactionId( long modifiedTransactionId ) + { + long highestModifier; + do + { + highestModifier = this.highestEvictedTransactionId; + if ( highestModifier >= modifiedTransactionId ) + { + return; + } + } + while ( !UnsafeUtil.compareAndSwapLong( this, evictedTransactionIdOffset, highestModifier, modifiedTransactionId ) ); + } + + long getHighestEvictedTransactionId() + { + return highestEvictedTransactionId; + } + /** * Expand the translation table such that it can include at least the given chunkId. * @param maxChunkId The new translation table must be big enough to include at least this chunkId. diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnReadPageCursor.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnReadPageCursor.java index 280aaf46f0d9c..3cd6bcd3853af 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnReadPageCursor.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnReadPageCursor.java @@ -23,6 +23,7 @@ import org.neo4j.io.pagecache.PageSwapper; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; final class MuninnReadPageCursor extends MuninnPageCursor { @@ -30,9 +31,10 @@ final class MuninnReadPageCursor extends MuninnPageCursor private long lockStamp; MuninnReadPageCursor nextCursor; - MuninnReadPageCursor( CursorPool.CursorSets cursorSets, long victimPage, PageCursorTracer pageCursorTracer ) + MuninnReadPageCursor( CursorPool.CursorSets cursorSets, long victimPage, PageCursorTracer pageCursorTracer, + VersionContext versionContext ) { - super( victimPage, pageCursorTracer ); + super( victimPage, pageCursorTracer, versionContext ); this.cursorSets = cursorSets; } @@ -60,10 +62,24 @@ public boolean next() throws IOException } pin( nextPageId, false ); currentPageId = nextPageId; + verifyContext(); nextPageId++; return true; } + private void verifyContext() + { + if ( versionContext.lastClosedTransactionId() == Long.MAX_VALUE ) + { + return; + } + if ( page.getLastModifiedTxId() > versionContext.lastClosedTransactionId() || + pagedFile.getHighestEvictedTransactionId() > versionContext.lastClosedTransactionId() ) + { + versionContext.markAsDirty(); + } + } + @Override protected boolean tryLockPage( MuninnPage page ) { diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnWritePageCursor.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnWritePageCursor.java index 74f05d4cd9244..edb0b16dd0df6 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnWritePageCursor.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/MuninnWritePageCursor.java @@ -24,15 +24,17 @@ import org.neo4j.io.pagecache.PageSwapper; import org.neo4j.io.pagecache.PagedFile; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; final class MuninnWritePageCursor extends MuninnPageCursor { private final CursorPool.CursorSets cursorSets; MuninnWritePageCursor nextCursor; - MuninnWritePageCursor( CursorPool.CursorSets cursorSets, long victimPage, PageCursorTracer pageCursorTracer ) + MuninnWritePageCursor( CursorPool.CursorSets cursorSets, long victimPage, PageCursorTracer pageCursorTracer, + VersionContext versionContext ) { - super( victimPage, pageCursorTracer ); + super( victimPage, pageCursorTracer, versionContext ); this.cursorSets = cursorSets; } @@ -101,6 +103,7 @@ protected void pinCursorToPage( MuninnPage page, long filePageId, PageSwapper sw // be closed and the page lock will be released. assertPagedFileStillMappedAndGetIdOfLastPage(); page.incrementUsage(); + page.setLastModifiedTxId( versionContext.committingTransactionId() ); } @Override diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/StandalonePageCacheFactory.java b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/StandalonePageCacheFactory.java index a6a089e8cc634..e028166779cd0 100644 --- a/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/StandalonePageCacheFactory.java +++ b/community/io/src/main/java/org/neo4j/io/pagecache/impl/muninn/StandalonePageCacheFactory.java @@ -26,6 +26,8 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; /* * This class is an helper to allow to construct properly a page cache in the few places we need it without all @@ -42,16 +44,18 @@ private StandalonePageCacheFactory() public static PageCache createPageCache( FileSystemAbstraction fileSystem ) { - return createPageCache( fileSystem, null, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE ); + return createPageCache( fileSystem, null, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE, + EmptyVersionContextSupplier.INSTANCE ); } public static PageCache createPageCache( FileSystemAbstraction fileSystem, Integer pageSize ) { - return createPageCache( fileSystem, pageSize, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE ); + return createPageCache( fileSystem, pageSize, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE, + EmptyVersionContextSupplier.INSTANCE ); } public static PageCache createPageCache( FileSystemAbstraction fileSystem, Integer pageSize, - PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier ) + PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier, VersionContextSupplier versionContextSupplier ) { SingleFilePageSwapperFactory factory = new SingleFilePageSwapperFactory(); factory.setFileSystemAbstraction( fileSystem ); @@ -59,6 +63,7 @@ public static PageCache createPageCache( FileSystemAbstraction fileSystem, Integ int cachePageSize = pageSize != null ? pageSize : factory.getCachePageSizeHint(); long pageCacheMemory = ByteUnit.mebiBytes( 8 ); long pageCount = pageCacheMemory / cachePageSize; - return new MuninnPageCache( factory, (int) pageCount, cachePageSize, tracer, cursorTracerSupplier ); + return new MuninnPageCache( factory, (int) pageCount, cachePageSize, tracer, cursorTracerSupplier, + versionContextSupplier ); } } diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContext.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContext.java new file mode 100644 index 0000000000000..fc3077d32e267 --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContext.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.tracing.cursor.context; + +/** + * {@link VersionContext} that does not perform any kind of version tracking for cases when its not required. + * @see VersionContext + */ +public class EmptyVersionContext implements VersionContext +{ + public static final VersionContext INSTANCE = new EmptyVersionContext(); + + private EmptyVersionContext() + { + } + + @Override + public void initRead() + { + } + + @Override + public void initWrite( long committingTransactionId ) + { + + } + + @Override + public long committingTransactionId() + { + return 0; + } + + @Override + public long lastClosedTransactionId() + { + return Long.MAX_VALUE; + } + + @Override + public void markAsDirty() + { + + } + + @Override + public boolean isDirty() + { + return false; + } + + @Override + public void clear() + { + } +} diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContextSupplier.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContextSupplier.java new file mode 100644 index 0000000000000..8ebb629927b82 --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/EmptyVersionContextSupplier.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.tracing.cursor.context; + +import java.util.function.LongSupplier; + +/** + * {@link VersionContextSupplier} version that supply same {@link EmptyVersionContext} each time for cases + * where version context mechanics is not required + */ +public class EmptyVersionContextSupplier implements VersionContextSupplier +{ + public static final VersionContextSupplier INSTANCE = new EmptyVersionContextSupplier(); + + private EmptyVersionContextSupplier() + { + } + + @Override + public void init( LongSupplier lastClosedTransactionIdSupplier ) + { + } + + @Override + public VersionContext getVersionContext() + { + return EmptyVersionContext.INSTANCE; + } +} diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContext.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContext.java new file mode 100644 index 0000000000000..5389e923e5021 --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContext.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.tracing.cursor.context; + +/** + * Context that contains state of ongoing versioned data read or write. + * + * Read context performing data read with a version that is equal to last closed transaction. + * Write context data modification with have version that is equal to transaction that is currently committing. + * + * As soon reader that associated with a context will observe data version that it should not see, context will be + * marked as dirty. + */ +public interface VersionContext +{ + /** + * Initialise read context with latest closed transaction id as it current version. + */ + void initRead(); + + /** + * Initialise write context with committingTxId as modification version. + * @param committingTxId currently committing transaction id + */ + void initWrite( long committingTxId ); + + /** + * Context currently committing transaction id + * @return committing transaction id + */ + long committingTransactionId(); + + /** + * Last closed transaction id that read context was initialised with + * @return last closed transaction id + */ + long lastClosedTransactionId(); + + /** + * Mark current context as dirty + */ + void markAsDirty(); + + /** + * Check whenever current context is dirty + * @return true if context is dirty, false otherwise + */ + boolean isDirty(); + + /** + * Clear transaction ids that read/write context was initialised with + */ + void clear(); + +} diff --git a/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContextSupplier.java b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContextSupplier.java new file mode 100644 index 0000000000000..be87b79a30e3e --- /dev/null +++ b/community/io/src/main/java/org/neo4j/io/pagecache/tracing/cursor/context/VersionContextSupplier.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.tracing.cursor.context; + +import java.util.function.LongSupplier; + +/** + * Supplier to create {@link VersionContext} used during version data read and write operations + */ +public interface VersionContextSupplier +{ + /** + * Initialise current supplier with provider of last closed transaction ids + * for future version context to be able to get version ids + * @param lastClosedTransactionIdSupplier closed transaction id supplier. + */ + void init( LongSupplier lastClosedTransactionIdSupplier ); + + /** + * Provide version context + * @return instance of version context + */ + VersionContext getVersionContext(); + +} diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java index 07415af7442c6..38559edcc8c38 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/PageCacheTestSupport.java @@ -42,8 +42,10 @@ import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.pagecache.impl.SingleFilePageSwapperFactory; import org.neo4j.io.pagecache.tracing.PageCacheTracer; -import org.neo4j.test.rule.RepeatRule; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; +import org.neo4j.test.rule.RepeatRule; import static org.junit.Assert.assertThat; import static org.neo4j.test.matchers.ByteArrayMatcher.byteArray; @@ -110,15 +112,31 @@ public void tearDown() throws Exception protected final T createPageCache( PageSwapperFactory swapperFactory, int maxPages, int pageSize, PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier ) { - return fixture.createPageCache( swapperFactory, maxPages, pageSize, tracer, cursorTracerSupplier ); + return createPageCache( swapperFactory, maxPages, pageSize, tracer, cursorTracerSupplier, + EmptyVersionContextSupplier.INSTANCE ); + } + + protected final T createPageCache( PageSwapperFactory swapperFactory, int maxPages, int pageSize, + PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier, VersionContextSupplier versionContextSupplier ) + { + return fixture.createPageCache( swapperFactory, maxPages, pageSize, tracer, cursorTracerSupplier, + versionContextSupplier ); } protected T createPageCache( FileSystemAbstraction fs, int maxPages, int pageSize, PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier ) + { + return createPageCache( fs, maxPages, pageSize, tracer, cursorTracerSupplier, + EmptyVersionContextSupplier.INSTANCE ); + } + + protected T createPageCache( FileSystemAbstraction fs, int maxPages, int pageSize, PageCacheTracer tracer, + PageCursorTracerSupplier cursorTracerSupplier, VersionContextSupplier versionContextSupplier ) { PageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory(); swapperFactory.setFileSystemAbstraction( fs ); - return createPageCache( swapperFactory, maxPages, pageSize, tracer, cursorTracerSupplier ); + return createPageCache( swapperFactory, maxPages, pageSize, tracer, cursorTracerSupplier, + versionContextSupplier ); } protected final T getPageCache( FileSystemAbstraction fs, int maxPages, int pageSize, PageCacheTracer tracer, @@ -318,7 +336,7 @@ protected void verifyRecordsInFile( ReadableByteChannel channel, int recordCount public abstract static class Fixture { public abstract T createPageCache( PageSwapperFactory swapperFactory, int maxPages, int pageSize, - PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier ); + PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier, VersionContextSupplier contextSupplier ); public abstract void tearDownPageCache( T pageCache ) throws IOException; diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java index 4e87d0f754c33..9a08bc60b7df6 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/impl/muninn/MuninnPageCacheFixture.java @@ -26,6 +26,7 @@ import org.neo4j.io.pagecache.PageSwapperFactory; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; public class MuninnPageCacheFixture extends PageCacheTestSupport.Fixture { @@ -33,9 +34,9 @@ public class MuninnPageCacheFixture extends PageCacheTestSupport.Fixture 0 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 7 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 1 ); + } + + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); + assertEquals( 7, ((MuninnPageCursor) cursor).page.getLastModifiedTxId() ); + assertEquals( 1, cursor.getLong() ); + } + } + + @Test + public void pareModificationTrackingNoticeWriteFromAnotherThread() throws Exception + { + TestVersionContext cursorContext = new TestVersionContext( () -> 0 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 7 ); + + Future future = executor.submit( () -> + { + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 1 ); + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + } ); + future.get(); + + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); + assertEquals( 7, ((MuninnPageCursor) cursor).page.getLastModifiedTxId() ); + assertEquals( 1, cursor.getLong() ); + } + } + + @Test + public void pageModificationTracksHighestModifierTransactionId() throws IOException + { + TestVersionContext cursorContext = new TestVersionContext( () -> 0 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 1 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 1 ); + } + cursorContext.initWrite( 12 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 2 ); + } + cursorContext.initWrite( 7 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 3 ); + } + + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); + assertEquals( 12, ((MuninnPageCursor) cursor).page.getLastModifiedTxId() ); + assertEquals( 3, cursor.getLong() ); + } + } + + @Test + public void markCursorContextAsDirtyWhenReadingDataFromMoreRecentTransactions() throws IOException + { + TestVersionContext cursorContext = new TestVersionContext( () -> 3 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 7 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 3 ); + } + + cursorContext.initRead(); + assertFalse( cursorContext.isDirty() ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); + assertEquals( 3, cursor.getLong() ); + assertTrue( cursorContext.isDirty() ); + } + } + + @Test + public void doNotMarkCursorContextAsDirtyWhenReadingDataFromOlderTransactions() throws IOException + { + TestVersionContext cursorContext = new TestVersionContext( () -> 23 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 17 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 3 ); + } + + cursorContext.initRead(); + assertFalse( cursorContext.isDirty() ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); + assertEquals( 3, cursor.getLong() ); + assertFalse( cursorContext.isDirty() ); + } + } + + @Test + public void markContextAsDirtyWhenAnyEvictedPageHaveModificationTransactionHigherThenReader() throws IOException + { + TestVersionContext cursorContext = new TestVersionContext( () -> 5 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 3 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 3 ); + } + + cursorContext.initWrite( 13 ); + try ( PageCursor cursor = pagedFile.io( 1, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 4 ); + } + pageCache.evictPages( 1, 1, EvictionRunEvent.NULL ); + + cursorContext.initRead(); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); + assertEquals( 3, cursor.getLong() ); + assertTrue( cursorContext.isDirty() ); + } + } + + @Test + public void doNotMarkContextAsDirtyWhenAnyEvictedPageHaveModificationTransactionLowerThenReader() throws IOException + { + TestVersionContext cursorContext = new TestVersionContext( () -> 15 ); + VersionContextSupplier versionContextSupplier = new ConfiguredVersionContextSupplier( cursorContext ); + MuninnPageCache pageCache = + createPageCache( fs, 2, 8, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, versionContextSupplier ); + + PagedFile pagedFile = pageCache.map( file( "a" ), 8 ); + cursorContext.initWrite( 3 ); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 3 ); + } + + cursorContext.initWrite( 13 ); + try ( PageCursor cursor = pagedFile.io( 1, PF_SHARED_WRITE_LOCK ) ) + { + assertTrue( cursor.next() ); + cursor.putLong( 4 ); + } + pageCache.evictPages( 1, 1, EvictionRunEvent.NULL ); + + cursorContext.initRead(); + try ( PageCursor cursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + assertTrue( cursor.next() ); + assertEquals( 3, cursor.getLong() ); + assertFalse( cursorContext.isDirty() ); + } + } + @Test public void closingTheCursorMustUnlockModifiedPage() throws Exception { @@ -349,4 +565,82 @@ public void mustThrowIfMappingFileWouldOverflowReferenceCount() throws Exception } } } + + private static class ConfiguredVersionContextSupplier implements VersionContextSupplier + { + + private final VersionContext versionContext; + + ConfiguredVersionContextSupplier( VersionContext versionContext ) + { + this.versionContext = versionContext; + } + + @Override + public void init( LongSupplier lastClosedTransactionIdSupplier ) + { + } + + @Override + public VersionContext getVersionContext() + { + return versionContext; + } + } + + private static class TestVersionContext implements VersionContext + { + + private final IntSupplier closedTxIdSupplier; + private long committingTxId; + private long lastClosedTxId; + private boolean dirty; + + TestVersionContext(IntSupplier closedTxIdSupplier) + { + this.closedTxIdSupplier = closedTxIdSupplier; + } + + @Override + public void initRead() + { + this.lastClosedTxId = closedTxIdSupplier.getAsInt(); + } + + @Override + public void initWrite( long committingTxId ) + { + this.committingTxId = committingTxId; + } + + @Override + public long committingTransactionId() + { + return committingTxId; + } + + @Override + public long lastClosedTransactionId() + { + return lastClosedTxId; + } + + @Override + public void markAsDirty() + { + dirty = true; + } + + @Override + public boolean isDirty() + { + return dirty; + } + + @Override + public void clear() + { + + } + } } diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/randomharness/RandomPageCacheTestHarness.java b/community/io/src/test/java/org/neo4j/io/pagecache/randomharness/RandomPageCacheTestHarness.java index 7306abf140a18..8814ef71da640 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/randomharness/RandomPageCacheTestHarness.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/randomharness/RandomPageCacheTestHarness.java @@ -46,6 +46,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.io.pagecache.tracing.linear.LinearHistoryPageCacheTracerTest; import org.neo4j.io.pagecache.tracing.linear.LinearTracers; @@ -384,7 +385,7 @@ private void runIteration( long timeout, TimeUnit unit ) throws Exception PageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory(); swapperFactory.setFileSystemAbstraction( fs ); MuninnPageCache cache = new MuninnPageCache( swapperFactory, cachePageCount, cachePageSize, tracer, - cursorTracerSupplier ); + cursorTracerSupplier, EmptyVersionContextSupplier.INSTANCE ); cache.setPrintExceptionsOnClose( false ); Map fileMap = new HashMap<>( files.length ); for ( int i = 0; i < Math.min( files.length, initialMappedFiles ); i++ ) diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/stress/PageCacheStressTest.java b/community/io/src/test/java/org/neo4j/io/pagecache/stress/PageCacheStressTest.java index 9e93329bd4fce..5ce051c93c409 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/stress/PageCacheStressTest.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/stress/PageCacheStressTest.java @@ -30,6 +30,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -90,7 +91,7 @@ public void run() throws Exception swapperFactory.setFileSystemAbstraction( fs ); try ( PageCache pageCacheUnderTest = new MuninnPageCache( swapperFactory, numberOfCachePages, cachePageSize, - tracer, pageCursorTracerSupplier ) ) + tracer, pageCursorTracerSupplier, EmptyVersionContextSupplier.INSTANCE ) ) { PageCacheStresser pageCacheStresser = new PageCacheStresser( numberOfPages, numberOfThreads, workingDirectory ); diff --git a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java index 4002f7fd0a2fb..99d2cc3db1547 100644 --- a/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java +++ b/community/io/src/test/java/org/neo4j/io/pagecache/tracing/recording/RecordingPageCursorTracer.java @@ -174,7 +174,7 @@ public void init( PageCacheTracer tracer ) @Override public void reportEvents() { - Objects.nonNull( tracer ); + Objects.requireNonNull( tracer ); tracer.pins( pins ); tracer.faults( faults ); } diff --git a/community/io/src/test/java/org/neo4j/test/rule/PageCacheRule.java b/community/io/src/test/java/org/neo4j/test/rule/PageCacheRule.java index d6c9da3f6e959..cdcc9089c26d8 100644 --- a/community/io/src/test/java/org/neo4j/test/rule/PageCacheRule.java +++ b/community/io/src/test/java/org/neo4j/test/rule/PageCacheRule.java @@ -33,6 +33,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; public class PageCacheRule extends ExternalResource { @@ -171,7 +172,8 @@ public PageCache getPageCache( FileSystemAbstraction fs, PageCacheConfig overrid pageCache = StandalonePageCacheFactory.createPageCache( fs, selectConfig( baseConfig.pageSize, overriddenConfig.pageSize, null ), selectConfig( baseConfig.tracer, overriddenConfig.tracer, PageCacheTracer.NULL ), - selectConfig( baseConfig.pageCursorTracerSupplier, overriddenConfig.pageCursorTracerSupplier, DefaultPageCursorTracerSupplier.INSTANCE )); + selectConfig( baseConfig.pageCursorTracerSupplier, overriddenConfig.pageCursorTracerSupplier, + DefaultPageCursorTracerSupplier.INSTANCE ), EmptyVersionContextSupplier.INSTANCE ); pageCachePostConstruct( overriddenConfig ); return pageCache; } diff --git a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java index 4670d54966a82..b56d2adb6b538 100644 --- a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java +++ b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java @@ -460,6 +460,17 @@ public class GraphDatabaseSettings implements LoadableConfig @Internal public static final Setting rebuild_idgenerators_fast = setting("unsupported.dbms.id_generator_fast_rebuild_enabled", BOOLEAN, TRUE ); + @Description( "Specifies if engine should run cypher query based on a snapshot of accessed data. " + + "Query will be restarted in case if concurrent modification of data will be detected." ) + @Internal + public static final Setting snapshot_query = setting( "unsupported.dbms.query.snapshot", BOOLEAN, FALSE ); + + @Description( "Specifies number or retries that query engine will do to execute query based on " + + "stable accessed data snapshot before giving up." ) + @Internal + public static final Setting snapshot_query_retries = setting( "unsupported.dbms.query.snapshot.retries", + INTEGER, "5", range( 1, Integer.MAX_VALUE ) ); + // Store memory settings @Description("Target size for pages of mapped memory. If set to 0, then a reasonable default is chosen, " + "depending on the storage device used.") diff --git a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java index 1f8d77b0ee7bc..e6754531e51c8 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java @@ -36,6 +36,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.KernelAPI; import org.neo4j.kernel.api.TokenNameLookup; import org.neo4j.kernel.api.exceptions.KernelException; @@ -273,6 +274,7 @@ boolean applicable( DiagnosticsPhase phase ) private File storeDir; private boolean readOnly; private final IdController idController; + private final VersionContextSupplier versionContextSupplier; private final AccessCapability accessCapability; private StorageEngine storageEngine; @@ -285,41 +287,20 @@ boolean applicable( DiagnosticsPhase phase ) * core API are now slowly accumulating in the Kernel implementation. Over time, these components should be * refactored into bigger components that wrap the very granular things we depend on here. */ - public NeoStoreDataSource( - File storeDir, - Config config, - IdGeneratorFactory idGeneratorFactory, - LogService logService, - JobScheduler scheduler, - TokenNameLookup tokenNameLookup, - DependencyResolver dependencyResolver, - PropertyKeyTokenHolder propertyKeyTokens, - LabelTokenHolder labelTokens, - RelationshipTypeTokenHolder relationshipTypeTokens, - StatementLocksFactory statementLocksFactory, - SchemaWriteGuard schemaWriteGuard, - TransactionEventHandlers transactionEventHandlers, - IndexingService.Monitor indexingServiceMonitor, - FileSystemAbstraction fs, - TransactionMonitor transactionMonitor, - DatabaseHealth databaseHealth, + public NeoStoreDataSource( File storeDir, Config config, IdGeneratorFactory idGeneratorFactory, + LogService logService, JobScheduler scheduler, TokenNameLookup tokenNameLookup, + DependencyResolver dependencyResolver, PropertyKeyTokenHolder propertyKeyTokens, + LabelTokenHolder labelTokens, RelationshipTypeTokenHolder relationshipTypeTokens, + StatementLocksFactory statementLocksFactory, SchemaWriteGuard schemaWriteGuard, + TransactionEventHandlers transactionEventHandlers, IndexingService.Monitor indexingServiceMonitor, + FileSystemAbstraction fs, TransactionMonitor transactionMonitor, DatabaseHealth databaseHealth, PhysicalLogFile.Monitor physicalLogMonitor, TransactionHeaderInformationFactory transactionHeaderInformationFactory, - StartupStatisticsProvider startupStatistics, - Guard guard, - CommitProcessFactory commitProcessFactory, - AutoIndexing autoIndexing, - PageCache pageCache, - ConstraintSemantics constraintSemantics, - Monitors monitors, - Tracers tracers, - Procedures procedures, - IOLimiter ioLimiter, - AvailabilityGuard availabilityGuard, - SystemNanoClock clock, - AccessCapability accessCapability, - StoreCopyCheckPointMutex storeCopyCheckPointMutex, - IdController idController ) + StartupStatisticsProvider startupStatistics, Guard guard, CommitProcessFactory commitProcessFactory, + AutoIndexing autoIndexing, PageCache pageCache, ConstraintSemantics constraintSemantics, Monitors monitors, + Tracers tracers, Procedures procedures, IOLimiter ioLimiter, AvailabilityGuard availabilityGuard, + SystemNanoClock clock, AccessCapability accessCapability, StoreCopyCheckPointMutex storeCopyCheckPointMutex, + IdController idController, VersionContextSupplier versionContextSupplier ) { this.storeDir = storeDir; this.config = config; @@ -356,6 +337,7 @@ public NeoStoreDataSource( readOnly = config.get( Configuration.read_only ); this.idController = idController; + this.versionContextSupplier = versionContextSupplier; msgLog = logProvider.getLog( getClass() ); this.lockService = new ReentrantLockService(); this.legacyIndexProviderLookup = new LegacyIndexProviderLookup() @@ -433,12 +415,16 @@ public void start() throws IOException storageEngine = buildStorageEngine( propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, legacyIndexProviderLookup, - indexConfigStore, updateableSchemaState::clear, legacyIndexTransactionOrdering ); + indexConfigStore, updateableSchemaState::clear, legacyIndexTransactionOrdering, + versionContextSupplier ); LogEntryReader logEntryReader = new VersionAwareLogEntryReader<>( storageEngine.commandReaderFactory(), STRICT ); TransactionIdStore transactionIdStore = dependencies.resolveDependency( TransactionIdStore.class ); + + versionContextSupplier.init( transactionIdStore::getLastClosedTransactionId ); + LogVersionRepository logVersionRepository = dependencies.resolveDependency( LogVersionRepository.class ); NeoStoreTransactionLogModule transactionLogModule = buildTransactionLogs( storeDir, config, logProvider, scheduler, fs, @@ -565,14 +551,15 @@ private StorageEngine buildStorageEngine( PropertyKeyTokenHolder propertyKeyTokenHolder, LabelTokenHolder labelTokens, RelationshipTypeTokenHolder relationshipTypeTokens, LegacyIndexProviderLookup legacyIndexProviderLookup, IndexConfigStore indexConfigStore, - Runnable schemaStateChangeCallback, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering ) + Runnable schemaStateChangeCallback, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering, + VersionContextSupplier versionContextSupplier ) { RecordStorageEngine storageEngine = new RecordStorageEngine( storeDir, config, pageCache, fs, logProvider, propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, schemaStateChangeCallback, constraintSemantics, scheduler, tokenNameLookup, lockService, schemaIndexProvider, indexingServiceMonitor, databaseHealth, labelScanStoreProvider, legacyIndexProviderLookup, indexConfigStore, legacyIndexTransactionOrdering, idGeneratorFactory, - idController ); + idController, versionContextSupplier ); // We pretend that the storage engine abstract hides all details within it. Whereas that's mostly // true it's not entirely true for the time being. As long as we need this call below, which @@ -725,7 +712,8 @@ private NeoStoreKernelModule buildKernel( TransactionAppender appender, KernelTransactions kernelTransactions = life.add( new KernelTransactions( statementLocksFactory, constraintIndexCreator, statementOperationContainer, schemaWriteGuard, transactionHeaderInformationFactory, transactionCommitProcess, indexConfigStore, legacyIndexProviderLookup, hooks, transactionMonitor, - availabilityGuard, tracers, storageEngine, procedures, transactionIdStore, clock, accessCapability ) ); + availabilityGuard, tracers, storageEngine, procedures, transactionIdStore, clock, accessCapability, + versionContextSupplier ) ); buildTransactionMonitor( kernelTransactions, clock, config ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelStatement.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelStatement.java index e7084230b84b5..eff68aa199fec 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelStatement.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelStatement.java @@ -25,6 +25,8 @@ import org.neo4j.graphdb.NotInTransactionException; import org.neo4j.graphdb.TransactionTerminatedException; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; import org.neo4j.kernel.api.AssertOpen; import org.neo4j.kernel.api.DataWriteOperations; import org.neo4j.kernel.api.ExecutionStatisticsOperations; @@ -55,14 +57,16 @@ *
    *
  1. Construct {@link KernelStatement} when {@link KernelTransactionImplementation} is constructed
    2. *
  2. For every transaction...
    3. - *
  3. Call {@link #initialize(StatementLocks, StatementOperationParts, PageCursorTracer)} which makes this instance + *
  4. Call {@link #initialize(StatementLocks, StatementOperationParts, PageCursorTracer, VersionContext)} which makes this + * instance * full available and ready to use. Call when the {@link KernelTransactionImplementation} is initialized.
    5. *
  5. Alternate {@link #acquire()} / {@link #close()} when acquiring / closing a statement for the transaction... * Temporarily asymmetric number of calls to {@link #acquire()} / {@link #close()} is supported, although in * the end an equal number of calls must have been issued.
    6. *
  6. To be safe call {@link #forceClose()} at the end of a transaction to force a close of the statement, * even if there are more than one current call to {@link #acquire()}. This instance is now again ready - * to be {@link #initialize(StatementLocks, StatementOperationParts, PageCursorTracer)} initialized} and used for the transaction + * to be {@link #initialize(StatementLocks, StatementOperationParts, PageCursorTracer, VersionContext)} initialized} and used for + * the transaction * instance again, when it's initialized.
    7. *
*/ @@ -78,6 +82,7 @@ public class KernelStatement implements TxStateHolder, Statement, AssertOpen private int referenceCount; private volatile ExecutingQueryList executingQueryList; private final LockTracer systemLockTracer; + private VersionContext versionContext = EmptyVersionContext.INSTANCE; public KernelStatement( KernelTransactionImplementation transaction, TxStateHolder txStateHolder, @@ -195,11 +200,12 @@ public void assertOpen() } public void initialize( StatementLocks statementLocks, StatementOperationParts operationParts, - PageCursorTracer pageCursorCounters ) + PageCursorTracer pageCursorCounters, VersionContext versionContext ) { this.statementLocks = statementLocks; this.pageCursorTracer = pageCursorCounters; facade.initialize( operationParts ); + this.versionContext = versionContext; } public StatementLocks locks() @@ -270,6 +276,7 @@ private void cleanupResources() { // closing is done by KTI storeStatement.release(); + versionContext.clear(); executingQueryList = ExecutingQueryList.EMPTY; } @@ -278,6 +285,11 @@ public KernelTransactionImplementation getTransaction() return transaction; } + public VersionContext getVersionContext() + { + return versionContext; + } + void assertAllows( Function allows, String mode ) { AccessMode accessMode = transaction.securityContext().mode(); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactionImplementation.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactionImplementation.java index 9cb8c084762d2..56b312793daf6 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactionImplementation.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactionImplementation.java @@ -35,6 +35,8 @@ import org.neo4j.collection.pool.Pool; import org.neo4j.graphdb.TransactionTerminatedException; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.KeyReadTokenNameLookup; import org.neo4j.kernel.api.exceptions.ConstraintViolationTransactionFailureException; @@ -105,6 +107,7 @@ public class KernelTransactionImplementation implements KernelTransaction, TxSta private final TransactionCommitProcess commitProcess; private final TransactionMonitor transactionMonitor; private final PageCursorTracerSupplier cursorTracerSupplier; + private final VersionContextSupplier versionContextSupplier; private final StoreReadLayer storeLayer; private final Clock clock; @@ -115,6 +118,7 @@ public class KernelTransactionImplementation implements KernelTransaction, TxSta private TransactionWriteState writeState; private TransactionHooks.TransactionHooksState hooksState; private StatementOperationParts currentTransactionOperations; + private VersionContext versionContext; private final KernelStatement currentStatement; private final StorageStatement storageStatement; private final List closeListeners = new ArrayList<>( 2 ); @@ -161,7 +165,8 @@ public KernelTransactionImplementation( StatementOperationContainer operationCon LockTracer lockTracer, PageCursorTracerSupplier cursorTracerSupplier, StorageEngine storageEngine, - AccessCapability accessCapability ) + AccessCapability accessCapability, + VersionContextSupplier versionContextSupplier ) { this.operationContainer = operationContainer; this.schemaWriteGuard = schemaWriteGuard; @@ -177,6 +182,7 @@ public KernelTransactionImplementation( StatementOperationContainer operationCon this.clock = clock; this.transactionTracer = transactionTracer; this.cursorTracerSupplier = cursorTracerSupplier; + this.versionContextSupplier = versionContextSupplier; this.storageStatement = storeLayer.newStatement(); this.currentStatement = new KernelStatement( this, this, storageStatement, procedures, accessCapability, lockTracer ); @@ -210,7 +216,9 @@ public KernelTransactionImplementation initialize( this.transactionId = NOT_COMMITTED_TRANSACTION_ID; this.commitTime = NOT_COMMITTED_TRANSACTION_COMMIT_TIME; this.currentTransactionOperations = timeoutMillis > 0 ? operationContainer.guardedParts() : operationContainer.nonGuarderParts(); - this.currentStatement.initialize( statementLocks, currentTransactionOperations, cursorTracerSupplier.get() ); + this.versionContext = this.versionContextSupplier.getVersionContext(); + this.currentStatement.initialize( statementLocks, currentTransactionOperations, cursorTracerSupplier.get(), + versionContext ); return this; } @@ -585,7 +593,7 @@ private long commit() throws TransactionFailureException // Commit the transaction success = true; - TransactionToApply batch = new TransactionToApply( transactionRepresentation ); + TransactionToApply batch = new TransactionToApply( transactionRepresentation, versionContext ); txId = transactionId = commitProcess.commit( batch, commitEvent, INTERNAL ); commitTime = timeCommitted; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactions.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactions.java index 60015b63313c3..a1fc278c56911 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactions.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactions.java @@ -31,6 +31,7 @@ import org.neo4j.function.Factory; import org.neo4j.graphdb.DatabaseShutdownException; import org.neo4j.graphdb.TransactionFailureException; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.AvailabilityGuard; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.KernelTransactionHandle; @@ -80,6 +81,7 @@ public class KernelTransactions extends LifecycleAdapter implements Supplier legacyIndexTxStateSupplier; + private final VersionContextSupplier versionContextSupplier; private final Clock clock; private final ReentrantReadWriteLock newTransactionsLock = new ReentrantReadWriteLock(); @@ -125,7 +127,8 @@ public KernelTransactions( StatementLocksFactory statementLocksFactory, StorageEngine storageEngine, Procedures procedures, TransactionIdStore transactionIdStore, - Clock clock, AccessCapability accessCapability ) + Clock clock, AccessCapability accessCapability, + VersionContextSupplier versionContextSupplier ) { this.statementLocksFactory = statementLocksFactory; this.constraintIndexCreator = constraintIndexCreator; @@ -143,6 +146,7 @@ public KernelTransactions( StatementLocksFactory statementLocksFactory, this.accessCapability = accessCapability; this.legacyIndexTxStateSupplier = () -> new CachingLegacyIndexTransactionState( new LegacyIndexTransactionStateImpl( indexConfigStore, legacyIndexProviderLookup ) ); + this.versionContextSupplier = versionContextSupplier; this.clock = clock; blockNewTransactions(); } @@ -337,7 +341,7 @@ public KernelTransactionImplementation newInstance() constraintIndexCreator, procedures, transactionHeaderInformationFactory, transactionCommitProcess, transactionMonitor, legacyIndexTxStateSupplier, localTxPool, clock, tracers.transactionTracer, tracers.lockTracer, tracers.pageCursorTracerSupplier, - storageEngine, accessCapability ); + storageEngine, accessCapability, versionContextSupplier ); this.transactions.add( tx ); return tx; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/TransactionToApply.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/TransactionToApply.java index c528507b98c7f..d1e432d156a73 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/TransactionToApply.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/TransactionToApply.java @@ -23,6 +23,8 @@ import java.util.function.LongConsumer; import org.neo4j.helpers.collection.Visitor; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; import org.neo4j.kernel.impl.transaction.TransactionRepresentation; import org.neo4j.kernel.impl.transaction.log.Commitment; import org.neo4j.kernel.impl.transaction.log.LogPosition; @@ -66,6 +68,7 @@ public class TransactionToApply implements CommandsToApply, AutoCloseable // These fields are provided by user private final TransactionRepresentation transactionRepresentation; private long transactionId; + private final VersionContext versionContext; private TransactionToApply nextTransactionInBatch; // These fields are provided by commit process, storage engine, or recovery process @@ -78,17 +81,28 @@ public class TransactionToApply implements CommandsToApply, AutoCloseable */ public TransactionToApply( TransactionRepresentation transactionRepresentation ) { - this( transactionRepresentation, TRANSACTION_ID_NOT_SPECIFIED ); + this( transactionRepresentation, EmptyVersionContext.INSTANCE ); } /** - * Used as convenience when committing a transaction that has already gotten a transaction id assigned, - * i.e. when replicating a transaction. + * Used when committing a transaction that hasn't already gotten a transaction id assigned. */ + public TransactionToApply( TransactionRepresentation transactionRepresentation, VersionContext versionContext ) + { + this( transactionRepresentation, TRANSACTION_ID_NOT_SPECIFIED, versionContext ); + } + public TransactionToApply( TransactionRepresentation transactionRepresentation, long transactionId ) + { + this( transactionRepresentation, transactionId, EmptyVersionContext.INSTANCE ); + } + + public TransactionToApply( TransactionRepresentation transactionRepresentation, long transactionId, + VersionContext versionContext ) { this.transactionRepresentation = transactionRepresentation; this.transactionId = transactionId; + this.versionContext = versionContext; } // These methods are called by the user when building a batch @@ -130,6 +144,7 @@ public void commitment( Commitment commitment, long transactionId ) { this.commitment = commitment; this.transactionId = transactionId; + this.versionContext.initWrite( transactionId ); } public void logPosition( LogPosition position ) @@ -151,6 +166,7 @@ public void onClose( LongConsumer closedCallback ) @Override public void close() { + versionContext.clear(); if ( closedCallback != null ) { closedCallback.accept( transactionId ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContext.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContext.java new file mode 100644 index 0000000000000..32be2977112b8 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContext.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.context; + +import java.util.function.LongSupplier; + +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; + +import static org.neo4j.kernel.impl.transaction.log.TransactionIdStore.BASE_TX_ID; + +/** + * Transactional version context that used by read transaction to read data of specific version. + * Or perform versioned data modification. + */ +public class TransactionVersionContext implements VersionContext +{ + private final LongSupplier lastClosedTxIdSupplier; + private long transactionId = BASE_TX_ID; + private long lastClosedTxId = Long.MAX_VALUE; + private boolean dirty; + + public TransactionVersionContext( LongSupplier lastClosedTxIdSupplier ) + { + this.lastClosedTxIdSupplier = lastClosedTxIdSupplier; + } + + @Override + public void initRead() + { + long txId = lastClosedTxIdSupplier.getAsLong(); + assert txId >= BASE_TX_ID; + lastClosedTxId = txId; + dirty = false; + } + + @Override + public void initWrite( long committingTxId ) + { + assert committingTxId >= BASE_TX_ID; + transactionId = committingTxId; + } + + @Override + public long committingTransactionId() + { + return transactionId; + } + + @Override + public long lastClosedTransactionId() + { + return lastClosedTxId; + } + + @Override + public void markAsDirty() + { + dirty = true; + } + + @Override + public boolean isDirty() + { + return dirty; + } + + @Override + public void clear() + { + transactionId = BASE_TX_ID; + lastClosedTxId = Long.MAX_VALUE; + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContextSupplier.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContextSupplier.java new file mode 100644 index 0000000000000..12f3ddff7a145 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/context/TransactionVersionContextSupplier.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.context; + +import java.util.function.LongSupplier; + +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; + +/** + * {@link VersionContextSupplier} that supplier thread bound version context that should be used in a context of + * transaction(Committing or reading). + */ +public class TransactionVersionContextSupplier implements VersionContextSupplier +{ + protected ThreadLocal cursorContext; + + @Override + public void init( LongSupplier lastClosedTransactionIdSupplier ) + { + this.cursorContext = ThreadLocal.withInitial( () -> new TransactionVersionContext( lastClosedTransactionIdSupplier ) ); + } + + @Override + public VersionContext getVersionContext() + { + return cursorContext == null ? EmptyVersionContext.INSTANCE : cursorContext.get(); + } + +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java index 39eb7506751eb..62332cb91b83d 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java @@ -219,7 +219,8 @@ public DataSourceModule( final PlatformModule platformModule, EditionModule edit platformModule.availabilityGuard, platformModule.clock, editionModule.accessCapability, platformModule.storeCopyCheckPointMutex, - editionModule.idController) ); + editionModule.idController, + platformModule.versionContextSupplier ) ); dataSourceManager.register( neoStoreDataSource ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java index f86d551c1a1dd..59aeeb3de2660 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java @@ -29,11 +29,14 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileSystemLifecycleAdapter; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.AvailabilityGuard; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.extension.KernelExtensions; import org.neo4j.kernel.extension.UnsatisfiedDependencyStrategies; import org.neo4j.kernel.impl.api.LogRotationMonitor; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.logging.StoreLogService; import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory; @@ -110,6 +113,7 @@ public class PlatformModule public final SystemNanoClock clock; public final StoreCopyCheckPointMutex storeCopyCheckPointMutex; + public final VersionContextSupplier versionContextSupplier; public PlatformModule( File providedStoreDir, Map params, DatabaseInfo databaseInfo, GraphDatabaseFacadeFactory.Dependencies externalDependencies, GraphDatabaseFacade graphDatabaseFacade ) @@ -171,7 +175,11 @@ public PlatformModule( File providedStoreDir, Config config, DatabaseInfo databa dependencies.satisfyDependency( firstImplementor( CheckPointerMonitor.class, tracers.checkPointTracer, CheckPointerMonitor.NULL ) ); - pageCache = dependencies.satisfyDependency( createPageCache( fileSystem, config, logging, tracers ) ); + versionContextSupplier = createCursorContextSupplier( config ); + dependencies.satisfyDependency( versionContextSupplier ); + pageCache = dependencies.satisfyDependency( createPageCache( fileSystem, config, logging, tracers, + versionContextSupplier ) ); + life.add( new PageCacheLifecycle( pageCache ) ); diagnosticsManager = life.add( dependencies @@ -200,6 +208,12 @@ public PlatformModule( File providedStoreDir, Config config, DatabaseInfo databa publishPlatformInfo( dependencies.resolveDependency( UsageData.class ) ); } + protected VersionContextSupplier createCursorContextSupplier( Config config ) + { + return config.get( GraphDatabaseSettings.snapshot_query ) ? new TransactionVersionContextSupplier() + : EmptyVersionContextSupplier.INSTANCE; + } + protected SystemNanoClock createClock() { return Clocks.nanoClock(); @@ -277,11 +291,12 @@ protected Neo4jJobScheduler createJobScheduler() } protected PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, LogService logging, - Tracers tracers ) + Tracers tracers, VersionContextSupplier versionContextSupplier ) { Log pageCacheLog = logging.getInternalLog( PageCache.class ); ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( - fileSystem, config, tracers.pageCacheTracer, tracers.pageCursorTracerSupplier, pageCacheLog ); + fileSystem, config, tracers.pageCacheTracer, tracers.pageCursorTracerSupplier, pageCacheLog, + versionContextSupplier ); PageCache pageCache = pageCacheFactory.getOrCreatePageCache(); if ( config.get( GraphDatabaseSettings.dump_configuration ) ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfigurableStandalonePageCacheFactory.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfigurableStandalonePageCacheFactory.java index 521b200c35b03..cd5bbc25334fe 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfigurableStandalonePageCacheFactory.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfigurableStandalonePageCacheFactory.java @@ -26,6 +26,8 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.logging.FormattedLogProvider; @@ -45,12 +47,13 @@ private ConfigurableStandalonePageCacheFactory() public static PageCache createPageCache( FileSystemAbstraction fileSystem ) { return createPageCache( fileSystem, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE, - Config.defaults() ); + Config.defaults(), EmptyVersionContextSupplier.INSTANCE ); } public static PageCache createPageCache( FileSystemAbstraction fileSystem, Config config ) { - return createPageCache( fileSystem, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE, config ); + return createPageCache( fileSystem, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE, config, + EmptyVersionContextSupplier.INSTANCE ); } /** @@ -60,17 +63,19 @@ public static PageCache createPageCache( FileSystemAbstraction fileSystem, Confi * @param pageCursorTracerSupplier supplier of thread local (transaction local) page cursor tracer that will provide * thread local page cache statistics * @param config page cache configuration + * @param versionContextSupplier version context supplier * @return created page cache instance */ public static PageCache createPageCache( FileSystemAbstraction fileSystem, PageCacheTracer pageCacheTracer, - PageCursorTracerSupplier pageCursorTracerSupplier, Config config ) + PageCursorTracerSupplier pageCursorTracerSupplier, Config config, + VersionContextSupplier versionContextSupplier ) { Config finalConfig = config.withDefaults( MapUtil.stringMap( GraphDatabaseSettings.pagecache_memory.name(), "8M" ) ); FormattedLogProvider logProvider = FormattedLogProvider.toOutputStream( System.err ); ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( fileSystem, finalConfig, pageCacheTracer, pageCursorTracerSupplier, - logProvider.getLog( PageCache.class ) ); + logProvider.getLog( PageCache.class ), versionContextSupplier ); return pageCacheFactory.getOrCreatePageCache(); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactory.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactory.java index 83814bb623d66..ecdecdc5a44c8 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactory.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactory.java @@ -28,6 +28,7 @@ import org.neo4j.io.pagecache.impl.muninn.MuninnPageCache; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.util.OsBeanUtil; import org.neo4j.logging.Log; @@ -45,6 +46,7 @@ public class ConfiguringPageCacheFactory private final Config config; private final PageCacheTracer pageCacheTracer; private final Log log; + private final VersionContextSupplier versionContextSupplier; private PageCache pageCache; private PageCursorTracerSupplier pageCursorTracerSupplier; @@ -54,12 +56,15 @@ public class ConfiguringPageCacheFactory * @param config page swapper configuration * @param pageCacheTracer global page cache tracer * @param pageCursorTracerSupplier supplier of thread local (transaction local) page cursor tracer that will provide - * thread local page cache statistics +* thread local page cache statistics * @param log page cache factory log + * @param versionContextSupplier cursor context factory */ public ConfiguringPageCacheFactory( FileSystemAbstraction fs, Config config, PageCacheTracer pageCacheTracer, - PageCursorTracerSupplier pageCursorTracerSupplier, Log log ) + PageCursorTracerSupplier pageCursorTracerSupplier, Log log, + VersionContextSupplier versionContextSupplier ) { + this.versionContextSupplier = versionContextSupplier; this.swapperFactory = createAndConfigureSwapperFactory( fs, config, log ); this.config = config; this.pageCacheTracer = pageCacheTracer; @@ -111,7 +116,7 @@ protected PageCache createPageCache() return new MuninnPageCache( swapperFactory, maxPages, - cachePageSize, pageCacheTracer, pageCursorTracerSupplier ); + cachePageSize, pageCacheTracer, pageCursorTracerSupplier, versionContextSupplier ); } public int calculateMaxPages( Config config, int cachePageSize ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java index 6d987f7c8b0fa..20e9e2c003df6 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java @@ -33,6 +33,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.TokenNameLookup; import org.neo4j.kernel.api.exceptions.TransactionApplyKernelException; import org.neo4j.kernel.api.exceptions.TransactionFailureException; @@ -170,7 +171,8 @@ public RecordStorageEngine( IndexConfigStore indexConfigStore, IdOrderingQueue legacyIndexTransactionOrdering, IdGeneratorFactory idGeneratorFactory, - IdController idController ) + IdController idController, + VersionContextSupplier versionContextSupplier ) { this.propertyKeyTokenHolder = propertyKeyTokenHolder; this.relationshipTypeTokenHolder = relationshipTypeTokens; @@ -184,7 +186,8 @@ public RecordStorageEngine( this.legacyIndexTransactionOrdering = legacyIndexTransactionOrdering; this.idController = idController; - StoreFactory factory = new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fs, logProvider ); + StoreFactory factory = new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fs, logProvider, + versionContextSupplier ); neoStores = factory.openAllNeoStores( true ); try diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java index 37a4d1dab960d..b5aec8a0f36b3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java @@ -37,6 +37,8 @@ import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PagedFile; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.NeoStoresDiagnostics; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.CountsAccessor; @@ -104,6 +106,7 @@ public boolean test( StoreType type ) private final IdGeneratorFactory idGeneratorFactory; private final PageCache pageCache; private final LogProvider logProvider; + private final VersionContextSupplier versionContextSupplier; private final boolean createIfNotExist; private final File storeDir; private final File neoStoreFileName; @@ -121,6 +124,7 @@ public boolean test( StoreType type ) PageCache pageCache, final LogProvider logProvider, FileSystemAbstraction fileSystemAbstraction, + VersionContextSupplier versionContextSupplier, RecordFormats recordFormats, boolean createIfNotExist, StoreType[] storeTypes, @@ -132,6 +136,7 @@ public boolean test( StoreType type ) this.pageCache = pageCache; this.logProvider = logProvider; this.fileSystemAbstraction = fileSystemAbstraction; + this.versionContextSupplier = versionContextSupplier; this.recordFormats = recordFormats; this.createIfNotExist = createIfNotExist; this.openOptions = openOptions; @@ -416,7 +421,8 @@ public CountsTracker getCounts() private CountsTracker createWritableCountsTracker( File fileName ) { - return new CountsTracker( logProvider, fileSystemAbstraction, pageCache, config, fileName ); + return new CountsTracker( logProvider, fileSystemAbstraction, pageCache, config, fileName, + versionContextSupplier ); } private ReadOnlyCountsTracker createReadOnlyCountsTracker( File fileName ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreAccess.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreAccess.java index 7583bdd491e77..01f8d05046bf2 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreAccess.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreAccess.java @@ -23,6 +23,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.CountsAccessor; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; @@ -79,7 +80,7 @@ public StoreAccess( FileSystemAbstraction fileSystem, PageCache pageCache, File public StoreAccess( FileSystemAbstraction fileSystem, PageCache pageCache, File storeDir, Config config ) { this( new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fileSystem ), pageCache, - fileSystem, NullLogProvider.getInstance() ).openAllNeoStores() ); + fileSystem, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ).openAllNeoStores() ); this.closeable = true; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreFactory.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreFactory.java index 7a437884c6398..e891892865692 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreFactory.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreFactory.java @@ -25,6 +25,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.format.RecordFormatPropertyConfigurator; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; @@ -69,42 +70,46 @@ public class StoreFactory private final PageCache pageCache; private final RecordFormats recordFormats; private final OpenOption[] openOptions; + private final VersionContextSupplier versionContextSupplier; - public StoreFactory( File storeDir, PageCache pageCache, FileSystemAbstraction fileSystem, LogProvider logProvider ) + public StoreFactory( File storeDir, PageCache pageCache, FileSystemAbstraction fileSystem, LogProvider + logProvider, VersionContextSupplier versionContextSupplier ) { this( storeDir, Config.defaults(), new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, - logProvider ); + logProvider, versionContextSupplier ); } public StoreFactory( File storeDir, PageCache pageCache, FileSystemAbstraction fileSystem, - RecordFormats recordFormats, LogProvider logProvider ) + RecordFormats recordFormats, LogProvider logProvider, VersionContextSupplier versionContextSupplier ) { this( storeDir, Config.defaults(), new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, - recordFormats, logProvider ); + recordFormats, logProvider, versionContextSupplier ); } public StoreFactory( File storeDir, Config config, IdGeneratorFactory idGeneratorFactory, PageCache pageCache, - FileSystemAbstraction fileSystemAbstraction, LogProvider logProvider ) + FileSystemAbstraction fileSystemAbstraction, LogProvider logProvider, VersionContextSupplier versionContextSupplier ) { this( storeDir, config, idGeneratorFactory, pageCache, fileSystemAbstraction, RecordFormatSelector.selectForStoreOrConfig( config, storeDir, fileSystemAbstraction, pageCache, logProvider ), - logProvider ); + logProvider, versionContextSupplier ); } public StoreFactory( File storeDir, Config config, IdGeneratorFactory idGeneratorFactory, PageCache pageCache, - FileSystemAbstraction fileSystemAbstraction, RecordFormats recordFormats, LogProvider logProvider ) + FileSystemAbstraction fileSystemAbstraction, RecordFormats recordFormats, LogProvider logProvider, + VersionContextSupplier versionContextSupplier ) { this( storeDir, MetaDataStore.DEFAULT_NAME, config, idGeneratorFactory, pageCache, fileSystemAbstraction, - recordFormats, logProvider ); + recordFormats, logProvider, versionContextSupplier ); } public StoreFactory( File storeDir, String storeName, Config config, IdGeneratorFactory idGeneratorFactory, PageCache pageCache, FileSystemAbstraction fileSystemAbstraction, RecordFormats recordFormats, - LogProvider logProvider, OpenOption... openOptions ) + LogProvider logProvider, VersionContextSupplier versionContextSupplier, OpenOption... openOptions ) { this.config = config; this.idGeneratorFactory = idGeneratorFactory; this.fileSystemAbstraction = fileSystemAbstraction; + this.versionContextSupplier = versionContextSupplier; this.recordFormats = recordFormats; this.openOptions = openOptions; new RecordFormatPropertyConfigurator( recordFormats, config ).configure(); @@ -167,6 +172,7 @@ public NeoStores openNeoStores( boolean createStoreIfNotExists, StoreType... sto } } return new NeoStores( neoStoreFileName, config, idGeneratorFactory, pageCache, logProvider, - fileSystemAbstraction, recordFormats, createStoreIfNotExists, storeTypes, openOptions ); + fileSystemAbstraction, versionContextSupplier, recordFormats, createStoreIfNotExists, storeTypes, + openOptions ); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/CountsTracker.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/CountsTracker.java index 10c3c7cc17500..e6827436e7909 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/CountsTracker.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/CountsTracker.java @@ -26,6 +26,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.CountsAccessor; import org.neo4j.kernel.impl.api.CountsVisitor; @@ -84,46 +85,16 @@ public class CountsTracker extends AbstractKeyValueStore public static final String TYPE_DESCRIPTOR = "CountsStore"; public CountsTracker( final LogProvider logProvider, FileSystemAbstraction fs, PageCache pages, Config config, - File baseFile ) + File baseFile, VersionContextSupplier versionContextSupplier ) { - this( logProvider, fs, pages, config, baseFile, Clocks.systemClock() ); + this( logProvider, fs, pages, config, baseFile, Clocks.systemClock(), versionContextSupplier ); } public CountsTracker( final LogProvider logProvider, FileSystemAbstraction fs, PageCache pages, Config config, - File baseFile, Clock clock ) + File baseFile, Clock clock, VersionContextSupplier versionContextSupplier ) { - super( fs, pages, baseFile, new RotationMonitor() - { - final Log log = logProvider.getLog( CountsTracker.class ); - - @Override - public void failedToOpenStoreFile( File path, Exception error ) - { - log.error( "Failed to open counts store file: " + path, error ); - } - - @Override - public void beforeRotation( File source, File target, Headers headers ) - { - log.info( format( "About to rotate counts store at transaction %d to [%s], from [%s].", - headers.get( FileVersion.FILE_VERSION ).txId, target, source ) ); - } - - @Override - public void rotationSucceeded( File source, File target, Headers headers ) - { - log.info( format( "Successfully rotated counts store at transaction %d to [%s], from [%s].", - headers.get( FileVersion.FILE_VERSION ).txId, target, source ) ); - } - - @Override - public void rotationFailed( File source, File target, Headers headers, Exception e ) - { - log.error( format( "Failed to rotate counts store at transaction %d to [%s], from [%s].", - headers.get( FileVersion.FILE_VERSION ).txId, target, source ), e ); - } - }, new RotationTimerFactory( clock, - config.get( counts_store_rotation_timeout ).toMillis() ), 16, 16, HEADER_FIELDS ); + super( fs, pages, baseFile, new CountsTrackerRotationMonitor( logProvider ), new RotationTimerFactory( clock, + config.get( counts_store_rotation_timeout ).toMillis() ), versionContextSupplier,16, 16, HEADER_FIELDS ); } public CountsTracker setInitializer( final DataInitializer initializer ) @@ -203,7 +174,7 @@ public Register.DoubleLongRegister indexSample( long indexId, Register.DoubleLon public Optional apply( long txId ) { - return updater( txId ).map( CountsUpdater::new ); + return updater( txId ).map( CountsUpdater::new ); } public CountsAccessor.IndexStatsUpdater updateIndexCounts() @@ -292,6 +263,43 @@ protected void writeFormatSpecifier( WritableBuffer formatSpecifier ) formatSpecifier.put( 0, FORMAT ); } + private static class CountsTrackerRotationMonitor implements RotationMonitor + { + final Log log; + + CountsTrackerRotationMonitor( LogProvider logProvider ) + { + log = logProvider.getLog( CountsTracker.class ); + } + + @Override + public void failedToOpenStoreFile( File path, Exception error ) + { + log.error( "Failed to open counts store file: " + path, error ); + } + + @Override + public void beforeRotation( File source, File target, Headers headers ) + { + log.info( format( "About to rotate counts store at transaction %d to [%s], from [%s].", + headers.get( FileVersion.FILE_VERSION ).txId, target, source ) ); + } + + @Override + public void rotationSucceeded( File source, File target, Headers headers ) + { + log.info( format( "Successfully rotated counts store at transaction %d to [%s], from [%s].", + headers.get( FileVersion.FILE_VERSION ).txId, target, source ) ); + } + + @Override + public void rotationFailed( File source, File target, Headers headers, Exception e ) + { + log.error( format( "Failed to rotate counts store at transaction %d to [%s], from [%s].", + headers.get( FileVersion.FILE_VERSION ).txId, target, source ), e ); + } + } + private class DelegatingVisitor extends Visitor implements MetadataVisitor { private final CountsVisitor visitor; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/ReadOnlyCountsTracker.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/ReadOnlyCountsTracker.java index 5934f19c2f5a9..1ed3de330130e 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/ReadOnlyCountsTracker.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/ReadOnlyCountsTracker.java @@ -24,6 +24,8 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.kvstore.State; import org.neo4j.logging.LogProvider; @@ -34,7 +36,7 @@ public class ReadOnlyCountsTracker extends CountsTracker public ReadOnlyCountsTracker( LogProvider logProvider, FileSystemAbstraction fileSystem, PageCache pageCache, Config config, File baseFile ) { - super( logProvider, fileSystem, pageCache, config, baseFile ); + super( logProvider, fileSystem, pageCache, config, baseFile, EmptyVersionContextSupplier.INSTANCE ); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStore.java index 41ec87311e4c6..b5920261dbe11 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStore.java @@ -29,6 +29,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.impl.locking.LockWrapper; import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.lifecycle.LifecycleAdapter; @@ -49,6 +50,7 @@ public abstract class AbstractKeyValueStore extends LifecycleAdapter private final ReadWriteLock updateLock = new ReentrantReadWriteLock( /*fair=*/true ); private final Format format; final RotationStrategy rotationStrategy; + private final VersionContextSupplier versionContextSupplier; private final RotationTimerFactory rotationTimerFactory; volatile ProgressiveState state; private DataInitializer> stateInitializer; @@ -58,9 +60,11 @@ public abstract class AbstractKeyValueStore extends LifecycleAdapter private volatile boolean stopped; public AbstractKeyValueStore( FileSystemAbstraction fs, PageCache pages, File base, RotationMonitor monitor, - RotationTimerFactory timerFactory, int keySize, int valueSize, HeaderField... headerFields ) + RotationTimerFactory timerFactory, VersionContextSupplier versionContextSupplier, int keySize, + int valueSize, HeaderField... headerFields ) { this.fs = fs; + this.versionContextSupplier = versionContextSupplier; this.keySize = keySize; this.valueSize = valueSize; Rotation rotation = getClass().getAnnotation( Rotation.class ); @@ -71,7 +75,8 @@ public AbstractKeyValueStore( FileSystemAbstraction fs, PageCache pages, File ba this.format = new Format( headerFields ); this.rotationStrategy = rotation.value().create( fs, pages, format, monitor, base, rotation.parameters() ); this.rotationTimerFactory = timerFactory; - this.state = new DeadState.Stopped<>( format, getClass().getAnnotation( State.class ).value() ); + this.state = new DeadState.Stopped<>( format, getClass().getAnnotation( State.class ).value(), + versionContextSupplier ); } protected final void setEntryUpdaterInitializer( DataInitializer> stateInitializer ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ActiveState.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ActiveState.java index 0623099eb9635..9c698113575bf 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ActiveState.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ActiveState.java @@ -24,18 +24,22 @@ import java.util.Optional; import java.util.concurrent.locks.Lock; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; + public abstract class ActiveState extends ProgressiveState { public interface Factory { - ActiveState open( ReadableState store, File file ); + ActiveState open( ReadableState store, File file, VersionContextSupplier versionContextSupplier ); } protected final ReadableState store; + protected final VersionContextSupplier versionContextSupplier; - public ActiveState( ReadableState store ) + public ActiveState( ReadableState store, VersionContextSupplier versionContextSupplier ) { this.store = store; + this.versionContextSupplier = versionContextSupplier; } @Override @@ -82,7 +86,7 @@ final EntryUpdater resetter( Lock lock, Runnable closeAction ) final ProgressiveState stop() throws IOException { close(); - return new DeadState.Stopped<>( keyFormat(), factory() ); + return new DeadState.Stopped<>( keyFormat(), factory(), versionContextSupplier ); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapState.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapState.java index abec06d35de5f..3fe766fd36493 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapState.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapState.java @@ -30,30 +30,37 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; +import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; + class ConcurrentMapState extends ActiveState { - private final ConcurrentMap changes; + private final ConcurrentMap changes; + private final VersionContextSupplier versionContextSupplier; private final File file; private final AtomicLong highestAppliedVersion; private final AtomicLong appliedChanges; private final AtomicBoolean hasTrackedChanges; private final long previousVersion; - ConcurrentMapState( ReadableState store, File file ) + ConcurrentMapState( ReadableState store, File file, VersionContextSupplier versionContextSupplier ) { - super( store ); + super( store, versionContextSupplier ); this.previousVersion = store.version(); this.file = file; + this.versionContextSupplier = versionContextSupplier; this.highestAppliedVersion = new AtomicLong( previousVersion ); this.changes = new ConcurrentHashMap<>(); this.appliedChanges = new AtomicLong(); hasTrackedChanges = new AtomicBoolean(); } - private ConcurrentMapState( Prototype prototype, ReadableState store, File file ) + private ConcurrentMapState( Prototype prototype, ReadableState store, File file, VersionContextSupplier versionContextSupplier ) { - super( store ); + super( store, versionContextSupplier ); this.previousVersion = store.version(); + this.versionContextSupplier = versionContextSupplier; this.file = file; this.hasTrackedChanges = prototype.hasTrackedChanges; this.changes = prototype.changes; @@ -76,35 +83,38 @@ public EntryUpdater updater( long version, Lock lock ) } update( highestAppliedVersion, version ); hasTrackedChanges.set( true ); - return new Updater<>( lock, store, changes, appliedChanges ); + return new Updater<>( lock, store, changes, appliedChanges, version ); } @Override public EntryUpdater unsafeUpdater( Lock lock ) { hasTrackedChanges.set( true ); - return new Updater<>( lock, store, changes, null ); + return new Updater<>( lock, store, changes, null, TransactionIdStore.BASE_TX_ID ); } private static class Updater extends EntryUpdater { private AtomicLong changeCounter; private final ReadableState store; - private final ConcurrentMap changes; + private final ConcurrentMap changes; + private final long version; - Updater( Lock lock, ReadableState store, ConcurrentMap changes, AtomicLong changeCounter ) + Updater( Lock lock, ReadableState store, ConcurrentMap changes, AtomicLong changeCounter, + long version ) { super( lock ); this.changeCounter = changeCounter; this.store = store; this.changes = changes; + this.version = version; } @Override public void apply( Key key, ValueUpdate update ) throws IOException { ensureOpenOnSameThread(); - applyUpdate( store, changes, key, update, false ); + applyUpdate( store, changes, key, update, false, version ); } @Override @@ -138,7 +148,7 @@ protected EntryUpdater resettingUpdater( Lock lock, final Runnable closeAct public void apply( Key key, ValueUpdate update ) throws IOException { ensureOpen(); - applyUpdate( store, changes, key, update, true ); + applyUpdate( store, changes, key, update, true, highestAppliedVersion.get() ); } @Override @@ -163,16 +173,16 @@ protected PrototypeState prototype( long version ) } @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") - static void applyUpdate( ReadableState store, ConcurrentMap changes, - Key key, ValueUpdate update, boolean reset ) throws IOException + static void applyUpdate( ReadableState store, ConcurrentMap changes, Key key, ValueUpdate + update, boolean reset, long version ) throws IOException { - byte[] value = changes.get( key ); + ChangeEntry value = changes.get( key ); if ( value == null ) { final byte[] proposal = new byte[store.keyFormat().valueSize()]; synchronized ( proposal ) { - value = changes.putIfAbsent( key, proposal ); + value = changes.putIfAbsent( key, ChangeEntry.of( proposal, version ) ); if ( value == null ) { BigEndianByteArrayBuffer buffer = new BigEndianByteArrayBuffer( proposal ); @@ -191,7 +201,8 @@ static void applyUpdate( ReadableState store, ConcurrentMap extends PrototypeState { - final ConcurrentMap changes = new ConcurrentHashMap<>(); + final ConcurrentMap changes = new ConcurrentHashMap<>(); final AtomicLong highestAppliedVersion, appliedChanges = new AtomicLong(); + final VersionContextSupplier versionContextSupplier; final AtomicBoolean hasTrackedChanges; private final long threshold; Prototype( ConcurrentMapState state, long version ) { super( state ); + this.versionContextSupplier = state.versionContextSupplier; threshold = version; hasTrackedChanges = new AtomicBoolean(); this.highestAppliedVersion = new AtomicLong( version ); } @Override - protected ActiveState create( ReadableState sub, File file ) + protected ActiveState create( ReadableState sub, File file, VersionContextSupplier versionContextSupplier ) { - return new ConcurrentMapState<>( this, sub, file ); + return new ConcurrentMapState<>( this, sub, file, versionContextSupplier ); } @Override @@ -244,11 +257,11 @@ protected EntryUpdater updater( long version, Lock lock ) if ( version > threshold ) { hasTrackedChanges.set( true ); - return new Updater<>( lock, store, changes, appliedChanges ); + return new Updater<>( lock, store, changes, appliedChanges, version ); } else { - return new Updater<>( lock, store, changes, null ); + return new Updater<>( lock, store, changes, null, version ); } } @@ -256,7 +269,7 @@ protected EntryUpdater updater( long version, Lock lock ) protected EntryUpdater unsafeUpdater( Lock lock ) { hasTrackedChanges.set( true ); - return new Updater<>( lock, store, changes, null ); + return new Updater<>( lock, store, changes, null, highestAppliedVersion.get() ); } @Override @@ -274,7 +287,7 @@ protected long version() @Override protected boolean lookup( Key key, ValueSink sink ) throws IOException { - return performLookup( store, changes, key, sink ); + return performLookup( store, versionContextSupplier.getVersionContext(), changes, key, sink ); } @Override @@ -339,16 +352,20 @@ protected Factory factory() @Override protected boolean lookup( Key key, ValueSink sink ) throws IOException { - return performLookup( store, changes, key, sink ); + return performLookup( store, versionContextSupplier.getVersionContext(), changes, key, sink ); } - private static boolean performLookup( ReadableState store, ConcurrentMap changes, - Key key, ValueSink sink ) throws IOException + private static boolean performLookup( ReadableState store, VersionContext versionContext, + ConcurrentMap changes, Key key, ValueSink sink ) throws IOException { - byte[] value = changes.get( key ); - if ( value != null ) + ChangeEntry change = changes.get( key ); + if ( change != null ) { - sink.value( new BigEndianByteArrayBuffer( value ) ); + if ( change.version > versionContext.lastClosedTransactionId() ) + { + versionContext.markAsDirty(); + } + sink.value( new BigEndianByteArrayBuffer( change.data ) ); return true; } return store.lookup( key, sink ); @@ -363,7 +380,7 @@ public DataProvider dataProvider() throws IOException return dataProvider( store, changes ); } - private static DataProvider dataProvider( ReadableState store, ConcurrentMap changes ) + private static DataProvider dataProvider( ReadableState store, ConcurrentMap changes ) throws IOException { if ( changes.isEmpty() ) @@ -378,16 +395,16 @@ private static DataProvider dataProvider( ReadableState store, Concur } } - private static byte[][] sortedUpdates( KeyFormat keys, ConcurrentMap changes ) + private static byte[][] sortedUpdates( KeyFormat keys, ConcurrentMap changes ) { Entry[] buffer = new Entry[changes.size()]; - Iterator> entries = changes.entrySet().iterator(); + Iterator> entries = changes.entrySet().iterator(); for ( int i = 0; i < buffer.length; i++ ) { - Map.Entry next = entries.next(); // we hold the lock, so this should succeed + Map.Entry next = entries.next(); // we hold the lock, so this should succeed byte[] key = new byte[keys.keySize()]; keys.writeKey( next.getKey(), new BigEndianByteArrayBuffer( key ) ); - buffer[i] = new Entry( key, next.getValue() ); + buffer[i] = new Entry( key, next.getValue().data ); } Arrays.sort( buffer ); assert !entries.hasNext() : "We hold the lock, so we should see 'size' entries."; @@ -445,4 +462,21 @@ public void close() throws IOException { } } + + private static class ChangeEntry + { + private byte[] data; + private long version; + + static ChangeEntry of( byte[] data, long version ) + { + return new ChangeEntry( data, version ); + } + + ChangeEntry( byte[] data, long version ) + { + this.data = data; + this.version = version; + } + } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/DeadState.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/DeadState.java index 728563c7285e4..47fc03b00910e 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/DeadState.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/DeadState.java @@ -28,6 +28,7 @@ import java.util.function.Function; import org.neo4j.helpers.collection.Pair; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; abstract class DeadState extends ProgressiveState { @@ -99,18 +100,20 @@ protected KeyFormat keyFormat() private final KeyFormat keys; final ActiveState.Factory stateFactory; + final VersionContextSupplier versionContextSupplier; - private DeadState( KeyFormat keys, ActiveState.Factory stateFactory ) + private DeadState( KeyFormat keys, ActiveState.Factory stateFactory, VersionContextSupplier versionContextSupplier ) { this.keys = keys; this.stateFactory = stateFactory; + this.versionContextSupplier = versionContextSupplier; } static class Stopped extends DeadState { - Stopped( KeyFormat keys, ActiveState.Factory stateFactory ) + Stopped( KeyFormat keys, ActiveState.Factory stateFactory, VersionContextSupplier versionContextSupplier ) { - super( keys, stateFactory ); + super( keys, stateFactory, versionContextSupplier ); } @Override @@ -125,10 +128,10 @@ ProgressiveState initialize( RotationStrategy rotation ) throws IOException Pair opened = rotation.open(); if ( opened == null ) { - return new NeedsCreation<>( keyFormat(), stateFactory, rotation ); + return new NeedsCreation<>( keyFormat(), stateFactory, rotation, versionContextSupplier ); } return new Prepared<>( stateFactory.open( ReadableState.store( keyFormat(), opened.other() ), - opened.first() ) ); + opened.first(), versionContextSupplier ) ); } @Override @@ -143,16 +146,17 @@ private static class NeedsCreation extends DeadState { private final RotationStrategy rotation; - private NeedsCreation( KeyFormat keys, ActiveState.Factory stateFactory, RotationStrategy rotation ) + private NeedsCreation( KeyFormat keys, ActiveState.Factory stateFactory, RotationStrategy rotation, + VersionContextSupplier versionContextSupplier ) { - super( keys, stateFactory ); + super( keys, stateFactory, versionContextSupplier ); this.rotation = rotation; } @Override ProgressiveState stop() throws IOException { - return new Stopped<>( keyFormat(), stateFactory ); + return new Stopped<>( keyFormat(), stateFactory, versionContextSupplier ); } @Override @@ -169,14 +173,16 @@ ActiveState start( DataInitializer> initializer ) throws throw new IllegalStateException( "Store needs to be created, and no initializer is given." ); } Pair created = initialState( initializer ); - return stateFactory.open( ReadableState.store( keyFormat(), created.other() ), created.first() ); + return stateFactory.open( ReadableState.store( keyFormat(), created.other() ), created.first(), + versionContextSupplier ); } private Pair initialState( DataInitializer> initializer ) throws IOException { long version = initializer.initialVersion(); - ActiveState creation = stateFactory.open( ReadableState.empty( keyFormat(), version ), null ); + ActiveState creation = stateFactory.open( ReadableState.empty( keyFormat(), version ), null, + versionContextSupplier ); try { try ( EntryUpdater updater = creation.resetter( new ReentrantLock(), new Runnable() @@ -250,7 +256,7 @@ private static class Prepared extends DeadState private Prepared( ActiveState state ) { - super( state.keyFormat(), state.factory() ); + super( state.keyFormat(), state.factory(), state.versionContextSupplier ); this.state = state; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/PrototypeState.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/PrototypeState.java index 1ad0c82683ff6..da57e6f68d689 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/PrototypeState.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/PrototypeState.java @@ -24,6 +24,8 @@ import java.util.Optional; import java.util.concurrent.locks.Lock; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; + public abstract class PrototypeState extends WritableState { protected final ActiveState store; @@ -33,7 +35,7 @@ public PrototypeState( ActiveState store ) this.store = store; } - protected abstract ActiveState create( ReadableState sub, File file ); + protected abstract ActiveState create( ReadableState sub, File file, VersionContextSupplier versionContextSupplier ); @Override protected final Headers headers() diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/RotationState.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/RotationState.java index b9d0892d49a54..c970d834b4b51 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/RotationState.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/RotationState.java @@ -90,7 +90,8 @@ ActiveState rotate( boolean force, RotationStrategy strategy, RotationTimer } Pair next = strategy .next( file(), updateHeaders( headersUpdater ), keyFormat().filter( preState.dataProvider() ) ); - return postState.create( ReadableState.store( preState.keyFormat(), next.other() ), next.first() ); + return postState.create( ReadableState.store( preState.keyFormat(), next.other() ), next.first(), + preState.versionContextSupplier ); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/State.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/State.java index d725202ee8c18..8c1bd532a7691 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/State.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/kvstore/State.java @@ -26,6 +26,9 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; + @Inherited @Target(ElementType.TYPE) @Retention(RetentionPolicy.RUNTIME) @@ -38,17 +41,17 @@ enum Strategy implements ActiveState.Factory CONCURRENT_HASH_MAP { @Override - public ActiveState open( ReadableState store, File file ) + public ActiveState open( ReadableState store, File file, VersionContextSupplier versionContextSupplier ) { - return new ConcurrentMapState<>( store, file ); + return new ConcurrentMapState<>( store, file, versionContextSupplier ); } }, READ_ONLY_CONCURRENT_HASH_MAP { @Override - public ActiveState open( ReadableState store, File file ) + public ActiveState open( ReadableState store, File file, VersionContextSupplier versionContextSupplier ) { - return new ConcurrentMapState( store, file ) + return new ConcurrentMapState( store, file, EmptyVersionContextSupplier.INSTANCE ) { @Override protected boolean hasChanges() diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DirectRecordStoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DirectRecordStoreMigrator.java index 2e3a250c3e2ac..1be39b91bd1c3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DirectRecordStoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DirectRecordStoreMigrator.java @@ -27,6 +27,7 @@ import org.neo4j.helpers.ArrayUtil; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.RecordStore; @@ -65,11 +66,12 @@ public void migrate( File fromStoreDir, RecordFormats fromFormat, File toStoreDi try ( NeoStores fromStores = new StoreFactory( fromStoreDir, config, new DefaultIdGeneratorFactory( fs ), - pageCache, fs, fromFormat, NullLogProvider.getInstance() ).openNeoStores( true, storesToOpen ); - NeoStores toStores = new StoreFactory( toStoreDir, - withPersistedStoreHeadersAsConfigFrom( fromStores, storesToOpen ), - new DefaultIdGeneratorFactory( fs ), - pageCache, fs, toFormat, NullLogProvider.getInstance() ).openNeoStores( true, storesToOpen ) ) + pageCache, fs, fromFormat, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ) + .openNeoStores( true, storesToOpen ); + NeoStores toStores = new StoreFactory( toStoreDir, withPersistedStoreHeadersAsConfigFrom( fromStores, storesToOpen ), + new DefaultIdGeneratorFactory( fs ), pageCache, fs, toFormat, NullLogProvider.getInstance(), + EmptyVersionContextSupplier.INSTANCE ) + .openNeoStores( true, storesToOpen ) ) { for ( StoreType type : types ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/PropertyDeduplicator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/PropertyDeduplicator.java index 0c1b48f20b710..8b29f85c5e2f0 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/PropertyDeduplicator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/PropertyDeduplicator.java @@ -31,6 +31,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileUtils; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeStore; @@ -70,7 +71,8 @@ public PropertyDeduplicator( FileSystemAbstraction fileSystem, File workingDir, public void deduplicateProperties() throws IOException { - StoreFactory factory = new StoreFactory( workingDir, pageCache, fileSystem, NullLogProvider.getInstance() ); + StoreFactory factory = new StoreFactory( workingDir, pageCache, fileSystem, NullLogProvider.getInstance(), + EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = factory.openNeoStores( StoreType.PROPERTY, StoreType.NODE, StoreType.SCHEMA) ) { PropertyStore propertyStore = neoStores.getPropertyStore(); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java index 85d38c05494c2..6c8efd3073f8b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java @@ -48,6 +48,7 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PagedFile; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.AssertOpen; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.configuration.Config; @@ -412,7 +413,7 @@ private void rebuildCountsFromScratch( File storeDir, long lastTxId, String vers RecordFormats recordFormats = selectForVersion( versionToMigrateTo ); StoreFactory storeFactory = new StoreFactory( storeDir, pageCache, fileSystem, recordFormats, - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = storeFactory.openAllNeoStores() ) { NodeStore nodeStore = neoStores.getNodeStore(); @@ -425,7 +426,8 @@ private void rebuildCountsFromScratch( File storeDir, long lastTxId, String vers lastTxId, nodeStore, relationshipStore, highLabelId, highRelationshipTypeId, NumberArrayFactory.auto( pageCache, storeDir, true ) ); life.add( new CountsTracker( - logService.getInternalLogProvider(), fileSystem, pageCache, config, storeFileBase ) + logService.getInternalLogProvider(), fileSystem, pageCache, config, storeFileBase, + EmptyVersionContextSupplier.INSTANCE ) .setInitializer( initializer ) ); } } @@ -512,7 +514,7 @@ private void migrateWithBatchImporter( File storeDir, File migrationDir, long la private NeoStores instantiateLegacyStore( RecordFormats format, File storeDir ) { return new StoreFactory( storeDir, config, new ReadOnlyIdGeneratorFactory(), pageCache, fileSystem, - format, NullLogProvider.getInstance() ).openAllNeoStores( true ); + format, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ).openAllNeoStores( true ); } private void prepareBatchImportMigration( File storeDir, File migrationDir, RecordFormats oldFormat, @@ -591,7 +593,7 @@ private void prepareBatchImportMigration( File storeDir, File migrationDir, Reco private void createStore( File migrationDir, RecordFormats newFormat ) { StoreFactory storeFactory = new StoreFactory( new File( migrationDir.getPath() ), pageCache, fileSystem, - newFormat, NullLogProvider.getInstance() ); + newFormat, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = storeFactory.openAllNeoStores( true ) ) { neoStores.getMetaDataStore(); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java index df3ab815c6f51..243c0adbbf059 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/internal/BatchInserterImpl.java @@ -54,6 +54,7 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.exceptions.KernelException; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.exceptions.schema.AlreadyConstrainedException; @@ -233,7 +234,8 @@ public BatchInserterImpl( final File storeDir, final FileSystemAbstraction fileS life = new LifeSupport(); this.storeDir = storeDir; ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( - fileSystem, config, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, NullLog.getInstance() ); + fileSystem, config, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, NullLog.getInstance(), + EmptyVersionContextSupplier.INSTANCE ); PageCache pageCache = pageCacheFactory.getOrCreatePageCache(); life.add( new PageCacheLifecycle( pageCache ) ); @@ -249,7 +251,7 @@ public BatchInserterImpl( final File storeDir, final FileSystemAbstraction fileS RecordFormats recordFormats = RecordFormatSelector.selectForStoreOrConfig( config, storeDir, fileSystem, pageCache, internalLogProvider ); StoreFactory sf = new StoreFactory( this.storeDir, config, idGeneratorFactory, pageCache, fileSystem, - recordFormats, internalLogProvider ); + recordFormats, internalLogProvider, EmptyVersionContextSupplier.INSTANCE ); maxNodeId = recordFormats.node().getMaxId(); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java index afc52a39c3743..55f68474582ae 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java @@ -31,6 +31,8 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.extension.KernelExtensionFactory; @@ -184,7 +186,7 @@ public static BatchingNeoStores batchingNeoStores( FileSystemAbstraction fileSys Config neo4jConfig = getNeo4jConfig( config, dbConfig ); final PageCacheTracer tracer = new DefaultPageCacheTracer(); PageCache pageCache = createPageCache( fileSystem, neo4jConfig, logService.getInternalLogProvider(), tracer, - DefaultPageCursorTracerSupplier.INSTANCE ); + DefaultPageCursorTracerSupplier.INSTANCE, EmptyVersionContextSupplier.INSTANCE ); BatchingNeoStores batchingNeoStores = new BatchingNeoStores( fileSystem, pageCache, storeDir, recordFormats, neo4jConfig, config, logService, @@ -212,10 +214,10 @@ protected static Config getNeo4jConfig( Configuration config, Config dbConfig ) } private static PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, LogProvider log, - PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier ) + PageCacheTracer tracer, PageCursorTracerSupplier cursorTracerSupplier, VersionContextSupplier contextSupplier ) { return new ConfiguringPageCacheFactory( fileSystem, config, tracer, cursorTracerSupplier, - log.getLog( BatchingNeoStores.class ) ).getOrCreatePageCache(); + log.getLog( BatchingNeoStores.class ), contextSupplier ).getOrCreatePageCache(); } private boolean alreadyContainsData( NeoStores neoStores ) @@ -227,7 +229,7 @@ private StoreFactory newStoreFactory( String name, OpenOption... openOptions ) { return new StoreFactory( storeDir, name, neo4jConfig, new BatchingIdGeneratorFactory( fileSystem ), pageCache, fileSystem, recordFormats, logProvider, - openOptions ); + EmptyVersionContextSupplier.INSTANCE, openOptions ); } /** diff --git a/community/kernel/src/test/java/org/neo4j/kernel/api/KernelTransactionFactory.java b/community/kernel/src/test/java/org/neo4j/kernel/api/KernelTransactionFactory.java index 097f6684f07b8..57f2764af92ab 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/api/KernelTransactionFactory.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/api/KernelTransactionFactory.java @@ -23,6 +23,7 @@ import org.neo4j.collection.pool.Pool; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.security.SecurityContext; import org.neo4j.kernel.impl.api.KernelTransactionImplementation; import org.neo4j.kernel.impl.api.SchemaWriteGuard; @@ -91,7 +92,7 @@ static Instances kernelTransactionWithInternals( SecurityContext securityContext NULL, LockTracer.NONE, PageCursorTracerSupplier.NULL, - storageEngine, new CanWrite() ); + storageEngine, new CanWrite(), EmptyVersionContextSupplier.INSTANCE ); StatementLocks statementLocks = new SimpleStatementLocks( new NoOpClient() ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTerminationTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTerminationTest.java index c6394e2df09a9..ae6deff7c0235 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTerminationTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTerminationTest.java @@ -31,6 +31,7 @@ import org.neo4j.collection.pool.Pool; import org.neo4j.graphdb.TransactionTerminatedException; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.exceptions.Status; import org.neo4j.kernel.api.exceptions.TransactionFailureException; @@ -337,7 +338,8 @@ private static class TestKernelTransaction extends KernelTransactionImplementati mock( TransactionCommitProcess.class ), monitor, () -> mock( LegacyIndexTransactionState.class ), mock( Pool.class ), Clocks.fakeClock(), TransactionTracer.NULL, LockTracer.NONE, PageCursorTracerSupplier.NULL, - mock( StorageEngine.class, RETURNS_MOCKS ), new CanWrite() ); + mock( StorageEngine.class, RETURNS_MOCKS ), new CanWrite(), + EmptyVersionContextSupplier.INSTANCE ); this.monitor = monitor; } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTestBase.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTestBase.java index 9c30b31447f07..19d9867611f59 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTestBase.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionTestBase.java @@ -27,6 +27,7 @@ import org.neo4j.collection.pool.Pool; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.KernelTransaction.Type; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.api.security.SecurityContext; @@ -143,7 +144,7 @@ public KernelTransactionImplementation newNotInitializedTransaction() return new KernelTransactionImplementation( operationContainer, schemaWriteGuard, hooks, null, null, headerInformationFactory, commitProcess, transactionMonitor, legacyIndexStateSupplier, txPool, clock, TransactionTracer.NULL, LockTracer.NONE, PageCursorTracerSupplier.NULL, storageEngine, - new CanWrite() ); + new CanWrite(), EmptyVersionContextSupplier.INSTANCE ); } public class CapturingCommitProcess implements TransactionCommitProcess diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionsTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionsTest.java index 48599b4193a4f..1e2443618b4b7 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionsTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/KernelTransactionsTest.java @@ -36,6 +36,8 @@ import org.neo4j.graphdb.DatabaseShutdownException; import org.neo4j.graphdb.security.AuthorizationExpiredException; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.AvailabilityGuard; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.KernelTransactionHandle; @@ -560,7 +562,8 @@ private static KernelTransactions createTransactions( StorageEngine storageEngin null, statementOperationsContianer, null, DEFAULT, commitProcess, null, null, new TransactionHooks(), mock( TransactionMonitor.class ), availabilityGuard, - tracers, storageEngine, new Procedures(), transactionIdStore, clock, new CanWrite() ); + tracers, storageEngine, new Procedures(), transactionIdStore, clock, new CanWrite(), + EmptyVersionContextSupplier.INSTANCE ); } private static TestKernelTransactions createTestTransactions( StorageEngine storageEngine, @@ -572,7 +575,7 @@ private static TestKernelTransactions createTestTransactions( StorageEngine stor null, DEFAULT, commitProcess, null, null, new TransactionHooks(), mock( TransactionMonitor.class ), availabilityGuard, tracers, storageEngine, new Procedures(), transactionIdStore, clock, - new CanWrite() ); + new CanWrite(), EmptyVersionContextSupplier.INSTANCE ); } private static TransactionCommitProcess newRememberingCommitProcess( final TransactionRepresentation[] slot ) @@ -621,13 +624,13 @@ private static class TestKernelTransactions extends KernelTransactions LegacyIndexProviderLookup legacyIndexProviderLookup, TransactionHooks hooks, TransactionMonitor transactionMonitor, AvailabilityGuard availabilityGuard, Tracers tracers, StorageEngine storageEngine, Procedures procedures, TransactionIdStore transactionIdStore, Clock clock, - AccessCapability accessCapability ) + AccessCapability accessCapability, VersionContextSupplier versionContextSupplier ) { super( statementLocksFactory, constraintIndexCreator, statementOperationsContianer, schemaWriteGuard, txHeaderFactory, transactionCommitProcess, indexConfigStore, legacyIndexProviderLookup, hooks, transactionMonitor, availabilityGuard, tracers, storageEngine, procedures, transactionIdStore, clock, - accessCapability ); + accessCapability, versionContextSupplier ); } @Override diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LockingStatementOperationsTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LockingStatementOperationsTest.java index 6db9181691f76..c904722f59e01 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LockingStatementOperationsTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LockingStatementOperationsTest.java @@ -27,6 +27,7 @@ import java.util.function.Function; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContext; import org.neo4j.kernel.api.exceptions.EntityNotFoundException; import org.neo4j.kernel.api.exceptions.InvalidTransactionTypeKernelException; import org.neo4j.kernel.api.exceptions.KernelException; @@ -104,7 +105,7 @@ public LockingStatementOperationsTest() lockingOps = new LockingStatementOperations( entityReadOps, entityWriteOps, schemaReadOps, schemaWriteOps, schemaStateOps ); - state.initialize( new SimpleStatementLocks( locks ), null, PageCursorTracer.NULL ); + state.initialize( new SimpleStatementLocks( locks ), null, PageCursorTracer.NULL, EmptyVersionContext.INSTANCE ); state.acquire(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/PropertyPhysicalToLogicalConverterTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/PropertyPhysicalToLogicalConverterTest.java index 182868a265d8b..be23c70f0e3a0 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/PropertyPhysicalToLogicalConverterTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/PropertyPhysicalToLogicalConverterTest.java @@ -28,6 +28,7 @@ import java.io.File; import org.neo4j.helpers.collection.Iterables; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.PropertyStore; @@ -221,7 +222,8 @@ public void before() throws Exception File storeDir = new File( "dir" ); fs.get().mkdirs( storeDir ); StoreFactory storeFactory = new StoreFactory( storeDir, Config.empty(), new DefaultIdGeneratorFactory( fs.get() ), - pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance(), + EmptyVersionContextSupplier.INSTANCE ); neoStores = storeFactory.openAllNeoStores( true ); store = neoStores.getPropertyStore(); converter = new PropertyPhysicalToLogicalConverter( store ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursorTest.java index ce7a15377611e..3e72fe3b0d5db 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursorTest.java @@ -34,6 +34,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory; import org.neo4j.kernel.impl.store.NeoStores; @@ -103,9 +104,9 @@ public static void setupStores() fs = new DefaultFileSystemAbstraction(); pageCache = new ConfiguringPageCacheFactory( fs, Config.defaults().augment( stringMap( pagecache_memory.name(), "8m" ) ), NULL, - PageCursorTracerSupplier.NULL, NullLog.getInstance() ) + PageCursorTracerSupplier.NULL, NullLog.getInstance(), EmptyVersionContextSupplier.INSTANCE ) .getOrCreatePageCache(); - StoreFactory storeFactory = new StoreFactory( storeDir, pageCache, fs, NullLogProvider.getInstance() ); + StoreFactory storeFactory = new StoreFactory( storeDir, pageCache, fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); neoStores = storeFactory.openAllNeoStores( true ); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StorePropertyCursorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StorePropertyCursorTest.java index ab56dbd3d111d..87dfb01149b22 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StorePropertyCursorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StorePropertyCursorTest.java @@ -38,6 +38,7 @@ import org.neo4j.cursor.Cursor; import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.impl.store.DynamicArrayStore; import org.neo4j.kernel.impl.store.DynamicRecordAllocator; import org.neo4j.kernel.impl.store.DynamicStringStore; @@ -283,7 +284,7 @@ public static void setUp() throws IOException fs.deleteRecursively( storeDir ); } fs.mkdirs( storeDir ); - neoStores = new StoreFactory( storeDir, pageCache, fs, NullLogProvider.getInstance() ) + neoStores = new StoreFactory( storeDir, pageCache, fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ) .openAllNeoStores( true ); propertyStore = neoStores.getPropertyStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreSingleRelationshipCursorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreSingleRelationshipCursorTest.java index d9268c65ad3a9..77c25f432901e 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreSingleRelationshipCursorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreSingleRelationshipCursorTest.java @@ -25,6 +25,7 @@ import org.junit.Test; import org.junit.rules.RuleChain; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.impl.locking.LockService; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.RecordCursors; @@ -109,7 +110,7 @@ private void createRelationshipRecord( RelationshipStore relationshipStore, bool private StoreFactory getStoreFactory() { return new StoreFactory( testDirectory.directory(), pageCacheRule.getPageCache( fileSystemRule.get() ), - fileSystemRule.get(), NullLogProvider.getInstance() ); + fileSystemRule.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); } private StoreSingleRelationshipCursor createRelationshipCursor() diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/core/ManyPropertyKeysIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/core/ManyPropertyKeysIT.java index 3d0131eb27a57..da527dc5a417a 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/core/ManyPropertyKeysIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/core/ManyPropertyKeysIT.java @@ -33,6 +33,7 @@ import org.neo4j.graphdb.Transaction; import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.KernelAPI; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.Statement; @@ -130,7 +131,7 @@ private GraphDatabaseAPI databaseWithManyPropertyKeys( int propertyKeyCount ) th PageCache pageCache = pageCacheRule.getPageCache( fileSystemRule.get() ); StoreFactory storeFactory = new StoreFactory( storeDir, pageCache, fileSystemRule.get(), - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = storeFactory.openAllNeoStores( true ); PropertyKeyTokenStore store = neoStores.getPropertyKeyTokenStore(); for ( int i = 0; i < propertyKeyCount; i++ ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactoryTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactoryTest.java index 65eea2d06978c..d4afbdc6acf06 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactoryTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/pagecache/ConfiguringPageCacheFactoryTest.java @@ -27,6 +27,7 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.logging.AssertableLogProvider; import org.neo4j.logging.Log; @@ -66,7 +67,7 @@ public void shouldFitAsManyPagesAsItCan() throws Throwable // When ConfiguringPageCacheFactory factory = new ConfiguringPageCacheFactory( fsRule.get(), config, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, - NullLog.getInstance() ); + NullLog.getInstance(), EmptyVersionContextSupplier.INSTANCE ); // Then try ( PageCache cache = factory.getOrCreatePageCache() ) @@ -88,7 +89,7 @@ public void shouldWarnWhenCreatedWithConfiguredPageCache() throws Exception // When ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( fsRule.get(), config, - PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, log ); + PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, log, EmptyVersionContextSupplier.INSTANCE ); // Then try ( PageCache pageCache = pageCacheFactory.getOrCreatePageCache() ) @@ -111,7 +112,7 @@ public void mustUseAndLogConfiguredPageSwapper() throws Exception // When new ConfiguringPageCacheFactory( fsRule.get(), config, PageCacheTracer.NULL, - PageCursorTracerSupplier.NULL, log ); + PageCursorTracerSupplier.NULL, log, EmptyVersionContextSupplier.INSTANCE ); // Then assertThat( PageSwapperFactoryForTesting.countCreatedPageSwapperFactories(), is( 1 ) ); @@ -129,7 +130,7 @@ public void mustThrowIfConfiguredPageSwapperCannotBeFound() throws Exception // When new ConfiguringPageCacheFactory( fsRule.get(), config, PageCacheTracer.NULL, - PageCursorTracerSupplier.NULL, NullLog.getInstance() ); + PageCursorTracerSupplier.NULL, NullLog.getInstance(), EmptyVersionContextSupplier.INSTANCE ); } @Test @@ -145,7 +146,7 @@ public void mustIgnoreExplicitlySpecifiedCachePageSizeIfPageSwapperHintIsStrict( // When ConfiguringPageCacheFactory factory = new ConfiguringPageCacheFactory( fsRule.get(), config, PageCacheTracer.NULL, PageCursorTracerSupplier.NULL, - NullLog.getInstance() ); + NullLog.getInstance(), EmptyVersionContextSupplier.INSTANCE ); // Then try ( PageCache cache = factory.getOrCreatePageCache() ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/FreeIdsAfterRecoveryTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/FreeIdsAfterRecoveryTest.java index 14b88f27371e7..c971283438599 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/FreeIdsAfterRecoveryTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/FreeIdsAfterRecoveryTest.java @@ -19,12 +19,13 @@ */ package org.neo4j.kernel.impl.store; -import java.io.File; - import org.junit.Rule; import org.junit.Test; import org.junit.rules.RuleChain; +import java.io.File; + +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.impl.store.id.IdGeneratorImpl; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.storemigration.StoreFile; @@ -52,7 +53,8 @@ public void shouldCompletelyRebuildIdGeneratorsAfterCrash() throws Exception { // GIVEN StoreFactory storeFactory = new StoreFactory( directory.directory(), - pageCacheRule.getPageCache( fileSystemRule.get() ), fileSystemRule.get(), NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fileSystemRule.get() ), fileSystemRule.get(), NullLogProvider.getInstance(), + EmptyVersionContextSupplier.INSTANCE ); long highId; try ( NeoStores stores = storeFactory.openAllNeoStores( true ) ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/IdGeneratorRebuildFailureEmulationTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/IdGeneratorRebuildFailureEmulationTest.java index 6a820cea95085..7fb418574cc13 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/IdGeneratorRebuildFailureEmulationTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/IdGeneratorRebuildFailureEmulationTest.java @@ -19,10 +19,6 @@ */ package org.neo4j.kernel.impl.store; -import java.io.File; -import java.util.HashMap; -import java.util.Map; - import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -32,6 +28,10 @@ import org.junit.runners.Suite; import org.junit.runners.Suite.SuiteClasses; +import java.io.File; +import java.util.HashMap; +import java.util.Map; + import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.PropertyContainer; @@ -41,6 +41,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.impl.factory.CommunityEditionModule; @@ -123,7 +124,7 @@ public void initialize() params.put( GraphDatabaseSettings.rebuild_idgenerators_fast.name(), Settings.FALSE ); Config config = Config.embeddedDefaults( params ); factory = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs ), - pageCacheRule.getPageCache( fs ), fs, NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs ), fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); } @After diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/MetaDataStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/MetaDataStoreTest.java index 0348237d1a31a..9371d30976c7e 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/MetaDataStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/MetaDataStoreTest.java @@ -39,6 +39,7 @@ import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PagedFile; import org.neo4j.io.pagecache.impl.DelegatingPageCursor; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.impl.store.format.standard.Standard; import org.neo4j.kernel.impl.store.record.MetaDataRecord; import org.neo4j.kernel.impl.store.record.RecordLoad; @@ -112,7 +113,8 @@ public boolean checkAndClearBoundsFlag() private MetaDataStore newMetaDataStore() throws IOException { LogProvider logProvider = NullLogProvider.getInstance(); - StoreFactory storeFactory = new StoreFactory( STORE_DIR, pageCacheWithFakeOverflow, fs, logProvider ); + StoreFactory storeFactory = new StoreFactory( STORE_DIR, pageCacheWithFakeOverflow, fs, logProvider, + EmptyVersionContextSupplier.INSTANCE ); return storeFactory.openNeoStores( true, StoreType.META_DATA ).getMetaDataStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java index 58c04504f1511..233a8aca485fb 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresTest.java @@ -47,6 +47,7 @@ import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.exceptions.EntityNotFoundException; @@ -143,7 +144,7 @@ public void setUpNeoStores() throws Exception Config config = Config.embeddedDefaults(); pageCache = pageCacheRule.getPageCache( fs.get() ); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), pageCache, - fs.get(), NullLogProvider.getInstance() ); + fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); sf.openAllNeoStores( true ).close(); } @@ -152,7 +153,7 @@ public void impossibleToGetStoreFromClosedNeoStoresContainer() { Config config = Config.embeddedDefaults(); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), pageCache, - fs.get(), NullLogProvider.getInstance() ); + fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = sf.openAllNeoStores( true ); assertNotNull( neoStores.getMetaDataStore() ); @@ -169,7 +170,7 @@ public void notAllowCreateDynamicStoreWithNegativeBlockSize() { Config config = Config.embeddedDefaults(); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), pageCache, - fs.get(), NullLogProvider.getInstance() ); + fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); exception.expect( IllegalArgumentException.class ); exception.expectMessage( "Block size of dynamic array store should be positive integer." ); @@ -185,7 +186,7 @@ public void impossibleToGetNotRequestedStore() { Config config = Config.embeddedDefaults(); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), pageCache, - fs.get(), NullLogProvider.getInstance() ); + fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = sf.openNeoStores( true, StoreType.NODE_LABEL ); exception.expect( IllegalStateException.class ); @@ -535,7 +536,7 @@ public void setVersion() throws Exception Config config = Config.embeddedDefaults(); StoreFactory sf = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fileSystem ), pageCache, - fileSystem, NullLogProvider.getInstance() ); + fileSystem, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = sf.openAllNeoStores(); assertEquals( 12, neoStores.getMetaDataStore().getCurrentLogVersion() ); @@ -592,7 +593,7 @@ public void testSetLatestConstraintTx() throws Exception // given Config config = Config.embeddedDefaults(); StoreFactory sf = new StoreFactory( dir.directory(), config, new DefaultIdGeneratorFactory( fs.get() ), - pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); // when NeoStores neoStores = sf.openAllNeoStores( true ); @@ -621,7 +622,8 @@ public void testSetLatestConstraintTx() throws Exception public void shouldInitializeTheTxIdToOne() { StoreFactory factory = - new StoreFactory( new File( "graph.db/neostore" ), pageCache, fs.get(), NullLogProvider.getInstance() ); + new StoreFactory( new File( "graph.db/neostore" ), pageCache, fs.get(), + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = factory.openAllNeoStores( true ) ) { @@ -640,7 +642,7 @@ public void shouldThrowUnderlyingStorageExceptionWhenFailingToLoadStorage() { FileSystemAbstraction fileSystem = fs.get(); File neoStoreDir = new File( "/tmp/graph.db/neostore" ).getAbsoluteFile(); - StoreFactory factory = new StoreFactory( neoStoreDir, pageCache, fileSystem, NullLogProvider.getInstance() ); + StoreFactory factory = new StoreFactory( neoStoreDir, pageCache, fileSystem, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = factory.openAllNeoStores( true ) ) { @@ -717,7 +719,7 @@ public void shouldSetHighestTransactionIdWhenNeeded() throws Throwable // GIVEN FileSystemAbstraction fileSystem = fs.get(); fileSystem.mkdirs( storeDir ); - StoreFactory factory = new StoreFactory( storeDir, pageCache, fileSystem, NullLogProvider.getInstance() ); + StoreFactory factory = new StoreFactory( storeDir, pageCache, fileSystem, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStore = factory.openAllNeoStores( true ) ) { @@ -741,7 +743,7 @@ public void shouldNotSetHighestTransactionIdWhenNeeded() throws Throwable // GIVEN FileSystemAbstraction fileSystem = fs.get(); fileSystem.mkdirs( storeDir ); - StoreFactory factory = new StoreFactory( storeDir, pageCache, fileSystem, NullLogProvider.getInstance() ); + StoreFactory factory = new StoreFactory( storeDir, pageCache, fileSystem, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStore = factory.openAllNeoStores( true ) ) { @@ -767,7 +769,7 @@ public void shouldCloseAllTheStoreEvenIfExceptionsAreThrown() throws Exception Config defaults = Config.embeddedDefaults( singletonMap( counts_store_rotation_timeout.name(), "60m" ) ); StoreFactory factory = new StoreFactory( storeDir, defaults, new DefaultIdGeneratorFactory( fileSystem ), pageCache, - fileSystem, NullLogProvider.getInstance() ); + fileSystem, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStore = factory.openAllNeoStores( true ); // let's hack the counts store so it fails to rotate and hence it fails to close as well... @@ -856,7 +858,7 @@ private static long defaultStoreVersion() private static StoreFactory newStoreFactory( File neoStoreDir, PageCache pageCache, FileSystemAbstraction fs ) { return new StoreFactory( neoStoreDir, pageCache, fs, RecordFormatSelector.defaultFormat(), - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); } private Token createDummyIndex( int id, String key ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeStoreTest.java index 7d69ceac42f1d..c3b94be73f86a 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeStoreTest.java @@ -19,6 +19,12 @@ */ package org.neo4j.kernel.impl.store; +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.junit.After; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; + import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; @@ -29,12 +35,6 @@ import java.util.function.Supplier; import java.util.stream.LongStream; -import org.apache.commons.lang3.mutable.MutableBoolean; -import org.junit.After; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; - import org.neo4j.collection.primitive.Primitive; import org.neo4j.collection.primitive.PrimitiveLongSet; import org.neo4j.graphdb.mockfs.DelegatingFileSystemAbstraction; @@ -45,6 +45,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.allocator.ReusableRecordsAllocator; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; @@ -384,7 +385,7 @@ protected IdGenerator instantiate( FileSystemAbstraction fs, File fileName, int } } ); StoreFactory factory = new StoreFactory( storeDir, Config.empty(), idGeneratorFactory, pageCache, fs, - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); neoStores = factory.openAllNeoStores( true ); nodeStore = neoStores.getNodeStore(); return nodeStore; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RecordStoreConsistentReadTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RecordStoreConsistentReadTest.java index 2f1a0ddaed779..a95816f2b1fba 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RecordStoreConsistentReadTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RecordStoreConsistentReadTest.java @@ -35,6 +35,7 @@ import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.impl.store.allocator.ReusableRecordsAllocator; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; import org.neo4j.kernel.impl.store.record.DynamicRecord; @@ -82,7 +83,7 @@ private NeoStores storeFixture() PageCache pageCache = pageCacheRule.getPageCache( fs, config().withInconsistentReads( nextReadIsInconsistent ) ); File storeDir = new File( "stores" ); - StoreFactory factory = new StoreFactory( storeDir, pageCache, fs, NullLogProvider.getInstance() ); + StoreFactory factory = new StoreFactory( storeDir, pageCache, fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = factory.openAllNeoStores( true ); S store = initialiseStore( neoStores ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RelationshipGroupStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RelationshipGroupStoreTest.java index 4700ac6c8bc26..6f38a8d6bac57 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RelationshipGroupStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RelationshipGroupStoreTest.java @@ -40,6 +40,7 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.MyRelTypes; import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine; @@ -170,7 +171,7 @@ private StoreFactory factory( Integer customThreshold, PageCache pageCache ) customConfig.put( GraphDatabaseSettings.dense_node_threshold.name(), "" + customThreshold ); } return new StoreFactory( directory, Config.embeddedDefaults( customConfig ), new DefaultIdGeneratorFactory( fs ), pageCache, - fs, NullLogProvider.getInstance() ); + fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); } @Test diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/SchemaStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/SchemaStoreTest.java index 2cbbcafb759f4..5dad174e5044f 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/SchemaStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/SchemaStoreTest.java @@ -31,6 +31,7 @@ import java.util.stream.IntStream; import org.neo4j.helpers.collection.Iterables; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.api.schema.constaints.ConstraintDescriptorFactory; import org.neo4j.kernel.api.schema.index.IndexDescriptorFactory; @@ -70,7 +71,7 @@ public void before() throws Exception config = Config.empty(); DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fs.get() ); storeFactory = new StoreFactory( storeDir, config, idGeneratorFactory, pageCacheRule.getPageCache( fs.get() ), - fs.get(), NullLogProvider.getInstance() ); + fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); neoStores = storeFactory.openAllNeoStores( true ); store = neoStores.getSchemaStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/StoreFactoryTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/StoreFactoryTest.java index 2ae9db248e2b5..c65c55e7baa9b 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/StoreFactoryTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/StoreFactoryTest.java @@ -32,6 +32,7 @@ import org.neo4j.helpers.collection.MapUtil; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.impl.store.format.RecordFormats; @@ -75,7 +76,7 @@ private StoreFactory storeFactory( Config config, OpenOption... openOptions ) LogProvider logProvider = NullLogProvider.getInstance(); RecordFormats recordFormats = selectForStoreOrConfig( config, storeDir, fsRule.get(), pageCache, logProvider ); return new StoreFactory( storeDir, DEFAULT_NAME, config, idGeneratorFactory, pageCache, fsRule.get(), - recordFormats, logProvider, openOptions ); + recordFormats, logProvider, EmptyVersionContextSupplier.INSTANCE, openOptions ); } private File directory( String name ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestArrayStore.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestArrayStore.java index 68e3256f821d3..4079325b48fcc 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestArrayStore.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestArrayStore.java @@ -35,6 +35,7 @@ import org.neo4j.helpers.collection.Pair; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; import org.neo4j.kernel.impl.store.record.DynamicRecord; @@ -68,7 +69,7 @@ public void before() throws Exception DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fs ); PageCache pageCache = pageCacheRule.getPageCache( fs ); StoreFactory factory = new StoreFactory( dir, Config.empty(), idGeneratorFactory, pageCache, fs, - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); neoStores = factory.openAllNeoStores( true ); arrayStore = neoStores.getPropertyStore().getArrayStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestDynamicStore.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestDynamicStore.java index e22a90c20d025..162e3db181951 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestDynamicStore.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestDynamicStore.java @@ -35,6 +35,7 @@ import org.neo4j.helpers.collection.Iterables; import org.neo4j.helpers.collection.MapUtil; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; import org.neo4j.kernel.impl.store.record.DynamicRecord; @@ -64,7 +65,7 @@ public void setUp() fs.get().mkdir( storeDir ); config = config(); storeFactory = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), - pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); } @After diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGraphProperties.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGraphProperties.java index 018531bba18ac..85984f2bfb6dd 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGraphProperties.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGraphProperties.java @@ -34,6 +34,7 @@ import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.graphdb.mockfs.UncloseableDelegatingFileSystemAbstraction; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.core.NodeManager; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; @@ -180,7 +181,7 @@ public void firstRecordOtherThanZeroIfNotFirst() throws Exception Config config = Config.embeddedDefaults( Collections.emptyMap() ); StoreFactory storeFactory = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), - pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = storeFactory.openAllNeoStores(); long prop = neoStores.getMetaDataStore().getGraphNextProp(); assertTrue( prop != 0 ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGrowingFileMemoryMapping.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGrowingFileMemoryMapping.java index 54e649d5e93c5..519d7869bffe9 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGrowingFileMemoryMapping.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestGrowingFileMemoryMapping.java @@ -29,6 +29,7 @@ import org.neo4j.io.ByteUnit; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.format.standard.NodeRecordFormat; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; @@ -72,7 +73,7 @@ public void shouldGrowAFileWhileContinuingToMemoryMapNewRegions() throws Excepti DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fileSystemAbstraction ); PageCache pageCache = pageCacheRule.getPageCache( fileSystemAbstraction, config ); StoreFactory storeFactory = new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, - fileSystemAbstraction, NullLogProvider.getInstance() ); + fileSystemAbstraction, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = storeFactory.openAllNeoStores( true ); NodeStore nodeStore = neoStores.getNodeStore(); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestIdGeneratorRebuilding.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestIdGeneratorRebuilding.java index 1a3f0fe852dd2..bf89ea5f396bc 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestIdGeneratorRebuilding.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/TestIdGeneratorRebuilding.java @@ -32,6 +32,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; import org.neo4j.helpers.collection.MapUtil; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.AbstractNeo4jTestCase; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; @@ -129,7 +130,7 @@ public void verifyDynamicSizedStoresCanRebuildIdGeneratorSlowly() throws Excepti GraphDatabaseSettings.rebuild_idgenerators_fast.name(), "false" ) ); StoreFactory storeFactory = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs ), - pageCacheRule.getPageCache( fs ), fs, NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs ), fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = storeFactory.openAllNeoStores( true ); DynamicStringStore store = neoStores.getPropertyStore().getStringStore(); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsComputerTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsComputerTest.java index 7dc1b4a8e673b..703d05e65cfb9 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsComputerTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsComputerTest.java @@ -37,6 +37,7 @@ import org.neo4j.graphdb.mockfs.UncloseableDelegatingFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.CountsComputer; import org.neo4j.kernel.impl.store.MetaDataStore; @@ -312,14 +313,14 @@ private void cleanupCountsForRebuilding() private CountsTracker createCountsTracker() { return new CountsTracker( NullLogProvider.getInstance(), fs, pageCache, - Config.empty(), new File( dir, COUNTS_STORE_BASE ) ); + Config.empty(), new File( dir, COUNTS_STORE_BASE ), EmptyVersionContextSupplier.INSTANCE ); } private void rebuildCounts( long lastCommittedTransactionId ) throws IOException { cleanupCountsForRebuilding(); - StoreFactory storeFactory = new StoreFactory( dir, pageCache, fs, NullLogProvider.getInstance() ); + StoreFactory storeFactory = new StoreFactory( dir, pageCache, fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); try ( Lifespan life = new Lifespan(); NeoStores neoStores = storeFactory.openAllNeoStores() ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsRotationTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsRotationTest.java index 200d4762c656d..d9150e661a8ac 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsRotationTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsRotationTest.java @@ -51,6 +51,7 @@ import org.neo4j.helpers.collection.Pair; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.CountsAccessor; import org.neo4j.kernel.impl.api.CountsVisitor; @@ -389,7 +390,7 @@ private CountsTracker createCountsTracker( PageCache pageCache ) private CountsTracker createCountsTracker( PageCache pageCache, Config config ) { return new CountsTracker( NullLogProvider.getInstance(), fs, pageCache, config, - new File( dir.getPath(), COUNTS_STORE_BASE ) ); + new File( dir.getPath(), COUNTS_STORE_BASE ), EmptyVersionContextSupplier.INSTANCE ); } private void checkPoint( GraphDatabaseAPI db ) throws IOException diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsTrackerTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsTrackerTest.java index 61b29f2ed9895..ca1dbaeb26024 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsTrackerTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/counts/CountsTrackerTest.java @@ -31,9 +31,13 @@ import org.neo4j.function.IOFunction; import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.CountsAccessor; import org.neo4j.kernel.impl.api.CountsVisitor; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; import org.neo4j.kernel.impl.store.CountsOracle; import org.neo4j.kernel.impl.store.counts.keys.CountsKey; import org.neo4j.kernel.impl.store.counts.keys.CountsKeyFactory; @@ -52,6 +56,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; @@ -189,6 +194,54 @@ public void shouldUpdateCountsOnExistingStore() throws Exception } } + @Test + public void detectInMemoryDirtyVersionRead() + { + int labelId = 1; + long lastClosedTransactionId = 11L; + long writeTransactionId = 22L; + TransactionVersionContextSupplier versionContextSupplier = new TransactionVersionContextSupplier(); + versionContextSupplier.init( () -> lastClosedTransactionId ); + VersionContext versionContext = versionContextSupplier.getVersionContext(); + + try ( Lifespan life = new Lifespan() ) + { + CountsTracker tracker = life.add( newTracker( versionContextSupplier ) ); + try ( CountsAccessor.Updater updater = tracker.apply( writeTransactionId ).get() ) + { + updater.incrementNodeCount( labelId, 1 ); + } + + versionContext.initRead(); + tracker.nodeCount( labelId, Registers.newDoubleLongRegister() ); + assertTrue( versionContext.isDirty() ); + } + } + + @Test + public void allowNonDirtyInMemoryDirtyVersionRead() + { + int labelId = 1; + long lastClosedTransactionId = 15L; + long writeTransactionId = 13L; + TransactionVersionContextSupplier versionContextSupplier = new TransactionVersionContextSupplier(); + versionContextSupplier.init( () -> lastClosedTransactionId ); + VersionContext versionContext = versionContextSupplier.getVersionContext(); + + try ( Lifespan life = new Lifespan() ) + { + CountsTracker tracker = life.add( newTracker( versionContextSupplier ) ); + try ( CountsAccessor.Updater updater = tracker.apply( writeTransactionId ).get() ) + { + updater.incrementNodeCount( labelId, 1 ); + } + + versionContext.initRead(); + tracker.nodeCount( labelId, Registers.newDoubleLongRegister() ); + assertFalse( versionContext.isDirty() ); + } + } + @Test public void shouldBeAbleToReadUpToDateValueWhileAnotherThreadIsPerformingRotation() throws Exception { @@ -217,7 +270,7 @@ public void shouldBeAbleToReadUpToDateValueWhileAnotherThreadIsPerformingRotatio final Barrier.Control barrier = new Barrier.Control(); CountsTracker tracker = life.add( new CountsTracker( resourceManager.logProvider(), resourceManager.fileSystem(), resourceManager.pageCache(), - Config.empty(), resourceManager.testPath() ) + Config.empty(), resourceManager.testPath(), EmptyVersionContextSupplier.INSTANCE ) { @Override protected boolean include( CountsKey countsKey, ReadableBuffer value ) @@ -357,7 +410,7 @@ public void shouldNotEndUpInBrokenStateAfterRotationFailure() throws Exception { // GIVEN FakeClock clock = Clocks.fakeClock(); - CountsTracker tracker = resourceManager.managed( newTracker( clock ) ); + CountsTracker tracker = resourceManager.managed( newTracker( clock, EmptyVersionContextSupplier.INSTANCE ) ); int labelId = 1; try ( CountsAccessor.Updater tx = tracker.apply( 2 ).get() ) { @@ -403,13 +456,18 @@ public void shouldNotEndUpInBrokenStateAfterRotationFailure() throws Exception private CountsTracker newTracker() { - return newTracker( Clocks.systemClock() ); + return newTracker( EmptyVersionContextSupplier.INSTANCE ); + } + + private CountsTracker newTracker( VersionContextSupplier versionContextSupplier ) + { + return newTracker( Clocks.systemClock(), versionContextSupplier ); } - private CountsTracker newTracker( Clock clock ) + private CountsTracker newTracker( Clock clock, VersionContextSupplier versionContextSupplier ) { return new CountsTracker( resourceManager.logProvider(), resourceManager.fileSystem(), - resourceManager.pageCache(), Config.empty(), resourceManager.testPath(), clock ) + resourceManager.pageCache(), Config.empty(), resourceManager.testPath(), clock, versionContextSupplier ) .setInitializer( new DataInitializer() { @Override diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStoreTest.java index a9bede4e50b2c..3b0b67d68cd5e 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/AbstractKeyValueStoreTest.java @@ -36,6 +36,7 @@ import org.neo4j.function.ThrowingConsumer; import org.neo4j.helpers.collection.Pair; import org.neo4j.io.fs.StoreChannel; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; import org.neo4j.kernel.lifecycle.Lifespan; import org.neo4j.test.rule.Resources; @@ -90,7 +91,7 @@ public String toString() public void retryLookupOnConcurrentStoreStateChange() throws IOException { Store testStore = resourceManager.managed( createTestStore( TimeUnit.DAYS.toMillis( 2 ) ) ); - ConcurrentMapState newState = new ConcurrentMapState<>( testStore.state, mock( File.class ) ); + ConcurrentMapState newState = new ConcurrentMapState<>( testStore.state, mock( File.class ), EmptyVersionContextSupplier.INSTANCE ); testStore.put( "test", "value" ); CountingErroneousReader countingErroneousReader = new CountingErroneousReader( testStore, newState ); @@ -615,7 +616,8 @@ private Store( HeaderField... headerFields ) private Store( long rotationTimeout, HeaderField... headerFields ) { super( resourceManager.fileSystem(), resourceManager.pageCache(), resourceManager.testPath(), null, - new RotationTimerFactory( Clocks.systemClock(), rotationTimeout ), 16, 16, headerFields ); + new RotationTimerFactory( Clocks.systemClock(), rotationTimeout ), + EmptyVersionContextSupplier.INSTANCE,16, 16, headerFields ); this.headerFields = headerFields; setEntryUpdaterInitializer( new DataInitializer>() { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapStateTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapStateTest.java index 964a3d33e743b..83addc5007a84 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapStateTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/kvstore/ConcurrentMapStateTest.java @@ -19,30 +19,48 @@ */ package org.neo4j.kernel.impl.store.kvstore; +import com.google.common.util.concurrent.Runnables; +import org.junit.Before; import org.junit.Test; import java.io.File; +import java.io.IOException; import java.util.concurrent.locks.Lock; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; + import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class ConcurrentMapStateTest { - private final ReadableState store = mock( ReadableState.class ); + private final ReadableState store = mock( ReadableState.class ); private final File file = mock( File.class ); private final Lock lock = mock( Lock.class ); + @Before + public void setUp() throws Exception + { + KeyFormat keyFormat = mock( KeyFormat.class ); + when( keyFormat.valueSize() ).thenReturn( Long.BYTES ); + when( store.keyFormat() ).thenReturn( keyFormat ); + } + @Test public void shouldCreateAnUpdaterForTheNextUnseenVersionUpdate() throws Exception { // given long initialVersion = 42; when( store.version() ).thenReturn( initialVersion ); - ConcurrentMapState state = new ConcurrentMapState<>( store, file ); + ConcurrentMapState state = createMapState(); // when long updateVersion = 43; @@ -60,7 +78,7 @@ public void shouldCreateAnUpdaterForAnUnseenVersionUpdateWithAGap() throws Excep // given long initialVersion = 42; when( store.version() ).thenReturn( initialVersion ); - ConcurrentMapState state = new ConcurrentMapState<>( store, file ); + ConcurrentMapState state = createMapState(); // when long updateVersion = 45; @@ -79,7 +97,7 @@ public void shouldCreateAnUpdaterForMultipleVersionUpdatesInOrder() throws Excep // given long initialVersion = 42; when( store.version() ).thenReturn( initialVersion ); - ConcurrentMapState state = new ConcurrentMapState<>( store, file ); + ConcurrentMapState state = createMapState(); // when EntryUpdater updater; @@ -108,7 +126,7 @@ public void shouldCreateAnUpdaterForMultipleVersionUpdatesNotInOrder() throws Ex // given long initialVersion = 42; when( store.version() ).thenReturn( initialVersion ); - ConcurrentMapState state = new ConcurrentMapState<>( store, file ); + ConcurrentMapState state = createMapState(); // when EntryUpdater updater; @@ -137,7 +155,7 @@ public void shouldUseEmptyUpdaterOnVersionLowerOrEqualToTheInitialVersion() thro // given long initialVersion = 42; when( store.version() ).thenReturn( initialVersion ); - ConcurrentMapState state = new ConcurrentMapState<>( store, file ); + ConcurrentMapState state = createMapState(); // when EntryUpdater updater = state.updater( initialVersion, lock ); @@ -146,4 +164,98 @@ public void shouldUseEmptyUpdaterOnVersionLowerOrEqualToTheInitialVersion() thro assertEquals( "Empty updater should be used for version less or equal to initial", EntryUpdater.noUpdates(), updater ); } + + @Test + public void markDirtyVersionLookupOnKeyUpdate() throws IOException + { + long updaterVersionTxId = 25; + long lastClosedTxId = 20; + TransactionVersionContextSupplier versionContextSupplier = new TransactionVersionContextSupplier(); + versionContextSupplier.init( () -> lastClosedTxId ); + ConcurrentMapState mapState = createMapState( versionContextSupplier ); + VersionContext versionContext = versionContextSupplier.getVersionContext(); + try ( EntryUpdater updater = mapState.updater( updaterVersionTxId, lock ) ) + { + updater.apply( "a", new SimpleValueUpdate( 1 ) ); + updater.apply( "b", new SimpleValueUpdate( 2 ) ); + } + + assertEquals( updaterVersionTxId, mapState.version() ); + versionContext.initRead(); + mapState.lookup( "a", new EmptyValueSink() ); + assertTrue( versionContext.isDirty() ); + } + + @Test + public void markDirtyVersionLookupOnKeyReset() throws IOException + { + long updaterVersionTxId = 25; + long lastClosedTxId = 20; + when( store.version() ).thenReturn( updaterVersionTxId ); + TransactionVersionContextSupplier versionContextSupplier = new TransactionVersionContextSupplier(); + versionContextSupplier.init( () -> lastClosedTxId ); + VersionContext versionContext = versionContextSupplier.getVersionContext(); + + ConcurrentMapState mapState = createMapState( versionContextSupplier ); + + versionContext.initRead(); + mapState.resettingUpdater( lock, Runnables.doNothing() ).apply( "a", new SimpleValueUpdate( 1 ) ); + mapState.lookup( "a", new EmptyValueSink() ); + assertTrue( versionContext.isDirty() ); + } + + @Test + public void doNotMarkVersionAsDirtyOnAnotherKeyUpdate() throws IOException + { + long updaterVersionTxId = 25; + long lastClosedTxId = 20; + TransactionVersionContextSupplier versionContextSupplier = new TransactionVersionContextSupplier(); + versionContextSupplier.init( () -> lastClosedTxId ); + ConcurrentMapState mapState = createMapState( versionContextSupplier ); + VersionContext versionContext = versionContextSupplier.getVersionContext(); + try ( EntryUpdater updater = mapState.updater( updaterVersionTxId, lock ) ) + { + updater.apply( "b", new SimpleValueUpdate( 2 ) ); + } + + assertEquals( updaterVersionTxId, mapState.version() ); + versionContext.initRead(); + mapState.lookup( "a", new EmptyValueSink() ); + assertFalse( versionContext.isDirty() ); + } + + private ConcurrentMapState createMapState() + { + return createMapState( EmptyVersionContextSupplier.INSTANCE ); + } + + private ConcurrentMapState createMapState( VersionContextSupplier versionContextSupplier ) + { + return new ConcurrentMapState<>( store, file, versionContextSupplier ); + } + + private static class SimpleValueUpdate implements ValueUpdate + { + private final long value; + + SimpleValueUpdate( long value ) + { + this.value = value; + } + + @Override + public void update( WritableBuffer target ) + { + target.putLong( 0, value ); + } + } + + private static class EmptyValueSink extends ValueSink + { + @Override + protected void value( ReadableBuffer value ) + { + + } + } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java index 0f29089b0f907..a6ae9a81b0078 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorIT.java @@ -33,6 +33,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.index.inmemory.InMemoryIndexProvider; @@ -149,7 +150,7 @@ public void shouldBeAbleToResumeMigrationOnMoving() throws Exception // THEN starting the new store should be successful StoreFactory storeFactory = - new StoreFactory( storeDirectory, pageCache, fs, logService.getInternalLogProvider() ); + new StoreFactory( storeDirectory, pageCache, fs, logService.getInternalLogProvider(), EmptyVersionContextSupplier.INSTANCE ); storeFactory.openAllNeoStores().close(); } @@ -184,7 +185,7 @@ public void shouldBeAbleToResumeMigrationOnRebuildingCounts() throws Exception // THEN starting the new store should be successful StoreFactory storeFactory = - new StoreFactory( storeDirectory, pageCache, fs, logService.getInternalLogProvider() ); + new StoreFactory( storeDirectory, pageCache, fs, logService.getInternalLogProvider(), EmptyVersionContextSupplier.INSTANCE ); storeFactory.openAllNeoStores().close(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/ApplyRecoveredTransactionsTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/ApplyRecoveredTransactionsTest.java index 2c3cd745532af..7b499d2aac071 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/ApplyRecoveredTransactionsTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/ApplyRecoveredTransactionsTest.java @@ -28,6 +28,7 @@ import java.util.Arrays; import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.TransactionToApply; import org.neo4j.kernel.impl.core.CacheAccessBackDoor; @@ -114,7 +115,7 @@ public void before() FileSystemAbstraction fs = fsr.get(); File storeDir = new File( "dir" ); StoreFactory storeFactory = new StoreFactory( storeDir, Config.empty(), new DefaultIdGeneratorFactory( fs ), - pageCacheRule.getPageCache( fs ), fs, NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs ), fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); neoStores = storeFactory.openAllNeoStores( true ); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeCommandTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeCommandTest.java index 816584dfc0e0c..c1cccd51af5fc 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeCommandTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeCommandTest.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Set; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeLabels; @@ -245,7 +246,8 @@ public void before() throws Exception fs.get().mkdirs( dir ); @SuppressWarnings("deprecation") StoreFactory storeFactory = new StoreFactory( dir, Config.empty(), new DefaultIdGeneratorFactory( fs.get() ), - pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance(), + EmptyVersionContextSupplier.INSTANCE ); neoStores = storeFactory.openAllNeoStores( true ); nodeStore = neoStores.getNodeStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeLabelsFieldTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeLabelsFieldTest.java index eb57490ba3c1e..b17b4944231ce 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeLabelsFieldTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeLabelsFieldTest.java @@ -38,6 +38,7 @@ import org.neo4j.helpers.CloneableInPublic; import org.neo4j.helpers.collection.Iterables; import org.neo4j.helpers.collection.Pair; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.DynamicNodeLabels; import org.neo4j.kernel.impl.store.NeoStores; @@ -506,7 +507,7 @@ public void startUp() fs.get().mkdirs( storeDir ); Config config = Config.embeddedDefaults( stringMap( GraphDatabaseSettings.label_block_size.name(), "60" ) ); StoreFactory storeFactory = new StoreFactory( storeDir, config, new DefaultIdGeneratorFactory( fs.get() ), - pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance() ); + pageCacheRule.getPageCache( fs.get() ), fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); neoStores = storeFactory.openAllNeoStores( true ); nodeStore = neoStores.getNodeStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetterTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetterTest.java index f504235e4e759..65f0baddb2d12 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetterTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetterTest.java @@ -26,6 +26,7 @@ import java.io.File; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.StoreFactory; @@ -62,7 +63,8 @@ public void shouldAbortLoadingGroupChainIfComeTooFar() throws Exception File dir = new File( "dir" ); fs.get().mkdirs( dir ); LogProvider logProvider = NullLogProvider.getInstance(); - StoreFactory storeFactory = new StoreFactory( dir, pageCache.getPageCache( fs.get() ), fs.get(), logProvider ); + StoreFactory storeFactory = new StoreFactory( dir, pageCache.getPageCache( fs.get() ), fs.get(), logProvider, + EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores stores = storeFactory.openNeoStores( true, StoreType.RELATIONSHIP_GROUP ) ) { RecordStore store = spy( stores.getRelationshipGroupStore() ); diff --git a/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java b/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java index b558bbd8fd744..8381409b178d5 100644 --- a/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java +++ b/community/kernel/src/test/java/org/neo4j/test/AdversarialPageCacheGraphDatabaseFactory.java @@ -20,7 +20,6 @@ package org.neo4j.test; import java.io.File; -import java.util.Map; import org.neo4j.adversaries.Adversary; import org.neo4j.adversaries.pagecache.AdversarialPageCache; @@ -28,6 +27,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseFactory; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.factory.CommunityEditionModule; import org.neo4j.kernel.impl.factory.DatabaseInfo; @@ -71,9 +71,10 @@ protected FileSystemAbstraction createFileSystemAbstraction() @Override protected PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, - LogService logging, Tracers tracers ) + LogService logging, Tracers tracers, VersionContextSupplier versionContextSupplier ) { - PageCache pageCache = super.createPageCache( fileSystem, config, logging, tracers ); + PageCache pageCache = super.createPageCache( fileSystem, config, logging, tracers, + versionContextSupplier ); return new AdversarialPageCache( pageCache, adversary ); } }; diff --git a/community/kernel/src/test/java/org/neo4j/test/rule/ConfigurablePageCacheRule.java b/community/kernel/src/test/java/org/neo4j/test/rule/ConfigurablePageCacheRule.java index 91b391328af7c..d98f0c05e7627 100644 --- a/community/kernel/src/test/java/org/neo4j/test/rule/ConfigurablePageCacheRule.java +++ b/community/kernel/src/test/java/org/neo4j/test/rule/ConfigurablePageCacheRule.java @@ -27,6 +27,7 @@ import org.neo4j.io.pagecache.tracing.PageCacheTracer; import org.neo4j.io.pagecache.tracing.cursor.DefaultPageCursorTracerSupplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory; import org.neo4j.logging.FormattedLogProvider; @@ -55,7 +56,8 @@ private PageCache createPageCache( FileSystemAbstraction fs, PageCacheConfig pag GraphDatabaseSettings.pagecache_memory.name(), "8M" ) ); FormattedLogProvider logProvider = FormattedLogProvider.toOutputStream( System.err ); ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( - fs, finalConfig, tracer, cursorTracerSupplier, logProvider.getLog( PageCache.class ) ) + fs, finalConfig, tracer, cursorTracerSupplier, logProvider.getLog( PageCache.class ), + EmptyVersionContextSupplier.INSTANCE ) { @Override public int calculatePageSize( Config config, PageSwapperFactory swapperFactory ) diff --git a/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoreDataSourceRule.java b/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoreDataSourceRule.java index 5cdb76f8cf656..8844524a350cc 100644 --- a/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoreDataSourceRule.java +++ b/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoreDataSourceRule.java @@ -21,6 +21,7 @@ import java.io.File; import java.util.function.Function; + import org.neo4j.graphdb.DependencyResolver; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.IOLimiter; @@ -37,6 +38,7 @@ import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider; import org.neo4j.kernel.impl.api.scan.NativeLabelScanStoreExtension; import org.neo4j.kernel.impl.constraints.StandardConstraintSemantics; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; import org.neo4j.kernel.impl.core.DatabasePanicEventGenerator; import org.neo4j.kernel.impl.core.LabelTokenHolder; import org.neo4j.kernel.impl.core.PropertyKeyTokenHolder; @@ -145,7 +147,7 @@ logService, mock( JobScheduler.class, RETURNS_MOCKS ), mock( TokenNameLookup.cla new CanWrite(), new StoreCopyCheckPointMutex(), new BufferedIdController( new BufferingIdGeneratorFactory( idGeneratorFactory, IdReuseEligibility.ALWAYS, - idConfigurationProvider ), jobScheduler )); + idConfigurationProvider ), jobScheduler ), new TransactionVersionContextSupplier() ); return dataSource; } diff --git a/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoresRule.java b/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoresRule.java index 147da7e0e749d..51de8629531a2 100644 --- a/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoresRule.java +++ b/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoresRule.java @@ -28,6 +28,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory; import org.neo4j.kernel.impl.store.NeoStores; @@ -80,7 +81,7 @@ private NeoStores open( FileSystemAbstraction fs, PageCache pageCache, RecordFor File storeDir = testDirectory.makeGraphDbDir(); Config configuration = configOf( config ); StoreFactory storeFactory = new StoreFactory( storeDir, configuration, idGeneratorFactory.apply( fs ), - pageCache, fs, format, NullLogProvider.getInstance() ); + pageCache, fs, format, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); return neoStores = stores.length == 0 ? storeFactory.openAllNeoStores( true ) : storeFactory.openNeoStores( true, stores ); @@ -105,7 +106,7 @@ private static PageCache getOrCreatePageCache( Config config, FileSystemAbstract { Log log = NullLog.getInstance(); ConfiguringPageCacheFactory pageCacheFactory = new ConfiguringPageCacheFactory( fs, config, NULL, - PageCursorTracerSupplier.NULL, log ); + PageCursorTracerSupplier.NULL, log, EmptyVersionContextSupplier.INSTANCE ); return pageCacheFactory.getOrCreatePageCache(); } diff --git a/community/kernel/src/test/java/org/neo4j/test/rule/RecordStorageEngineRule.java b/community/kernel/src/test/java/org/neo4j/test/rule/RecordStorageEngineRule.java index 79fe84e1f59dd..2987f2038dcbc 100644 --- a/community/kernel/src/test/java/org/neo4j/test/rule/RecordStorageEngineRule.java +++ b/community/kernel/src/test/java/org/neo4j/test/rule/RecordStorageEngineRule.java @@ -25,6 +25,7 @@ import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.TokenNameLookup; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.configuration.Config; @@ -214,7 +215,7 @@ private class ExtendedRecordStorageEngine extends RecordStorageEngine relationshipTypeTokens, schemaStateChangeCallback, constraintSemantics, scheduler, tokenNameLookup, lockService, indexProvider, indexingServiceMonitor, databaseHealth, labelScanStoreProvider, legacyIndexProviderLookup, indexConfigStore, legacyIndexTransactionOrdering, idGeneratorFactory, - idController ); + idController, EmptyVersionContextSupplier.INSTANCE ); this.transactionApplierTransformer = transactionApplierTransformer; } diff --git a/community/neo4j/src/test/java/db/DatabaseShutdownTest.java b/community/neo4j/src/test/java/db/DatabaseShutdownTest.java index 1711c73b00730..039bc8714f348 100644 --- a/community/neo4j/src/test/java/db/DatabaseShutdownTest.java +++ b/community/neo4j/src/test/java/db/DatabaseShutdownTest.java @@ -32,6 +32,7 @@ import org.neo4j.io.pagecache.DelegatingPageCache; import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.NeoStoreDataSource; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.factory.CommunityEditionModule; @@ -107,10 +108,11 @@ protected PlatformModule createPlatform( File storeDir, Config config, return new PlatformModule( storeDir, config, databaseInfo, dependencies, graphDatabaseFacade ) { @Override - protected PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, - LogService logging, Tracers tracers ) + protected PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, LogService logging, + Tracers tracers, VersionContextSupplier versionContextSupplier ) { - PageCache pageCache = super.createPageCache( fileSystem, config, logging, tracers ); + PageCache pageCache = super.createPageCache( fileSystem, config, logging, tracers, + versionContextSupplier ); return new DelegatingPageCache( pageCache ) { @Override diff --git a/community/neo4j/src/test/java/db/QueryRestartIT.java b/community/neo4j/src/test/java/db/QueryRestartIT.java new file mode 100644 index 0000000000000..257cddcb28108 --- /dev/null +++ b/community/neo4j/src/test/java/db/QueryRestartIT.java @@ -0,0 +1,288 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package db; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.util.Map; +import java.util.function.LongSupplier; + +import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.QueryExecutionException; +import org.neo4j.graphdb.Result; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.factory.GraphDatabaseBuilder; +import org.neo4j.graphdb.factory.GraphDatabaseFactoryState; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; +import org.neo4j.kernel.GraphDatabaseDependencies; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.configuration.Settings; +import org.neo4j.kernel.impl.context.TransactionVersionContext; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; +import org.neo4j.kernel.impl.factory.CommunityEditionModule; +import org.neo4j.kernel.impl.factory.DatabaseInfo; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; +import org.neo4j.kernel.impl.factory.PlatformModule; +import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; +import org.neo4j.kernel.internal.GraphDatabaseAPI; +import org.neo4j.test.TestGraphDatabaseFactory; +import org.neo4j.test.rule.TestDirectory; + +import static org.junit.Assert.assertEquals; + +public class QueryRestartIT +{ + @Rule + public final TestDirectory testDirectory = TestDirectory.testDirectory(); + private GraphDatabaseService database; + private TestTransactionVersionContextSupplier testContextSupplier; + private File storeDir; + private TestVersionContext testCursorContext; + + @Before + public void setUp() + { + storeDir = testDirectory.directory(); + testContextSupplier = new TestTransactionVersionContextSupplier(); + database = startSnapshotQueryDb(); + createData(); + + testCursorContext = testCursorContext(); + testContextSupplier.setCursorContext( testCursorContext ); + } + + @After + public void tearDown() + { + if ( database != null ) + { + database.shutdown(); + } + } + + @Test + public void executeQueryWithoutRestarts() + { + testCursorContext.setWrongLastClosedTxId( false ); + + Result result = database.execute( "MATCH (n:label) RETURN n.c" ); + while ( result.hasNext() ) + { + assertEquals( "d", result.next().get( "n.c" ) ); + } + assertEquals( 0, testCursorContext.getAdditionalAttempts() ); + } + + @Test + public void executeQueryWithSingleRetry() + { + Result result = database.execute( "MATCH (n) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + while ( result.hasNext() ) + { + assertEquals( "d", result.next().get( "n.c" ) ); + } + } + + @Test + public void executeCountStoreQueryWithSingleRetry() + { + Result result = database.execute( "MATCH (n:toRetry) RETURN count(n)" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + while ( result.hasNext() ) + { + assertEquals( 1L, result.next().get( "count(n)" ) ); + } + } + + @Test + public void executeLabelScanQueryWithSingleRetry() + { + Result result = database.execute( "MATCH (n:toRetry) RETURN n.c" ); + assertEquals( 1, testCursorContext.getAdditionalAttempts() ); + while ( result.hasNext() ) + { + assertEquals( "d", result.next().get( "n.c" ) ); + } + } + + @Test + public void queryThatModifyDataAndSeeUnstableSnapshotThrowException() + { + try + { + database.execute( "MATCH (n:toRetry) CREATE () RETURN n.c" ); + } + catch ( QueryExecutionException e ) + { + assertEquals( "Unable to get clean data snapshot for query " + + "'MATCH (n:toRetry) CREATE () RETURN n.c' that perform updates.", e.getMessage() ); + } + } + + private GraphDatabaseService startSnapshotQueryDb() + { + return new CustomGraphDatabaseFactory( new CustomFacadeFactory() ) + .newEmbeddedDatabaseBuilder( storeDir ) + .setConfig( GraphDatabaseSettings.snapshot_query, Settings.TRUE ) + .newGraphDatabase(); + } + + private void createData() + { + Label label = Label.label( "toRetry" ); + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode( label ); + node.setProperty( "c", "d" ); + transaction.success(); + } + } + + private TestVersionContext testCursorContext() + { + TransactionIdStore transactionIdStore = getTransactionIdStore(); + return new TestVersionContext( transactionIdStore::getLastClosedTransactionId ); + } + + private TransactionIdStore getTransactionIdStore() + { + DependencyResolver dependencyResolver = ((GraphDatabaseAPI) database).getDependencyResolver(); + return dependencyResolver.resolveDependency( TransactionIdStore.class ); + } + + private class CustomGraphDatabaseFactory extends TestGraphDatabaseFactory + { + private GraphDatabaseFacadeFactory customFacadeFactory; + + CustomGraphDatabaseFactory( GraphDatabaseFacadeFactory customFacadeFactory ) + { + this.customFacadeFactory = customFacadeFactory; + } + + @Override + protected GraphDatabaseBuilder.DatabaseCreator createDatabaseCreator( File storeDir, + GraphDatabaseFactoryState state ) + { + return new GraphDatabaseBuilder.DatabaseCreator() + { + @Override + public GraphDatabaseService newDatabase( Map config ) + { + return newDatabase( Config.embeddedDefaults( config ) ); + } + + @Override + public GraphDatabaseService newDatabase( Config config ) + { + return customFacadeFactory.newFacade( storeDir, config, + GraphDatabaseDependencies.newDependencies( state.databaseDependencies() ) ); + } + }; + } + } + + private class CustomFacadeFactory extends GraphDatabaseFacadeFactory + { + + CustomFacadeFactory() + { + super( DatabaseInfo.COMMUNITY, CommunityEditionModule::new ); + } + + @Override + protected PlatformModule createPlatform( File storeDir, Config config, Dependencies dependencies, + GraphDatabaseFacade graphDatabaseFacade ) + { + return new PlatformModule( storeDir, config, databaseInfo, dependencies, graphDatabaseFacade ) + { + @Override + protected VersionContextSupplier createCursorContextSupplier( Config config ) + { + return testContextSupplier != null ? testContextSupplier : super.createCursorContextSupplier(config); + } + }; + } + } + + private class TestVersionContext extends TransactionVersionContext + { + + private boolean wrongLastClosedTxId = true; + private int additionalAttempts; + + TestVersionContext( LongSupplier transactionIdSupplier ) + { + super( transactionIdSupplier ); + } + + @Override + public long lastClosedTransactionId() + { + return wrongLastClosedTxId ? TransactionIdStore.BASE_TX_ID : super.lastClosedTransactionId(); + } + + @Override + public void markAsDirty() + { + super.markAsDirty(); + wrongLastClosedTxId = false; + } + + void setWrongLastClosedTxId( boolean wrongLastClosedTxId ) + { + this.wrongLastClosedTxId = wrongLastClosedTxId; + } + + @Override + public boolean isDirty() + { + boolean dirty = super.isDirty(); + if ( dirty ) + { + additionalAttempts++; + } + return dirty; + } + + int getAdditionalAttempts() + { + return additionalAttempts; + } + } + + private class TestTransactionVersionContextSupplier extends TransactionVersionContextSupplier + { + void setCursorContext( VersionContext versionContext ) + { + this.cursorContext.set( versionContext ); + } + } +} diff --git a/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java b/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java index 28347066b40e7..26d70d3564f57 100644 --- a/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java +++ b/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java @@ -45,6 +45,7 @@ import org.neo4j.helpers.collection.MapUtil; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.index.inmemory.InMemoryIndexProvider; @@ -413,7 +414,7 @@ public void upgradedNeoStoreShouldHaveNewUpgradeTimeAndUpgradeId() throws Except newUpgrader( upgradableDatabase, allowMigrateConfig, pageCache ).migrateIfNeeded( dbDirectory ); // Then - StoreFactory factory = new StoreFactory( dbDirectory, pageCache, fileSystem, NullLogProvider.getInstance() ); + StoreFactory factory = new StoreFactory( dbDirectory, pageCache, fileSystem, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = factory.openAllNeoStores() ) { assertThat( neoStores.getMetaDataStore().getUpgradeTransaction(), diff --git a/community/server/src/test/java/org/neo4j/server/rest/transactional/ExecutionResultSerializerTest.java b/community/server/src/test/java/org/neo4j/server/rest/transactional/ExecutionResultSerializerTest.java index 23e93e11a18d7..fab92566c2e42 100644 --- a/community/server/src/test/java/org/neo4j/server/rest/transactional/ExecutionResultSerializerTest.java +++ b/community/server/src/test/java/org/neo4j/server/rest/transactional/ExecutionResultSerializerTest.java @@ -42,6 +42,7 @@ import java.util.TreeMap; import java.util.TreeSet; +import org.neo4j.cypher.internal.javacompat.MapRow; import org.neo4j.graphdb.ExecutionPlanDescription; import org.neo4j.graphdb.InputPosition; import org.neo4j.graphdb.Node; @@ -77,7 +78,10 @@ import static org.neo4j.helpers.collection.MapUtil.map; import static org.neo4j.server.rest.domain.JsonHelper.jsonNode; import static org.neo4j.server.rest.domain.JsonHelper.readJson; -import static org.neo4j.server.rest.transactional.Neo4jJsonCodecTest.*; +import static org.neo4j.server.rest.transactional.Neo4jJsonCodecTest.MockGeometry; +import static org.neo4j.server.rest.transactional.Neo4jJsonCodecTest.MockPoint; +import static org.neo4j.server.rest.transactional.Neo4jJsonCodecTest.mockCartesian; +import static org.neo4j.server.rest.transactional.Neo4jJsonCodecTest.mockWGS84; import static org.neo4j.test.Property.property; import static org.neo4j.test.mockito.mock.GraphMock.link; import static org.neo4j.test.mockito.mock.GraphMock.node; diff --git a/community/server/src/test/java/org/neo4j/server/rest/transactional/GraphExtractionWriterTest.java b/community/server/src/test/java/org/neo4j/server/rest/transactional/GraphExtractionWriterTest.java index 3983bbaf14cf4..4d436b93b9786 100644 --- a/community/server/src/test/java/org/neo4j/server/rest/transactional/GraphExtractionWriterTest.java +++ b/community/server/src/test/java/org/neo4j/server/rest/transactional/GraphExtractionWriterTest.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Map; +import org.neo4j.cypher.internal.javacompat.MapRow; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Relationship; import org.neo4j.server.rest.domain.JsonHelper; diff --git a/community/server/src/test/java/org/neo4j/server/rest/transactional/RestRepresentationWriterTest.java b/community/server/src/test/java/org/neo4j/server/rest/transactional/RestRepresentationWriterTest.java index 73417f42cf8ce..42ea0052490c3 100644 --- a/community/server/src/test/java/org/neo4j/server/rest/transactional/RestRepresentationWriterTest.java +++ b/community/server/src/test/java/org/neo4j/server/rest/transactional/RestRepresentationWriterTest.java @@ -19,15 +19,17 @@ */ package org.neo4j.server.rest.transactional; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.net.URI; - import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.JsonNode; import org.hamcrest.MatcherAssert; import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; + +import org.neo4j.cypher.internal.javacompat.MapRow; import org.neo4j.server.rest.domain.JsonParseException; import static java.util.Arrays.asList; diff --git a/community/server/src/test/java/org/neo4j/server/rest/transactional/RowWriterTest.java b/community/server/src/test/java/org/neo4j/server/rest/transactional/RowWriterTest.java index bd0247854b8fa..192c875fac667 100644 --- a/community/server/src/test/java/org/neo4j/server/rest/transactional/RowWriterTest.java +++ b/community/server/src/test/java/org/neo4j/server/rest/transactional/RowWriterTest.java @@ -19,14 +19,16 @@ */ package org.neo4j.server.rest.transactional; -import java.io.ByteArrayOutputStream; -import java.io.IOException; - import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.JsonNode; import org.hamcrest.MatcherAssert; import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import org.neo4j.cypher.internal.javacompat.MapRow; import org.neo4j.server.rest.domain.JsonParseException; import static java.util.Arrays.asList; diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplier.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplier.java index 732a8a5d0f94b..ebcc031e0440f 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplier.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplier.java @@ -22,6 +22,7 @@ import java.util.function.Supplier; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionQueue; import org.neo4j.kernel.impl.api.TransactionToApply; @@ -46,6 +47,7 @@ public class BatchingTxApplier extends LifecycleAdapter private final PullRequestMonitor monitor; private final PageCursorTracerSupplier pageCursorTracerSupplier; + private final VersionContextSupplier versionContextSupplier; private final Log log; private TransactionQueue txQueue; @@ -56,7 +58,8 @@ public class BatchingTxApplier extends LifecycleAdapter public BatchingTxApplier( int maxBatchSize, Supplier txIdStoreSupplier, Supplier commitProcessSupplier, Monitors monitors, - PageCursorTracerSupplier pageCursorTracerSupplier, LogProvider logProvider ) + PageCursorTracerSupplier pageCursorTracerSupplier, + VersionContextSupplier versionContextSupplier, LogProvider logProvider ) { this.maxBatchSize = maxBatchSize; this.txIdStoreSupplier = txIdStoreSupplier; @@ -64,6 +67,7 @@ public BatchingTxApplier( int maxBatchSize, Supplier txIdSto this.pageCursorTracerSupplier = pageCursorTracerSupplier; this.log = logProvider.getLog( getClass() ); this.monitor = monitors.newMonitor( PullRequestMonitor.class ); + this.versionContextSupplier = versionContextSupplier; } @Override @@ -107,7 +111,7 @@ public void queue( CommittedTransactionRepresentation tx ) throws Exception return; } - txQueue.queue( new TransactionToApply( tx.getTransactionRepresentation(), receivedTxId ) ); + txQueue.queue( new TransactionToApply( tx.getTransactionRepresentation(), receivedTxId, versionContextSupplier.getVersionContext() ) ); if ( !stopped ) { diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionApplier.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionApplier.java index 81a2bf55e1857..bc27306944d81 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionApplier.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/catchup/tx/TransactionApplier.java @@ -24,6 +24,7 @@ import org.neo4j.com.TransactionStreamResponse; import org.neo4j.com.storecopy.TransactionObligationFulfiller; import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionRepresentationCommitProcess; @@ -47,17 +48,19 @@ public class TransactionApplier { private final TransactionRepresentationCommitProcess commitProcess; + private final VersionContextSupplier versionContextSupplier; public TransactionApplier( DependencyResolver resolver ) { commitProcess = new TransactionRepresentationCommitProcess( resolver.resolveDependency( TransactionAppender.class ), resolver.resolveDependency( StorageEngine.class ) ); + versionContextSupplier = resolver.resolveDependency( VersionContextSupplier.class ); } public void appendToLogAndApplyToStore( CommittedTransactionRepresentation tx ) throws TransactionFailureException { commitProcess.commit( new TransactionToApply( tx.getTransactionRepresentation(), - tx.getCommitEntry().getTxId() ), NULL, EXTERNAL ); + tx.getCommitEntry().getTxId(), versionContextSupplier.getVersionContext() ), NULL, EXTERNAL ); } } diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java index 5b68f1102e714..85b851ba8ac11 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/CoreBootstrapper.java @@ -35,6 +35,7 @@ import org.neo4j.causalclustering.identity.MemberId; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.NeoStores; @@ -109,7 +110,7 @@ public class CoreBootstrapper public CoreSnapshot bootstrap( Set members ) throws IOException { StoreFactory factory = new StoreFactory( storeDir, config, - new DefaultIdGeneratorFactory( fs ), pageCache, fs, logProvider ); + new DefaultIdGeneratorFactory( fs ), pageCache, fs, logProvider, EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = factory.openAllNeoStores( true ); neoStores.close(); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/CoreStateMachinesModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/CoreStateMachinesModule.java index fd02301d55831..fd8d23c1c99f6 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/CoreStateMachinesModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/CoreStateMachinesModule.java @@ -30,6 +30,7 @@ import org.neo4j.causalclustering.core.consensus.RaftMachine; import org.neo4j.causalclustering.core.replication.RaftReplicator; import org.neo4j.causalclustering.core.replication.Replicator; +import org.neo4j.causalclustering.core.state.machines.dummy.DummyMachine; import org.neo4j.causalclustering.core.state.machines.id.CommandIndexTracker; import org.neo4j.causalclustering.core.state.machines.id.IdAllocationState; import org.neo4j.causalclustering.core.state.machines.id.IdReusabilityCondition; @@ -39,7 +40,6 @@ import org.neo4j.causalclustering.core.state.machines.locks.LeaderOnlyLockManager; import org.neo4j.causalclustering.core.state.machines.locks.ReplicatedLockTokenState; import org.neo4j.causalclustering.core.state.machines.locks.ReplicatedLockTokenStateMachine; -import org.neo4j.causalclustering.core.state.machines.dummy.DummyMachine; import org.neo4j.causalclustering.core.state.machines.token.ReplicatedLabelTokenHolder; import org.neo4j.causalclustering.core.state.machines.token.ReplicatedPropertyKeyTokenHolder; import org.neo4j.causalclustering.core.state.machines.token.ReplicatedRelationshipTypeTokenHolder; @@ -53,6 +53,7 @@ import org.neo4j.causalclustering.identity.MemberId; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.CommitProcessFactory; import org.neo4j.kernel.impl.core.LabelTokenHolder; @@ -163,20 +164,24 @@ public CoreStateMachinesModule( MemberId myself, PlatformModule platformModule, ReplicatedLockTokenStateMachine replicatedLockTokenStateMachine = new ReplicatedLockTokenStateMachine( lockTokenState ); + VersionContextSupplier versionContextSupplier = platformModule.versionContextSupplier; ReplicatedTokenStateMachine labelTokenStateMachine = - new ReplicatedTokenStateMachine<>( labelTokenRegistry, new Token.Factory(), logProvider ); + new ReplicatedTokenStateMachine<>( labelTokenRegistry, new Token.Factory(), logProvider, + versionContextSupplier ); ReplicatedTokenStateMachine propertyKeyTokenStateMachine = - new ReplicatedTokenStateMachine<>( propertyKeyTokenRegistry, new Token.Factory(), logProvider ); + new ReplicatedTokenStateMachine<>( propertyKeyTokenRegistry, new Token.Factory(), logProvider, + versionContextSupplier ); ReplicatedTokenStateMachine relationshipTypeTokenStateMachine = new ReplicatedTokenStateMachine<>( relationshipTypeTokenRegistry, new RelationshipTypeToken.Factory(), - logProvider ); + logProvider, versionContextSupplier ); PageCursorTracerSupplier cursorTracerSupplier = platformModule.tracers.pageCursorTracerSupplier; ReplicatedTransactionStateMachine replicatedTxStateMachine = new ReplicatedTransactionStateMachine( commandIndexTracker, replicatedLockTokenStateMachine, - config.get( state_machine_apply_max_batch_size ), logProvider, cursorTracerSupplier ); + config.get( state_machine_apply_max_batch_size ), logProvider, cursorTracerSupplier, + versionContextSupplier ); dependencies.satisfyDependencies( replicatedTxStateMachine ); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachine.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachine.java index d788111b1a9e3..4c36e20a8c3fe 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachine.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachine.java @@ -25,6 +25,8 @@ import org.neo4j.causalclustering.core.state.Result; import org.neo4j.causalclustering.core.state.machines.StateMachine; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionToApply; @@ -50,15 +52,17 @@ public class ReplicatedTokenStateMachine implements StateMa private final TokenRegistry tokenRegistry; private final TokenFactory tokenFactory; + private final VersionContext versionContext; private final Log log; private long lastCommittedIndex = -1; public ReplicatedTokenStateMachine( TokenRegistry tokenRegistry, TokenFactory tokenFactory, - LogProvider logProvider ) + LogProvider logProvider, VersionContextSupplier versionContextSupplier ) { this.tokenRegistry = tokenRegistry; this.tokenFactory = tokenFactory; + this.versionContext = versionContextSupplier.getVersionContext(); this.log = logProvider.getLog( getClass() ); } @@ -108,7 +112,7 @@ private int applyToStore( Collection commands, long logIndex ) t try ( LockGroup ignored = new LockGroup() ) { - commitProcess.commit( new TransactionToApply( representation ), CommitEvent.NULL, + commitProcess.commit( new TransactionToApply( representation, versionContext ), CommitEvent.NULL, TransactionApplicationMode.EXTERNAL ); } catch ( TransactionFailureException e ) diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachine.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachine.java index 38b316f46f193..d538db5babbb6 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachine.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachine.java @@ -27,6 +27,7 @@ import org.neo4j.causalclustering.core.state.machines.id.CommandIndexTracker; import org.neo4j.causalclustering.core.state.machines.locks.ReplicatedLockTokenStateMachine; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionQueue; @@ -49,6 +50,7 @@ public class ReplicatedTransactionStateMachine implements StateMachine { callback.accept( Result.of( txId ) ); diff --git a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java index d009e282d21bf..fc602fa167670 100644 --- a/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java +++ b/enterprise/causal-clustering/src/main/java/org/neo4j/causalclustering/readreplica/EnterpriseReadReplicaEditionModule.java @@ -211,7 +211,8 @@ public class EnterpriseReadReplicaEditionModule extends EditionModule int maxBatchSize = config.get( CausalClusteringSettings.read_replica_transaction_applier_batch_size ); BatchingTxApplier batchingTxApplier = new BatchingTxApplier( maxBatchSize, dependencies.provideDependency( TransactionIdStore.class ), writableCommitProcess, - platformModule.monitors, platformModule.tracers.pageCursorTracerSupplier, logProvider ); + platformModule.monitors, platformModule.tracers.pageCursorTracerSupplier, + platformModule.versionContextSupplier, logProvider ); TimerService timerService = new TimerService( platformModule.jobScheduler, logProvider ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplierTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplierTest.java index 82692e47f8ead..982caeb1eee26 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplierTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/catchup/tx/BatchingTxApplierTest.java @@ -28,6 +28,7 @@ import org.neo4j.helpers.collection.Iterables; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionToApply; @@ -56,7 +57,7 @@ public class BatchingTxApplierTest private final BatchingTxApplier txApplier = new BatchingTxApplier( maxBatchSize, () -> idStore, () -> commitProcess, new Monitors(), PageCursorTracerSupplier.NULL, - NullLogProvider.getInstance() ); + EmptyVersionContextSupplier.INSTANCE, NullLogProvider.getInstance() ); @Before public void before() throws Throwable diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java index b5c14d3b1a8a4..efaed8dae2864 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/id/RebuildReplicatedIdGeneratorsTest.java @@ -28,6 +28,7 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.FileUtils; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.enterprise.id.EnterpriseIdTypeConfigurationProvider; import org.neo4j.kernel.impl.store.MetaDataStore; @@ -66,7 +67,7 @@ public void rebuildReplicatedIdGeneratorsOnRecovery() throws Exception StoreFactory storeFactory = new StoreFactory( testDirectory.graphDbDir(), Config.defaults(), getIdGenerationFactory( fileSystem ), pageCacheRule.getPageCache( fileSystem ), fileSystem, - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = storeFactory.openAllNeoStores( true ) ) { NodeStore nodeStore = neoStores.getNodeStore(); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachineTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachineTest.java index c6f787108ad0e..aca5df9a2e85a 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachineTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/token/ReplicatedTokenStateMachineTest.java @@ -19,12 +19,13 @@ */ package org.neo4j.causalclustering.core.state.machines.token; +import org.junit.Test; + import java.util.ArrayList; import java.util.List; -import org.junit.Test; - import org.neo4j.graphdb.TransactionFailureException; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionRepresentationCommitProcess; import org.neo4j.kernel.impl.api.TransactionToApply; @@ -40,10 +41,8 @@ import org.neo4j.storageengine.api.TransactionApplicationMode; import static java.util.Collections.singletonList; - import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; - import static org.neo4j.causalclustering.core.state.machines.token.ReplicatedTokenRequestSerializer.commandBytes; import static org.neo4j.causalclustering.core.state.machines.token.TokenType.LABEL; import static org.neo4j.causalclustering.core.state.machines.tx.LogIndexTxHeaderEncoding.decodeLogIndexFromTxHeader; @@ -59,7 +58,7 @@ public void shouldCreateTokenId() throws Exception // given TokenRegistry registry = new TokenRegistry<>( "Label" ); ReplicatedTokenStateMachine stateMachine = new ReplicatedTokenStateMachine<>( registry, - new Token.Factory(), NullLogProvider.getInstance() ); + new Token.Factory(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( mock( TransactionCommitProcess.class ), -1 ); // when @@ -76,7 +75,7 @@ public void shouldAllocateTokenIdToFirstReplicateRequest() throws Exception // given TokenRegistry registry = new TokenRegistry<>( "Label" ); ReplicatedTokenStateMachine stateMachine = new ReplicatedTokenStateMachine<>( registry, - new Token.Factory(), NullLogProvider.getInstance() ); + new Token.Factory(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( mock( TransactionCommitProcess.class ), -1 ); @@ -102,7 +101,7 @@ public void shouldStoreRaftLogIndexInTransactionHeader() throws Exception StubTransactionCommitProcess commitProcess = new StubTransactionCommitProcess( null, null ); ReplicatedTokenStateMachine stateMachine = new ReplicatedTokenStateMachine<>( new TokenRegistry<>( "Token" ), new Token.Factory(), - NullLogProvider.getInstance() ); + NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( commitProcess, -1 ); // when diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/CommitProcessStateMachineCollaborationTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/CommitProcessStateMachineCollaborationTest.java index 56499ccd3043c..0895998ae4a4d 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/CommitProcessStateMachineCollaborationTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/CommitProcessStateMachineCollaborationTest.java @@ -26,6 +26,7 @@ import org.neo4j.causalclustering.core.state.machines.locks.ReplicatedLockTokenRequest; import org.neo4j.causalclustering.core.state.machines.locks.ReplicatedLockTokenStateMachine; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionToApply; @@ -52,7 +53,7 @@ public void shouldFailTransactionIfLockSessionChanges() throws Exception ReplicatedTransactionStateMachine stateMachine = new ReplicatedTransactionStateMachine( mock( CommandIndexTracker.class ), lockState( finalLockSessionId ), 16, NullLogProvider.getInstance(), - PageCursorTracerSupplier.NULL ); + PageCursorTracerSupplier.NULL, EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( localCommitProcess, -1L ); DirectReplicator replicator = new DirectReplicator<>( stateMachine ); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachineTest.java b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachineTest.java index 38590d30d39c7..587ce0cc5554b 100644 --- a/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachineTest.java +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/causalclustering/core/state/machines/tx/ReplicatedTransactionStateMachineTest.java @@ -28,6 +28,7 @@ import org.neo4j.causalclustering.core.state.machines.locks.ReplicatedLockTokenStateMachine; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.exceptions.Status; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.TransactionCommitProcess; @@ -68,7 +69,8 @@ public void shouldCommitTransaction() throws Exception PageCursorTracer cursorTracer = mock( PageCursorTracer.class ); ReplicatedTransactionStateMachine stateMachine = new ReplicatedTransactionStateMachine( - commandIndexTracker, lockState( lockSessionId ), batchSize, logProvider, () -> cursorTracer ); + commandIndexTracker, lockState( lockSessionId ), batchSize, logProvider, () -> cursorTracer, + EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( localCommitProcess, -1L ); // when @@ -96,7 +98,7 @@ public void shouldFailFutureForTransactionCommittedUnderWrongLockSession() throw final ReplicatedTransactionStateMachine stateMachine = new ReplicatedTransactionStateMachine( commandIndexTracker, lockState( currentLockSessionId ), batchSize, logProvider, - PageCursorTracerSupplier.NULL ); + PageCursorTracerSupplier.NULL, EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( localCommitProcess, -1L ); AtomicBoolean called = new AtomicBoolean(); @@ -139,7 +141,7 @@ public void shouldAcceptTransactionCommittedWithNoLockManager() throws Exception ReplicatedTransactionStateMachine stateMachine = new ReplicatedTransactionStateMachine( commandIndexTracker, lockState( currentLockSessionId ), batchSize, logProvider, - PageCursorTracerSupplier.NULL ); + PageCursorTracerSupplier.NULL, EmptyVersionContextSupplier.INSTANCE ); stateMachine.installCommitProcess( localCommitProcess, -1L ); AtomicBoolean called = new AtomicBoolean(); diff --git a/enterprise/causal-clustering/src/test/java/org/neo4j/io/pagecache/impl/muninn/VersionContextTrackingIT.java b/enterprise/causal-clustering/src/test/java/org/neo4j/io/pagecache/impl/muninn/VersionContextTrackingIT.java new file mode 100644 index 0000000000000..e600c050707c9 --- /dev/null +++ b/enterprise/causal-clustering/src/test/java/org/neo4j/io/pagecache/impl/muninn/VersionContextTrackingIT.java @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.io.pagecache.impl.muninn; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import org.neo4j.causalclustering.core.CoreGraphDatabase; +import org.neo4j.causalclustering.discovery.Cluster; +import org.neo4j.causalclustering.discovery.CoreClusterMember; +import org.neo4j.causalclustering.readreplica.ReadReplicaGraphDatabase; +import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.PageSwapper; +import org.neo4j.io.pagecache.PagedFile; +import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracer; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContext; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; +import org.neo4j.test.causalclustering.ClusterRule; + +import static org.junit.Assert.assertEquals; +import static org.neo4j.causalclustering.discovery.Cluster.dataMatchesEventually; +import static org.neo4j.kernel.configuration.Settings.TRUE; + +public class VersionContextTrackingIT +{ + @Rule + public final ClusterRule clusterRule = new ClusterRule( getClass() ); + private static final int NUMBER_OF_TRANSACTIONS = 3; + private Cluster cluster; + + @Before + public void setup() throws Exception + { + cluster = clusterRule.withSharedCoreParam( GraphDatabaseSettings.snapshot_query, TRUE ) + .withSharedReadReplicaParam( GraphDatabaseSettings.snapshot_query, TRUE ) + .startCluster(); + } + + @Test + public void coreMemberTransactionIdPageTracking() throws Exception + { + long baseTxId = getBaseTransactionId(); + for ( int i = 1; i < 4; i++ ) + { + generateData(); + dataMatchesEventually( anyCoreClusterMember(), cluster.coreMembers() ); + assertEquals( getExpectedLatestPageVersion( baseTxId, i ), getLatestPageVersion( getAnyCore() ) ); + } + } + + @Test + public void readReplicateTransactionIdPageTracking() throws Exception + { + long baseTxId = getBaseTransactionId(); + for ( int i = 1; i < 4; i++ ) + { + generateData(); + dataMatchesEventually( anyCoreClusterMember(), cluster.readReplicas() ); + assertEquals( getExpectedLatestPageVersion( baseTxId, i ), getLatestPageVersion( getAnyReadReplica() ) ); + } + } + + private long getExpectedLatestPageVersion( long baseTxId, int round ) + { + return baseTxId + round * NUMBER_OF_TRANSACTIONS; + } + + private long getBaseTransactionId() + { + DependencyResolver dependencyResolver = getAnyCore().getDependencyResolver(); + TransactionIdStore transactionIdStore = dependencyResolver.resolveDependency( TransactionIdStore.class ); + return transactionIdStore.getLastClosedTransactionId(); + } + + private CoreClusterMember anyCoreClusterMember() + { + return cluster.coreMembers().iterator().next(); + } + + private CoreGraphDatabase getAnyCore() + { + return anyCoreClusterMember().database(); + } + + private ReadReplicaGraphDatabase getAnyReadReplica() + { + return cluster.findAnyReadReplica().database(); + } + + private long getLatestPageVersion( GraphDatabaseFacade databaseFacade ) throws IOException + { + DependencyResolver dependencyResolver = databaseFacade.getDependencyResolver(); + PageCache pageCache = dependencyResolver.resolveDependency( PageCache.class ); + NeoStores neoStores = dependencyResolver.resolveDependency( RecordStorageEngine.class ).testAccessNeoStores(); + File storeFile = neoStores.getNodeStore().getStorageFileName(); + long maxTransactionId = Long.MIN_VALUE; + try ( PagedFile pageFile = pageCache.getExistingMapping( storeFile ).get() ) + { + long lastPageId = pageFile.getLastPageId(); + for ( int i = 0; i <= lastPageId; i++ ) + { + try ( CursorPageAccessor pageCursor = new CursorPageAccessor( + (MuninnPageCursor) pageFile.io( i, PagedFile.PF_SHARED_READ_LOCK ) ) ) + { + if ( pageCursor.next() ) + { + maxTransactionId = Math.max( maxTransactionId, pageCursor.lastTxModifierId() ); + } + } + } + } + return maxTransactionId; + } + + private void generateData() throws Exception + { + for ( int i = 0; i < NUMBER_OF_TRANSACTIONS; i++ ) + { + cluster.coreTx( ( coreGraphDatabase, transaction ) -> + { + coreGraphDatabase.createNode(); + transaction.success(); + } ); + } + } + + private class CursorPageAccessor extends MuninnPageCursor + { + + private MuninnPageCursor delegate; + + CursorPageAccessor( MuninnPageCursor delegate ) + { + super( -1, PageCursorTracer.NULL, EmptyVersionContext.INSTANCE ); + this.delegate = delegate; + } + + long lastTxModifierId() + { + return delegate.page.getLastModifiedTxId(); + } + + @Override + protected void unpinCurrentPage() + { + delegate.unpinCurrentPage(); + } + + @Override + protected void convertPageFaultLock( MuninnPage page ) + { + delegate.convertPageFaultLock( page ); + } + + @Override + protected void pinCursorToPage( MuninnPage page, long filePageId, PageSwapper swapper ) + { + delegate.pinCursorToPage( page, filePageId, swapper ); + } + + @Override + protected boolean tryLockPage( MuninnPage page ) + { + return delegate.tryLockPage( page ); + } + + @Override + protected void unlockPage( MuninnPage page ) + { + delegate.unlockPage( page ); + } + + @Override + protected void releaseCursor() + { + delegate.releaseCursor(); + } + + @Override + public boolean next() throws IOException + { + return delegate.next(); + } + + @Override + public boolean shouldRetry() throws IOException + { + return delegate.shouldRetry(); + } + } +} diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/BatchingResponseHandler.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/BatchingResponseHandler.java index 7068787665baf..82caf0d5f4574 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/BatchingResponseHandler.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/BatchingResponseHandler.java @@ -26,6 +26,7 @@ import org.neo4j.com.Response.Handler; import org.neo4j.com.storecopy.ResponseUnpacker.TxHandler; import org.neo4j.helpers.collection.Visitor; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.impl.api.TransactionQueue; import org.neo4j.kernel.impl.api.TransactionToApply; import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation; @@ -42,14 +43,17 @@ class BatchingResponseHandler implements Response.Handler, { private final TransactionQueue queue; private final TxHandler txHandler; + private final VersionContextSupplier versionContextSupplier; private final TransactionObligationFulfiller obligationFulfiller; private final Log log; BatchingResponseHandler( int maxBatchSize, TransactionQueue.Applier applier, - TransactionObligationFulfiller obligationFulfiller, TxHandler txHandler, Log log ) + TransactionObligationFulfiller obligationFulfiller, TxHandler txHandler, + VersionContextSupplier versionContextSupplier, Log log ) { this.obligationFulfiller = obligationFulfiller; this.txHandler = txHandler; + this.versionContextSupplier = versionContextSupplier; this.queue = new TransactionQueue( maxBatchSize, applier ); this.log = log; } @@ -88,7 +92,8 @@ public boolean visit( CommittedTransactionRepresentation transaction ) throws Ex { queue.queue( new TransactionToApply( transaction.getTransactionRepresentation(), - transaction.getCommitEntry().getTxId() ) + transaction.getCommitEntry().getTxId(), + versionContextSupplier.getVersionContext() ) { @Override public void commitment( Commitment commitment, long transactionId ) diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java index d37756d0bc4c3..4dc924488cf20 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/ExternallyManagedPageCache.java @@ -22,7 +22,6 @@ import java.io.File; import java.io.IOException; import java.nio.file.OpenOption; -import java.util.Map; import java.util.Optional; import java.util.stream.Stream; @@ -33,13 +32,13 @@ import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PagedFile; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.extension.KernelExtensionFactory; import org.neo4j.kernel.impl.enterprise.EnterpriseEditionModule; import org.neo4j.kernel.impl.factory.DatabaseInfo; import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; -import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory.Dependencies; import org.neo4j.kernel.impl.factory.PlatformModule; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.monitoring.tracing.Tracers; @@ -136,9 +135,9 @@ protected PlatformModule createPlatform( File storeDir, Config config, Dependenc { return new PlatformModule( storeDir, config, databaseInfo, dependencies, graphDatabaseFacade ) { - @Override - protected PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, LogService logging, Tracers tracers ) + protected PageCache createPageCache( FileSystemAbstraction fileSystem, Config config, LogService logging, + Tracers tracers, VersionContextSupplier versionContextSupplier ) { return new ExternallyManagedPageCache( delegatePageCache ); } diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpacker.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpacker.java index c2028abe68781..52817e86f7133 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpacker.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpacker.java @@ -23,6 +23,7 @@ import org.neo4j.com.TransactionStream; import org.neo4j.com.TransactionStreamResponse; import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; import org.neo4j.kernel.impl.api.KernelTransactions; import org.neo4j.kernel.impl.api.TransactionCommitProcess; import org.neo4j.kernel.impl.api.TransactionRepresentationCommitProcess; @@ -200,6 +201,11 @@ public interface Dependencies LogService logService(); KernelTransactions kernelTransactions(); + + /** + * Version context supplier + */ + VersionContextSupplier versionContextSupplier(); } /** @@ -253,13 +259,16 @@ public KernelTransactions kernelTransactions() { return resolver.resolveDependency( KernelTransactions.class ); } + + @Override + public VersionContextSupplier versionContextSupplier() + { + return resolver.resolveDependency( VersionContextSupplier.class ); + } } public static final int DEFAULT_BATCH_SIZE = 100; - static final String msg = "Kernel panic detected: pulled transactions cannot be applied to a non-healthy database. " - + "In order to resolve this issue a manual restart of this instance is required."; - // Assigned in constructor private final Dependencies dependencies; private final int maxBatchSize; @@ -268,6 +277,7 @@ public KernelTransactions kernelTransactions() // Assigned in start() private TransactionObligationFulfiller obligationFulfiller; private TransactionBatchCommitter batchCommitter; + private VersionContextSupplier versionContextSupplier; private Log log; // Assigned in stop() private volatile boolean stopped; @@ -295,7 +305,7 @@ public void unpackResponse( Response response, TxHandler txHandler ) throws E } BatchingResponseHandler responseHandler = new BatchingResponseHandler( maxBatchSize, - batchCommitter, obligationFulfiller, txHandler, log ); + batchCommitter, obligationFulfiller, txHandler, versionContextSupplier, log ); try { response.accept( responseHandler ); @@ -311,6 +321,7 @@ public void start() { this.obligationFulfiller = dependencies.obligationFulfiller(); this.log = dependencies.logService().getInternalLog( BatchingResponseHandler.class ); + this.versionContextSupplier = dependencies.versionContextSupplier(); this.batchCommitter = new TransactionBatchCommitter( dependencies.kernelTransactions(), idReuseSafeZoneTime, dependencies.commitProcess(), log ); this.stopped = false; diff --git a/enterprise/com/src/test/java/org/neo4j/com/storecopy/ResponsePackerIT.java b/enterprise/com/src/test/java/org/neo4j/com/storecopy/ResponsePackerIT.java index bf5836d7f5e5a..99b3b1c66acf6 100644 --- a/enterprise/com/src/test/java/org/neo4j/com/storecopy/ResponsePackerIT.java +++ b/enterprise/com/src/test/java/org/neo4j/com/storecopy/ResponsePackerIT.java @@ -32,6 +32,7 @@ import org.neo4j.helpers.collection.Visitor; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.StoreFactory; @@ -106,7 +107,7 @@ private NeoStores createNeoStore( FileSystemAbstraction fs, PageCache pageCache { File storeDir = new File( "/store/" ); fs.mkdirs( storeDir ); - StoreFactory storeFactory = new StoreFactory( storeDir, pageCache, fs, NullLogProvider.getInstance() ); + StoreFactory storeFactory = new StoreFactory( storeDir, pageCache, fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); return storeFactory.openAllNeoStores( true ); } } diff --git a/enterprise/com/src/test/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpackerTest.java b/enterprise/com/src/test/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpackerTest.java index 87eba0bb404ce..5a8cfc0b8b343 100644 --- a/enterprise/com/src/test/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpackerTest.java +++ b/enterprise/com/src/test/java/org/neo4j/com/storecopy/TransactionCommittingResponseUnpackerTest.java @@ -32,6 +32,7 @@ import org.neo4j.com.TransactionStream; import org.neo4j.com.TransactionStreamResponse; import org.neo4j.com.storecopy.TransactionCommittingResponseUnpacker.Dependencies; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.impl.api.KernelTransactions; import org.neo4j.kernel.impl.api.TransactionCommitProcess; @@ -78,6 +79,7 @@ public void shouldUnfreezeKernelTransactionsAfterApplyIfBatchIsLarge() throws Th TransactionObligationFulfiller fulfiller = mock( TransactionObligationFulfiller.class ); when( dependencies.obligationFulfiller() ).thenReturn( fulfiller ); when( dependencies.logService() ).thenReturn( NullLogService.getInstance() ); + when( dependencies.versionContextSupplier() ).thenReturn( EmptyVersionContextSupplier.INSTANCE ); KernelTransactions kernelTransactions = mock( KernelTransactions.class ); when( dependencies.kernelTransactions() ).thenReturn( kernelTransactions ); TransactionCommitProcess commitProcess = mock( TransactionCommitProcess.class ); @@ -124,6 +126,7 @@ public void shouldCommitTransactionsInBatches() throws Exception TransactionCountingTransactionCommitProcess commitProcess = new TransactionCountingTransactionCommitProcess(); when( dependencies.commitProcess() ).thenReturn( commitProcess ); when( dependencies.logService() ).thenReturn( NullLogService.getInstance() ); + when( dependencies.versionContextSupplier() ).thenReturn( EmptyVersionContextSupplier.INSTANCE ); KernelTransactions kernelTransactions = mock( KernelTransactions.class ); when( dependencies.kernelTransactions() ).thenReturn( kernelTransactions ); TransactionCommittingResponseUnpacker unpacker = diff --git a/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EnterpriseCypherEngineProvider.java b/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EnterpriseCypherEngineProvider.java index a5c4c835f3105..838de9a1dd975 100644 --- a/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EnterpriseCypherEngineProvider.java +++ b/enterprise/cypher/cypher/src/main/java/org/neo4j/cypher/internal/javacompat/EnterpriseCypherEngineProvider.java @@ -23,8 +23,10 @@ import org.neo4j.cypher.internal.EnterpriseCompatibilityFactory; import org.neo4j.cypher.javacompat.internal.GraphDatabaseCypherService; import org.neo4j.graphdb.DependencyResolver; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Service; import org.neo4j.kernel.api.KernelAPI; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.query.QueryEngineProvider; import org.neo4j.kernel.impl.query.QueryExecutionEngine; @@ -57,6 +59,7 @@ protected QueryExecutionEngine createEngine( Dependencies deps, GraphDatabaseAPI LogService logService = resolver.resolveDependency( LogService.class ); KernelAPI kernelAPI = resolver.resolveDependency( KernelAPI.class ); Monitors monitors = resolver.resolveDependency( Monitors.class ); + Config config = resolver.resolveDependency( Config.class ); LogProvider logProvider = logService.getInternalLogProvider(); CommunityCompatibilityFactory inner = new CommunityCompatibilityFactory( queryService, kernelAPI, monitors, logProvider ); @@ -64,6 +67,26 @@ protected QueryExecutionEngine createEngine( Dependencies deps, GraphDatabaseAPI EnterpriseCompatibilityFactory compatibilityFactory = new EnterpriseCompatibilityFactory( inner, queryService, kernelAPI, monitors, logProvider ); deps.satisfyDependency( compatibilityFactory ); + return createEngine( queryService, config, logProvider, compatibilityFactory ); + } + + private QueryExecutionEngine createEngine( GraphDatabaseCypherService queryService, Config config, + LogProvider logProvider, EnterpriseCompatibilityFactory compatibilityFactory ) + { + return config.get( GraphDatabaseSettings.snapshot_query ) ? + snapshotEngine( queryService, config, logProvider, compatibilityFactory ) : + standardEngine( queryService, logProvider, compatibilityFactory ); + } + + private SnapshotExecutionEngine snapshotEngine( GraphDatabaseCypherService queryService, Config config, + LogProvider logProvider, EnterpriseCompatibilityFactory compatibilityFactory ) + { + return new SnapshotExecutionEngine( queryService, config, logProvider, compatibilityFactory); + } + + private ExecutionEngine standardEngine( GraphDatabaseCypherService queryService, LogProvider logProvider, + EnterpriseCompatibilityFactory compatibilityFactory ) + { return new ExecutionEngine( queryService, logProvider, compatibilityFactory ); } } diff --git a/enterprise/cypher/spec-suite-tools/src/test/java/cypher/MapRow.java b/enterprise/cypher/spec-suite-tools/src/test/java/cypher/MapRow.java deleted file mode 100644 index 32a97dcdeca99..0000000000000 --- a/enterprise/cypher/spec-suite-tools/src/test/java/cypher/MapRow.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (c) 2002-2018 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package cypher; - -import java.util.Map; -import java.util.NoSuchElementException; - -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Path; -import org.neo4j.graphdb.Relationship; -import org.neo4j.graphdb.Result; - -import static org.neo4j.helpers.Exceptions.withCause; - -public class MapRow implements Result.ResultRow -{ - private final Map map; - - public MapRow( Map map ) - { - this.map = map; - } - - private T get( String key, Class type ) - { - Object value = map.get( key ); - if ( value == null ) - { - if ( !map.containsKey( key ) ) - { - throw new NoSuchElementException( "No such entry: " + key ); - } - } - try - { - return type.cast( value ); - } - catch ( ClassCastException e ) - { - throw withCause( new NoSuchElementException( "Element '" + key + "' is not a " + type.getSimpleName() ), e ); - } - } - - @Override - public Node getNode( String key ) - { - return get( key, Node.class ); - } - - @Override - public Relationship getRelationship( String key ) - { - return get( key, Relationship.class ); - } - - @Override - public Object get( String key ) - { - return get( key, Object.class ); - } - - @Override - public String getString( String key ) - { - return get( key, String.class ); - } - - @Override - public Number getNumber( String key ) - { - return get( key, Number.class ); - } - - @Override - public Boolean getBoolean( String key ) - { - return get( key, Boolean.class ); - } - - @Override - public Path getPath( String key ) - { - return get( key, Path.class ); - } -} diff --git a/enterprise/cypher/spec-suite-tools/src/test/scala/cypher/feature/parser/ParsingTestSupport.scala b/enterprise/cypher/spec-suite-tools/src/test/scala/cypher/feature/parser/ParsingTestSupport.scala index 86e716344d96c..5628f8d5e8977 100644 --- a/enterprise/cypher/spec-suite-tools/src/test/scala/cypher/feature/parser/ParsingTestSupport.scala +++ b/enterprise/cypher/spec-suite-tools/src/test/scala/cypher/feature/parser/ParsingTestSupport.scala @@ -21,12 +21,12 @@ package cypher.feature.parser import java.util.Collections -import cypher.MapRow import cypher.feature.parser.matchers.ResultMatcher import org.junit.runner.RunWith import org.mockito.Mockito._ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer +import org.neo4j.cypher.internal.javacompat.MapRow import org.neo4j.graphdb.Result.ResultVisitor import org.neo4j.graphdb._ import org.scalatest.junit.JUnitRunner diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/upgrade/MasterClientTest.java b/enterprise/ha/src/test/java/org/neo4j/ha/upgrade/MasterClientTest.java index bb7b90b4ee0e3..4251964356203 100644 --- a/enterprise/ha/src/test/java/org/neo4j/ha/upgrade/MasterClientTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/ha/upgrade/MasterClientTest.java @@ -39,6 +39,7 @@ import org.neo4j.com.storecopy.TransactionCommittingResponseUnpacker.Dependencies; import org.neo4j.helpers.HostnamePort; import org.neo4j.helpers.collection.Visitor; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.ha.MasterClient320; import org.neo4j.kernel.ha.com.master.ConversationManager; @@ -130,6 +131,7 @@ public void clientShouldReadAndApplyTransactionLogsOnNewLockSessionRequest() thr TransactionCommitProcess commitProcess = mock( TransactionCommitProcess.class ); when( deps.commitProcess() ).thenReturn( commitProcess ); when( deps.logService() ).thenReturn( NullLogService.getInstance() ); + when( deps.versionContextSupplier() ).thenReturn( EmptyVersionContextSupplier.INSTANCE ); when( deps.kernelTransactions() ).thenReturn( mock( KernelTransactions.class ) ); ResponseUnpacker unpacker = life.add( diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/OnDiskLastTxIdGetterTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/OnDiskLastTxIdGetterTest.java index d11b9729db711..c6f9a0ea7f69e 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/OnDiskLastTxIdGetterTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/OnDiskLastTxIdGetterTest.java @@ -25,6 +25,7 @@ import java.io.File; import java.util.function.LongSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.ha.transaction.OnDiskLastTxIdGetter; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.StoreFactory; @@ -55,7 +56,7 @@ public void testGetLastTxIdNoFilePresent() throws Exception public void lastTransactionIdIsBaseTxIdWhileNeoStoresAreStopped() { final StoreFactory storeFactory = new StoreFactory( new File( "store" ), pageCacheRule.getPageCache( fs.get() ), - fs.get(), NullLogProvider.getInstance() ); + fs.get(), NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); final NeoStores neoStores = storeFactory.openAllNeoStores( true ); neoStores.close(); diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/UpdatePullerTriggersPageTransactionTrackingIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/UpdatePullerTriggersPageTransactionTrackingIT.java new file mode 100644 index 0000000000000..19a4803e05135 --- /dev/null +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/UpdatePullerTriggersPageTransactionTrackingIT.java @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.ha; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.util.Map; +import java.util.function.LongSupplier; + +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.QueryExecutionException; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.factory.GraphDatabaseBuilder; +import org.neo4j.graphdb.factory.GraphDatabaseFactoryState; +import org.neo4j.graphdb.factory.GraphDatabaseSettings; +import org.neo4j.graphdb.factory.TestHighlyAvailableGraphDatabaseFactory; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContext; +import org.neo4j.io.pagecache.tracing.cursor.context.VersionContextSupplier; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.ha.factory.HighlyAvailableEditionModule; +import org.neo4j.kernel.impl.context.TransactionVersionContextSupplier; +import org.neo4j.kernel.impl.factory.DatabaseInfo; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory; +import org.neo4j.kernel.impl.factory.PlatformModule; +import org.neo4j.kernel.impl.ha.ClusterManager; +import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; +import org.neo4j.test.ha.ClusterRule; + +import static org.junit.Assert.assertEquals; +import static org.neo4j.helpers.collection.MapUtil.stringMap; +import static org.neo4j.kernel.configuration.Settings.TRUE; + +public class UpdatePullerTriggersPageTransactionTrackingIT +{ + @Rule + public final ClusterRule clusterRule = new ClusterRule( getClass() ); + private final Label NODE_LABEL = Label.label( "mark" ); + private final TestTransactionVersionContextSupplier contextSupplier = new TestTransactionVersionContextSupplier(); + private ClusterManager.ManagedCluster cluster; + + @Before + public void setup() throws Exception + { + CustomGraphDatabaseFactory customGraphDatabaseFactory = new CustomGraphDatabaseFactory(); + cluster = clusterRule.withSharedSetting( GraphDatabaseSettings.snapshot_query, TRUE ) + .withDbFactory( customGraphDatabaseFactory ) + .startCluster(); + HighlyAvailableGraphDatabase master = cluster.getMaster(); + for ( int i = 0; i < 3; i++ ) + { + try ( Transaction tx = master.beginTx() ) + { + master.createNode( NODE_LABEL ); + tx.success(); + } + } + cluster.sync(); + } + + @Test + public void updatePullerTriggerPageTransactionTracking() + { + HighlyAvailableGraphDatabase slave = cluster.getAnySlave(); + TransactionIdStore slaveTransactionIdStore = + slave.getDependencyResolver().resolveDependency( TransactionIdStore.class ); + assertEquals( 5, slaveTransactionIdStore.getLastClosedTransactionId() ); + + ByzantineLongSupplier byzantineIdSupplier = contextSupplier.getByzantineIdSupplier(); + byzantineIdSupplier.useWrongTxId(); + try ( Transaction ignored = slave.beginTx() ) + { + slave.execute( "match (n) return n" ); + } + catch ( QueryExecutionException executionException ) + { + assertEquals( "Unable to get clean data snapshot for query 'match (n) return n' after 5 attempts.", executionException.getMessage()); + } + byzantineIdSupplier.useCorrectTxId(); + slave.execute( "match (n) return n" ).close(); + } + + private class CustomGraphDatabaseFactory extends TestHighlyAvailableGraphDatabaseFactory + { + @Override + protected GraphDatabaseBuilder.DatabaseCreator createDatabaseCreator( File storeDir, + GraphDatabaseFactoryState state ) + { + return new GraphDatabaseBuilder.DatabaseCreator() + { + @Override + public GraphDatabaseService newDatabase( Map config ) + { + return newDatabase( Config.embeddedDefaults( config ) ); + } + + @Override + public GraphDatabaseService newDatabase( Config config ) + { + return new CustomHighlyAvailableGraphDatabase( storeDir, + config.with( stringMap( "unsupported.dbms.ephemeral", "false" ) ), + state.databaseDependencies() ); + } + }; + } + } + + private class CustomHighlyAvailableGraphDatabase extends HighlyAvailableGraphDatabase + { + + CustomHighlyAvailableGraphDatabase( File storeDir, Config config, + GraphDatabaseFacadeFactory.Dependencies dependencies ) + { + super( storeDir, config, dependencies ); + } + + @Override + protected GraphDatabaseFacadeFactory newHighlyAvailableFacadeFactory() + { + return new CustomFacadeFactory(); + } + } + + private class CustomFacadeFactory extends GraphDatabaseFacadeFactory + { + CustomFacadeFactory() + { + super( DatabaseInfo.HA, HighlyAvailableEditionModule::new ); + } + + @Override + public GraphDatabaseFacade newFacade( File storeDir, Config config, Dependencies dependencies ) + { + return initFacade( storeDir, config, dependencies, new HighlyAvailableGraphDatabase( storeDir, config, dependencies ) ); + } + + @Override + protected PlatformModule createPlatform( File storeDir, Config config, Dependencies dependencies, + GraphDatabaseFacade graphDatabaseFacade ) + { + return new PlatformModule( storeDir, config, databaseInfo, dependencies, graphDatabaseFacade ) + { + @Override + protected VersionContextSupplier createCursorContextSupplier( Config config ) + { + return contextSupplier; + } + }; + } + } + + private class TestTransactionVersionContextSupplier extends TransactionVersionContextSupplier + { + + private volatile ByzantineLongSupplier byzantineLongSupplier; + + @Override + public void init( LongSupplier lastClosedTransactionIdSupplier ) + { + byzantineLongSupplier = new ByzantineLongSupplier( lastClosedTransactionIdSupplier ); + super.init( byzantineLongSupplier ); + } + + @Override + public VersionContext getVersionContext() + { + return super.getVersionContext(); + } + + ByzantineLongSupplier getByzantineIdSupplier() + { + return byzantineLongSupplier; + } + } + + private class ByzantineLongSupplier implements LongSupplier + { + + private volatile boolean wrongTxId; + private final LongSupplier originalIdSupplier; + + ByzantineLongSupplier( LongSupplier originalIdSupplier ) + { + this.originalIdSupplier = originalIdSupplier; + } + + @Override + public long getAsLong() + { + return wrongTxId ? 1 : originalIdSupplier.getAsLong(); + } + + void useWrongTxId() + { + wrongTxId = true; + } + + void useCorrectTxId() + { + wrongTxId = false; + } + } +} diff --git a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java index 66ab162241f68..f6e1d1cd76cd7 100644 --- a/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java +++ b/enterprise/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigratorTest.java @@ -30,6 +30,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.tracing.cursor.PageCursorTracerSupplier; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.LogService; @@ -74,7 +75,7 @@ public void shouldNotDoActualStoreMigrationBetween3_0_5_and_next() throws Except try ( FileSystemAbstraction fs = new DefaultFileSystemAbstraction(); PageCache pageCache = new ConfiguringPageCacheFactory( fs, config, NULL, - PageCursorTracerSupplier.NULL, NullLog.getInstance() ) + PageCursorTracerSupplier.NULL, NullLog.getInstance(), EmptyVersionContextSupplier.INSTANCE ) .getOrCreatePageCache() ) { // For test code sanity diff --git a/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom20IT.java b/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom20IT.java index 712b38722931d..8a336b2a53d9a 100644 --- a/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom20IT.java +++ b/enterprise/neo4j-enterprise/src/test/java/org/neo4j/upgrade/StoreMigratorFrom20IT.java @@ -44,6 +44,7 @@ import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory; import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProvider; import org.neo4j.kernel.api.index.SchemaIndexProvider; @@ -166,7 +167,7 @@ public void shouldMigrate() throws IOException, ConsistencyCheckIncompleteExcept } LogProvider logProvider = NullLogProvider.getInstance(); - StoreFactory storeFactory = new StoreFactory( storeDir.directory(), pageCache, fs, logProvider ); + StoreFactory storeFactory = new StoreFactory( storeDir.directory(), pageCache, fs, logProvider, EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = storeFactory.openAllNeoStores( true ) ) { verifyNeoStore( neoStores ); diff --git a/tools/src/main/java/org/neo4j/tools/dump/DumpCountsStore.java b/tools/src/main/java/org/neo4j/tools/dump/DumpCountsStore.java index 95535a4c66803..6d9c8bd4da418 100644 --- a/tools/src/main/java/org/neo4j/tools/dump/DumpCountsStore.java +++ b/tools/src/main/java/org/neo4j/tools/dump/DumpCountsStore.java @@ -31,6 +31,7 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.api.ReadOperations; import org.neo4j.kernel.api.schema.index.IndexDescriptor; import org.neo4j.kernel.configuration.Config; @@ -79,7 +80,7 @@ public static void dumpCountsStore( FileSystemAbstraction fs, File path, PrintSt { if ( fs.isDirectory( path ) ) { - StoreFactory factory = new StoreFactory( path, pages, fs, NullLogProvider.getInstance() ); + StoreFactory factory = new StoreFactory( path, pages, fs, NullLogProvider.getInstance(), EmptyVersionContextSupplier.INSTANCE ); NeoStores neoStores = factory.openAllNeoStores(); SchemaStorage schemaStorage = new SchemaStorage( neoStores.getSchemaStore() ); @@ -268,7 +269,7 @@ private static class VisitableCountsTracker extends CountsTracker VisitableCountsTracker( LogProvider logProvider, FileSystemAbstraction fs, PageCache pages, Config config, File baseFile ) { - super( logProvider, fs, pages, config, baseFile ); + super( logProvider, fs, pages, config, baseFile, EmptyVersionContextSupplier.INSTANCE ); } @Override diff --git a/tools/src/main/java/org/neo4j/tools/dump/DumpStore.java b/tools/src/main/java/org/neo4j/tools/dump/DumpStore.java index 29d21fe8bea17..ebb1cf89b4d44 100644 --- a/tools/src/main/java/org/neo4j/tools/dump/DumpStore.java +++ b/tools/src/main/java/org/neo4j/tools/dump/DumpStore.java @@ -26,6 +26,7 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.CommonAbstractStore; import org.neo4j.kernel.impl.store.NeoStores; @@ -72,7 +73,7 @@ public static void main( String... args ) throws Exception { final DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fs ); Function createStoreFactory = file -> new StoreFactory( file.getParentFile(), - Config.defaults(), idGeneratorFactory, pageCache, fs, logProvider() ); + Config.defaults(), idGeneratorFactory, pageCache, fs, logProvider(), EmptyVersionContextSupplier.INSTANCE ); for ( String arg : args ) { diff --git a/tools/src/main/java/org/neo4j/tools/dump/DumpStoreChain.java b/tools/src/main/java/org/neo4j/tools/dump/DumpStoreChain.java index afe8eefd4b321..990630bdff44a 100644 --- a/tools/src/main/java/org/neo4j/tools/dump/DumpStoreChain.java +++ b/tools/src/main/java/org/neo4j/tools/dump/DumpStoreChain.java @@ -28,6 +28,7 @@ import org.neo4j.helpers.Args; import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeStore; @@ -118,7 +119,7 @@ void dump( File storeDir ) throws IOException DefaultIdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fs ); Config config = Config.defaults(); StoreFactory storeFactory = new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fs, - logProvider() ); + logProvider(), EmptyVersionContextSupplier.INSTANCE ); try ( NeoStores neoStores = storeFactory.openNeoStores( getStoreTypes() ) ) { diff --git a/tools/src/main/java/org/neo4j/tools/rawstorereader/RsdrMain.java b/tools/src/main/java/org/neo4j/tools/rawstorereader/RsdrMain.java index 71f9b463f34b5..fa39f9327e612 100644 --- a/tools/src/main/java/org/neo4j/tools/rawstorereader/RsdrMain.java +++ b/tools/src/main/java/org/neo4j/tools/rawstorereader/RsdrMain.java @@ -35,6 +35,7 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.tracing.cursor.context.EmptyVersionContextSupplier; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.InvalidRecordException; import org.neo4j.kernel.impl.store.MetaDataStore; @@ -112,7 +113,7 @@ private static StoreFactory openStore( FileSystemAbstraction fileSystem, File st { IdGeneratorFactory idGeneratorFactory = new DefaultIdGeneratorFactory( fileSystem ); NullLogProvider logProvider = NullLogProvider.getInstance(); - return new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fileSystem, logProvider ); + return new StoreFactory( storeDir, config, idGeneratorFactory, pageCache, fileSystem, logProvider, EmptyVersionContextSupplier.INSTANCE ); } private static void interact( FileSystemAbstraction fileSystem, NeoStores neoStores ) throws IOException