diff --git a/community/community-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorMatchers.scala b/community/community-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorMatchers.scala index 47d3538165e1..d504f24e44e3 100644 --- a/community/community-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorMatchers.scala +++ b/community/community-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorMatchers.scala @@ -199,7 +199,7 @@ object DataCollectorMatchers { errors += s"Expected value '$expectedValue' at position $i, but list was too small" } if (values.size > expected.size) - errors += s"Expected list of maxSize ${expected.size}, but got additional elements ${values.slice(expected.size, values.size)}" + errors += s"Expected list of ${expected.size} elements, but got additional elements ${values.slice(expected.size, values.size)}" case x => errors += s"Expected list but got '$x'" diff --git a/community/community-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorQueriesAcceptanceTest.scala b/community/community-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorQueriesAcceptanceTest.scala index 929cee361e6f..7508b38e4905 100644 --- a/community/community-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorQueriesAcceptanceTest.scala +++ b/community/community-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorQueriesAcceptanceTest.scala @@ -298,13 +298,10 @@ class DataCollectorQueriesAcceptanceTest extends DataCollectorTestSupport { test("[retrieveAllAnonymized] should anonymize tokens inside queries") { // given - execute("CALL db.stats.stop('QUERIES')").single execute("CREATE (:User {age: 99})-[:KNOWS]->(:Buddy {p: 42})-[:WANTS]->(:Raccoon)") // create tokens - execute("CALL db.stats.collect('QUERIES')").single execute("MATCH (:User)-[:KNOWS]->(:Buddy)-[:WANTS]->(:Raccoon) RETURN 1") execute("MATCH ({p: 42}), ({age: 43}) RETURN 1") - execute("CALL db.stats.stop('QUERIES')").single // when val res = execute("CALL db.stats.retrieveAllAnonymized('myToken')") @@ -319,10 +316,9 @@ class DataCollectorQueriesAcceptanceTest extends DataCollectorTestSupport { test("[retrieveAllAnonymized] should handle pre-parser options") { // given execute("CREATE (:User {age: 99})-[:KNOWS]->(:Buddy {p: 42})-[:WANTS]->(:Raccoon)") // create tokens - execute("CALL db.stats.collect('QUERIES')").single + execute("EXPLAIN MATCH (:User)-[:KNOWS]->(:Buddy)-[:WANTS]->(:Raccoon) RETURN 1") execute("CYPHER 3.4 runtime=interpreted PROFILE CREATE ()") - execute("CALL db.stats.stop('QUERIES')").single // when val res = execute("CALL db.stats.retrieveAllAnonymized('myToken')") @@ -338,10 +334,9 @@ class DataCollectorQueriesAcceptanceTest extends DataCollectorTestSupport { // given val path = Files.createTempFile("data", ".csv") val url = path.toUri.toURL.toString - execute("CALL db.stats.collect('QUERIES')").single + execute(s"LOAD CSV FROM '$url' AS row CREATE ({key: row[0]})") execute(s"USING PERIODIC COMMIT 30 LOAD CSV FROM '$url' AS row CREATE ({key: row[0]})") - execute("CALL db.stats.stop('QUERIES')").single // when val res = execute("CALL db.stats.retrieveAllAnonymized('myToken')") diff --git a/community/data-collector/src/main/java/org/neo4j/internal/collector/DataCollectorProcedures.java b/community/data-collector/src/main/java/org/neo4j/internal/collector/DataCollectorProcedures.java index 19add41a408c..d706657c71d4 100644 --- a/community/data-collector/src/main/java/org/neo4j/internal/collector/DataCollectorProcedures.java +++ b/community/data-collector/src/main/java/org/neo4j/internal/collector/DataCollectorProcedures.java @@ -19,9 +19,7 @@ */ package org.neo4j.internal.collector; -import java.time.ZonedDateTime; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; @@ -63,7 +61,7 @@ public Stream retrieve( @Name( value = "section" ) String sectio return TokensSection.retrieve( dataCollector.kernel ); case Sections.META: - return MetaSection.retrieve( null, dataCollector.kernel, dataCollector.queryCollector.nSilentQueryDrops() ); + return MetaSection.retrieve( null, dataCollector.kernel, dataCollector.queryCollector.numSilentQueryDrops() ); case Sections.QUERIES: return QueriesSection.retrieve( dataCollector.queryCollector.getData(), @@ -87,7 +85,7 @@ public Stream retrieveAllAnonymized( @Name( value = "graphToken" throw new InvalidArgumentsException( "Graph token must be a non-empty string" ); } - return Stream.of( MetaSection.retrieve( graphToken, dataCollector.kernel, dataCollector.queryCollector.nSilentQueryDrops() ), + return Stream.of( MetaSection.retrieve( graphToken, dataCollector.kernel, dataCollector.queryCollector.numSilentQueryDrops() ), GraphCountsSection.retrieve( dataCollector.kernel, Anonymizer.IDS ), QueriesSection.retrieve( dataCollector.queryCollector.getData(), new IdAnonymizer( transaction.tokenRead() ), diff --git a/community/data-collector/src/main/java/org/neo4j/internal/collector/MetaSection.java b/community/data-collector/src/main/java/org/neo4j/internal/collector/MetaSection.java index 37ab36d3b831..0943ab97999f 100644 --- a/community/data-collector/src/main/java/org/neo4j/internal/collector/MetaSection.java +++ b/community/data-collector/src/main/java/org/neo4j/internal/collector/MetaSection.java @@ -45,7 +45,9 @@ private MetaSection() { // only static methods } - static Stream retrieve( String graphToken, Kernel kernel, long nSilentQueryDrops ) throws TransactionFailureException + static Stream retrieve( String graphToken, + Kernel kernel, + long numSilentQueryDrops ) throws TransactionFailureException { Map systemData = new HashMap<>(); systemData.put( "jvmMemoryFree", Runtime.getRuntime().freeMemory() ); @@ -77,7 +79,7 @@ static Stream retrieve( String graphToken, Kernel kernel, long n systemData.put( "fileEncoding", System.getProperty( "file.encoding" ) ); Map internalData = new HashMap<>(); - internalData.put( "numSilentQueryCollectionMisses", nSilentQueryDrops ); + internalData.put( "numSilentQueryCollectionMisses", numSilentQueryDrops ); Map metaData = new HashMap<>(); metaData.put( "graphToken", graphToken ); diff --git a/community/data-collector/src/main/java/org/neo4j/internal/collector/QueryCollector.java b/community/data-collector/src/main/java/org/neo4j/internal/collector/QueryCollector.java index 45e0ef9d89dd..6047a81ef4a5 100644 --- a/community/data-collector/src/main/java/org/neo4j/internal/collector/QueryCollector.java +++ b/community/data-collector/src/main/java/org/neo4j/internal/collector/QueryCollector.java @@ -48,18 +48,27 @@ class QueryCollector extends CollectorStateMachine> impl private volatile boolean isCollecting; private final RingRecentBuffer queries; private final JobScheduler jobScheduler; + /** + * We retain at max 2^13 = 8192 queries in memory at any given time. This number + * was chosen as a trade-off between getting a useful amount of queries, and not + * wasting too much heap. Even with a buffer full of unique queries, the estimated + * footprint lies in tens of MBs. If the buffer is full of cached queries, the + * retained size was measured to 265 kB. + */ + private static final int QUERY_BUFFER_SIZE_IN_BITS = 13; QueryCollector( JobScheduler jobScheduler ) { super( true ); this.jobScheduler = jobScheduler; isCollecting = false; - queries = new RingRecentBuffer<>( 13 ); + + queries = new RingRecentBuffer<>( QUERY_BUFFER_SIZE_IN_BITS ); } - long nSilentQueryDrops() + long numSilentQueryDrops() { - return queries.nSilentQueryDrops(); + return queries.numSilentQueryDrops(); } // CollectorStateMachine diff --git a/community/data-collector/src/main/java/org/neo4j/internal/collector/RingRecentBuffer.java b/community/data-collector/src/main/java/org/neo4j/internal/collector/RingRecentBuffer.java index a6acb8219802..7e299fc73f2b 100644 --- a/community/data-collector/src/main/java/org/neo4j/internal/collector/RingRecentBuffer.java +++ b/community/data-collector/src/main/java/org/neo4j/internal/collector/RingRecentBuffer.java @@ -53,7 +53,7 @@ public RingRecentBuffer( int bitSize ) dropEvents = new AtomicLong( 0 ); } - long nSilentQueryDrops() + long numSilentQueryDrops() { return dropEvents.get(); } diff --git a/community/data-collector/src/test/scala/org/neo4j/internal/collector/RingRecentBufferTest.java b/community/data-collector/src/test/scala/org/neo4j/internal/collector/RingRecentBufferTest.java index cf2c2d106bbf..ae44e6d481f5 100644 --- a/community/data-collector/src/test/scala/org/neo4j/internal/collector/RingRecentBufferTest.java +++ b/community/data-collector/src/test/scala/org/neo4j/internal/collector/RingRecentBufferTest.java @@ -62,7 +62,7 @@ void shouldJustWork() } buffer.foreach( Assertions::assertNotNull ); - assertEquals( 0, buffer.nSilentQueryDrops() ); + assertEquals( 0, buffer.numSilentQueryDrops() ); } @Test @@ -100,7 +100,7 @@ void shouldNotReadSameElementTwice() throws ExecutionException, InterruptedExcep { executor.shutdown(); } - assertEquals( 0, buffer.nSilentQueryDrops() ); + assertEquals( 0, buffer.numSilentQueryDrops() ); } @Test @@ -133,7 +133,7 @@ void shouldNeverReadUnwrittenElements() throws ExecutionException, InterruptedEx { executor.shutdown(); } - assertEquals( 0, buffer.nSilentQueryDrops() ); + assertEquals( 0, buffer.numSilentQueryDrops() ); } @Test @@ -170,7 +170,7 @@ void shouldWorkWithManyConcurrentProducers() throws ExecutionException, Interrup { executor.shutdown(); } - assertEquals( 0, buffer.nSilentQueryDrops() ); + assertEquals( 0, buffer.numSilentQueryDrops() ); } private Runnable stress( int n, LongConsumer action )