From b732d5f31187d9fca553a2434373a5ca772d2f6e Mon Sep 17 00:00:00 2001 From: fickludd Date: Tue, 14 May 2019 14:42:26 +0200 Subject: [PATCH] Add unsupported.datacollector.max_query_text_size configuration option... This gives data admins the possibility to control the maximum number query text size retained by the data collector, including query text completely by setting max_query_text_size=0. This can be an important tuning mechanism to optimize the collected query texts against the used memory footprint, in particular for generated queries. --- .../DataCollectorQueriesAcceptanceTest.scala | 76 ++++++++++++++++++- .../internal/collector/DataCollector.java | 3 +- .../internal/collector/QueryCollector.java | 8 +- .../collector/TruncatedQuerySnapshot.java | 9 ++- .../internal/collector/QueriesSection.scala | 4 +- .../factory/GraphDatabaseSettings.java | 11 +++ 6 files changed, 102 insertions(+), 9 deletions(-) diff --git a/community/community-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorQueriesAcceptanceTest.scala b/community/community-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorQueriesAcceptanceTest.scala index 53996d13daafb..4f43491223bd3 100644 --- a/community/community-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorQueriesAcceptanceTest.scala +++ b/community/community-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorQueriesAcceptanceTest.scala @@ -300,7 +300,7 @@ class DataCollectorQueriesAcceptanceTest extends DataCollectorTestSupport { test("should limit the collected query text size") { // given - val largeQuery = (0 until 10000).map(i => s"CREATE (n$i) ").mkString("\n") + val largeQuery = (0 until 1000).map(i => s"CREATE (n$i) ").mkString("\n") execute(largeQuery) // when @@ -317,6 +317,80 @@ class DataCollectorQueriesAcceptanceTest extends DataCollectorTestSupport { ) } + test("should limit the collected query text size by configured max_query_text_size") { + // given + restartWithConfig(Map(GraphDatabaseSettings.data_collector_max_query_text_size -> "33")) + val largeQuery = (0 until 10).map(i => s"CREATE (n$i) ").mkString("\n") + execute(largeQuery) + + // when + val res = execute("CALL db.stats.retrieve('QUERIES')").toList + + // then + res should beListWithoutOrder( + beMapContaining( + "section" -> "QUERIES", + "data" -> beMapContaining( + "query" -> largeQuery.take(33) + ) + ) + ) + } + + test("should drop query text on max_query_text_size=0") { + // given + restartWithConfig(Map(GraphDatabaseSettings.data_collector_max_query_text_size -> "0")) + execute("RETURN 1") + + // when + val res = execute("CALL db.stats.retrieve('QUERIES')").toList + + // then + res should beListWithoutOrder( + beMapContaining( + "section" -> "QUERIES", + "data" -> beMapContaining("query" -> "") + ) + ) + } + + test("should distinguish queries even though dropped query text is not unique") { + // given + restartWithConfig(Map(GraphDatabaseSettings.data_collector_max_query_text_size -> "6")) + execute("RETURN 1+1 AS a", params = Map("p" -> 1)) + execute("RETURN 1+2 AS a", params = Map("p" -> 2)) + execute("RETURN 1+3 AS a", params = Map("p" -> 3)) + + // when + val res = execute("CALL db.stats.retrieve('QUERIES')").toList + + // then + res.size shouldBe 3 + res should beListWithoutOrder( + beMapContaining( + "section" -> "QUERIES", + "data" -> beMapContaining( + "query" -> "RETURN", + "invocations" -> beInvocationsInOrder(Map("p" -> 1)) + ) + ), + beMapContaining( + "section" -> "QUERIES", + "data" -> beMapContaining( + "query" -> "RETURN", + "invocations" -> beInvocationsInOrder(Map("p" -> 2)) + ) + ), + beMapContaining( + "section" -> "QUERIES", + "data" -> beMapContaining( + "query" -> "RETURN", + "invocations" -> beInvocationsInOrder(Map("p" -> 3)) + ) + ) + ) + } + test("should limit the collected query parameter sizes") { // given val entities = execute( diff --git a/community/data-collector/src/main/java/org/neo4j/internal/collector/DataCollector.java b/community/data-collector/src/main/java/org/neo4j/internal/collector/DataCollector.java index bb51ead666418..816e332edc69e 100644 --- a/community/data-collector/src/main/java/org/neo4j/internal/collector/DataCollector.java +++ b/community/data-collector/src/main/java/org/neo4j/internal/collector/DataCollector.java @@ -46,7 +46,8 @@ public class DataCollector implements AutoCloseable this.jobScheduler = jobScheduler; this.valueMapper = valueMapper; this.queryCollector = new QueryCollector( jobScheduler, - config.get( GraphDatabaseSettings.data_collector_max_recent_query_count ) ); + config.get( GraphDatabaseSettings.data_collector_max_recent_query_count ), + config.get( GraphDatabaseSettings.data_collector_max_query_text_size ) ); try { this.queryCollector.collect( Collections.emptyMap() ); diff --git a/community/data-collector/src/main/java/org/neo4j/internal/collector/QueryCollector.java b/community/data-collector/src/main/java/org/neo4j/internal/collector/QueryCollector.java index 5d7da25b7fc0f..16f96e9e7e605 100644 --- a/community/data-collector/src/main/java/org/neo4j/internal/collector/QueryCollector.java +++ b/community/data-collector/src/main/java/org/neo4j/internal/collector/QueryCollector.java @@ -42,12 +42,15 @@ class QueryCollector extends CollectorStateMachine queries; private final JobScheduler jobScheduler; + private final int maxQueryTextSize; QueryCollector( JobScheduler jobScheduler, - int maxRecentQueryCount ) + int maxRecentQueryCount, + int maxQueryTextSize ) { super( true ); this.jobScheduler = jobScheduler; + this.maxQueryTextSize = maxQueryTextSize; isCollecting = false; // Round down to the nearest power of 2 @@ -115,7 +118,8 @@ public void endSuccess( ExecutingQuery query ) snapshot.queryParameters(), snapshot.elapsedTimeMicros(), snapshot.compilationTimeMicros(), - snapshot.startTimestampMillis() ) ); + snapshot.startTimestampMillis(), + maxQueryTextSize ) ); } } } diff --git a/community/data-collector/src/main/java/org/neo4j/internal/collector/TruncatedQuerySnapshot.java b/community/data-collector/src/main/java/org/neo4j/internal/collector/TruncatedQuerySnapshot.java index 4aa992b6ebd7c..24950aa8a4d4c 100644 --- a/community/data-collector/src/main/java/org/neo4j/internal/collector/TruncatedQuerySnapshot.java +++ b/community/data-collector/src/main/java/org/neo4j/internal/collector/TruncatedQuerySnapshot.java @@ -49,6 +49,7 @@ */ class TruncatedQuerySnapshot { + final int fullQueryTextHash; final String queryText; final ExecutionPlanDescription queryPlan; final MapValue queryParameters; @@ -56,14 +57,16 @@ class TruncatedQuerySnapshot final Long compilationTimeMicros; final Long startTimestampMillis; - TruncatedQuerySnapshot( String queryText, + TruncatedQuerySnapshot( String fullQueryText, ExecutionPlanDescription queryPlan, MapValue queryParameters, Long elapsedTimeMicros, Long compilationTimeMicros, - Long startTimestampMillis ) + Long startTimestampMillis, + int maxQueryTextLength ) { - this.queryText = truncateQueryText( queryText, 10000 ); + this.fullQueryTextHash = fullQueryText.hashCode(); + this.queryText = truncateQueryText( fullQueryText, maxQueryTextLength ); this.queryPlan = queryPlan; this.queryParameters = truncateParameters( queryParameters ); this.elapsedTimeMicros = elapsedTimeMicros; diff --git a/community/data-collector/src/main/scala/org/neo4j/internal/collector/QueriesSection.scala b/community/data-collector/src/main/scala/org/neo4j/internal/collector/QueriesSection.scala index c387ca4bec71d..19ada683a3aab 100644 --- a/community/data-collector/src/main/scala/org/neo4j/internal/collector/QueriesSection.scala +++ b/community/data-collector/src/main/scala/org/neo4j/internal/collector/QueriesSection.scala @@ -43,7 +43,7 @@ object QueriesSection { case class ProfileData(dbHits: util.ArrayList[Long], rows: util.ArrayList[Long], params: util.Map[String, AnyRef]) - case class QueryKey(queryText: String, plan: ExecutionPlanDescription) + case class QueryKey(queryText: String, fullQueryTextHash: Int, plan: ExecutionPlanDescription) class QueryData() { val invocations = new ArrayBuffer[SingleInvocation] @@ -60,7 +60,7 @@ object QueriesSection { val snapshot = querySnapshots.next() val queryString = snapshot.queryText if (QUERY_FILTER.findFirstMatchIn(queryString).isEmpty) { - val snapshotList = queries.getOrElseUpdate(QueryKey(queryString, snapshot.queryPlan), new QueryData()) + val snapshotList = queries.getOrElseUpdate(QueryKey(queryString, snapshot.fullQueryTextHash, snapshot.queryPlan), new QueryData()) snapshotList.invocations += SingleInvocation(snapshot.queryParameters, snapshot.elapsedTimeMicros, snapshot.compilationTimeMicros, diff --git a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java index 5249313c77f60..4457fe2aeea08 100644 --- a/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java +++ b/community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java @@ -426,6 +426,17 @@ public class GraphDatabaseSettings implements LoadableConfig buildSetting( "unsupported.datacollector.max_recent_query_count", INTEGER, "8192" ) .constraint( min( 0 ) ).build(); + @Description( "Sets the upper limit for how much of the query text that will be retained by the query collector." + + " For queries longer than the limit, only a prefix of size limit will be retained by the collector." + + " Lowering this value will reduce the memory footprint of collected query invocations under loads with" + + " many queries with long query texts, which could occur for generated queries. The downside is that" + + " on retrieving queries by `db.stats.retrieve`, queries longer than this max size would be returned" + + " incomplete. Setting this to 0 will completely drop query texts from the collected queries.") + @Internal + public static final Setting data_collector_max_query_text_size = + buildSetting( "unsupported.datacollector.max_query_text_size", INTEGER, "10000" ) + .constraint( min( 0 ) ).build(); + @Description( "The maximum amount of time to wait for the database to become available, when " + "starting a new transaction." ) @Internal