Add unsupported.datacollector.max_query_text_size configuration optio…

…n...

This gives data admins the possibility to control the maximum number query
text size retained by the data collector, including query text completely
by setting max_query_text_size=0. This can be an important tuning
mechanism to optimize the collected query texts against the used memory
footprint, in particular for generated queries.

community/community-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorQueriesAcceptanceTest.scala

@@ -300,7 +300,7 @@ class DataCollectorQueriesAcceptanceTest extends DataCollectorTestSupport {
 
   test("should limit the collected query text size") {
     // given
-    val largeQuery = (0 until 10000).map(i => s"CREATE (n$i) ").mkString("\n")
+    val largeQuery = (0 until 1000).map(i => s"CREATE (n$i) ").mkString("\n")
     execute(largeQuery)
 
     // when
@@ -317,6 +317,80 @@ class DataCollectorQueriesAcceptanceTest extends DataCollectorTestSupport {
     )
   }
 
+  test("should limit the collected query text size by configured max_query_text_size") {
+    // given
+    restartWithConfig(Map(GraphDatabaseSettings.data_collector_max_query_text_size -> "33"))
+    val largeQuery = (0 until 10).map(i => s"CREATE (n$i) ").mkString("\n")
+    execute(largeQuery)
+
+    // when
+    val res = execute("CALL db.stats.retrieve('QUERIES')").toList
+
+    // then
+    res should beListWithoutOrder(
+      beMapContaining(
+        "section" -> "QUERIES",
+        "data" -> beMapContaining(
+          "query" -> largeQuery.take(33)
+        )
+      )
+    )
+  }
+
+  test("should drop query text on max_query_text_size=0") {
+    // given
+    restartWithConfig(Map(GraphDatabaseSettings.data_collector_max_query_text_size -> "0"))
+    execute("RETURN 1")
+
+    // when
+    val res = execute("CALL db.stats.retrieve('QUERIES')").toList
+
+    // then
+    res should beListWithoutOrder(
+      beMapContaining(
+        "section" -> "QUERIES",
+        "data" -> beMapContaining("query" -> "")
+      )
+    )
+  }
+
+  test("should distinguish queries even though dropped query text is not unique") {
+    // given
+    restartWithConfig(Map(GraphDatabaseSettings.data_collector_max_query_text_size -> "6"))
+    execute("RETURN 1+1 AS a", params = Map("p" -> 1))
+    execute("RETURN 1+2 AS a", params = Map("p" -> 2))
+    execute("RETURN 1+3 AS a", params = Map("p" -> 3))
+
+    // when
+    val res = execute("CALL db.stats.retrieve('QUERIES')").toList
+
+    // then
+    res.size shouldBe 3
+    res should beListWithoutOrder(
+      beMapContaining(
+        "section" -> "QUERIES",
+        "data" -> beMapContaining(
+          "query" -> "RETURN",
+          "invocations" -> beInvocationsInOrder(Map("p" -> 1))
+        )
+      ),
+      beMapContaining(
+        "section" -> "QUERIES",
+        "data" -> beMapContaining(
+          "query" -> "RETURN",
+          "invocations" -> beInvocationsInOrder(Map("p" -> 2))
+        )
+      ),
+      beMapContaining(
+        "section" -> "QUERIES",
+        "data" -> beMapContaining(
+          "query" -> "RETURN",
+          "invocations" -> beInvocationsInOrder(Map("p" -> 3))
+        )
+      )
+    )
+  }
+
   test("should limit the collected query parameter sizes") {
     // given
     val entities = execute(

community/data-collector/src/main/java/org/neo4j/internal/collector/DataCollector.java

@@ -46,7 +46,8 @@ public class DataCollector implements AutoCloseable
         this.jobScheduler = jobScheduler;
         this.valueMapper = valueMapper;
         this.queryCollector = new QueryCollector( jobScheduler,
-                                                  config.get( GraphDatabaseSettings.data_collector_max_recent_query_count ) );
+                                                  config.get( GraphDatabaseSettings.data_collector_max_recent_query_count ),
+                                                  config.get( GraphDatabaseSettings.data_collector_max_query_text_size ) );
         try
         {
             this.queryCollector.collect( Collections.emptyMap() );

community/data-collector/src/main/java/org/neo4j/internal/collector/QueryCollector.java

@@ -42,12 +42,15 @@ class QueryCollector extends CollectorStateMachine<Iterator<TruncatedQuerySnapsh
     private volatile boolean isCollecting;
     private final RingRecentBuffer<TruncatedQuerySnapshot> queries;
     private final JobScheduler jobScheduler;
+    private final int maxQueryTextSize;
 
     QueryCollector( JobScheduler jobScheduler,
-                    int maxRecentQueryCount )
+                    int maxRecentQueryCount,
+                    int maxQueryTextSize )
     {
         super( true );
         this.jobScheduler = jobScheduler;
+        this.maxQueryTextSize = maxQueryTextSize;
         isCollecting = false;
 
         // Round down to the nearest power of 2
@@ -115,7 +118,8 @@ public void endSuccess( ExecutingQuery query )
                                                 snapshot.queryParameters(),
                                                 snapshot.elapsedTimeMicros(),
                                                 snapshot.compilationTimeMicros(),
-                                                snapshot.startTimestampMillis() ) );
+                                                snapshot.startTimestampMillis(),
+                                                maxQueryTextSize ) );
         }
     }
 }

community/data-collector/src/main/java/org/neo4j/internal/collector/TruncatedQuerySnapshot.java

@@ -49,21 +49,24 @@
  */
 class TruncatedQuerySnapshot
 {
+    final int fullQueryTextHash;
     final String queryText;
     final ExecutionPlanDescription queryPlan;
     final MapValue queryParameters;
     final Long elapsedTimeMicros;
     final Long compilationTimeMicros;
     final Long startTimestampMillis;
 
-    TruncatedQuerySnapshot( String queryText,
+    TruncatedQuerySnapshot( String fullQueryText,
                             ExecutionPlanDescription queryPlan,
                             MapValue queryParameters,
                             Long elapsedTimeMicros,
                             Long compilationTimeMicros,
-                            Long startTimestampMillis )
+                            Long startTimestampMillis,
+                            int maxQueryTextLength )
     {
-        this.queryText = truncateQueryText( queryText, 10000 );
+        this.fullQueryTextHash = fullQueryText.hashCode();
+        this.queryText = truncateQueryText( fullQueryText, maxQueryTextLength );
         this.queryPlan = queryPlan;
         this.queryParameters = truncateParameters( queryParameters );
         this.elapsedTimeMicros = elapsedTimeMicros;

community/data-collector/src/main/scala/org/neo4j/internal/collector/QueriesSection.scala

@@ -43,7 +43,7 @@ object QueriesSection {
 
   case class ProfileData(dbHits: util.ArrayList[Long], rows: util.ArrayList[Long], params: util.Map[String, AnyRef])
 
-  case class QueryKey(queryText: String, plan: ExecutionPlanDescription)
+  case class QueryKey(queryText: String, fullQueryTextHash: Int, plan: ExecutionPlanDescription)
 
   class QueryData() {
     val invocations = new ArrayBuffer[SingleInvocation]
@@ -60,7 +60,7 @@ object QueriesSection {
       val snapshot = querySnapshots.next()
       val queryString = snapshot.queryText
       if (QUERY_FILTER.findFirstMatchIn(queryString).isEmpty) {
-        val snapshotList = queries.getOrElseUpdate(QueryKey(queryString, snapshot.queryPlan), new QueryData())
+        val snapshotList = queries.getOrElseUpdate(QueryKey(queryString, snapshot.fullQueryTextHash, snapshot.queryPlan), new QueryData())
         snapshotList.invocations += SingleInvocation(snapshot.queryParameters,
                                                      snapshot.elapsedTimeMicros,
                                                      snapshot.compilationTimeMicros,

community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java

@@ -426,6 +426,17 @@ public class GraphDatabaseSettings implements LoadableConfig
             buildSetting( "unsupported.datacollector.max_recent_query_count", INTEGER, "8192" )
                     .constraint( min( 0 ) ).build();
 
+    @Description( "Sets the upper limit for how much of the query text that will be retained by the query collector." +
+            " For queries longer than the limit, only a prefix of size limit will be retained by the collector." +
+            " Lowering this value will reduce the memory footprint of collected query invocations under loads with" +
+            " many queries with long query texts, which could occur for generated queries. The downside is that" +
+            " on retrieving queries by `db.stats.retrieve`, queries longer than this max size would be returned" +
+            " incomplete. Setting this to 0 will completely drop query texts from the collected queries.")
+    @Internal
+    public static final Setting<Integer> data_collector_max_query_text_size =
+            buildSetting( "unsupported.datacollector.max_query_text_size", INTEGER, "10000" )
+                    .constraint( min( 0 ) ).build();
+
     @Description( "The maximum amount of time to wait for the database to become available, when " +
                   "starting a new transaction." )
     @Internal