Skip to content

Commit

Permalink
Turn query collection on by default...
Browse files Browse the repository at this point in the history 
With this change we constantly collect queries in the query collector
unless explicitly stopped by db.stats.stop(). We also change so there is
no default timeout of db.stats.collect('QUERIES'). To facilite minimum
latency increase of query execution, we use the new RingRecentBuffer to
back QueryCollector.
  • Loading branch information
@fickludd
fickludd committed Mar 29, 2019
1 parent a9b1a0d commit e71a554
Show file tree
Hide file tree
Showing 10 changed files with 156 additions and 97 deletions.
2 changes: 1 addition & 1 deletion ...nity-it/cypher-it/src/test/scala/org/neo4j/internal/collector/DataCollectorMatchers.scala
View file
Expand Up @@ -199,7 +199,7 @@ object DataCollectorMatchers {
errors += s"Expected value '$expectedValue' at position $i, but list was too small" errors += s"Expected value '$expectedValue' at position $i, but list was too small"
} }
if (values.size > expected.size) if (values.size > expected.size)
errors += s"Expected list of size ${expected.size}, but got additional elements ${values.slice(expected.size, values.size)}" errors += s"Expected list of maxSize ${expected.size}, but got additional elements ${values.slice(expected.size, values.size)}"


case x => case x =>
errors += s"Expected list but got '$x'" errors += s"Expected list but got '$x'"
Expand Down
71 changes: 34 additions & 37 deletions ...r-it/src/test/scala/org/neo4j/internal/collector/DataCollectorQueriesAcceptanceTest.scala
View file
Expand Up @@ -21,27 +21,20 @@ package org.neo4j.internal.collector


import java.nio.file.Files import java.nio.file.Files


import org.neo4j.cypher._
import org.scalatest.matchers.{MatchResult, Matcher} import org.scalatest.matchers.{MatchResult, Matcher}


import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.ArrayBuffer


class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite { class DataCollectorQueriesAcceptanceTest extends DataCollectorTestSupport {


import DataCollectorMatchers._ import DataCollectorMatchers._


test("should collect and retrieve queries") { test("should collect and retrieve queries") {
// given // given
execute("RETURN 'not collected!'")
execute("CALL db.stats.collect('QUERIES')").single

execute("MATCH (n) RETURN count(n)") execute("MATCH (n) RETURN count(n)")
execute("MATCH (n)-->(m) RETURN n,m") execute("MATCH (n)-->(m) RETURN n,m")
execute("WITH 'hi' AS x RETURN x+'-ho'") execute("WITH 'hi' AS x RETURN x+'-ho'")


execute("CALL db.stats.stop('QUERIES')").single
execute("RETURN 'not collected!'")

// when // when
val res = execute("CALL db.stats.retrieve('QUERIES')").toList val res = execute("CALL db.stats.retrieve('QUERIES')").toList


Expand All @@ -68,11 +61,35 @@ class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite {
) )
} }


test("should clear queries") { test("should allow explicit control over query collection") {
// given // given
execute("CALL db.stats.stop('QUERIES')").single

execute("RETURN 'not collected!'")
execute("CALL db.stats.collect('QUERIES')").single execute("CALL db.stats.collect('QUERIES')").single
execute("MATCH (n) RETURN count(n)") execute("MATCH (n) RETURN count(n)")
execute("CALL db.stats.stop('QUERIES')").single execute("CALL db.stats.stop('QUERIES')").single
execute("RETURN 'not collected!'")

// when
val res = execute("CALL db.stats.retrieve('QUERIES')").toList

// then
res should beListWithoutOrder(
beMapContaining(
"section" -> "QUERIES",
"data" -> beMapContaining(
"query" -> "MATCH (n) RETURN count(n)"
)
)
)
}

test("should clear queries") {
// given
assertCollecting("QUERIES")
execute("MATCH (n) RETURN count(n)")
execute("CALL db.stats.stop('QUERIES')").single
execute("CALL db.stats.retrieve('QUERIES')").toList should have size 1 execute("CALL db.stats.retrieve('QUERIES')").toList should have size 1


// when // when
Expand All @@ -84,7 +101,7 @@ class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite {


test("should append queries if restarted collection") { test("should append queries if restarted collection") {
// given // given
execute("CALL db.stats.collect('QUERIES')").single assertCollecting("QUERIES")
execute("MATCH (n) RETURN count(n)") execute("MATCH (n) RETURN count(n)")
execute("CALL db.stats.stop('QUERIES')").single execute("CALL db.stats.stop('QUERIES')").single
execute("CALL db.stats.retrieve('QUERIES')").toList should have size 1 execute("CALL db.stats.retrieve('QUERIES')").toList should have size 1
Expand All @@ -99,15 +116,13 @@ class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite {


test("should stop collection after specified time") { test("should stop collection after specified time") {
// given // given
execute("CALL db.stats.stop('QUERIES')").single
execute("CALL db.stats.collect('QUERIES', {durationSeconds: 3})").single execute("CALL db.stats.collect('QUERIES', {durationSeconds: 3})").single
execute("MATCH (n) RETURN count(n)") execute("MATCH (n) RETURN count(n)")
Thread.sleep(4000) Thread.sleep(4000)


// then // then
execute("CALL db.stats.status()").single should beMapContaining( assertIdle("QUERIES")
"status" -> "idle",
"section" -> "QUERIES"
)


// and when // and when
execute("RETURN 'late query'") execute("RETURN 'late query'")
Expand All @@ -118,21 +133,20 @@ class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite {


test("should not stop later collection event after initial timeout") { test("should not stop later collection event after initial timeout") {
// given // given
execute("CALL db.stats.stop('QUERIES')").single
execute("CALL db.stats.collect('QUERIES', {durationSeconds: 3})").single execute("CALL db.stats.collect('QUERIES', {durationSeconds: 3})").single
execute("MATCH (n) RETURN count(n)") execute("MATCH (n) RETURN count(n)")
execute("CALL db.stats.stop('QUERIES')").single execute("CALL db.stats.stop('QUERIES')").single
execute("CALL db.stats.collect('QUERIES')").single execute("CALL db.stats.collect('QUERIES')").single
Thread.sleep(4000) Thread.sleep(4000)


// then // then
execute("CALL db.stats.status()").single should beMapContaining( assertCollecting("QUERIES")
"status" -> "collecting",
"section" -> "QUERIES"
)
} }


test("should retrieve query execution plan and estimated rows") { test("should retrieve query execution plan and estimated rows") {
// given // given
execute("CALL db.stats.stop('QUERIES')").single
execute("CREATE (a), (b), (c)") execute("CREATE (a), (b), (c)")


execute("CALL db.stats.collect('QUERIES')").single execute("CALL db.stats.collect('QUERIES')").single
Expand Down Expand Up @@ -184,13 +198,11 @@ class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite {


test("should retrieve invocations of query") { test("should retrieve invocations of query") {
// given // given
execute("CALL db.stats.collect('QUERIES')").single
execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> "BrassLeg")) execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> "BrassLeg"))
execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> 2)) execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> 2))
execute("WITH 42 AS x RETURN x") execute("WITH 42 AS x RETURN x")
execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> List(3.1, 3.2))) execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> List(3.1, 3.2)))
execute("WITH 42 AS x RETURN x") execute("WITH 42 AS x RETURN x")
execute("CALL db.stats.stop('QUERIES')").single


// when // when
val res = execute("CALL db.stats.retrieve('QUERIES')").toList val res = execute("CALL db.stats.retrieve('QUERIES')").toList
Expand Down Expand Up @@ -220,13 +232,11 @@ class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite {


test("should retrieve invocation summary of query") { test("should retrieve invocation summary of query") {
// given // given
execute("CALL db.stats.collect('QUERIES')").single
execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> "BrassLeg")) execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> "BrassLeg"))
execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> 2)) execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> 2))
execute("WITH 42 AS x RETURN x") execute("WITH 42 AS x RETURN x")
execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> List(3.1, 3.2))) execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> List(3.1, 3.2)))
execute("WITH 42 AS x RETURN x") execute("WITH 42 AS x RETURN x")
execute("CALL db.stats.stop('QUERIES')").single


// when // when
val res = execute("CALL db.stats.retrieve('QUERIES')").toList val res = execute("CALL db.stats.retrieve('QUERIES')").toList
Expand All @@ -246,15 +256,13 @@ class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite {


test("should limit number of retrieved invocations of query") { test("should limit number of retrieved invocations of query") {
// given // given
execute("CALL db.stats.collect('QUERIES')").single
execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> "BrassLeg")) execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> "BrassLeg"))
execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> 2)) execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> 2))
execute("WITH 42 AS x RETURN x") execute("WITH 42 AS x RETURN x")
execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> List(3.1, 3.2))) execute("MATCH (n {p: $param}) RETURN count(n)", Map("param" -> List(3.1, 3.2)))
execute("WITH 42 AS x RETURN x") execute("WITH 42 AS x RETURN x")
execute("WITH 42 AS x RETURN x") execute("WITH 42 AS x RETURN x")
execute("WITH 42 AS x RETURN x") execute("WITH 42 AS x RETURN x")
execute("CALL db.stats.stop('QUERIES')").single


// when // when
val res = execute("CALL db.stats.retrieve('QUERIES', {maxInvocations: 2})").toList val res = execute("CALL db.stats.retrieve('QUERIES', {maxInvocations: 2})").toList
Expand Down Expand Up @@ -290,7 +298,9 @@ class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite {


test("[retrieveAllAnonymized] should anonymize tokens inside queries") { test("[retrieveAllAnonymized] should anonymize tokens inside queries") {
// given // given
execute("CALL db.stats.stop('QUERIES')").single
execute("CREATE (:User {age: 99})-[:KNOWS]->(:Buddy {p: 42})-[:WANTS]->(:Raccoon)") // create tokens execute("CREATE (:User {age: 99})-[:KNOWS]->(:Buddy {p: 42})-[:WANTS]->(:Raccoon)") // create tokens

execute("CALL db.stats.collect('QUERIES')").single execute("CALL db.stats.collect('QUERIES')").single
execute("MATCH (:User)-[:KNOWS]->(:Buddy)-[:WANTS]->(:Raccoon) RETURN 1") execute("MATCH (:User)-[:KNOWS]->(:Buddy)-[:WANTS]->(:Raccoon) RETURN 1")
execute("MATCH ({p: 42}), ({age: 43}) RETURN 1") execute("MATCH ({p: 42}), ({age: 43}) RETURN 1")
Expand Down Expand Up @@ -346,10 +356,8 @@ class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite {


test("[retrieveAllAnonymized] should anonymize unknown tokens inside queries") { test("[retrieveAllAnonymized] should anonymize unknown tokens inside queries") {
// given // given
execute("CALL db.stats.collect('QUERIES')").single
execute("MATCH (:User)-[:KNOWS]->(:Buddy)-[:WANTS]->(:Raccoon) RETURN 1") execute("MATCH (:User)-[:KNOWS]->(:Buddy)-[:WANTS]->(:Raccoon) RETURN 1")
execute("MATCH ({p: 42}), ({age: 43}) RETURN 1") execute("MATCH ({p: 42}), ({age: 43}) RETURN 1")
execute("CALL db.stats.stop('QUERIES')").single


// when // when
val res = execute("CALL db.stats.retrieveAllAnonymized('myToken')") val res = execute("CALL db.stats.retrieveAllAnonymized('myToken')")
Expand All @@ -363,9 +371,7 @@ class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite {


test("[retrieveAllAnonymized] should anonymize string literals inside queries") { test("[retrieveAllAnonymized] should anonymize string literals inside queries") {
// given // given
execute("CALL db.stats.collect('QUERIES')").single
execute("RETURN 'Scrooge' AS uncle, 'Donald' AS name") execute("RETURN 'Scrooge' AS uncle, 'Donald' AS name")
execute("CALL db.stats.stop('QUERIES')").single


// when // when
val res = execute("CALL db.stats.retrieveAllAnonymized('myToken')") val res = execute("CALL db.stats.retrieveAllAnonymized('myToken')")
Expand All @@ -378,11 +384,9 @@ class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite {


test("[retrieveAllAnonymized] should anonymize variables and return names") { test("[retrieveAllAnonymized] should anonymize variables and return names") {
// given // given
execute("CALL db.stats.collect('QUERIES')").single
execute("RETURN 42 AS x") execute("RETURN 42 AS x")
execute("WITH 42 AS x RETURN x") execute("WITH 42 AS x RETURN x")
execute("WITH 1 AS k RETURN k + k") execute("WITH 1 AS k RETURN k + k")
execute("CALL db.stats.stop('QUERIES')").single


// when // when
val res = execute("CALL db.stats.retrieveAllAnonymized('myToken')") val res = execute("CALL db.stats.retrieveAllAnonymized('myToken')")
Expand All @@ -397,12 +401,10 @@ class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite {


test("[retrieveAllAnonymized] should anonymize parameters") { test("[retrieveAllAnonymized] should anonymize parameters") {
// given // given
execute("CALL db.stats.collect('QUERIES')").single
execute("RETURN 42 = $user, $name", Map("user" -> "BrassLeg", "name" -> "George")) execute("RETURN 42 = $user, $name", Map("user" -> "BrassLeg", "name" -> "George"))
execute("RETURN 42 = $user, $name", Map("user" -> 2, "name" -> "Glinda")) execute("RETURN 42 = $user, $name", Map("user" -> 2, "name" -> "Glinda"))
execute("RETURN 42 = $user, $name", Map("user" -> List(3.1, 3.2), "name" -> "Kim")) execute("RETURN 42 = $user, $name", Map("user" -> List(3.1, 3.2), "name" -> "Kim"))
execute("RETURN $user, $name, $user + $name", Map("user" -> List(3.1, 3.2), "name" -> "Kim")) execute("RETURN $user, $name, $user + $name", Map("user" -> List(3.1, 3.2), "name" -> "Kim"))
execute("CALL db.stats.stop('QUERIES')").single


// when // when
val res = execute("CALL db.stats.retrieveAllAnonymized('myToken')") val res = execute("CALL db.stats.retrieveAllAnonymized('myToken')")
Expand Down Expand Up @@ -440,11 +442,6 @@ class DataCollectorQueriesAcceptanceTest extends ExecutionEngineFunSuite {
) )
) )


private def assertInvalidArgument(query: String): Unit = {
val e = intercept[CypherExecutionException](execute(query))
e.status should be(org.neo4j.kernel.api.exceptions.Status.General.InvalidArguments)
}

case class QueryWithInvocationSummary(expectedQuery: String) extends Matcher[AnyRef] { case class QueryWithInvocationSummary(expectedQuery: String) extends Matcher[AnyRef] {
override def apply(left: AnyRef): MatchResult = { override def apply(left: AnyRef): MatchResult = {
left match { left match {
Expand Down
58 changes: 22 additions & 36 deletions ...her-it/src/test/scala/org/neo4j/internal/collector/DataCollectorStateAcceptanceTest.scala
View file
Expand Up @@ -19,44 +19,51 @@
*/ */
package org.neo4j.internal.collector package org.neo4j.internal.collector


import org.neo4j.cypher._ class DataCollectorStateAcceptanceTest extends DataCollectorTestSupport {

class DataCollectorStateAcceptanceTest extends ExecutionEngineFunSuite {


import DataCollectorMatchers._ import DataCollectorMatchers._


private val IDLE = "idle"
private val COLLECTING = "collecting"

test("QUERIES: happy path collect cycle") { test("QUERIES: happy path collect cycle") {
assertStatus(IDLE) assertCollecting("QUERIES")

execute("CALL db.stats.stop('QUERIES')").single should beMap(
"section" -> "QUERIES",
"success" -> true,
"message" -> "Collection stopped."
)

assertIdle("QUERIES")


execute("CALL db.stats.collect('QUERIES')").single should beMap( execute("CALL db.stats.collect('QUERIES')").single should beMap(
"section" -> "QUERIES", "section" -> "QUERIES",
"success" -> true, "success" -> true,
"message" -> "Collection started." "message" -> "Collection started."
) )


assertStatus(COLLECTING) assertCollecting("QUERIES")


execute("CALL db.stats.stop('QUERIES')").single should beMap( execute("CALL db.stats.stop('QUERIES')").single should beMap(
"section" -> "QUERIES", "section" -> "QUERIES",
"success" -> true, "success" -> true,
"message" -> "Collection stopped." "message" -> "Collection stopped."
) )


assertStatus(IDLE) assertIdle("QUERIES")


execute("CALL db.stats.collect('QUERIES')").single should beMap( execute("CALL db.stats.collect('QUERIES')").single should beMap(
"section" -> "QUERIES", "section" -> "QUERIES",
"success" -> true, "success" -> true,
"message" -> "Collection started." "message" -> "Collection started."
) )


assertStatus(COLLECTING) assertCollecting("QUERIES")
} }


test("QUERIES: stop while idle is idempotent") { test("QUERIES: stop while idle is idempotent") {
// given
execute("CALL db.stats.stop('QUERIES')").single
assertIdle("QUERIES")

// when // when
execute("CALL db.stats.stop('QUERIES')").single should beMap( execute("CALL db.stats.stop('QUERIES')").single should beMap(
"section" -> "QUERIES", "section" -> "QUERIES",
Expand All @@ -65,13 +72,12 @@ class DataCollectorStateAcceptanceTest extends ExecutionEngineFunSuite {
) )


// then // then
assertStatus(IDLE) assertIdle("QUERIES")
} }


test("QUERIES: collect while collecting is idempotent") { test("QUERIES: collect while collecting is idempotent") {
// given // given
execute("CALL db.stats.collect('QUERIES')") assertCollecting("QUERIES")
assertStatus(COLLECTING)


// when // when
execute("CALL db.stats.collect('QUERIES')").single should beMap( execute("CALL db.stats.collect('QUERIES')").single should beMap(
Expand All @@ -81,13 +87,12 @@ class DataCollectorStateAcceptanceTest extends ExecutionEngineFunSuite {
) )


// then // then
assertStatus(COLLECTING) assertCollecting("QUERIES")
} }


test("QUERIES: clear while collecting is not allowed") { test("QUERIES: clear while collecting is not allowed") {
// given // given
execute("CALL db.stats.collect('QUERIES')") assertCollecting("QUERIES")
assertStatus(COLLECTING)


// when // when
execute("CALL db.stats.clear('QUERIES')").single should beMap( execute("CALL db.stats.clear('QUERIES')").single should beMap(
Expand All @@ -97,7 +102,7 @@ class DataCollectorStateAcceptanceTest extends ExecutionEngineFunSuite {
) )


// then // then
assertStatus(COLLECTING) assertCollecting("QUERIES")
} }


test("collect/stop/clear of invalid section should throw") { test("collect/stop/clear of invalid section should throw") {
Expand All @@ -111,23 +116,4 @@ class DataCollectorStateAcceptanceTest extends ExecutionEngineFunSuite {
assertInvalidArgument("CALL db.stats.stop('GRAPH COUNTS')") assertInvalidArgument("CALL db.stats.stop('GRAPH COUNTS')")
assertInvalidArgument("CALL db.stats.clear('GRAPH COUNTS')") assertInvalidArgument("CALL db.stats.clear('GRAPH COUNTS')")
} }

private def assertInvalidArgument(query: String): Unit = {
try {
execute(query)
} catch {
case e: CypherExecutionException =>
e.status should be(org.neo4j.kernel.api.exceptions.Status.General.InvalidArguments)
case x =>
x shouldBe a[CypherExecutionException]
}
}

private def assertStatus(status: String): Unit = {
val res = execute("CALL db.stats.status()").single
res should beMapContaining(
"status" -> status,
"section" -> "QUERIES"
)
}
} }

0 comments on commit e71a554

Please sign in to comment.