diff --git a/modules/manage/pages/monitor/monitoring-n1ql-query.adoc b/modules/manage/pages/monitor/monitoring-n1ql-query.adoc index 7eef529454..593c8031c9 100644 --- a/modules/manage/pages/monitor/monitoring-n1ql-query.adoc +++ b/modules/manage/pages/monitor/monitoring-n1ql-query.adoc @@ -45,7 +45,8 @@ a| * <> * <> * <> -* <> +* <> +* <> * <> * <> * <> @@ -53,6 +54,7 @@ a| | Security Catalogs a| +* <> * <> * <> * <> @@ -62,6 +64,7 @@ a| * <> NOTE: These are only available using REST APIs. + |=== == Authentication and Client Privileges @@ -1846,6 +1849,139 @@ This will result in a list similar to: For more examples, take a look at the blog: https://blog.couchbase.com/optimize-n1ql-performance-using-request-profiling/[Optimize N1QL performance using request profiling^]. +[#sys-dictionary] +== system:dictionary + +This catalog maintains a list of the on-disk optimizer statistics stored in the `N1QL_CBO_STATS` collection. + +If you have multiple query nodes, the data retrieved from this catalog will be the same, regardless of the node on which you run the query. + +To see the list of on-disk optimizer statistics, use: + +[source,n1ql] +---- +SELECT * FROM system:dictionary; +---- + +This will result in a list similar to: + +[source,json] +---- +[ + { + "dictionary": { + "avgDocKeySize": 12, + "avgDocSize": 278, + "bucket": "travel-sample", // <.> + "distributionKeys": [ // <.> + "airportname", + "faa", + "city" + ], + "docCount": 1968, + "indexes": [ // <.> + { + "indexId": "bc3048e87bf84828", + "indexName": "def_inventory_airport_primary", + "indexStats": [ // <.> + { + "avgItemSize": 24, + "avgPageSize": 11760, + "numItems": 1968, + "numPages": 4, + "resRatio": 1 + } + ] + }, + // ... + ], + "keyspace": "airport", + "namespace": "default", + "scope": "inventory" + } + }, + // ... +] +---- + +This query returns an array of dictionaries, one for each keyspace for which optimizer statistics are available. +Each dictionary gives the following information: + +<.> Basic information about the keyspace +<.> Distribution keys for which histograms are available +<.> An array of indexes in this keyspace for which statistics are available +<.> An array of statistics for each index, with one element for each index partition + +For further details, refer to xref:n1ql:n1ql-language-reference/updatestatistics.adoc[UPDATE STATISTICS]. + +[#sys-dictionary-cache] +== system:dictionary_cache + +This catalog maintains a list of the in-memory cached subset of the optimizer statistics. + +If you have multiple query nodes, the data retrieved from this node shows cached optimizer statistics from all nodes. +Individual nodes may have a different subset of cached information. + +To see the list of in-memory optimizer statistics, use: + +[source,n1ql] +---- +SELECT * FROM system:dictionary_cache; +---- + +This will result in a list similar to: + +[source,json] +---- +[ + { + "dictionary_cache": { + "avgDocKeySize": 12, + "avgDocSize": 278, + "bucket": "travel-sample", // <.> + "distributionKeys": [ // <.> + "airportname", + "faa", + "city" + ], + "docCount": 1968, + "indexes": [ // <.> + { + "indexId": "bc3048e87bf84828", + "indexName": "def_inventory_airport_primary", + "indexStats": [ // <.> + { + "avgItemSize": 24, + "avgPageSize": 11760, + "numItems": 1968, + "numPages": 4, + "resRatio": 1 + } + ] + }, + // ... + ], + "keyspace": "airport", + "namespace": "default", + "node": "172.23.0.3:8091", // <.> + "scope": "inventory" + } + }, + // ... +] +---- + +This query returns an array of dictionary caches, one for each keyspace for which optimizer statistics are available. +Each dictionary cache gives the following information: + +<.> Basic information about the keyspace +<.> Distribution keys for which histograms are available +<.> An array of indexes in this keyspace for which statistics are available +<.> An array of statistics for each index, with one element for each index partition +<.> The query node where this dictionary cache is resident + +For further details, refer to xref:n1ql:n1ql-language-reference/updatestatistics.adoc[UPDATE STATISTICS]. + [#sys-functions] == system:functions @@ -1865,17 +2001,19 @@ This will result in a list similar to: { "functions": { "definition": { - "#language": "inline", // <1> - "expression": "substr0(`vString`, (length(`vString`) - `vLen`), `vLen`)", // <2> - "parameters": [ // <3> - "vString", - "vLen" - ] + "#language": "inline", // <.> + "expression": "(((`fahrenheit` - 32) * 5) / 9)", // <.> + "parameters": [ // <.> + "fahrenheit" + ], + "text": "((fahrenheit - 32) * 5/9)" // <.> }, "identity": { - "name": "rstr", - "namespace": "default", - "type": "global" + "bucket": "travel-sample", // <.> + "name": "celsius", // <.> + "namespace": "default", // <.> + "scope": "inventory", // <.> + "type": "scope" // <.> } } }, @@ -1883,29 +2021,39 @@ This will result in a list similar to: "functions": { "definition": { "#language": "javascript", - "library": "math77", // <4> - "object": "add" // <5> + "library": "geohash-js", // <.> + "name": "geohash-js", // <.> + "object": "calculateAdjacent", // <.> + "parameters": [ + "src", + "dir" + ] }, "identity": { - "name": "javaScriptAdd", // <6> - "namespace": "default", // <7> - "type": "global" // <8> + "name": "adjacent", + "namespace": "default", + "type": "global" } } - } + }, + // ... ] ---- This query returns the following attributes: -<1> [.out]`definition.#language`: string (the language of the function, for example, inline) -<2> [.out]`definition.expression`: string (for inline functions only: the expression defining the function) -<3> [.out]`definition.parameters`: array of strings (the parameters required by the function) -<4> [.out]`definition.library`: string (for JavaScript functions only: the library containing the function) -<5> [.out]`definition.object`: string (for JavaScript functions only: the object defining the function) -<6> [.out]`identity.name`: string (the name of the function) -<7> [.out]`identity.namespace`: string (the namespace of the function, for example, default) -<8> [.out]`identity.type`: string (the type of the function, for example, global) +<.> `definition.#language`: string (the language of the function, for example, inline) +<.> `definition.expression`: string (for inline functions only: the expression defining the function) +<.> `definition.parameters`: array of strings (the parameters required by the function) +<.> `definition.text`: string (for inline functions only: the verbatim text of the function) +<.> `identity.bucket`: string (for scope functions only: the bucket containing the function) +<.> `identity.name`: string (the name of the function) +<.> `identity.namespace`: string (the namespace of the function, for example, default) +<.> `identity.scope`: string (for scope functions only: the scope containing the function) +<.> `identity.type`: string (the type of the function, for example, global) +<.> `definition.library`: string (for JavaScript functions only: the library containing the function) +<.> `definition.name`: string (for JavaScript functions only: the relative name of the library) +<.> `definition.object`: string (for JavaScript functions only: the object defining the function) [#sys-functions-cache] == system:functions_cache @@ -1926,51 +2074,56 @@ This will result in a list similar to: [ { "functions_cache": { - "#language": "inline", // <1> - "avgServiceTime": "8.926µs", - "expression": "substr0(`vString`, (length(`vString`) - `vLen`), `vLen`)", // <2> - "lastUse": "2019-08-06 06:08:35.37498882 -0700 PDT m=+2236.523899555", - "maxServiceTime": "8.926µs", + "#language": "inline", // <.> + "avgServiceTime": "3.066847ms", + "expression": "(((`fahrenheit` - 32) * 5) / 9)", // <.> + "lastUse": "2022-03-09 00:17:59.60659793 +0000 UTC m=+35951.429537902", + "maxServiceTime": "3.066847ms", "minServiceTime": "0s", - "name": "rstr", // <3> - "namespace": "default", // <4> + "name": "celsius", // <.> + "namespace": "default", // <.> "node": "127.0.0.1:8091", - "parameters": [ // <5> - "vString", - "vLen" + "parameters": [ // <.> + "fahrenheit" ], - "type": "global", + "scope": "inventory", // <.> + "text": "((fahrenheit - 32) * 5/9)", // <.> + "type": "scope", "uses": 1 } }, { "functions_cache": { "#language": "javascript", - "avgServiceTime": "674.264µs", - "lastUse": "2019-08-06 06:10:50.681845642 -0700 PDT m=+2371.830756442", - "library": "math77", // <6> - "maxServiceTime": "674.264µs", + "avgServiceTime": "56.892636ms", + "lastUse": "2022-03-09 00:15:46.289934029 +0000 UTC m=+35818.007560703", + "library": "geohash-js", // <.> + "maxServiceTime": "146.025426ms", "minServiceTime": "0s", - "name": "javaScriptAdd", + "name": "geohash-js", "namespace": "default", "node": "127.0.0.1:8091", - "object": "add", // <7> - "type": "global", // <8> - "uses": 1 + "object": "calculateAdjacent", // <.> + "parameters": [ + "src", + "dir" + ], + "type": "global", // <.> + "uses": 4 } }, { "functions_cache": { - "avgServiceTime": "22.79µs", // <9> - "lastUse": "2019-08-06 16:59:51.630317391 +0100 BST m=+17.966707065", // <10> - "maxServiceTime": "22.79µs", // <11> - "minServiceTime": "0s", // <12> + "avgServiceTime": "3.057421ms", // <.> + "lastUse": "2022-03-09 00:17:25.396840275 +0000 UTC m=+35917.199008929", // <.> + "maxServiceTime": "3.057421ms", // <.> + "minServiceTime": "0s", // <.> "name": "notFound", "namespace": "default", - "node": "127.0.0.1:8091", // <13> + "node": "127.0.0.1:8091", // <.> "type": "global", - "undefined_function": true, // <14> - "uses": 1 // <15> + "undefined_function": true, // <.> + "uses": 1 // <.> } } ] @@ -1978,21 +2131,23 @@ This will result in a list similar to: This query returns the following attributes: -<1> [.out]`#language`: string (the language of the function, for example, inline) -<2> [.out]`expression`: string (for inline functions only: the expression defining the function) -<3> [.out]`name`: string (the name of the function) -<4> [.out]`namespace`: string (the namespace of the function, for example, default) -<5> [.out]`parameters`: array of strings (the parameters required by the function) -<6> [.out]`library`: string (for JavaScript functions only: the library containing the function) -<7> [.out]`object`: string (for JavaScript functions only: the object defining the function) -<8> [.out]`type`: string (the type of the function, for example, global) -<9> [.out]`avgServiceTime`: string (the mean service time for the function) -<10> [.out]`lastUse`: string (the date and time when the function was last used) -<11> [.out]`maxServiceTime`: string (the maximum service time for the function) -<12> [.out]`minServiceTime`: string (the minimum service time for the function) -<13> [.out]`node`: string (the query node where the function is cached) -<14> [.out]`undefined_function`: boolean (whether the function exists or is undefined) -<15> [.out]`uses`: number (the number of uses of the function) +<.> `#language`: string (the language of the function, for example, inline) +<.> `expression`: string (for inline functions only: the expression defining the function) +<.> `name`: string (the name of the function) +<.> `namespace`: string (the namespace of the function, for example, default) +<.> `parameters`: array of strings (the parameters required by the function) +<.> `scope`: string (for scope functions only: the scope containing the function) +<.> `text`: string (for inline functions only: the verbatim text of the function) +<.> `library`: string (for JavaScript functions only: the library containing the function) +<.> `object`: string (for JavaScript functions only: the object defining the function) +<.> `type`: string (the type of the function, for example, global) +<.> `avgServiceTime`: string (the mean service time for the function) +<.> `lastUse`: string (the date and time when the function was last used) +<.> `maxServiceTime`: string (the maximum service time for the function) +<.> `minServiceTime`: string (the minimum service time for the function) +<.> `node`: string (the query node where the function is cached) +<.> `undefined_function`: boolean (whether the function exists or is undefined) +<.> `uses`: number (the number of uses of the function) Each query node keeps its own cache of recently-used user-defined functions, so you may see the same function listed for multiple nodes. diff --git a/modules/n1ql/pages/n1ql-language-reference/updatestatistics.adoc b/modules/n1ql/pages/n1ql-language-reference/updatestatistics.adoc index 5722b8493b..41bda3cfa9 100644 --- a/modules/n1ql/pages/n1ql-language-reference/updatestatistics.adoc +++ b/modules/n1ql/pages/n1ql-language-reference/updatestatistics.adoc @@ -6,6 +6,7 @@ :imagesdir: ../../assets/images // Cross-references +:monitor: xref:manage:monitor/monitoring-n1ql-query.adoc :n1ql: xref:n1ql-language-reference :cbo: {n1ql}/cost-based-optimizer.adoc :select: {n1ql}/selectintro.adoc @@ -65,7 +66,27 @@ For a query which filters on an array or array of objects, you must collect the [[result]] == Result -The statement returns an empty array. +If successful, the statement returns an empty array. + +=== Index Residency + +Optimizer statistics for an index can only be gathered when the index is at least partially memory-resident. +The optimizer statistics are gathered from the memory-resident portion of an index. +A higher memory-resident ratio produces more accurate optimizer statistics for that index. + +If an index has a memory-resident ratio of zero, then the statement returns the following warning: + +[source,json] +---- +[ + { + "code": 5390, + "msg": "Index def_inventory_airport_faa is not in memory" + } +] +---- + +== Optimizer Statistics The `UPDATE STATISTICS` stores statistics in a system-like collection called `N1QL_CBO_STATS`. This is stored in a scope called `N1QL_SYSTEM_SCOPE` within a bucket called `N1QL_SYSTEM_BUCKET`. @@ -91,6 +112,11 @@ For large clusters, when this bucket gets big, increase the memory quota for thi When a cluster has multiple Query nodes, statistics gathered from one Query node via the `UPDATE STATISTICS` statement are automatically propagated to the other Query nodes. It is not necessary to run `UPDATE STATISTICS` with the same index expression for multiple query nodes. +=== Monitoring Optimizer Statistics + +You can monitor the optimizer statistics by querying the `system:dictionary` and `system:dictionary_cache` keyspaces. +For further details, refer to {monitor}[Monitor Queries]. + == Related Links * {cbo}[Cost-Based Optimizer] \ No newline at end of file