diff --git a/core/src/main/java/com/exadel/etoolbox/querykit/core/models/query/QueryInfo.java b/core/src/main/java/com/exadel/etoolbox/querykit/core/models/query/QueryInfo.java new file mode 100644 index 0000000..9985085 --- /dev/null +++ b/core/src/main/java/com/exadel/etoolbox/querykit/core/models/query/QueryInfo.java @@ -0,0 +1,68 @@ +package com.exadel.etoolbox.querykit.core.models.query; + +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.StringUtils; + +import javax.jcr.query.Query; +import javax.jcr.query.QueryManager; +import javax.jcr.query.QueryResult; +import javax.jcr.query.Row; +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +public class QueryInfo { + + private static final Pattern PROPERTY_INDEX_PATTERN = Pattern.compile("/\\*\\sproperty\\s([^\\s=]+)[=\\s]"); + private static final Pattern LUCENE_INDEX_PATTERN = Pattern.compile("/\\*\\slucene:([^\\s*]+)[\\s*]"); + + private QueryManager queryManager; + @Setter + @Getter + private String statement; + @Setter + @Getter + private String language; + @Setter + @Getter + private String threadName; + @Setter + @Getter + private String lastExecuted; + @Setter + @Getter + private int executeCount; + private Set indexes; + + public QueryInfo(QueryManager queryManager) { + this.queryManager = queryManager; + } + + public Set getIndexes() throws IOException { + if (indexes != null) { + return indexes; + } + try { + indexes = new HashSet<>(); + Query query = queryManager.createQuery("explain " + statement, language); + QueryResult queryResult = query.execute(); + Row row = queryResult.getRows().nextRow(); + String queryPlan = row.getValue("plan").getString(); + + Stream.of(PROPERTY_INDEX_PATTERN, LUCENE_INDEX_PATTERN).forEach(pattern -> { + Matcher matcher = pattern.matcher(queryPlan); + while (matcher.find()) { + indexes.add(matcher.group(1).trim().replaceAll("\\(.+\\)", StringUtils.EMPTY)); + } + }); + } catch (Exception e) { + indexes = Collections.emptySet(); + } + return indexes; + } +} diff --git a/core/src/main/java/com/exadel/etoolbox/querykit/core/models/query/QueryStatisticsModel.java b/core/src/main/java/com/exadel/etoolbox/querykit/core/models/query/QueryStatisticsModel.java new file mode 100644 index 0000000..9393c42 --- /dev/null +++ b/core/src/main/java/com/exadel/etoolbox/querykit/core/models/query/QueryStatisticsModel.java @@ -0,0 +1,198 @@ +package com.exadel.etoolbox.querykit.core.models.query; + +import com.exadel.etoolbox.querykit.core.utils.Constants; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.sling.api.SlingHttpServletRequest; +import org.apache.sling.api.resource.Resource; +import org.apache.sling.api.resource.ResourceResolver; +import org.apache.sling.models.annotations.Model; +import org.apache.sling.models.annotations.injectorspecific.SlingObject; + +import javax.annotation.PostConstruct; +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import javax.jcr.query.Query; +import javax.jcr.query.QueryManager; +import javax.jcr.query.QueryResult; +import javax.jcr.query.Row; +import javax.management.MBeanServerConnection; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +@Model(adaptables = SlingHttpServletRequest.class) +@Slf4j +public class QueryStatisticsModel { + + private static final Pattern PROPERTY_INDEX_PATTERN = Pattern.compile("/\\*\\sproperty\\s([^\\s=]+)[=\\s]"); + private static final Pattern LUCENE_INDEX_PATTERN = Pattern.compile("/\\*\\slucene:([^\\s*]+)[\\s*]"); + + private static final String OAK_INDEX_PREFIX = "/oak:index"; + private static final String STRIP_CHARS = " /"; + + @SlingObject + private ResourceResolver resourceResolver; + + @Getter + private List allQueries; + @Getter + private Map indexesByUsage = new LinkedHashMap<>(); + @Getter + private Map> queriesByIndex = new HashMap<>(); + @Getter + private LinkedHashMap> queriesByIndexSorted = new LinkedHashMap<>(); + + @PostConstruct + private void init() throws IOException { + + QueryManager queryManager = getQueryManager(resourceResolver); + MBeanServerConnection server = ManagementFactory.getPlatformMBeanServer(); + + allQueries = getQueryInfo(server, queryManager); + + for (QueryInfo queryInfo : allQueries) { + if (queryInfo.getIndexes().isEmpty()) { + continue; + } + queryInfo.getIndexes().forEach(index -> queriesByIndex.computeIfAbsent(index, k -> new ArrayList<>()).add(queryInfo)); + } + + queriesByIndex.entrySet().stream() + .sorted((a, b) -> Integer.compare(a.getValue().size(), b.getValue().size()) * -1) + .forEach(entry -> queriesByIndexSorted.put(entry.getKey(), entry.getValue())); + + getIndexes(resourceResolver).forEach(index -> indexesByUsage.put( + index, + queriesByIndex.entrySet().stream() + .filter(entry -> { + String indexId = getIndexId(index); + return entry.getKey().equals(indexId) || entry.getKey().endsWith(indexId + Constants.CLOSING_BRACKET); + }) + .map(entry -> entry.getValue().size()) + .findFirst().orElse(0))); + } + + private QueryManager getQueryManager(ResourceResolver resourceResolver) { + Session session = resourceResolver.adaptTo(Session.class); + if (session == null) { + return null; + } + try { + return session.getWorkspace().getQueryManager(); + } catch (RepositoryException e) { + log.error("Can't get QueryManager from the session", e); + return null; + } + } + + private List getQueryInfo(MBeanServerConnection server, QueryManager queryManager) { + ObjectName queryStatMbean = getQueryStatMBean(server); + if (queryStatMbean == null || queryManager == null) { + return Collections.emptyList(); + } + String jsonRaw; + List result = new ArrayList<>(); + + try { + jsonRaw = server.invoke(queryStatMbean, "asJson", null, null).toString(); + + } catch (Exception e) { + return Collections.emptyList(); + } + + JsonParser jsonParser = new JsonParser(); + JsonArray queries = jsonParser.parse(jsonRaw).getAsJsonArray(); + + for(int i = 0; i < queries.size(); i++) { + JsonObject jsonObject = queries.get(i).getAsJsonObject(); + QueryInfo queryInfo = getQueryInfo(queryManager, jsonObject); + if (queryInfo != null) { + result.add(queryInfo); + } + } + result.sort((a, b) -> Integer.compare(a.getExecuteCount(), b.getExecuteCount()) * -1); + return result; + } + + private QueryInfo getQueryInfo(QueryManager queryManager, JsonObject jsonObject) { + String statement = getAsString(jsonObject, "query"); + if (StringUtils.contains(statement, "oak-internal") || StringUtils.startsWithIgnoreCase(statement, "explain ")) { + return null; + } + String language = getAsString(jsonObject, "language"); + String thread = getAsString(jsonObject, "lastThreadName"); + String lastExecuted = getAsString(jsonObject, "lastExecutedMillis"); + int executeCount = getAsInt(jsonObject, "executeCount"); + QueryInfo queryInfo = new QueryInfo(queryManager); + queryInfo.setLanguage(language); + queryInfo.setStatement(statement); + queryInfo.setThreadName(thread); + queryInfo.setExecuteCount(executeCount); + queryInfo.setLastExecuted(lastExecuted); + return queryInfo; + } + + private ObjectName getQueryStatMBean(MBeanServerConnection server) { + try { + Set names = server.queryNames(new ObjectName("org.apache.jackrabbit.oak:type=QueryStats,*"), null); + return names.iterator().next(); + } catch (IOException | MalformedObjectNameException | NoSuchElementException e) { + log.warn("Can't get 'org.apache.jackrabbit.oak:type=QueryStats,*'", e); + return null; + } + } + + private List getIndexes(ResourceResolver resourceResolver) { + Resource oakIndex = resourceResolver.getResource(OAK_INDEX_PREFIX); + if (oakIndex == null || !oakIndex.hasChildren()) { + return Collections.emptyList(); + } + return StreamSupport.stream(oakIndex.getChildren().spliterator(), false) + .map(Resource::getPath) + .sorted() + .collect(Collectors.toList()); + } + + private String getIndexId(String value) { + if (value.contains(Constants.OPENING_BRACKET) && value.contains(Constants.CLOSING_BRACKET)) { + return StringUtils.substringBefore(value, Constants.OPENING_BRACKET).trim(); + } else if (StringUtils.containsIgnoreCase(value, OAK_INDEX_PREFIX)) { + return StringUtils.strip(StringUtils.substringAfter(value, OAK_INDEX_PREFIX), STRIP_CHARS); + } + return StringUtils.strip(value, STRIP_CHARS); + } + + private String getAsString(JsonObject object, String property) { + try { + return object.get(property).getAsString(); + } catch (UnsupportedOperationException e) { + return StringUtils.EMPTY; + } + } + + private int getAsInt(JsonObject object, String property) { + try { + return object.get(property).getAsInt(); + } catch (UnsupportedOperationException e) { + return 0; + } + } + + public Map getIndexesByUsage() { + Map map = new LinkedHashMap<>(); + indexesByUsage.forEach((key, value) -> map.put(StringUtils.removeStart(key, "/oak:index/"), value)); + return map; + } +} diff --git a/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/.content.xml b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/.content.xml new file mode 100644 index 0000000..dfeefd9 --- /dev/null +++ b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/.content.xml @@ -0,0 +1,41 @@ + + + + + + + + <actions jcr:primaryType="nt:unstructured"> + <primary jcr:primaryType="nt:unstructured"> + <back + jcr:primaryType="nt:unstructured" + sling:resourceType="granite/ui/components/coral/foundation/anchorbutton" + href="/etoolbox-query-kit.html" + text="Back" + variant="primary"/> + </primary> + </actions> + <content + jcr:primaryType="nt:unstructured" + sling:resourceType="granite/ui/components/coral/foundation/container" + margin="{Boolean}true"> + <items jcr:primaryType="nt:unstructured"> + <result + jcr:primaryType="nt:unstructured" + sling:resourceType="/apps/etoolbox-query-kit/components/console/queryStatistics"/> + </items> + </content> + </jcr:content> +</jcr:root> \ No newline at end of file diff --git a/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/.content.xml b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/.content.xml new file mode 100644 index 0000000..179fffa --- /dev/null +++ b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/.content.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8"?> +<jcr:root xmlns:cq="http://www.day.com/jcr/cq/1.0" xmlns:jcr="http://www.jcp.org/jcr/1.0" + jcr:primaryType="cq:ClientLibraryFolder" + categories="[etoolbox-query-kit.queryStatistics]" + dependencies="[coralui2,moment]" + jsProcessor="[min:gcc;languageIn=ECMASCRIPT_2015,compilationLevel=advanced,languageOut=ECMASCRIPT5]"/> diff --git a/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/css.txt b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/css.txt new file mode 100644 index 0000000..0959e43 --- /dev/null +++ b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/css.txt @@ -0,0 +1,2 @@ +#base=css +main-layout.css \ No newline at end of file diff --git a/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/css/main-layout.css b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/css/main-layout.css new file mode 100644 index 0000000..8bab66a --- /dev/null +++ b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/css/main-layout.css @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +.foundation-layout-panel-content { + padding-left: 20px; + padding-right: 20px; +} + +.query-statistics-headlines { + text-align: center; +} \ No newline at end of file diff --git a/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/js.txt b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/js.txt new file mode 100644 index 0000000..1390d31 --- /dev/null +++ b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/js.txt @@ -0,0 +1,2 @@ +#base=js +query-statistics.js \ No newline at end of file diff --git a/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/js/query-statistics.js b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/js/query-statistics.js new file mode 100644 index 0000000..a665b27 --- /dev/null +++ b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/clientlibs/js/query-statistics.js @@ -0,0 +1,11 @@ +(function (document, $, ns) { + 'use strict'; + + $(document).on('click', '.active-index', function (index) { + $('#tab-queries-by-index')[0].click(); + let indexId = this.getAttribute('value'); + setTimeout((function() { + $(`#${indexId}`)[0].scrollIntoView(); + }), 0) + }); +})(document, Granite.$, Granite.Eqk = (Granite.Eqk || {})); diff --git a/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/queryStatistics.html b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/queryStatistics.html new file mode 100644 index 0000000..7feb0a4 --- /dev/null +++ b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/components/console/queryStatistics/queryStatistics.html @@ -0,0 +1,87 @@ +<sly data-sly-use.queryStatisticsModel="com.exadel.etoolbox.querykit.core.models.query.QueryStatisticsModel"></sly> + +<coral-tablist target="#queries-coral-panel" orientation="horizontal"> + <coral-tab aria-controls="all-queries">All queries</coral-tab> + <coral-tab id="tab-queries-by-index" aria-controls="queries-by-index">Queries by index</coral-tab> + <coral-tab aria-controls="indexes-by-usage">Indexes by usage</coral-tab> +</coral-tablist> + +<coral-panelstack id="queries-coral-panel"> + <coral-panel id="all-queries"> + <div class="query-statistics-headlines"> + <h1>All queries</h1> + </div> + + <table is="coral-table"> + <thead is="coral-table-head" sticky> + <tr is="coral-table-row"> + <th is="coral-table-headercell">#</th> + <th is="coral-table-headercell">Count</th> + <th is="coral-table-headercell">Language</th> + <th is="coral-table-headercell">Statement</th> + <th is="coral-table-headercell">Indexes</th> + </tr> + </thead> + <tbody is="coral-table-body" data-sly-list.queryInfo="${queryStatisticsModel.allQueries}"> + <tr is="coral-table-row"> + <td is="coral-table-cell">${queryInfoList.count}</td> + <td is="coral-table-cell">${queryInfo.executeCount}</td> + <td is="coral-table-cell">${queryInfo.language.toUpperCase}</td> + <td is="coral-table-cell"> + ${queryInfo.statement}<br/> + <span class="thread">${queryInfo.threadName} at ${queryInfo.lastExecuted}</span> + </td> + <td is="coral-table-cell" data-sly-list.index="${queryInfo.indexes}"> + <button value="${index}" is="coral-button" variant="minimal" iconsize="S" class="active-index"> + ${index} + </button><br> + </td> + <td is="coral-table-cell" data-sly-test="${queryInfo.indexes.empty}"> + </td> + </tr> + </tbody> + </table> + </coral-panel> + <coral-panel id="queries-by-index"> + <div class="query-statistics-headlines"> + <h1>Queries by index</h1> + </div> + + <sly data-sly-list.queryByIndex="${queryStatisticsModel.queriesByIndexSorted}"> + <h4 id="${queryByIndex}">${queryByIndex} <span class="quantity">${queryStatisticsModel.queriesByIndexSorted[queryByIndex].size}</span></h4> + <sly data-sly-set.calcHeight="${'height:calc(37px + 44px * {0})' @format=queryStatisticsModel.queriesByIndexSorted[queryByIndex]}"></sly> + <table is="coral-table" style="${queryStatisticsModel.queriesByIndexSorted[queryByIndex].size > 7 ? 'height:345px' : calcHeight @ context='styleString'}"> + <thead is="coral-table-head" sticky> + <tr is="coral-table-row"> + <th is="coral-table-headercell" style="width: 14px">#</th> + <th is="coral-table-headercell" style="width: 14px">Count</th> + <th is="coral-table-headercell" style="width: 35px">Language</th> + <th is="coral-table-headercell">Statement</th> + </tr> + </thead> + <tbody is="coral-table-body" data-sly-list.queryInfo="${queryStatisticsModel.queriesByIndexSorted[queryByIndex]}"> + <tr is="coral-table-row"> + <td is="coral-table-cell">${queryInfoList.count}</td> + <td is="coral-table-cell">${queryInfo.executeCount}</td> + <td is="coral-table-cell">${queryInfo.language.toUpperCase}</td> + <td is="coral-table-cell"> + ${queryInfo.statement}<br/> + <span class="thread">${queryInfo.threadName} at ${queryInfo.lastExecuted}</span> + </td> + </tr> + </tbody> + </table> + </sly> + </coral-panel> + <coral-panel id="indexes-by-usage"> + <div class="query-statistics-headlines"> + <h1>Indexes by usage</h1> + </div> + <br/> + <sly data-sly-list.index="${queryStatisticsModel.indexesByUsage}"> + <button value="${index}" data-sly-attribute.disabled="${queryStatisticsModel.indexesByUsage[index] == 0}" is="coral-button" variant="minimal" iconsize="S" class="quantity active-index"> + ${index} ${queryStatisticsModel.indexesByUsage[index]}    + </button> + </sly> + </coral-panel> +</coral-panelstack> \ No newline at end of file diff --git a/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/console/query/.content.xml b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/console/query/.content.xml index dfdf7bd..8acc2e7 100644 --- a/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/console/query/.content.xml +++ b/ui.apps/src/main/content/jcr_root/apps/etoolbox-query-kit/console/query/.content.xml @@ -96,6 +96,12 @@ nesting="hide" src.url="/mnt/overlay/etoolbox-query-kit/console/dialogs/settings.html"/> </settings> + <statistics + jcr:primaryType="nt:unstructured" + sling:resourceType="granite/ui/components/coral/foundation/collection/actionlink" + action="foundation.link" + href="/etoolbox-query-kit/oak-query-index-usage.html" + text="Query Statistics"/> </items> </tools> </primary>