From 1f9acbbbd546cfef3a4d8e7936b4f6c3c2ce0ee8 Mon Sep 17 00:00:00 2001 From: "binbin.zheng" Date: Fri, 28 Oct 2022 23:51:04 +0800 Subject: [PATCH 1/2] KYLIN-5375 sql performance enhancement --- .../metadata/query/JdbcQueryHistoryStore.java | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java index f5175e26192..a0f7896d00a 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java @@ -54,10 +54,10 @@ import org.apache.ibatis.session.SqlSessionFactory; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.StorageURL; -import org.apache.kylin.common.util.Pair; import org.apache.kylin.common.logging.LogOutputStream; import org.apache.kylin.common.persistence.metadata.JdbcDataSource; import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil; +import org.apache.kylin.common.util.Pair; import org.apache.kylin.metadata.query.util.QueryHisStoreUtil; import org.mybatis.dynamic.sql.BasicColumn; import org.mybatis.dynamic.sql.SqlBuilder; @@ -66,6 +66,7 @@ import org.mybatis.dynamic.sql.render.RenderingStrategies; import org.mybatis.dynamic.sql.select.QueryExpressionDSL; import org.mybatis.dynamic.sql.select.SelectModel; +import org.mybatis.dynamic.sql.select.join.EqualTo; import org.mybatis.dynamic.sql.select.render.SelectStatementProvider; import org.mybatis.dynamic.sql.update.render.UpdateStatementProvider; @@ -274,10 +275,14 @@ public List queryQueryHistoriesByIdOffset(long id, int batchSize, QueryHistoryMapper mapper = session.getMapper(QueryHistoryMapper.class); SelectStatementProvider statementProvider = select(getSelectFields(queryHistoryTable)) // .from(queryHistoryTable) // - .where(queryHistoryTable.id, isGreaterThan(id)) // - .and(queryHistoryTable.projectName, isEqualTo(project)) // + .join(select(BasicColumn.columnList(queryHistoryTable.id)).from(queryHistoryTable) + .where(queryHistoryTable.id, isGreaterThan(id)) // + .and(queryHistoryTable.projectName, isEqualTo(project)) // + .orderBy(queryHistoryTable.id) // + .limit(batchSize), "idTable") // + .on(queryHistoryTable.id.qualifiedWith(queryHistoryTable.tableNameAtRuntime()), + new EqualTo(queryHistoryTable.id.qualifiedWith("idTable"))) .orderBy(queryHistoryTable.id) // - .limit(batchSize) // .build().render(RenderingStrategies.MYBATIS3); return mapper.selectMany(statementProvider); } @@ -558,10 +563,15 @@ InsertStatementProvider getInsertQhRealizationP private SelectStatementProvider queryQueryHistoriesByConditionsProvider(QueryHistoryRequest request, int limit, int offset) { - return filterByConditions(select(getSelectFields(queryHistoryTable)).from(queryHistoryTable), request) + return select(getSelectFields(queryHistoryTable)).from(queryHistoryTable) + .join(filterByConditions(select(BasicColumn.columnList(queryHistoryTable.id)).from(queryHistoryTable), + request).orderBy(queryHistoryTable.queryTime.descending()) // + .limit(limit) // + .offset(offset), + "idTable") // + .on(queryHistoryTable.id.qualifiedWith(queryHistoryTable.tableNameAtRuntime()), + new EqualTo(queryHistoryTable.id.qualifiedWith("idTable"))) .orderBy(queryHistoryTable.queryTime.descending()) // - .limit(limit) // - .offset(offset) // .build().render(RenderingStrategies.MYBATIS3); } @@ -743,13 +753,13 @@ private SelectStatementProvider queryAvgDurationByTimeProvider(long startTime, l } private BasicColumn[] getSelectFields(QueryHistoryTable queryHistoryTable) { - return BasicColumn.columnList(queryHistoryTable.id, queryHistoryTable.cacheHit, queryHistoryTable.duration, - queryHistoryTable.engineType, queryHistoryTable.errorType, queryHistoryTable.hostName, - queryHistoryTable.indexHit, queryHistoryTable.projectName, queryHistoryTable.queryHistoryInfo, - queryHistoryTable.queryId, queryHistoryTable.queryRealizations, queryHistoryTable.queryStatus, - queryHistoryTable.querySubmitter, queryHistoryTable.queryTime, queryHistoryTable.resultRowCount, - queryHistoryTable.sql, queryHistoryTable.sqlPattern, queryHistoryTable.totalScanBytes, - queryHistoryTable.totalScanCount); + return BasicColumn.columnList(queryHistoryTable.id.qualifiedWith(queryHistoryTable.tableNameAtRuntime()), + queryHistoryTable.cacheHit, queryHistoryTable.duration, queryHistoryTable.engineType, + queryHistoryTable.errorType, queryHistoryTable.hostName, queryHistoryTable.indexHit, + queryHistoryTable.projectName, queryHistoryTable.queryHistoryInfo, queryHistoryTable.queryId, + queryHistoryTable.queryRealizations, queryHistoryTable.queryStatus, queryHistoryTable.querySubmitter, + queryHistoryTable.queryTime, queryHistoryTable.resultRowCount, queryHistoryTable.sql, + queryHistoryTable.sqlPattern, queryHistoryTable.totalScanBytes, queryHistoryTable.totalScanCount); } } From c3891c3f5b5ecfd55545f2e78f3944804a4ffd8e Mon Sep 17 00:00:00 2001 From: "jiawei.li" <1019037687@qq.com> Date: Mon, 26 Dec 2022 14:22:57 +0800 Subject: [PATCH 2/2] minor fix code smell --- .../metadata/query/JdbcQueryHistoryStore.java | 57 ++++++++++--------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java index a0f7896d00a..a4b3194f34f 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java @@ -84,6 +84,7 @@ public class JdbcQueryHistoryStore { public static final String WEEK = "week"; public static final String DAY = "day"; public static final String COUNT = "count"; + public static final String ID_TABLE_ALIAS = "idTable"; public static final String DELETE_REALIZATION_LOG = "Delete {} row query history realization takes {} ms"; private final QueryHistoryTable queryHistoryTable; @@ -184,7 +185,7 @@ public List queryHistoryDailyStatistic(long startTime, long return mapper.selectDaily(qhTableName, startTime, endTime); } } - + public List queryQueryHistoriesSubmitters(QueryHistoryRequest request, int size) { try (SqlSession session = sqlSessionFactory.openSession()) { QueryHistoryMapper mapper = session.getMapper(QueryHistoryMapper.class); @@ -279,9 +280,9 @@ public List queryQueryHistoriesByIdOffset(long id, int batchSize, .where(queryHistoryTable.id, isGreaterThan(id)) // .and(queryHistoryTable.projectName, isEqualTo(project)) // .orderBy(queryHistoryTable.id) // - .limit(batchSize), "idTable") // + .limit(batchSize), ID_TABLE_ALIAS) // .on(queryHistoryTable.id.qualifiedWith(queryHistoryTable.tableNameAtRuntime()), - new EqualTo(queryHistoryTable.id.qualifiedWith("idTable"))) + new EqualTo(queryHistoryTable.id.qualifiedWith(ID_TABLE_ALIAS))) .orderBy(queryHistoryTable.id) // .build().render(RenderingStrategies.MYBATIS3); return mapper.selectMany(statementProvider); @@ -293,11 +294,11 @@ public List queryCountAndAvgDuration(long startTime, long endTi QueryStatisticsMapper mapper = session.getMapper(QueryStatisticsMapper.class); SelectStatementProvider statementProvider = select(count(queryHistoryTable.queryId).as(COUNT), avg(queryHistoryTable.duration).as("mean")) // - .from(queryHistoryTable) // - .where(queryHistoryTable.queryTime, isGreaterThanOrEqualTo(startTime)) // - .and(queryHistoryTable.queryTime, isLessThan(endTime)) // - .and(queryHistoryTable.projectName, isEqualTo(project)) // - .build().render(RenderingStrategies.MYBATIS3); + .from(queryHistoryTable) // + .where(queryHistoryTable.queryTime, isGreaterThanOrEqualTo(startTime)) // + .and(queryHistoryTable.queryTime, isLessThan(endTime)) // + .and(queryHistoryTable.projectName, isEqualTo(project)) // + .build().render(RenderingStrategies.MYBATIS3); return mapper.selectMany(statementProvider); } } @@ -307,12 +308,12 @@ public List queryCountByModel(long startTime, long endTime, Str QueryStatisticsMapper mapper = session.getMapper(QueryStatisticsMapper.class); SelectStatementProvider statementProvider = select(queryHistoryRealizationTable.model, count(queryHistoryRealizationTable.queryId).as(COUNT)) // - .from(queryHistoryRealizationTable) // - .where(queryHistoryRealizationTable.queryTime, isGreaterThanOrEqualTo(startTime)) // - .and(queryHistoryRealizationTable.queryTime, isLessThan(endTime)) // - .and(queryHistoryRealizationTable.projectName, isEqualTo(project)) // - .groupBy(queryHistoryRealizationTable.model) // - .build().render(RenderingStrategies.MYBATIS3); + .from(queryHistoryRealizationTable) // + .where(queryHistoryRealizationTable.queryTime, isGreaterThanOrEqualTo(startTime)) // + .and(queryHistoryRealizationTable.queryTime, isLessThan(endTime)) // + .and(queryHistoryRealizationTable.projectName, isEqualTo(project)) // + .groupBy(queryHistoryRealizationTable.model) // + .build().render(RenderingStrategies.MYBATIS3); return mapper.selectMany(statementProvider); } } @@ -363,18 +364,18 @@ public List queryAvgDurationByModel(long startTime, long endTim QueryStatisticsMapper mapper = session.getMapper(QueryStatisticsMapper.class); SelectStatementProvider statementProvider = select(queryHistoryRealizationTable.model, avg(queryHistoryRealizationTable.duration).as("mean")) // - .from(queryHistoryRealizationTable) // - .where(queryHistoryRealizationTable.queryTime, isGreaterThanOrEqualTo(startTime)) // - .and(queryHistoryRealizationTable.queryTime, isLessThan(endTime)) // - .and(queryHistoryRealizationTable.projectName, isEqualTo(project)) // - .groupBy(queryHistoryRealizationTable.model) // - .build().render(RenderingStrategies.MYBATIS3); + .from(queryHistoryRealizationTable) // + .where(queryHistoryRealizationTable.queryTime, isGreaterThanOrEqualTo(startTime)) // + .and(queryHistoryRealizationTable.queryTime, isLessThan(endTime)) // + .and(queryHistoryRealizationTable.projectName, isEqualTo(project)) // + .groupBy(queryHistoryRealizationTable.model) // + .build().render(RenderingStrategies.MYBATIS3); return mapper.selectMany(statementProvider); } } public List queryAvgDurationByTime(long startTime, long endTime, String timeDimension, - String project) { + String project) { try (SqlSession session = sqlSessionFactory.openSession()) { QueryStatisticsMapper mapper = session.getMapper(QueryStatisticsMapper.class); SelectStatementProvider statementProvider = queryAvgDurationByTimeProvider(startTime, endTime, @@ -562,15 +563,15 @@ InsertStatementProvider getInsertQhRealizationP } private SelectStatementProvider queryQueryHistoriesByConditionsProvider(QueryHistoryRequest request, int limit, - int offset) { + int offset) { return select(getSelectFields(queryHistoryTable)).from(queryHistoryTable) .join(filterByConditions(select(BasicColumn.columnList(queryHistoryTable.id)).from(queryHistoryTable), request).orderBy(queryHistoryTable.queryTime.descending()) // .limit(limit) // .offset(offset), - "idTable") // + ID_TABLE_ALIAS) // .on(queryHistoryTable.id.qualifiedWith(queryHistoryTable.tableNameAtRuntime()), - new EqualTo(queryHistoryTable.id.qualifiedWith("idTable"))) + new EqualTo(queryHistoryTable.id.qualifiedWith(ID_TABLE_ALIAS))) .orderBy(queryHistoryTable.queryTime.descending()) // .build().render(RenderingStrategies.MYBATIS3); } @@ -658,8 +659,8 @@ private QueryExpressionDSL.QueryExpressionWhereBuilder filterQueryH } else if (request.getFilterModelIds() != null && !request.getFilterModelIds().isEmpty()) { // Process CONSTANTS, HIVE, RDBMS and model1, model2, model3... filterSql = filterSql.and(queryHistoryTable.engineType, isIn(realizations), or(queryHistoryTable.queryId, - isIn(selectDistinct(queryHistoryRealizationTable.queryId).from(queryHistoryRealizationTable) - .where(queryHistoryRealizationTable.model, isIn(request.getFilterModelIds()))))); + isIn(selectDistinct(queryHistoryRealizationTable.queryId).from(queryHistoryRealizationTable) + .where(queryHistoryRealizationTable.model, isIn(request.getFilterModelIds()))))); } else { // Process CONSTANTS, HIVE, RDBMS filterSql = filterSql.and(queryHistoryTable.engineType, isIn(realizations)); @@ -691,7 +692,7 @@ private SelectStatementProvider queryCountByTimeProvider(long startTime, long en } private SelectStatementProvider queryCountByTimeProvider(long startTime, long endTime, String timeDimension, - String project) { + String project) { if (timeDimension.equalsIgnoreCase(MONTH)) { return select(queryHistoryTable.queryFirstDayOfMonth.as("time"), count(queryHistoryTable.id).as(COUNT)) // .from(queryHistoryTable) // @@ -722,7 +723,7 @@ private SelectStatementProvider queryCountByTimeProvider(long startTime, long en } private SelectStatementProvider queryAvgDurationByTimeProvider(long startTime, long endTime, String timeDimension, - String project) { + String project) { if (timeDimension.equalsIgnoreCase(MONTH)) { return select(queryHistoryTable.queryFirstDayOfMonth.as("time"), avg(queryHistoryTable.duration).as("mean")) // .from(queryHistoryTable) //