From 05387b60e853886fe46d91156dbf55079cdf8e96 Mon Sep 17 00:00:00 2001 From: hncboy <619452863@qq.com> Date: Sun, 3 Sep 2023 00:47:04 +0800 Subject: [PATCH] feat: increase query table data (issue #82) --- .../DorisClient.java | 21 +++- .../solidui/plugin/jdbc/BaseJdbcClient.java | 5 +- .../solidui/plugin/jdbc/JdbcClient.java | 5 + .../solidui/plugin/jdbc/MysqlClient.java | 21 +++- .../controller/MetadataQueryController.java | 38 ++++-- .../service/MetadataQueryService.java | 1 + .../impl/MetadataQueryServiceImpl.java | 111 +++++++++++++----- 7 files changed, 155 insertions(+), 47 deletions(-) diff --git a/solidui-datasource-plugin/solidui-datasource-doris/src/main/java/com.cloudorc.solidui.plugin.jdbc/DorisClient.java b/solidui-datasource-plugin/solidui-datasource-doris/src/main/java/com.cloudorc.solidui.plugin.jdbc/DorisClient.java index fd32959..65122f9 100644 --- a/solidui-datasource-plugin/solidui-datasource-doris/src/main/java/com.cloudorc.solidui.plugin.jdbc/DorisClient.java +++ b/solidui-datasource-plugin/solidui-datasource-doris/src/main/java/com.cloudorc.solidui.plugin.jdbc/DorisClient.java @@ -18,9 +18,15 @@ package com.cloudorc.solidui.plugin.jdbc; -import java.sql.*; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.Optional; public class DorisClient extends BaseJdbcClient { @@ -29,7 +35,12 @@ public DorisClient(Connection conn) { } @Override - public List getAllDatabase() throws SQLException{ + public String generateSelectAllDataSql(String database, String tableName) { + return "SELECT * FROM " + tableName; + } + + @Override + public List getAllDatabase() throws SQLException { List dataBaseName = new ArrayList<>(); Statement stmt = null; ResultSet rs = null; @@ -47,7 +58,7 @@ public List getAllDatabase() throws SQLException{ } @Override - public List getAllTables(String database) throws SQLException{ + public List getAllTables(String database) throws SQLException { List tableNames = new ArrayList<>(); Statement stmt = null; ResultSet rs = null; @@ -64,7 +75,7 @@ public List getAllTables(String database) throws SQLException{ } @Override - public List> getSelectResult(String sql) throws SQLException{ + public List> getSelectResult(String sql) throws SQLException { PreparedStatement pstmt = null; ResultSet rs = null; List> lists = new ArrayList<>(); @@ -83,7 +94,7 @@ public List> getSelectResult(String sql) throws SQLException{ while (rs.next()) { List stringArrayList = new ArrayList<>(); for (int i = 1; i < columnCount + 1; i++) { - stringArrayList.add(rs.getString(i).trim()); + stringArrayList.add(Optional.ofNullable(rs.getString(i)).map(String::trim).orElse(null)); } lists.add(stringArrayList); } diff --git a/solidui-datasource-plugin/solidui-datasource-jdbc/src/main/java/com/cloudorc/solidui/plugin/jdbc/BaseJdbcClient.java b/solidui-datasource-plugin/solidui-datasource-jdbc/src/main/java/com/cloudorc/solidui/plugin/jdbc/BaseJdbcClient.java index 21c4462..d92d53e 100644 --- a/solidui-datasource-plugin/solidui-datasource-jdbc/src/main/java/com/cloudorc/solidui/plugin/jdbc/BaseJdbcClient.java +++ b/solidui-datasource-plugin/solidui-datasource-jdbc/src/main/java/com/cloudorc/solidui/plugin/jdbc/BaseJdbcClient.java @@ -27,7 +27,7 @@ import java.sql.Statement; -public abstract class BaseJdbcClient implements JdbcClient{ +public abstract class BaseJdbcClient implements JdbcClient { public final Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -55,8 +55,9 @@ public void closeResource(Connection connection, Statement statement, ResultSet @Override public void close() throws IOException { - closeResource(conn,null,null); + closeResource(conn, null, null); } + @Override public Connection getConn() { return conn; diff --git a/solidui-datasource-plugin/solidui-datasource-jdbc/src/main/java/com/cloudorc/solidui/plugin/jdbc/JdbcClient.java b/solidui-datasource-plugin/solidui-datasource-jdbc/src/main/java/com/cloudorc/solidui/plugin/jdbc/JdbcClient.java index 21b2b67..22b48c6 100644 --- a/solidui-datasource-plugin/solidui-datasource-jdbc/src/main/java/com/cloudorc/solidui/plugin/jdbc/JdbcClient.java +++ b/solidui-datasource-plugin/solidui-datasource-jdbc/src/main/java/com/cloudorc/solidui/plugin/jdbc/JdbcClient.java @@ -24,6 +24,11 @@ public interface JdbcClient { + /** + * Generates a SQL query string to select all data from a specified table. + */ + String generateSelectAllDataSql(String database, String tableName); + List getAllDatabase() throws SQLException; List getAllTables(String database) throws SQLException ; diff --git a/solidui-datasource-plugin/solidui-datasource-mysql/src/main/java/com/cloudorc/solidui/plugin/jdbc/MysqlClient.java b/solidui-datasource-plugin/solidui-datasource-mysql/src/main/java/com/cloudorc/solidui/plugin/jdbc/MysqlClient.java index add1cd8..ae89bb2 100644 --- a/solidui-datasource-plugin/solidui-datasource-mysql/src/main/java/com/cloudorc/solidui/plugin/jdbc/MysqlClient.java +++ b/solidui-datasource-plugin/solidui-datasource-mysql/src/main/java/com/cloudorc/solidui/plugin/jdbc/MysqlClient.java @@ -18,9 +18,15 @@ package com.cloudorc.solidui.plugin.jdbc; -import java.sql.*; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.Optional; public class MysqlClient extends BaseJdbcClient { @@ -29,7 +35,12 @@ public MysqlClient(Connection conn) { } @Override - public List getAllDatabase() throws SQLException{ + public String generateSelectAllDataSql(String database, String tableName) { + return "SELECT * FROM " + tableName; + } + + @Override + public List getAllDatabase() throws SQLException { List dataBaseName = new ArrayList<>(); Statement stmt = null; ResultSet rs = null; @@ -47,7 +58,7 @@ public List getAllDatabase() throws SQLException{ } @Override - public List getAllTables(String database) throws SQLException{ + public List getAllTables(String database) throws SQLException { List tableNames = new ArrayList<>(); Statement stmt = null; ResultSet rs = null; @@ -64,7 +75,7 @@ public List getAllTables(String database) throws SQLException{ } @Override - public List> getSelectResult(String sql) throws SQLException{ + public List> getSelectResult(String sql) throws SQLException { PreparedStatement pstmt = null; ResultSet rs = null; List> lists = new ArrayList<>(); @@ -82,7 +93,7 @@ public List> getSelectResult(String sql) throws SQLException{ while (rs.next()) { List stringArrayList = new ArrayList<>(); for (int i = 1; i < columnCount + 1; i++) { - stringArrayList.add(rs.getString(i).trim()); + stringArrayList.add(Optional.ofNullable(rs.getString(i)).map(String::trim).orElse(null)); } lists.add(stringArrayList); } diff --git a/solidui-entrance/src/main/java/com/cloudorc/solidui/entrance/controller/MetadataQueryController.java b/solidui-entrance/src/main/java/com/cloudorc/solidui/entrance/controller/MetadataQueryController.java index deac3ce..4a874e6 100644 --- a/solidui-entrance/src/main/java/com/cloudorc/solidui/entrance/controller/MetadataQueryController.java +++ b/solidui-entrance/src/main/java/com/cloudorc/solidui/entrance/controller/MetadataQueryController.java @@ -25,11 +25,18 @@ import io.swagger.annotations.ApiOperation; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; import javax.servlet.http.HttpServletRequest; -import static com.cloudorc.solidui.entrance.enums.Status.*; +import static com.cloudorc.solidui.entrance.enums.Status.QUERY_METADATA_DB_ERROR; +import static com.cloudorc.solidui.entrance.enums.Status.QUERY_METADATA_SQL_ERROR; +import static com.cloudorc.solidui.entrance.enums.Status.QUERY_METADATA_TABLE_ERROR; @Api(tags = "metadata_query") @RestController @@ -51,7 +58,7 @@ public Result getDatabases( @RequestParam("dataSourceName") String dataSourceName, @RequestParam("typeName") String typeName, HttpServletRequest request) { - return metadataQueryService.queryDatabasesByDsName(dataSourceName,typeName); + return metadataQueryService.queryDatabasesByDsName(dataSourceName, typeName); } @@ -69,7 +76,25 @@ public Result getTables( @RequestParam("database") String database, @RequestParam("typeName") String typeName, HttpServletRequest request) { - return metadataQueryService.queryTablesByDsName(dataSourceName,database,typeName); + return metadataQueryService.queryTablesByDsName(dataSourceName, database, typeName); + } + + @ApiOperation(value = "query table data") + @ApiImplicitParams({ + @ApiImplicitParam(name = "dataSourceName", required = true, dataType = "String", value = "data source name"), + @ApiImplicitParam(name = "typeName", required = true, dataType = "String", value = "typeName"), + @ApiImplicitParam(name = "database", required = true, dataType = "String", value = "database"), + @ApiImplicitParam(name = "tableName", required = true, dataType = "String", value = "tableName") + }) + @ResponseStatus(HttpStatus.OK) + @ApiException(QUERY_METADATA_SQL_ERROR) + @GetMapping(value = "/queryTableData") + public Result getTables( + @RequestParam String dataSourceName, + @RequestParam String database, + @RequestParam String typeName, + @RequestParam String tableName) { + return metadataQueryService.queryTableData(dataSourceName, database, typeName, tableName); } @ApiOperation(value = "querySql", notes = "query sql") @@ -86,10 +111,9 @@ public Result querySelectSql( @RequestParam("sql") String sql, @RequestParam("typeName") String typeName, HttpServletRequest request) { - return metadataQueryService.queryBySql(dataSourceName,sql,typeName); + return metadataQueryService.queryBySql(dataSourceName, sql, typeName); } - @ApiOperation(value = "querySql/id", notes = "query sql") @ApiImplicitParams({ @ApiImplicitParam(name = "dataSourceId", required = true, dataType = "Long", value = "data source id"), @@ -104,7 +128,7 @@ public Result querySelectSql( @RequestParam("sql") String sql, @RequestParam("typeId") Long typeId, HttpServletRequest request) { - return metadataQueryService.queryBySql(dataSourceId,sql,typeId); + return metadataQueryService.queryBySql(dataSourceId, sql, typeId); } diff --git a/solidui-entrance/src/main/java/com/cloudorc/solidui/entrance/service/MetadataQueryService.java b/solidui-entrance/src/main/java/com/cloudorc/solidui/entrance/service/MetadataQueryService.java index 706fe14..680918d 100644 --- a/solidui-entrance/src/main/java/com/cloudorc/solidui/entrance/service/MetadataQueryService.java +++ b/solidui-entrance/src/main/java/com/cloudorc/solidui/entrance/service/MetadataQueryService.java @@ -31,4 +31,5 @@ public interface MetadataQueryService { Result queryConnection(String dataSourceName,String typeName); + Result queryTableData(String dataSourceName, String database, String typeName, String tableName); } diff --git a/solidui-entrance/src/main/java/com/cloudorc/solidui/entrance/service/impl/MetadataQueryServiceImpl.java b/solidui-entrance/src/main/java/com/cloudorc/solidui/entrance/service/impl/MetadataQueryServiceImpl.java index bd504c4..e279047 100644 --- a/solidui-entrance/src/main/java/com/cloudorc/solidui/entrance/service/impl/MetadataQueryServiceImpl.java +++ b/solidui-entrance/src/main/java/com/cloudorc/solidui/entrance/service/impl/MetadataQueryServiceImpl.java @@ -33,7 +33,11 @@ import org.springframework.stereotype.Service; import java.sql.Connection; +import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; @Service public class MetadataQueryServiceImpl extends BaseServiceImpl implements MetadataQueryService { @@ -45,20 +49,20 @@ public class MetadataQueryServiceImpl extends BaseServiceImpl implements Metadat @Override public Result queryDatabasesByDsName(String dataSourceName, String typeName) { - DataSource dataSource = dataSourceMapper.queryByName(dataSourceName,null); + DataSource dataSource = dataSourceMapper.queryByName(dataSourceName, null); Result result = new Result<>(); - if(dataSource == null) { + if (dataSource == null) { putMsg(result, Status.QUERY_METADATA_DB_ERROR); return result; } try { JdbcClient jdbcClient = DataSourceUtils.queryJdbcClient(typeName, dataSource); - if(jdbcClient != null) { + if (jdbcClient != null) { List allDatabase = jdbcClient.getAllDatabase(); - if(CollectionUtils.isEmpty(allDatabase)){ + if (CollectionUtils.isEmpty(allDatabase)) { putMsg(result, Status.QUERY_METADATA_DB_ERROR); - }else{ + } else { putMsg(result, Status.SUCCESS); result.setData(allDatabase); } @@ -76,20 +80,20 @@ public Result queryDatabasesByDsName(String dataSourceName, String typeName) { @Override public Result queryTablesByDsName(String dataSourceName, String dbName, String typeName) { - DataSource dataSource = dataSourceMapper.queryByName(dataSourceName,null); + DataSource dataSource = dataSourceMapper.queryByName(dataSourceName, null); Result result = new Result<>(); - if(dataSource == null) { + if (dataSource == null) { putMsg(result, Status.QUERY_METADATA_TABLE_ERROR); return result; } try { JdbcClient jdbcClient = DataSourceUtils.queryJdbcClient(typeName, dataSource); - if(jdbcClient != null) { + if (jdbcClient != null) { List allTables = jdbcClient.getAllTables(dbName); - if(CollectionUtils.isEmpty(allTables)){ + if (CollectionUtils.isEmpty(allTables)) { putMsg(result, Status.QUERY_METADATA_TABLE_ERROR); - }else{ + } else { putMsg(result, Status.SUCCESS); result.setData(allTables); } @@ -108,25 +112,25 @@ public Result queryTablesByDsName(String dataSourceName, String dbName, String t @Override public Result queryBySql(String dataSourceName, String sql, String typeName) { Result result = new Result<>(); - if(dataSourceName == null || typeName == null){ + if (dataSourceName == null || typeName == null) { putMsg(result, Status.QUERY_METADATA_SQL_ERROR); return result; } - DataSource dataSource = dataSourceMapper.queryByName(dataSourceName,null); + DataSource dataSource = dataSourceMapper.queryByName(dataSourceName, null); - if(dataSource == null) { + if (dataSource == null) { putMsg(result, Status.QUERY_METADATA_SQL_ERROR); return result; } try { JdbcClient jdbcClient = DataSourceUtils.queryJdbcClient(typeName, dataSource); - if(jdbcClient != null) { + if (jdbcClient != null) { List> selectResult = jdbcClient.getSelectResult(sql); - if(CollectionUtils.isEmpty(selectResult)){ + if (CollectionUtils.isEmpty(selectResult)) { putMsg(result, Status.QUERY_METADATA_SQL_ERROR); - }else{ + } else { putMsg(result, Status.SUCCESS); result.setData(selectResult); } @@ -145,29 +149,29 @@ public Result queryBySql(String dataSourceName, String sql, String typeName) { @Override public Result queryBySql(Long dataSourceId, String sql, Long typeId) { Result result = new Result<>(); - if(dataSourceId == null || typeId == null){ + if (dataSourceId == null || typeId == null) { putMsg(result, Status.QUERY_METADATA_SQL_ERROR); return result; } - DataSource dataSource = dataSourceMapper.queryByName(null,dataSourceId); + DataSource dataSource = dataSourceMapper.queryByName(null, dataSourceId); - if(dataSource == null) { + if (dataSource == null) { putMsg(result, Status.QUERY_METADATA_SQL_ERROR); return result; } DataSourceType dataSourceType = dataSourceTypeMapper.selectById(typeId); - if(dataSourceType == null){ + if (dataSourceType == null) { putMsg(result, Status.QUERY_DATASOURCE_ERROR); return result; } try { JdbcClient jdbcClient = DataSourceUtils.queryJdbcClient(dataSourceType.getName(), dataSource); - if(jdbcClient != null) { + if (jdbcClient != null) { List> selectResult = jdbcClient.getSelectResult(sql); - if(CollectionUtils.isEmpty(selectResult)){ + if (CollectionUtils.isEmpty(selectResult)) { putMsg(result, Status.QUERY_METADATA_SQL_ERROR); - }else{ + } else { putMsg(result, Status.SUCCESS); result.setData(selectResult); } @@ -185,20 +189,20 @@ public Result queryBySql(Long dataSourceId, String sql, Long typeId) { @Override public Result queryConnection(String dataSourceName, String typeName) { - DataSource dataSource = dataSourceMapper.queryByName(dataSourceName,null); + DataSource dataSource = dataSourceMapper.queryByName(dataSourceName, null); Result result = new Result<>(); - if(dataSource == null) { + if (dataSource == null) { putMsg(result, Status.QUERY_METADATA_CONN_ERROR); return result; } try { JdbcClient jdbcClient = DataSourceUtils.queryJdbcClient(typeName, dataSource); - if(jdbcClient != null) { + if (jdbcClient != null) { Connection conn = jdbcClient.getConn(); - if(conn == null){ + if (conn == null) { putMsg(result, Status.QUERY_METADATA_CONN_ERROR); - }else{ + } else { putMsg(result, Status.SUCCESS); } } else { @@ -212,4 +216,55 @@ public Result queryConnection(String dataSourceName, String typeName) { return result; } + + @Override + public Result queryTableData(String dataSourceName, String database, String typeName, String tableName) { + Result result = new Result<>(); + DataSource dataSource = dataSourceMapper.queryByName(dataSourceName, null); + if (Objects.isNull(dataSource)) { + putMsg(result, Status.DATASOURCE_NOT_EXISTS_ERROR); + return result; + } + try { + JdbcClient jdbcClient = DataSourceUtils.queryJdbcClient(typeName, dataSource); + if (Objects.isNull(jdbcClient)) { + putMsg(result, Status.DATASOURCE_NOT_EXISTS_ERROR); + return result; + } + + String selectAllDataSql = jdbcClient.generateSelectAllDataSql(database, tableName); + List> selectResult = jdbcClient.getSelectResult(selectAllDataSql); + // it cannot be empty because it contains at least columns + if (CollectionUtils.isEmpty(selectResult)) { + putMsg(result, Status.QUERY_METADATA_SQL_ERROR); + return result; + } + + // empty data + if (selectResult.size() == 1) { + putMsg(result, Status.SUCCESS); + result.setData(CollectionUtils.EMPTY_COLLECTION); + } + + List> fieldValueResults = new ArrayList<>(selectResult.size()); + List columns = selectResult.get(0); + // starting from the second line + for (int i = 1; i < selectResult.size(); i++) { + List values = selectResult.get(i); + Map fieldValueMap = new LinkedHashMap<>(columns.size()); + for (int j = 0; j < columns.size() && j < values.size(); j++) { + fieldValueMap.put(columns.get(j), values.get(j)); + } + fieldValueResults.add(fieldValueMap); + } + + putMsg(result, Status.SUCCESS); + result.setData(fieldValueResults); + } catch (Exception e) { + logger.error("queryTableData error", e); + putMsg(result, Status.QUERY_METADATA_SQL_ERROR); + } + + return result; + } }