Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Refactor] Remove CreateTableStmt in old parser #8824

Merged
merged 3 commits into from Jul 18, 2022
Merged

[Refactor] Remove CreateTableStmt in old parser #8824

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
688bd74
Remove create table statement in old parser
xlwh Jul 17, 2022
0981539
Add test case
xlwh Jul 17, 2022
ba342b3
Fix old test cases
xlwh Jul 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
39 changes: 0 additions & 39 deletions fe/fe-core/src/main/cup/sql_parser.cup
View file
Open in desktop
Expand Up @@ -1081,45 +1081,6 @@ create_stmt ::=
{:
RESULT = new CreateFunctionStmt(functionType, functionName, args, returnType, intermediateType, properties);
:}
/* Table */
| KW_CREATE externalable_table:isExternal opt_if_not_exists:ifNotExists table_name:name
LPAREN column_definition_list:columns RPAREN opt_engine:engineName opt_charset_name:charsetName
opt_keys:keys
opt_comment:tableComment
opt_partition:partition
opt_distribution:distribution
opt_rollup:index
opt_properties:tblProperties
opt_ext_properties:extProperties
{:
RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, engineName, charsetName, keys, partition,
distribution, tblProperties, extProperties, tableComment, index);
:}
| KW_CREATE externalable_table:isExternal opt_if_not_exists:ifNotExists table_name:name
LPAREN column_definition_list:columns COMMA index_definition_list:indexes RPAREN opt_engine:engineName opt_charset_name:charsetName
opt_keys:keys
opt_comment:tableComment
opt_partition:partition
opt_distribution:distribution
opt_rollup:index
opt_properties:tblProperties
opt_ext_properties:extProperties
{:
RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, indexes, engineName, charsetName, keys, partition,
distribution, tblProperties, extProperties, tableComment, index);
:}
| KW_CREATE externalable_table:isExternal opt_if_not_exists:ifNotExists table_name:name
opt_col_list:columns
opt_comment:comment
opt_partition:partition
opt_distribution:distribution
opt_properties:tblProperties
KW_AS select_stmt:queryStmt
{:
CreateTableStmt create_stmt = new CreateTableStmt(ifNotExists, isExternal, name, null,
"olap", null, partition, distribution, tblProperties, null, comment);
RESULT = new CreateTableAsSelectStmt(create_stmt, columns, queryStmt);
:}
/* User */
| KW_CREATE KW_USER opt_if_not_exists:ifNotExists grant_user:user opt_user_role:userRole
{:
Expand Down
265 changes: 0 additions & 265 deletions fe/fe-core/src/main/java/com/starrocks/analysis/CreateTableStmt.java
View file
Open in desktop
Expand Up @@ -23,42 +23,19 @@

import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.starrocks.catalog.AggregateType;
import com.starrocks.catalog.Column;
import com.starrocks.catalog.Index;
import com.starrocks.catalog.KeysType;
import com.starrocks.catalog.PartitionType;
import com.starrocks.catalog.PrimitiveType;
import com.starrocks.common.AnalysisException;
import com.starrocks.common.Config;
import com.starrocks.common.ErrorCode;
import com.starrocks.common.ErrorReport;
import com.starrocks.common.FeConstants;
import com.starrocks.common.FeNameFormat;
import com.starrocks.common.UserException;
import com.starrocks.common.util.PrintableMap;
import com.starrocks.common.util.PropertyAnalyzer;
import com.starrocks.external.elasticsearch.EsUtil;
import com.starrocks.mysql.privilege.PrivPredicate;
import com.starrocks.qe.ConnectContext;
import com.starrocks.server.GlobalStateMgr;
import com.starrocks.sql.ast.AstVisitor;
import org.apache.commons.collections.CollectionUtils;

import java.io.DataInput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;

import static com.starrocks.catalog.AggregateType.BITMAP_UNION;
import static com.starrocks.catalog.AggregateType.HLL_UNION;

public class CreateTableStmt extends DdlStmt {

Expand Down Expand Up @@ -315,218 +292,6 @@ public List<Index> getIndexes() {
return indexes;
}

@Override
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
tableName.analyze(analyzer);
FeNameFormat.checkTableName(tableName.getTbl());

if (!GlobalStateMgr.getCurrentState().getAuth().checkTblPriv(ConnectContext.get(), tableName.getDb(),
tableName.getTbl(), PrivPredicate.CREATE)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "CREATE");
}

analyzeEngineName();
analyzeCharsetName();

// analyze key desc
if (!(engineName.equals("mysql") || engineName.equals("broker") ||
engineName.equals("hive") || engineName.equals("iceberg") ||
engineName.equals("hudi") || engineName.equals("jdbc"))) {
// olap table or lake table
if (keysDesc == null) {
List<String> keysColumnNames = Lists.newArrayList();
int keyLength = 0;
boolean hasAggregate = false;
for (ColumnDef columnDef : columnDefs) {
if (columnDef.getAggregateType() != null) {
hasAggregate = true;
break;
}
}
if (hasAggregate) {
for (ColumnDef columnDef : columnDefs) {
if (columnDef.getAggregateType() == null) {
keysColumnNames.add(columnDef.getName());
}
}
keysDesc = new KeysDesc(KeysType.AGG_KEYS, keysColumnNames);
} else {
for (ColumnDef columnDef : columnDefs) {
keyLength += columnDef.getType().getIndexSize();
if (keysColumnNames.size() >= FeConstants.shortkey_max_column_count
|| keyLength > FeConstants.shortkey_maxsize_bytes) {
if (keysColumnNames.size() == 0
&& columnDef.getType().getPrimitiveType().isCharFamily()) {
keysColumnNames.add(columnDef.getName());
}
break;
}
if (!columnDef.getType().isKeyType()) {
break;
}
if (columnDef.getType().getPrimitiveType() == PrimitiveType.VARCHAR) {
keysColumnNames.add(columnDef.getName());
break;
}
keysColumnNames.add(columnDef.getName());
}
if (columnDefs.isEmpty()) {
throw new AnalysisException("Empty schema");
}
// The OLAP table must has at least one short key and the float and double should not be short key.
// So the float and double could not be the first column in OLAP table.
if (keysColumnNames.isEmpty()) {
throw new AnalysisException(
"Data type of first column cannot be " + columnDefs.get(0).getType());
}
keysDesc = new KeysDesc(KeysType.DUP_KEYS, keysColumnNames);
}
}

keysDesc.analyze(columnDefs);
for (int i = 0; i < keysDesc.keysColumnSize(); ++i) {
columnDefs.get(i).setIsKey(true);
}
if (keysDesc.getKeysType() != KeysType.AGG_KEYS) {
AggregateType type = AggregateType.REPLACE;
// note: PRIMARY_KEYS uses REPLACE aggregate type for now
if (keysDesc.getKeysType() == KeysType.DUP_KEYS) {
type = AggregateType.NONE;
}
for (int i = keysDesc.keysColumnSize(); i < columnDefs.size(); ++i) {
columnDefs.get(i).setAggregateType(type);
}
}
} else {
// mysql, broker, iceberg, hudi and hive do not need key desc
if (keysDesc != null) {
throw new AnalysisException("Create " + engineName + " table should not contain keys desc");
}

for (ColumnDef columnDef : columnDefs) {
if (engineName.equals("mysql") && columnDef.getType().isComplexType()) {
throw new AnalysisException(engineName + " external table don't support complex type");
}
columnDef.setIsKey(true);
}
}

// analyze column def
if (columnDefs == null || columnDefs.isEmpty()) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLE_MUST_HAVE_COLUMNS);
}

boolean hasHll = false;
boolean hasBitmap = false;
Set<String> columnSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
for (ColumnDef columnDef : columnDefs) {
columnDef.analyze(isOlapOrLakeEngine());

if (columnDef.getAggregateType() == HLL_UNION) {
hasHll = true;
}

if (columnDef.getAggregateType() == BITMAP_UNION) {
hasBitmap = columnDef.getType().isBitmapType();
}

if (!columnSet.add(columnDef.getName())) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, columnDef.getName());
}
}

if (hasHll && keysDesc.getKeysType() != KeysType.AGG_KEYS) {
throw new AnalysisException("HLL_UNION must be used in AGG_KEYS");
}

if (hasBitmap && keysDesc.getKeysType() != KeysType.AGG_KEYS) {
throw new AnalysisException("BITMAP_UNION must be used in AGG_KEYS");
}

if (isOlapOrLakeEngine()) {
// analyze partition
if (partitionDesc != null) {
if (partitionDesc.getType() == PartitionType.RANGE || partitionDesc.getType() == PartitionType.LIST) {
partitionDesc.analyze(columnDefs, properties);
} else {
throw new AnalysisException(
"Currently only support range and list partition with engine type olap");
}
}

// analyze distribution
if (distributionDesc == null) {
if (ConnectContext.get().getSessionVariable().isAllowDefaultPartition()) {
if (properties == null) {
properties = Maps.newHashMap();
properties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, "1");
}
distributionDesc = new HashDistributionDesc(Config.default_bucket_num,
Lists.newArrayList(columnDefs.get(0).getName()));
} else {
throw new AnalysisException("Create olap table should contain distribution desc");
}
}
distributionDesc.analyze(columnSet);
} else if (engineName.equals("elasticsearch")) {
EsUtil.analyzePartitionAndDistributionDesc(partitionDesc, distributionDesc);
} else {
if (partitionDesc != null || distributionDesc != null) {
throw new AnalysisException("Create " + engineName
+ " table should not contain partition or distribution desc");
}
}

for (ColumnDef columnDef : columnDefs) {
Column col = columnDef.toColumn();
if (keysDesc != null && (keysDesc.getKeysType() == KeysType.UNIQUE_KEYS
|| keysDesc.getKeysType() == KeysType.PRIMARY_KEYS ||
keysDesc.getKeysType() == KeysType.DUP_KEYS)) {
if (!col.isKey()) {
col.setAggregationTypeImplicit(true);
}
}
columns.add(col);
}

if (CollectionUtils.isNotEmpty(indexDefs)) {
Set<String> distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
Set<List<String>> distinctCol = new HashSet<>();

for (IndexDef indexDef : indexDefs) {
indexDef.analyze();
if (!isOlapOrLakeEngine()) {
throw new AnalysisException("index only support in olap engine at current version.");
}
for (String indexColName : indexDef.getColumns()) {
boolean found = false;
for (Column column : columns) {
if (column.getName().equalsIgnoreCase(indexColName)) {
indexDef.checkColumn(column, getKeysDesc().getKeysType());
found = true;
break;
}
}
if (!found) {
throw new AnalysisException("BITMAP column does not exist in table. invalid column: "
+ indexColName);
}
}
indexes.add(new Index(indexDef.getIndexName(), indexDef.getColumns(), indexDef.getIndexType(),
indexDef.getComment()));
distinct.add(indexDef.getIndexName());
distinctCol.add(indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList()));
}
if (distinct.size() != indexes.size()) {
throw new AnalysisException("index name must be unique.");
}
if (distinctCol.size() != indexes.size()) {
throw new AnalysisException("same index columns have multiple index name is not allowed.");
}
}
}

public List<ColumnDef> getColumnDefs() {
return columnDefs;
}
Expand All @@ -547,36 +312,6 @@ public void setDistributionDesc(DistributionDesc distributionDesc) {
this.distributionDesc = distributionDesc;
}

private void analyzeEngineName() throws AnalysisException {
if (Strings.isNullOrEmpty(engineName)) {
engineName = "olap";
}
engineName = engineName.toLowerCase();

if (!engineNames.contains(engineName)) {
throw new AnalysisException("Unknown engine name: " + engineName);
}

if (isLakeEngine() && !Config.use_staros) {
throw new AnalysisException("Engine " + engineName + " needs 'use_staros = true' config in fe.conf");
}
}

private void analyzeCharsetName() throws AnalysisException {
if (Strings.isNullOrEmpty(charsetName)) {
charsetName = "utf8";
}
charsetName = charsetName.toLowerCase();

if (!charsetNames.contains(charsetName)) {
throw new AnalysisException("Unknown charset name: " + charsetName);
}
// be is not supported yet,so Display unsupported information to the user
if (!charsetName.equals(DEFAULT_CHARSET_NAME)){
throw new AnalysisException("charset name " + charsetName + " is not supported yet");
}
}

public static CreateTableStmt read(DataInput in) throws IOException {
throw new RuntimeException("CreateTableStmt serialization is not supported anymore.");
}
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/test/java/com/starrocks/analysis/CreateTableWithPartitionTest.java
View file
Open in desktop
Expand Up @@ -138,7 +138,7 @@ public void testCreateTableBatchPartitionWithDynamicPrefix() throws Exception {
" \"replication_num\" = \"1\",\n" +
" \"dynamic_partition.prefix\" = \"p_\"\n" +
");";
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTableSql, ctx);
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(createTableSql, ctx);
PartitionDesc partitionDesc = createTableStmt.getPartitionDesc();
Assert.assertTrue(
partitionDesc.toSql().contains("PARTITION p_20140101 VALUES [('2014-01-01'), ('2014-01-02'))"));
Expand Down
6 changes: 3 additions & 3 deletions fe/fe-core/src/test/java/com/starrocks/catalog/ColocateTableIndexTest.java
View file
Open in desktop
Expand Up @@ -35,7 +35,7 @@ public void testDropTable() throws Exception {
"DISTRIBUTED BY HASH(k1)\n" +
"BUCKETS 4\n" +
"PROPERTIES(\"colocate_with\"=\"group1\", \"replication_num\" = \"1\");\n";
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
GlobalStateMgr.getCurrentState().createTable(createTableStmt);
List<List<String>> infos = GlobalStateMgr.getCurrentColocateIndex().getInfos();
// group1->table1_1
Expand All @@ -51,7 +51,7 @@ public void testDropTable() throws Exception {
"DISTRIBUTED BY HASH(k1)\n" +
"BUCKETS 4\n" +
"PROPERTIES(\"colocate_with\"=\"group1\", \"replication_num\" = \"1\");\n";
createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
GlobalStateMgr.getCurrentState().createTable(createTableStmt);
// group1 -> table1_1, table1_2
infos = GlobalStateMgr.getCurrentColocateIndex().getInfos();
Expand All @@ -71,7 +71,7 @@ public void testDropTable() throws Exception {
"DISTRIBUTED BY HASH(k1)\n" +
"BUCKETS 4\n" +
"PROPERTIES(\"colocate_with\"=\"group2\", \"replication_num\" = \"1\");\n";
createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
GlobalStateMgr.getCurrentState().createTable(createTableStmt);
// group1 -> table1_1, table1_2
// group2 -> table2_l
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/test/java/com/starrocks/lake/CreateLakeTableTest.java
View file
Open in desktop
Expand Up @@ -47,7 +47,7 @@ public static void afterClass() {
}

private static void createTable(String sql) throws Exception {
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
GlobalStateMgr.getCurrentState().createTable(createTableStmt);
}

Expand Down