Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Table Model] Table model Device Schema Management #12437

Open
wants to merge 51 commits into
base: ty/TableModelGrammar
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
f2c80eb
implement table insert via tree model
MarcosZyk Apr 16, 2024
e0dc92b
refactor insert parse
MarcosZyk Apr 17, 2024
45df4d6
Merge branch 'ty/TableModelGrammar' into table-insert
MarcosZyk Apr 17, 2024
a3e8685
set up device creation
MarcosZyk Apr 17, 2024
d098bf7
implement device creation and attribute store
MarcosZyk Apr 17, 2024
9417229
remove comment
MarcosZyk Apr 17, 2024
238e2a7
implement getTableSchema
MarcosZyk Apr 17, 2024
7a1728c
implement getTableSchema
MarcosZyk Apr 18, 2024
fa77727
Merge branch 'ty/TableModelGrammar' into table-model-debug
MarcosZyk Apr 18, 2024
7636d9b
mock
MarcosZyk Apr 18, 2024
79fe9c0
Merge branch 'ty/TableModelGrammar' into table-model-debug
MarcosZyk Apr 18, 2024
59321ec
fix snapshot
MarcosZyk Apr 18, 2024
aca76ca
Merge branch 'table-insert' into table-model-debug
MarcosZyk Apr 18, 2024
e21e98b
Merge branch 'ty/TableModelGrammar' into table-model-debug
MarcosZyk Apr 18, 2024
8f86290
Merge branch 'ty/TableModelGrammar' into table-model-debug
MarcosZyk Apr 20, 2024
323ceb1
merge
MarcosZyk Apr 20, 2024
5757e8f
Merge branch 'ty/TableModelGrammar' into table-model-debug
MarcosZyk Apr 21, 2024
34e5b0e
implement schema filter
MarcosZyk Apr 21, 2024
e1345d7
Merge branch 'ty/TableModelGrammar' into table-model-debug
MarcosZyk Apr 22, 2024
0dfdb30
add warn
MarcosZyk Apr 22, 2024
e0e3b5c
Merge branch 'ty/TableModelGrammar' into table-model-debug
MarcosZyk Apr 22, 2024
691823c
Merge branch 'ty/TableModelGrammar' into table-model-debug
MarcosZyk Apr 22, 2024
f3efdc2
Merge branch 'ty/TableModelGrammar' into table-model-debug
MarcosZyk Apr 28, 2024
2adf50b
refactor expression to filter via visitor
MarcosZyk Apr 28, 2024
80c6b3f
refactor schemafilter to device pattern
MarcosZyk Apr 28, 2024
1c407ba
implement ser deser
MarcosZyk Apr 28, 2024
60e237b
Merge branch 'ty/TableModelGrammar' into table-model-debug
MarcosZyk Apr 28, 2024
07f3b01
rollback config
MarcosZyk Apr 28, 2024
5a9a23e
delete show device
MarcosZyk Apr 28, 2024
8557956
set up TableDeviceSchemaCache
MarcosZyk May 5, 2024
c1dac7f
set up validate schema
MarcosZyk May 5, 2024
9c47034
implement schema fetch and auto create
MarcosZyk May 5, 2024
e7c9663
Merge branch 'ty/TableModelGrammar' into table-model-debug
MarcosZyk May 9, 2024
3b27baa
implement query using device cache
MarcosZyk May 9, 2024
b0a536b
improve code smell
MarcosZyk May 10, 2024
a867ffb
support null device value
MarcosZyk May 12, 2024
687ee69
Merge branch 'ty/TableModelGrammar' into table-model-debug
MarcosZyk May 19, 2024
1b5f923
implement is null
MarcosZyk May 19, 2024
987f06f
set up table validate
MarcosZyk May 19, 2024
66fc5f4
implement drop table
MarcosZyk May 19, 2024
5564225
fix OrFilter
MarcosZyk May 20, 2024
1edcc41
add table column
MarcosZyk May 20, 2024
861c4b3
fix serialize bug during device fetch
MarcosZyk May 21, 2024
4b9cd18
insert null
MarcosZyk May 22, 2024
cefafcd
Merge branch 'ty/TableModelGrammar' into table-model-debug
MarcosZyk May 22, 2024
0daae57
fix
MarcosZyk May 22, 2024
4de5ed1
use linkedHashSet for deduplicate
MarcosZyk May 22, 2024
83541b6
fix invalidate cache during droping table
MarcosZyk May 22, 2024
586b419
spotless apply
MarcosZyk May 22, 2024
da315b9
fix multi region
MarcosZyk May 27, 2024
a4fc709
fix serialization
MarcosZyk Jun 11, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public class ConfigNodeConfig {
* SchemaRegionGroups for each Database. When set schema_region_group_extension_policy=AUTO, this
* parameter is the default minimal number of SchemaRegionGroups for each Database.
*/
private int defaultSchemaRegionGroupNumPerDatabase = 1;
private int defaultSchemaRegionGroupNumPerDatabase = 2;

/** The maximum number of SchemaRegions expected to be managed by each DataNode. */
private double schemaRegionPerDataNode = schemaReplicationFactor;
Expand All @@ -101,7 +101,7 @@ public class ConfigNodeConfig {
* DataRegionGroups for each Database. When set data_region_group_extension_policy=AUTO, this
* parameter is the default minimal number of DataRegionGroups for each Database.
*/
private int defaultDataRegionGroupNumPerDatabase = 2;
private int defaultDataRegionGroupNumPerDatabase = 1;

/** The maximum number of DataRegions expected to be managed by each DataNode. */
private double dataRegionPerDataNode = 5.0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,13 @@
import org.apache.iotdb.confignode.consensus.request.write.sync.RecordPipeMessagePlan;
import org.apache.iotdb.confignode.consensus.request.write.sync.SetPipeStatusPlanV1;
import org.apache.iotdb.confignode.consensus.request.write.sync.ShowPipePlanV1;
import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan;
import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan;
import org.apache.iotdb.confignode.consensus.request.write.table.CommitDropTablePlan;
import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan;
import org.apache.iotdb.confignode.consensus.request.write.table.PreDropTablePlan;
import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan;
import org.apache.iotdb.confignode.consensus.request.write.table.RollbackDropTablePlan;
import org.apache.iotdb.confignode.consensus.request.write.template.CommitSetSchemaTemplatePlan;
import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan;
import org.apache.iotdb.confignode.consensus.request.write.template.DropSchemaTemplatePlan;
Expand Down Expand Up @@ -408,6 +412,18 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException {
case CommitCreateTable:
plan = new CommitCreateTablePlan();
break;
case PreDropTable:
plan = new PreDropTablePlan();
break;
case RollbackPreDropTable:
plan = new RollbackDropTablePlan();
break;
case CommitDropTable:
plan = new CommitDropTablePlan();
break;
case AddTableColumn:
plan = new AddTableColumnPlan();
break;
case GetNodePathsPartition:
plan = new GetNodePathsPartitionPlan();
break;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,10 @@ public enum ConfigPhysicalPlanType {
PreCreateTable((short) 850),
RollbackCreateTable((short) 851),
CommitCreateTable((short) 852),
PreDropTable((short) 853),
RollbackPreDropTable((short) 854),
CommitDropTable((short) 855),
AddTableColumn((short) 856),

/** Deprecated types for sync, restored them for upgrade. */
@Deprecated
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.confignode.consensus.request.write.table;

import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil;
import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;

import org.apache.tsfile.utils.ReadWriteIOUtils;

import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;

public class AddTableColumnPlan extends ConfigPhysicalPlan {

private String database;

private String tableName;

private List<TsTableColumnSchema> columnSchemaList;

private boolean isRollback;

public AddTableColumnPlan() {
super(ConfigPhysicalPlanType.AddTableColumn);
}

public AddTableColumnPlan(
String database,
String tableName,
List<TsTableColumnSchema> columnSchemaList,
boolean isRollback) {
super(ConfigPhysicalPlanType.AddTableColumn);
this.database = database;
this.tableName = tableName;
this.columnSchemaList = columnSchemaList;
this.isRollback = isRollback;
}

public String getDatabase() {
return database;
}

public String getTableName() {
return tableName;
}

public List<TsTableColumnSchema> getColumnSchemaList() {
return columnSchemaList;
}

public boolean isRollback() {
return isRollback;
}

@Override
protected void serializeImpl(DataOutputStream stream) throws IOException {
stream.writeShort(getType().getPlanType());

ReadWriteIOUtils.write(database, stream);
ReadWriteIOUtils.write(tableName, stream);
TsTableColumnSchemaUtil.serialize(columnSchemaList, stream);
ReadWriteIOUtils.write(isRollback, stream);
}

@Override
protected void deserializeImpl(ByteBuffer buffer) throws IOException {
this.database = ReadWriteIOUtils.readString(buffer);
this.tableName = ReadWriteIOUtils.readString(buffer);
this.columnSchemaList = TsTableColumnSchemaUtil.deserializeColumnSchemaList(buffer);
this.isRollback = ReadWriteIOUtils.readBool(buffer);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.confignode.consensus.request.write.table;

import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;

import org.apache.tsfile.utils.ReadWriteIOUtils;

import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;

public class CommitDropTablePlan extends ConfigPhysicalPlan {

private String database;

private String tableName;

public CommitDropTablePlan() {
super(ConfigPhysicalPlanType.CommitDropTable);
}

public CommitDropTablePlan(String database, String tableName) {
super(ConfigPhysicalPlanType.CommitDropTable);
this.database = database;
this.tableName = tableName;
}

public String getDatabase() {
return database;
}

public String getTableName() {
return tableName;
}

@Override
protected void serializeImpl(DataOutputStream stream) throws IOException {
stream.writeShort(getType().getPlanType());
ReadWriteIOUtils.write(database, stream);
ReadWriteIOUtils.write(tableName, stream);
}

@Override
protected void deserializeImpl(ByteBuffer buffer) throws IOException {
this.database = ReadWriteIOUtils.readString(buffer);
this.tableName = ReadWriteIOUtils.readString(buffer);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.confignode.consensus.request.write.table;

import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;

import org.apache.tsfile.utils.ReadWriteIOUtils;

import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;

public class PreDropTablePlan extends ConfigPhysicalPlan {

private String database;

private String tableName;

public PreDropTablePlan() {
super(ConfigPhysicalPlanType.PreDropTable);
}

public PreDropTablePlan(String database, String tableName) {
super(ConfigPhysicalPlanType.PreDropTable);
this.database = database;
this.tableName = tableName;
}

public String getDatabase() {
return database;
}

public String getTableName() {
return tableName;
}

@Override
protected void serializeImpl(DataOutputStream stream) throws IOException {
stream.writeShort(getType().getPlanType());
ReadWriteIOUtils.write(database, stream);
ReadWriteIOUtils.write(tableName, stream);
}

@Override
protected void deserializeImpl(ByteBuffer buffer) throws IOException {
this.database = ReadWriteIOUtils.readString(buffer);
this.tableName = ReadWriteIOUtils.readString(buffer);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iotdb.confignode.consensus.request.write.table;

import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;

import org.apache.tsfile.utils.ReadWriteIOUtils;

import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;

public class RollbackDropTablePlan extends ConfigPhysicalPlan {

private String database;

private String tableName;

public RollbackDropTablePlan() {
super(ConfigPhysicalPlanType.RollbackPreDropTable);
}

public RollbackDropTablePlan(String database, String tableName) {
super(ConfigPhysicalPlanType.RollbackPreDropTable);
this.database = database;
this.tableName = tableName;
}

public String getDatabase() {
return database;
}

public String getTableName() {
return tableName;
}

@Override
protected void serializeImpl(DataOutputStream stream) throws IOException {
stream.writeShort(getType().getPlanType());
ReadWriteIOUtils.write(database, stream);
ReadWriteIOUtils.write(tableName, stream);
}

@Override
protected void deserializeImpl(ByteBuffer buffer) throws IOException {
this.database = ReadWriteIOUtils.readString(buffer);
this.tableName = ReadWriteIOUtils.readString(buffer);
}
}