From ac95928c32332a9ce4ba38579c55eb18963b3323 Mon Sep 17 00:00:00 2001 From: abc982627271 Date: Mon, 18 Jul 2022 20:24:35 +0800 Subject: [PATCH 1/9] update codes --- .../src/main/java/com/starrocks/lake/ShardDeleter.java | 2 ++ .../main/java/com/starrocks/server/LocalMetastore.java | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/ShardDeleter.java b/fe/fe-core/src/main/java/com/starrocks/lake/ShardDeleter.java index 08220c8bcc870..d982c4e655660 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/ShardDeleter.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/ShardDeleter.java @@ -118,6 +118,8 @@ private void deleteUnusedShard() { shardIds.removeAll(deletedShards); } GlobalStateMgr.getCurrentState().getEditLog().logDeleteUnusedShard(deletedShards); + // for debug + LOG.info("delete shards {} succ", deletedShards); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index c9479410477d1..8b664f92594d9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -3785,7 +3785,7 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti ErrorReport.reportDdlException(ErrorCode.ERR_BAD_TABLE_ERROR, dbTbl.getTbl()); } - if (table.getType() != Table.TableType.OLAP) { + if (!table.isOlapOrLakeTable()) { throw new DdlException("Only support truncate OLAP table"); } @@ -3928,6 +3928,14 @@ private void truncateTableInternal(OlapTable olapTable, List newParti for (Long tabletId : oldTabletIds) { GlobalStateMgr.getCurrentInvertedIndex().deleteTablet(tabletId); } + + // if it is lake table, need to delete shard and drop tablet + if (olapTable.isLakeTable()) { + // for debug + LOG.info("add oldTabletIds {} in truncateTableInternal", oldTabletIds); + stateMgr.getShardManager().getShardDeleter().addUnusedShardId(oldTabletIds); + editLog.logAddUnusedShard(oldTabletIds); + } } public void replayTruncateTable(TruncateTableInfo info) { From a0a9068ea0d40bcdf0952ddb5d05f3970ded1552 Mon Sep 17 00:00:00 2001 From: abc982627271 Date: Tue, 19 Jul 2022 10:43:06 +0800 Subject: [PATCH 2/9] a succ version --- .../main/java/com/starrocks/catalog/OlapTable.java | 12 +++++++++++- .../main/java/com/starrocks/lake/ShardDeleter.java | 2 -- .../java/com/starrocks/server/LocalMetastore.java | 4 +--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java index e9bfe90074b69..ff99f44e885a1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java @@ -49,6 +49,7 @@ import com.starrocks.common.util.PropertyAnalyzer; import com.starrocks.common.util.RangeUtils; import com.starrocks.common.util.Util; +import com.starrocks.lake.LakeTable; import com.starrocks.persist.gson.GsonPostProcessable; import com.starrocks.qe.OriginStatement; import com.starrocks.server.GlobalStateMgr; @@ -1254,10 +1255,19 @@ public void gsonPostProcess() throws IOException { public OlapTable selectiveCopy(Collection reservedPartitions, boolean resetState, IndexExtState extState) { OlapTable copied = new OlapTable(); - if (!DeepCopy.copy(this, copied, OlapTable.class)) { + + TableType type = getType(); + if (type == TableType.LAKE) { + copied = DeepCopy.copyWithGson(this, LakeTable.class); + if (copied == null) { + LOG.warn("failed to copy lake table: " + getName()); + return null; + } + } else if (!DeepCopy.copy(this, copied, OlapTable.class)) { LOG.warn("failed to copy olap table: " + getName()); return null; } + return selectiveCopyInternal(copied, reservedPartitions, resetState, extState); } diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/ShardDeleter.java b/fe/fe-core/src/main/java/com/starrocks/lake/ShardDeleter.java index d982c4e655660..08220c8bcc870 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/ShardDeleter.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/ShardDeleter.java @@ -118,8 +118,6 @@ private void deleteUnusedShard() { shardIds.removeAll(deletedShards); } GlobalStateMgr.getCurrentState().getEditLog().logDeleteUnusedShard(deletedShards); - // for debug - LOG.info("delete shards {} succ", deletedShards); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index 8b664f92594d9..7bab8661be387 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -3786,7 +3786,7 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti } if (!table.isOlapOrLakeTable()) { - throw new DdlException("Only support truncate OLAP table"); + throw new DdlException("Only support truncate OLAP table or LAKE table"); } OlapTable olapTable = (OlapTable) table; @@ -3931,8 +3931,6 @@ private void truncateTableInternal(OlapTable olapTable, List newParti // if it is lake table, need to delete shard and drop tablet if (olapTable.isLakeTable()) { - // for debug - LOG.info("add oldTabletIds {} in truncateTableInternal", oldTabletIds); stateMgr.getShardManager().getShardDeleter().addUnusedShardId(oldTabletIds); editLog.logAddUnusedShard(oldTabletIds); } From 3658ef41c0188eb412041871434bc62fb21bc216 Mon Sep 17 00:00:00 2001 From: abc982627271 Date: Tue, 19 Jul 2022 17:07:23 +0800 Subject: [PATCH 3/9] modify according to comments --- .../java/com/starrocks/lake/LakeTable.java | 23 +++++++++++++++++++ .../com/starrocks/server/LocalMetastore.java | 9 ++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java b/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java index c8908c90d67d8..dedd5b644f716 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java @@ -7,20 +7,25 @@ import com.starrocks.catalog.Column; import com.starrocks.catalog.DistributionInfo; import com.starrocks.catalog.KeysType; +import com.starrocks.catalog.MaterializedIndex; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.PartitionInfo; import com.starrocks.catalog.TableIndexes; import com.starrocks.catalog.TableProperty; import com.starrocks.common.DdlException; +import com.starrocks.common.io.DeepCopy; import com.starrocks.common.io.Text; import com.starrocks.persist.gson.GsonUtils; import com.starrocks.server.GlobalStateMgr; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -34,6 +39,8 @@ */ public class LakeTable extends OlapTable { + private static final Logger LOG = LogManager.getLogger(LakeTable.class); + public LakeTable(long id, String tableName, List baseSchema, KeysType keysType, PartitionInfo partitionInfo, DistributionInfo defaultDistributionInfo, TableIndexes indexes) { super(id, tableName, baseSchema, keysType, partitionInfo, defaultDistributionInfo, @@ -82,6 +89,22 @@ public void setStorageInfo(ShardStorageInfo shardStorageInfo, boolean enableCach .setStorageInfo(new StorageInfo(newShardStorageInfo, new StorageCacheInfo(enableCache, cacheTtlS))); } + public OlapTable selectiveCopy(Collection reservedPartitions, boolean resetState, + MaterializedIndex.IndexExtState extState) { + LakeTable copied = (LakeTable) new OlapTable(); + + TableType type = getType(); + if (type == TableType.LAKE) { + copied = DeepCopy.copyWithGson(this, LakeTable.class); + if (copied == null) { + LOG.warn("failed to copy lake table: " + getName()); + return null; + } + } + + return super.selectiveCopyInternal(copied, reservedPartitions, resetState, extState); + } + @Override public void write(DataOutput out) throws IOException { // write type first diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java index 7bab8661be387..67cbf1bdbaead 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java @@ -3892,7 +3892,7 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti } // replace - truncateTableInternal(olapTable, newPartitions, truncateEntireTable); + truncateTableInternal(olapTable, newPartitions, truncateEntireTable, false); // write edit log TruncateTableInfo info = new TruncateTableInfo(db.getId(), olapTable.getId(), newPartitions, @@ -3906,7 +3906,8 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti tblRef.getName().toSql(), tblRef.getPartitionNames()); } - private void truncateTableInternal(OlapTable olapTable, List newPartitions, boolean isEntireTable) { + private void truncateTableInternal(OlapTable olapTable, List newPartitions, + boolean isEntireTable, boolean isReplay) { // use new partitions to replace the old ones. Set oldTabletIds = Sets.newHashSet(); for (Partition newPartition : newPartitions) { @@ -3930,7 +3931,7 @@ private void truncateTableInternal(OlapTable olapTable, List newParti } // if it is lake table, need to delete shard and drop tablet - if (olapTable.isLakeTable()) { + if (olapTable.isLakeTable() && !isReplay) { stateMgr.getShardManager().getShardDeleter().addUnusedShardId(oldTabletIds); editLog.logAddUnusedShard(oldTabletIds); } @@ -3941,7 +3942,7 @@ public void replayTruncateTable(TruncateTableInfo info) { db.writeLock(); try { OlapTable olapTable = (OlapTable) db.getTable(info.getTblId()); - truncateTableInternal(olapTable, info.getPartitions(), info.isEntireTable()); + truncateTableInternal(olapTable, info.getPartitions(), info.isEntireTable(), true); if (!GlobalStateMgr.isCheckpointThread()) { // add tablet to inverted index From f35550c7392ece4ff475001a37dffdaa1fb3400d Mon Sep 17 00:00:00 2001 From: abc982627271 Date: Tue, 19 Jul 2022 17:13:38 +0800 Subject: [PATCH 4/9] upt --- .../main/java/com/starrocks/catalog/OlapTable.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java index ff99f44e885a1..dac1134a9d958 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java @@ -49,7 +49,6 @@ import com.starrocks.common.util.PropertyAnalyzer; import com.starrocks.common.util.RangeUtils; import com.starrocks.common.util.Util; -import com.starrocks.lake.LakeTable; import com.starrocks.persist.gson.GsonPostProcessable; import com.starrocks.qe.OriginStatement; import com.starrocks.server.GlobalStateMgr; @@ -1255,15 +1254,7 @@ public void gsonPostProcess() throws IOException { public OlapTable selectiveCopy(Collection reservedPartitions, boolean resetState, IndexExtState extState) { OlapTable copied = new OlapTable(); - - TableType type = getType(); - if (type == TableType.LAKE) { - copied = DeepCopy.copyWithGson(this, LakeTable.class); - if (copied == null) { - LOG.warn("failed to copy lake table: " + getName()); - return null; - } - } else if (!DeepCopy.copy(this, copied, OlapTable.class)) { + if (!DeepCopy.copy(this, copied, OlapTable.class)) { LOG.warn("failed to copy olap table: " + getName()); return null; } From 06fa45682ca182a51a0a5ae6feff86887fb887e7 Mon Sep 17 00:00:00 2001 From: abc982627271 Date: Tue, 19 Jul 2022 17:14:49 +0800 Subject: [PATCH 5/9] remove useless blank line --- fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java index dac1134a9d958..e9bfe90074b69 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java @@ -1258,7 +1258,6 @@ public OlapTable selectiveCopy(Collection reservedPartitions, boolean re LOG.warn("failed to copy olap table: " + getName()); return null; } - return selectiveCopyInternal(copied, reservedPartitions, resetState, extState); } From 146ee6944edbdcdc8b8f8272f05c7a6cdf909e78 Mon Sep 17 00:00:00 2001 From: abc982627271 Date: Tue, 19 Jul 2022 17:57:12 +0800 Subject: [PATCH 6/9] update codes --- .../main/java/com/starrocks/lake/LakeTable.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java b/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java index dedd5b644f716..e7e38933132a5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java @@ -88,21 +88,19 @@ public void setStorageInfo(ShardStorageInfo shardStorageInfo, boolean enableCach tableProperty .setStorageInfo(new StorageInfo(newShardStorageInfo, new StorageCacheInfo(enableCache, cacheTtlS))); } - + + @Override public OlapTable selectiveCopy(Collection reservedPartitions, boolean resetState, MaterializedIndex.IndexExtState extState) { LakeTable copied = (LakeTable) new OlapTable(); TableType type = getType(); - if (type == TableType.LAKE) { - copied = DeepCopy.copyWithGson(this, LakeTable.class); - if (copied == null) { - LOG.warn("failed to copy lake table: " + getName()); - return null; - } + copied = DeepCopy.copyWithGson(this, LakeTable.class); + if (copied == null) { + LOG.warn("failed to copy lake table: " + getName()); + return null; } - - return super.selectiveCopyInternal(copied, reservedPartitions, resetState, extState); + return selectiveCopyInternal(copied, reservedPartitions, resetState, extState); } @Override From 222bb0add952881883d51013f994a1c2c4fbcfec Mon Sep 17 00:00:00 2001 From: abc982627271 Date: Tue, 19 Jul 2022 17:59:09 +0800 Subject: [PATCH 7/9] xx --- fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java b/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java index e7e38933132a5..bea6480cf423f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java @@ -88,13 +88,11 @@ public void setStorageInfo(ShardStorageInfo shardStorageInfo, boolean enableCach tableProperty .setStorageInfo(new StorageInfo(newShardStorageInfo, new StorageCacheInfo(enableCache, cacheTtlS))); } - + @Override public OlapTable selectiveCopy(Collection reservedPartitions, boolean resetState, MaterializedIndex.IndexExtState extState) { LakeTable copied = (LakeTable) new OlapTable(); - - TableType type = getType(); copied = DeepCopy.copyWithGson(this, LakeTable.class); if (copied == null) { LOG.warn("failed to copy lake table: " + getName()); From 7dcfe5d3f89baade05688355dc863f18f328a3c2 Mon Sep 17 00:00:00 2001 From: abc982627271 Date: Tue, 19 Jul 2022 19:30:29 +0800 Subject: [PATCH 8/9] fix bug --- fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java b/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java index bea6480cf423f..146f4a2e1bfb5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java @@ -92,7 +92,7 @@ public void setStorageInfo(ShardStorageInfo shardStorageInfo, boolean enableCach @Override public OlapTable selectiveCopy(Collection reservedPartitions, boolean resetState, MaterializedIndex.IndexExtState extState) { - LakeTable copied = (LakeTable) new OlapTable(); + OlapTable copied = new OlapTable(); copied = DeepCopy.copyWithGson(this, LakeTable.class); if (copied == null) { LOG.warn("failed to copy lake table: " + getName()); From 448e3ba3e4dc427c07fde34a8f99e26d8b4c7bca Mon Sep 17 00:00:00 2001 From: abc982627271 Date: Tue, 19 Jul 2022 19:33:26 +0800 Subject: [PATCH 9/9] upt --- fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java b/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java index 146f4a2e1bfb5..e3239f5bb3be5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/lake/LakeTable.java @@ -92,10 +92,9 @@ public void setStorageInfo(ShardStorageInfo shardStorageInfo, boolean enableCach @Override public OlapTable selectiveCopy(Collection reservedPartitions, boolean resetState, MaterializedIndex.IndexExtState extState) { - OlapTable copied = new OlapTable(); - copied = DeepCopy.copyWithGson(this, LakeTable.class); + LakeTable copied = DeepCopy.copyWithGson(this, LakeTable.class); if (copied == null) { - LOG.warn("failed to copy lake table: " + getName()); + LOG.warn("failed to copy lake table: {}", getName()); return null; } return selectiveCopyInternal(copied, reservedPartitions, resetState, extState);