Skip to content

[bug](cloud restore) rewrite table properties and partition info in cloud restore #63696

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
xy720 wants to merge 9 commits into
base: master
Choose a base branch
from
Open

[bug](cloud restore) rewrite table properties and partition info in cloud restore #63696

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
61bb7bd
save
xy720 May 26, 2026
aec6599
save
xy720 May 26, 2026
ccaf3d3
save
xy720 May 26, 2026
8dcf75f
save
xy720 May 26, 2026
05aef72
save
xy720 May 27, 2026
b621e18
save
xy720 May 27, 2026
46f309f
save
xy720 May 27, 2026
fdc39c2
save
xy720 May 27, 2026
13c513e
save
xy720 May 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1162,9 +1162,20 @@ protected void allReplicasCreated() {
if (reserveReplica) {
restoreReplicaAlloc = remotePartitionInfo.getReplicaAllocation(remotePartId);
}
boolean isInMemory = remotePartitionInfo.getIsInMemory(remotePartId);
if (Config.isCloudMode()) {
// In cloud mode, storage_medium, cooldown_time, storage_policy and in_memory
// from the source cluster are not applicable. Reset them to defaults.
remoteDataProperty = new DataProperty(
DataProperty.DEFAULT_STORAGE_MEDIUM,
DataProperty.MAX_COOLDOWN_TIME_MS,
"",
remoteDataProperty.isMutable());
isInMemory = false;
}
localPartitionInfo.addPartition(restoredPart.getId(), false, remoteItem,
remoteDataProperty, restoreReplicaAlloc,
remotePartitionInfo.getIsInMemory(remotePartId),
isInMemory,
remotePartitionInfo.getIsMutable(remotePartId));
}
localTbl.addPartition(restoredPart);
Expand Down Expand Up @@ -1707,9 +1718,20 @@ private void replayCheckAndPrepareMeta() {
if (reserveReplica) {
restoreReplicaAlloc = remotePartitionInfo.getReplicaAllocation(remotePartId);
}
boolean isInMemory = remotePartitionInfo.getIsInMemory(remotePartId);
if (Config.isCloudMode()) {
// In cloud mode, storage_medium, cooldown_time, storage_policy and in_memory
// from the source cluster are not applicable. Reset them to defaults.
remoteDataProperty = new DataProperty(
DataProperty.DEFAULT_STORAGE_MEDIUM,
DataProperty.MAX_COOLDOWN_TIME_MS,
"",
remoteDataProperty.isMutable());
isInMemory = false;
}
localPartitionInfo.addPartition(restorePart.getId(), false, remotePartitionInfo.getItem(remotePartId),
remoteDataProperty, restoreReplicaAlloc,
remotePartitionInfo.getIsInMemory(remotePartId),
isInMemory,
remotePartitionInfo.getIsMutable(remotePartId));
localTbl.addPartition(restorePart);

Expand Down
31 changes: 25 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.SinglePartitionDesc;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TTabletType;
Expand Down Expand Up @@ -427,15 +428,33 @@ public void resetPartitionIdForRestore(
idToStoragePolicy = Maps.newHashMap();

for (Map.Entry<Long, Long> entry : partitionIdMap.entrySet()) {
idToDataProperty.put(entry.getKey(), origIdToDataProperty.get(entry.getValue()));
idToReplicaAllocation.put(entry.getKey(),
restoreReplicaAlloc == null ? origIdToReplicaAllocation.get(entry.getValue())
long newPartId = entry.getKey();
long origPartId = entry.getValue();

if (Config.isCloudMode()) {
// In cloud mode, storage_medium, cooldown_time, and storage_policy are not applicable.
Comment thread
xy720 marked this conversation as resolved.
// Reset DataProperty to default and clear storage policy to avoid carrying over
// source cluster's storage settings that have no meaning in cloud mode.
DataProperty origDataProperty = origIdToDataProperty.get(origPartId);
idToDataProperty.put(newPartId, new DataProperty(
Comment thread
xy720 marked this conversation as resolved.
DataProperty.DEFAULT_STORAGE_MEDIUM,
DataProperty.MAX_COOLDOWN_TIME_MS,
"",
origDataProperty != null ? origDataProperty.isMutable() : true));
idToStoragePolicy.put(newPartId, "");
idToInMemory.put(newPartId, false);
} else {
idToDataProperty.put(newPartId, origIdToDataProperty.get(origPartId));
idToStoragePolicy.put(newPartId, origIdToStoragePolicy.getOrDefault(origPartId, ""));
idToInMemory.put(newPartId, origIdToInMemory.get(origPartId));
}

idToReplicaAllocation.put(newPartId,
restoreReplicaAlloc == null ? origIdToReplicaAllocation.get(origPartId)
: restoreReplicaAlloc);
if (!isSinglePartitioned) {
idToItem.put(entry.getKey(), origIdToItem.get(entry.getValue()));
idToItem.put(newPartId, origIdToItem.get(origPartId));
}
idToInMemory.put(entry.getKey(), origIdToInMemory.get(entry.getValue()));
idToStoragePolicy.put(entry.getKey(), origIdToStoragePolicy.getOrDefault(entry.getValue(), ""));
}
}

Expand Down
12 changes: 12 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import org.apache.doris.analysis.DataSortInfo;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.persist.OperationType;
import org.apache.doris.persist.gson.GsonPostProcessable;
Expand Down Expand Up @@ -194,6 +195,17 @@ public TableProperty buildProperty(short opCode) {
*/
public TableProperty resetPropertiesForRestore(boolean reserveDynamicPartitionEnable, boolean reserveReplica,
ReplicaAllocation replicaAlloc) {
if (Config.isCloudMode()) {
// In cloud mode, rewrite all unsupported or forced properties from the source cluster.
// These properties (e.g., replication_num, replication_allocation, storage_policy,
Comment thread
xy720 marked this conversation as resolved.
// storage_medium, in_memory, etc.) are not applicable in cloud mode. If kept, they would
// cause some critical problems.
PropertyAnalyzer.getInstance().rewriteForceProperties(properties);
Comment thread
xy720 marked this conversation as resolved.
buildInMemory();
buildStorageMedium();
buildStoragePolicy();
buildMinLoadReplicaNum();
}
Comment thread
xy720 marked this conversation as resolved.
// disable dynamic partition
if (properties.containsKey(DynamicPartitionProperty.ENABLE)) {
if (!reserveDynamicPartitionEnable) {
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/cloud/common/util/CloudPropertyAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ public CloudPropertyAnalyzer() {
RewriteProperty.delete("default." + PropertyAnalyzer.PROPERTIES_REPLICATION_NUM),
RewriteProperty.delete("default." + PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION),
RewriteProperty.delete(DynamicPartitionProperty.STORAGE_MEDIUM),
RewriteProperty.delete(DynamicPartitionProperty.STORAGE_POLICY),
RewriteProperty.replace(DynamicPartitionProperty.REPLICATION_NUM,
String.valueOf(ReplicaAllocation.DEFAULT_ALLOCATION.getTotalReplicaNum())),
RewriteProperty.replace(DynamicPartitionProperty.REPLICATION_ALLOCATION,
Expand Down
Loading