From bd8fdd1e0d2de548565cde75fe4d11819b138872 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 10:42:40 +0800 Subject: [PATCH 001/213] Update DeviceAttributeStore.java --- .../schemaregion/attribute/DeviceAttributeStore.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java index 471f3be8c61b5..e3f39528a9555 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java @@ -185,18 +185,18 @@ public Map alterAttribute( } @Override - public String getAttribute(int pointer, String name) { + public String getAttribute(final int pointer, final String name) { return deviceAttributeList.get(pointer).get(name); } - private void serialize(OutputStream outputStream) throws IOException { + private void serialize(final OutputStream outputStream) throws IOException { ReadWriteIOUtils.write(deviceAttributeList.size(), outputStream); - for (Map attributeMap : deviceAttributeList) { + for (final Map attributeMap : deviceAttributeList) { ReadWriteIOUtils.write(attributeMap, outputStream); } } - private void deserialize(InputStream inputStream) throws IOException { + private void deserialize(final InputStream inputStream) throws IOException { int size = ReadWriteIOUtils.readInt(inputStream); for (int i = 0; i < size; i++) { deviceAttributeList.add(ReadWriteIOUtils.readMap(inputStream)); From 83570706f3279a18a881e16738429f0926e107f5 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:02:23 +0800 Subject: [PATCH 002/213] init --- .../update/DeviceAttributeRemoteUpdater.java | 26 +++++++++++++++++++ .../attribute/update/UpdateContainer.java | 24 +++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java new file mode 100644 index 0000000000000..0b6ab1226298c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; + +import java.util.concurrent.atomic.AtomicLong; + +public class DeviceAttributeRemoteUpdater { + private final AtomicLong version = new AtomicLong(0); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java new file mode 100644 index 0000000000000..064ff6c7be5dd --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; + +public interface UpdateContainer { + +} From 479dc17594869669314391d7053ab9769815a025 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:06:45 +0800 Subject: [PATCH 003/213] little enrich --- .../update/DeviceAttributeRemoteUpdater.java | 7 +++ .../update/UpdateClearContainer.java | 52 +++++++++++++++++++ .../attribute/update/UpdateContainer.java | 19 ++++++- .../update/UpdateDetailContainer.java | 52 +++++++++++++++++++ 4 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 0b6ab1226298c..12497640ffbf6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -19,8 +19,15 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; +import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; + import java.util.concurrent.atomic.AtomicLong; public class DeviceAttributeRemoteUpdater { private final AtomicLong version = new AtomicLong(0); + private final MemSchemaRegionStatistics regionStatistics; + + public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatistics) { + this.regionStatistics = regionStatistics; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java new file mode 100644 index 0000000000000..efbcc49d90da8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; + +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.Pair; + +import java.io.FileInputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Map; + +public class UpdateClearContainer implements UpdateContainer { + @Override + public int updateAttribute( + final IDeviceID deviceId, final Map updatedAttributes) { + return 0; + } + + @Override + public ByteBuffer getUpdateBuffer() { + return null; + } + + @Override + public Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer) { + return null; + } + + @Override + public void serialize(final OutputStream outputstream) {} + + @Override + public void deserialize(final FileInputStream fileInputStream) {} +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 064ff6c7be5dd..0f8df202e9782 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -19,6 +19,23 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.Pair; + +import java.io.FileInputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Map; + public interface UpdateContainer { - + + int updateAttribute(final IDeviceID deviceId, final Map updatedAttributes); + + ByteBuffer getUpdateBuffer(); + + Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer); + + void serialize(final OutputStream outputstream); + + void deserialize(final FileInputStream fileInputStream); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java new file mode 100644 index 0000000000000..d6e31772942c1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; + +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.Pair; + +import java.io.FileInputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Map; + +public class UpdateDetailContainer implements UpdateContainer { + @Override + public int updateAttribute( + final IDeviceID deviceId, final Map updatedAttributes) { + return 0; + } + + @Override + public ByteBuffer getUpdateBuffer() { + return null; + } + + @Override + public Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer) { + return null; + } + + @Override + public void serialize(final OutputStream outputstream) {} + + @Override + public void deserialize(final FileInputStream fileInputStream) {} +} From 4f6bfdd09420a9d35fe2981a38dd6f53fe49b641 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:09:56 +0800 Subject: [PATCH 004/213] little enrich --- .../schemaregion/attribute/update/UpdateClearContainer.java | 6 ++++++ .../attribute/update/UpdateDetailContainer.java | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index efbcc49d90da8..bbaf34c752c9b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -25,9 +25,15 @@ import java.io.FileInputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; public class UpdateClearContainer implements UpdateContainer { + + private Set tableNames = Collections.newSetFromMap(new ConcurrentHashMap<>()); + @Override public int updateAttribute( final IDeviceID deviceId, final Map updatedAttributes) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index d6e31772942c1..b13c79f733d10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -26,8 +26,12 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; public class UpdateDetailContainer implements UpdateContainer { + + private Map>> updateMap = new ConcurrentHashMap<>(); + @Override public int updateAttribute( final IDeviceID deviceId, final Map updatedAttributes) { From 74119a764f88b90c429e8b4649eb3fc4ae23daf5 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:13:34 +0800 Subject: [PATCH 005/213] little enrich too --- .../attribute/update/UpdateClearContainer.java | 18 +++++++++++++++--- .../attribute/update/UpdateContainer.java | 5 +++-- .../update/UpdateDetailContainer.java | 3 ++- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index bbaf34c752c9b..05e2adc8232de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -21,8 +21,10 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.FileInputStream; +import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Collections; @@ -32,7 +34,7 @@ public class UpdateClearContainer implements UpdateContainer { - private Set tableNames = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Set tableNames = Collections.newSetFromMap(new ConcurrentHashMap<>()); @Override public int updateAttribute( @@ -51,8 +53,18 @@ public Pair updateSelfByCommitBuffer(final ByteBuffer commitBu } @Override - public void serialize(final OutputStream outputstream) {} + public void serialize(final OutputStream outputstream) throws IOException { + ReadWriteIOUtils.write(tableNames.size(), outputstream); + for (final String tableName : tableNames) { + ReadWriteIOUtils.write(tableName, outputstream); + } + } @Override - public void deserialize(final FileInputStream fileInputStream) {} + public void deserialize(final FileInputStream fileInputStream) throws IOException { + final int size = ReadWriteIOUtils.readInt(fileInputStream); + for (int i = 0; i < size; ++i) { + tableNames.add(ReadWriteIOUtils.readString(fileInputStream)); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 0f8df202e9782..73d7b7bc92824 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -23,6 +23,7 @@ import org.apache.tsfile.utils.Pair; import java.io.FileInputStream; +import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Map; @@ -35,7 +36,7 @@ public interface UpdateContainer { Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer); - void serialize(final OutputStream outputstream); + void serialize(final OutputStream outputstream) throws IOException; - void deserialize(final FileInputStream fileInputStream); + void deserialize(final FileInputStream fileInputStream) throws IOException; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index b13c79f733d10..85b522f0f4798 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -30,7 +30,8 @@ public class UpdateDetailContainer implements UpdateContainer { - private Map>> updateMap = new ConcurrentHashMap<>(); + private final Map>> updateMap = + new ConcurrentHashMap<>(); @Override public int updateAttribute( From 88fbf5dcbab2a337691484675813a6bd0f4bae7d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:23:01 +0800 Subject: [PATCH 006/213] partial snapshot --- .../update/DeviceAttributeRemoteUpdater.java | 12 +++++ .../impl/SchemaRegionMemoryImpl.java | 49 +++++++++++++------ 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 12497640ffbf6..e8ea4e8129c2c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -21,6 +21,8 @@ import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; +import java.io.File; +import java.io.IOException; import java.util.concurrent.atomic.AtomicLong; public class DeviceAttributeRemoteUpdater { @@ -30,4 +32,14 @@ public class DeviceAttributeRemoteUpdater { public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatistics) { this.regionStatistics = regionStatistics; } + + public synchronized boolean createSnapshot(final File targetDir) { + // TODO + return true; + } + + public void loadFromSnapshot(final File snapshotDir, final String sgSchemaDirPath) + throws IOException { + // TODO + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 9530e2eb7755b..7ef7b58a3ffab 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -68,6 +68,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionUtils; import org.apache.iotdb.db.schemaengine.schemaregion.attribute.DeviceAttributeStore; import org.apache.iotdb.db.schemaengine.schemaregion.attribute.IDeviceAttributeStore; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeRemoteUpdater; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.FakeCRC32Deserializer; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.FakeCRC32Serializer; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.SchemaLogReader; @@ -191,6 +192,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { private MTreeBelowSGMemoryImpl mtree; private TagManager tagManager; private IDeviceAttributeStore deviceAttributeStore; + private DeviceAttributeRemoteUpdater deviceAttributeRemoteUpdater; // region Interfaces and Implementation of initialization、snapshot、recover and clear public SchemaRegionMemoryImpl(ISchemaRegionParams schemaRegionParams) throws MetadataException { @@ -241,6 +243,7 @@ public synchronized void init() throws MetadataException { isRecovering = true; deviceAttributeStore = new DeviceAttributeStore(regionStatistics); + deviceAttributeRemoteUpdater = new DeviceAttributeRemoteUpdater(regionStatistics); tagManager = new TagManager(schemaRegionDirPath, regionStatistics); mtree = new MTreeBelowSGMemoryImpl( @@ -461,7 +464,7 @@ public synchronized void deleteSchemaRegion() throws MetadataException { // currently, this method is only used for cluster-ratis mode @Override - public synchronized boolean createSnapshot(File snapshotDir) { + public synchronized boolean createSnapshot(final File snapshotDir) { if (!initialized) { logger.warn( "Failed to create snapshot of schemaRegion {}, because the schemaRegion has not been initialized.", @@ -470,34 +473,52 @@ public synchronized boolean createSnapshot(File snapshotDir) { } logger.info("Start create snapshot of schemaRegion {}", schemaRegionId); boolean isSuccess; - long startTime = System.currentTimeMillis(); + boolean currentResult; + final long startTime = System.currentTimeMillis(); - long mtreeSnapshotStartTime = System.currentTimeMillis(); + long snapshotStartTime = System.currentTimeMillis(); isSuccess = mtree.createSnapshot(snapshotDir); logger.info( - "MTree snapshot creation of schemaRegion {} costs {}ms.", + "MTree snapshot creation of schemaRegion {} costs {}ms. Status: {}", + schemaRegionId, + System.currentTimeMillis() - snapshotStartTime, + isSuccess); + + snapshotStartTime = System.currentTimeMillis(); + currentResult = tagManager.createSnapshot(snapshotDir); + isSuccess = isSuccess && currentResult; + logger.info( + "Tag snapshot creation of schemaRegion {} costs {}ms. Status: {}", schemaRegionId, - System.currentTimeMillis() - mtreeSnapshotStartTime); + System.currentTimeMillis() - snapshotStartTime, + currentResult); - long tagSnapshotStartTime = System.currentTimeMillis(); - isSuccess = isSuccess && tagManager.createSnapshot(snapshotDir); + snapshotStartTime = System.currentTimeMillis(); + currentResult = deviceAttributeStore.createSnapshot(snapshotDir); + isSuccess = isSuccess && currentResult; logger.info( - "Tag snapshot creation of schemaRegion {} costs {}ms.", + "Device attribute snapshot creation of schemaRegion {} costs {}ms. Status: {}", schemaRegionId, - System.currentTimeMillis() - tagSnapshotStartTime); + System.currentTimeMillis() - snapshotStartTime, + currentResult); - long deviceAttributeSnapshotStartTime = System.currentTimeMillis(); - isSuccess = isSuccess && deviceAttributeStore.createSnapshot(snapshotDir); + snapshotStartTime = System.currentTimeMillis(); + currentResult = deviceAttributeRemoteUpdater.createSnapshot(snapshotDir); + isSuccess = isSuccess && currentResult; logger.info( - "Device attribute snapshot creation of schemaRegion {} costs {}ms", + "Device attribute remote updater snapshot creation of schemaRegion {} costs {}ms. Status: {}", schemaRegionId, - System.currentTimeMillis() - deviceAttributeSnapshotStartTime); + System.currentTimeMillis() - snapshotStartTime, + currentResult); logger.info( "Snapshot creation of schemaRegion {} costs {}ms.", schemaRegionId, System.currentTimeMillis() - startTime); - logger.info("Successfully create snapshot of schemaRegion {}", schemaRegionId); + + if (isSuccess) { + logger.info("Successfully create snapshot of schemaRegion {}", schemaRegionId); + } return isSuccess; } From 83e41c634dcd4258c9e58c05df18f5f15f0161f2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:25:49 +0800 Subject: [PATCH 007/213] Update SchemaRegionMemoryImpl.java --- .../impl/SchemaRegionMemoryImpl.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 7ef7b58a3ffab..d35c6543ae04c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -536,23 +536,31 @@ public void loadSnapshot(final File latestSnapshotRootDir) { isRecovering = true; - final long deviceAttributeSnapshotStartTime = System.currentTimeMillis(); + long snapshotStartTime = System.currentTimeMillis(); deviceAttributeStore = new DeviceAttributeStore(regionStatistics); deviceAttributeStore.loadFromSnapshot(latestSnapshotRootDir, schemaRegionDirPath); logger.info( "Device attribute snapshot loading of schemaRegion {} costs {}ms.", schemaRegionId, - System.currentTimeMillis() - deviceAttributeSnapshotStartTime); + System.currentTimeMillis() - snapshotStartTime); - final long tagSnapshotStartTime = System.currentTimeMillis(); + snapshotStartTime = System.currentTimeMillis(); + deviceAttributeRemoteUpdater = new DeviceAttributeRemoteUpdater(regionStatistics); + deviceAttributeRemoteUpdater.loadFromSnapshot(latestSnapshotRootDir, schemaRegionDirPath); + logger.info( + "Device attribute remote updater snapshot loading of schemaRegion {} costs {}ms.", + schemaRegionId, + System.currentTimeMillis() - snapshotStartTime); + + snapshotStartTime = System.currentTimeMillis(); tagManager = TagManager.loadFromSnapshot(latestSnapshotRootDir, schemaRegionDirPath, regionStatistics); logger.info( "Tag snapshot loading of schemaRegion {} costs {}ms.", schemaRegionId, - System.currentTimeMillis() - tagSnapshotStartTime); + System.currentTimeMillis() - snapshotStartTime); - final long mTreeSnapshotStartTime = System.currentTimeMillis(); + snapshotStartTime = System.currentTimeMillis(); mtree = MTreeBelowSGMemoryImpl.loadFromSnapshot( latestSnapshotRootDir, @@ -588,7 +596,7 @@ public void loadSnapshot(final File latestSnapshotRootDir) { logger.info( "MTree snapshot loading of schemaRegion {} costs {}ms.", schemaRegionId, - System.currentTimeMillis() - mTreeSnapshotStartTime); + System.currentTimeMillis() - snapshotStartTime); isRecovering = false; initialized = true; From 19af79a46874e0d36660f4e9e9203330b437a0d6 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:42:29 +0800 Subject: [PATCH 008/213] Update DeviceAttributeRemoteUpdater.java --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index e8ea4e8129c2c..300ab131082f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -19,13 +19,18 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; import java.io.File; import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; public class DeviceAttributeRemoteUpdater { + + private ConcurrentMap attributeUpdateMap = new ConcurrentHashMap<>(); private final AtomicLong version = new AtomicLong(0); private final MemSchemaRegionStatistics regionStatistics; From 16ecc276744159b862b35b5814dde8cb2d29ee1c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:46:53 +0800 Subject: [PATCH 009/213] Update DeviceAttributeRemoteUpdater.java --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 300ab131082f1..da4d6e6f673c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -22,14 +22,19 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; +import org.apache.tsfile.utils.Pair; + import java.io.File; import java.io.IOException; +import java.util.Collections; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; public class DeviceAttributeRemoteUpdater { - + private Set> targetDataNodeLocations = + Collections.newSetFromMap(new ConcurrentHashMap<>()); private ConcurrentMap attributeUpdateMap = new ConcurrentHashMap<>(); private final AtomicLong version = new AtomicLong(0); private final MemSchemaRegionStatistics regionStatistics; From fb225c1d3749bb0cf828b156bd414e0b1e84ab78 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:18:21 +0800 Subject: [PATCH 010/213] partial logic --- .../attribute/DeviceAttributeStore.java | 2 +- .../update/DeviceAttributeRemoteUpdater.java | 116 ++++++++++++++++-- .../update/UpdateClearContainer.java | 4 +- .../attribute/update/UpdateContainer.java | 4 +- .../update/UpdateDetailContainer.java | 4 +- .../impl/SchemaRegionMemoryImpl.java | 2 +- .../iotdb/commons/schema/SchemaConstant.java | 6 +- .../utils/ThriftCommonsSerDeUtils.java | 28 ++++- 8 files changed, 144 insertions(+), 22 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java index e3f39528a9555..7ac2618e114d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java @@ -96,7 +96,7 @@ public synchronized boolean createSnapshot(final File targetDir) { return true; } catch (final IOException e) { - logger.error("Failed to create mtree snapshot due to {}", e.getMessage(), e); + logger.error("Failed to create device attribute snapshot due to {}", e.getMessage(), e); FileUtils.deleteFileIfExist(snapshot); return false; } finally { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index da4d6e6f673c6..43bee14d3d06b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -20,22 +20,39 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.file.SystemFileFactory; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; import java.util.Collections; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; public class DeviceAttributeRemoteUpdater { - private Set> targetDataNodeLocations = + private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeRemoteUpdater.class); + + private final Set> targetDataNodeLocations = Collections.newSetFromMap(new ConcurrentHashMap<>()); - private ConcurrentMap attributeUpdateMap = new ConcurrentHashMap<>(); + private final ConcurrentMap attributeUpdateMap = + new ConcurrentHashMap<>(); private final AtomicLong version = new AtomicLong(0); private final MemSchemaRegionStatistics regionStatistics; @@ -44,12 +61,97 @@ public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatis } public synchronized boolean createSnapshot(final File targetDir) { - // TODO - return true; + final File snapshotTmp = + SystemFileFactory.INSTANCE.getFile( + targetDir, SchemaConstant.DEVICE_ATTRIBUTE_REMOTE_UPDATER_SNAPSHOT_TMP); + final File snapshot = + SystemFileFactory.INSTANCE.getFile( + targetDir, SchemaConstant.DEVICE_ATTRIBUTE_REMOTE_UPDATER_SNAPSHOT); + + try { + final FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); + final BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream); + try { + serialize(outputStream); + } finally { + outputStream.flush(); + fileOutputStream.getFD().sync(); + outputStream.close(); + } + if (snapshot.exists() && !FileUtils.deleteFileIfExist(snapshot)) { + logger.error( + "Failed to delete old snapshot {} while creating device attribute remote updater snapshot.", + snapshot.getName()); + return false; + } + if (!snapshotTmp.renameTo(snapshot)) { + logger.error( + "Failed to rename {} to {} while creating device attribute remote updater snapshot.", + snapshotTmp.getName(), + snapshot.getName()); + FileUtils.deleteFileIfExist(snapshot); + return false; + } + + return true; + } catch (final IOException e) { + logger.error( + "Failed to create device attribute remote updater snapshot due to {}", e.getMessage(), e); + FileUtils.deleteFileIfExist(snapshot); + return false; + } finally { + FileUtils.deleteFileIfExist(snapshotTmp); + } + } + + private void serialize(final OutputStream outputStream) throws IOException { + ReadWriteIOUtils.write(targetDataNodeLocations.size(), outputStream); + for (final Pair targetDataNodeLocation : targetDataNodeLocations) { + ThriftCommonsSerDeUtils.serializeTEndPoint(targetDataNodeLocation.getLeft(), outputStream); + ReadWriteIOUtils.write(targetDataNodeLocation.getRight(), outputStream); + } + + ReadWriteIOUtils.write(attributeUpdateMap.size(), outputStream); + for (final Map.Entry entry : attributeUpdateMap.entrySet()) { + ThriftCommonsSerDeUtils.serializeTEndPoint(entry.getKey(), outputStream); + ReadWriteIOUtils.write(entry.getValue() instanceof UpdateDetailContainer, outputStream); + entry.getValue().serialize(outputStream); + } + + ReadWriteIOUtils.write(version.get(), outputStream); } - public void loadFromSnapshot(final File snapshotDir, final String sgSchemaDirPath) - throws IOException { - // TODO + public void loadFromSnapshot(final File snapshotDir) throws IOException { + final File snapshot = + SystemFileFactory.INSTANCE.getFile( + snapshotDir, SchemaConstant.DEVICE_ATTRIBUTE_REMOTE_UPDATER_SNAPSHOT); + try (final BufferedInputStream inputStream = + new BufferedInputStream(Files.newInputStream(snapshot.toPath()))) { + deserialize(inputStream); + } catch (final Exception e) { + logger.warn( + "Load device attribute remote updater snapshot from {} failed, continue...", snapshotDir); + } + } + + private void deserialize(final InputStream inputStream) throws IOException { + int size = ReadWriteIOUtils.readInt(inputStream); + for (int i = 0; i < size; i++) { + targetDataNodeLocations.add( + new Pair<>( + ThriftCommonsSerDeUtils.deserializeTEndPoint(inputStream), + ReadWriteIOUtils.readInt(inputStream))); + } + + size = ReadWriteIOUtils.readInt(inputStream); + for (int i = 0; i < size; ++i) { + final TEndPoint endPoint = ThriftCommonsSerDeUtils.deserializeTEndPoint(inputStream); + final UpdateContainer container = + ReadWriteIOUtils.readBool(inputStream) + ? new UpdateDetailContainer() + : new UpdateClearContainer(); + container.deserialize(inputStream); + attributeUpdateMap.put(endPoint, container); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 05e2adc8232de..f31d09e4e0c7b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -23,8 +23,8 @@ import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; -import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Collections; @@ -61,7 +61,7 @@ public void serialize(final OutputStream outputstream) throws IOException { } @Override - public void deserialize(final FileInputStream fileInputStream) throws IOException { + public void deserialize(final InputStream fileInputStream) throws IOException { final int size = ReadWriteIOUtils.readInt(fileInputStream); for (int i = 0; i < size; ++i) { tableNames.add(ReadWriteIOUtils.readString(fileInputStream)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 73d7b7bc92824..0aa52336d7d05 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -22,8 +22,8 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Pair; -import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Map; @@ -38,5 +38,5 @@ public interface UpdateContainer { void serialize(final OutputStream outputstream) throws IOException; - void deserialize(final FileInputStream fileInputStream) throws IOException; + void deserialize(final InputStream fileInputStream) throws IOException; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 85b522f0f4798..265d6c181244c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -22,7 +22,7 @@ import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Pair; -import java.io.FileInputStream; +import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Map; @@ -53,5 +53,5 @@ public Pair updateSelfByCommitBuffer(final ByteBuffer commitBu public void serialize(final OutputStream outputstream) {} @Override - public void deserialize(final FileInputStream fileInputStream) {} + public void deserialize(final InputStream fileInputStream) {} } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index d35c6543ae04c..615f09d0fc74a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -546,7 +546,7 @@ public void loadSnapshot(final File latestSnapshotRootDir) { snapshotStartTime = System.currentTimeMillis(); deviceAttributeRemoteUpdater = new DeviceAttributeRemoteUpdater(regionStatistics); - deviceAttributeRemoteUpdater.loadFromSnapshot(latestSnapshotRootDir, schemaRegionDirPath); + deviceAttributeRemoteUpdater.loadFromSnapshot(latestSnapshotRootDir); logger.info( "Device attribute remote updater snapshot loading of schemaRegion {} costs {}ms.", schemaRegionId, diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/SchemaConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/SchemaConstant.java index d8e2d6a33ff7f..8497da8f8b5a4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/SchemaConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/SchemaConstant.java @@ -59,9 +59,13 @@ private SchemaConstant() { public static final String MTREE_SNAPSHOT_TMP = "mtree.snapshot.tmp"; public static final String DEVICE_ATTRIBUTE_SNAPSHOT = "device_attribute.snapshot"; - public static final String DEVICE_ATTRIBUTE_SNAPSHOT_TMP = "device_attribute.snapshot.tmp"; + public static final String DEVICE_ATTRIBUTE_REMOTE_UPDATER_SNAPSHOT = + "device_attribute_remote_updater.snapshot"; + public static final String DEVICE_ATTRIBUTE_REMOTE_UPDATER_SNAPSHOT_TMP = + "device_attribute_remote_updater.snapshot.tmp"; + public static final String SYSTEM_DATABASE = "root.__system"; public static final String[] ALL_RESULT_NODES = new String[] {"root", "**"}; public static final PartialPath ALL_MATCH_PATTERN = new PartialPath(ALL_RESULT_NODES); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/ThriftCommonsSerDeUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/ThriftCommonsSerDeUtils.java index 5d336febbde61..aa6a8d09d8290 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/ThriftCommonsSerDeUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/ThriftCommonsSerDeUtils.java @@ -41,6 +41,8 @@ import org.apache.thrift.transport.TTransportException; import java.io.DataOutputStream; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; import static org.apache.iotdb.rpc.TConfigurationConst.defaultTConfiguration; @@ -52,10 +54,9 @@ private ThriftCommonsSerDeUtils() { // Empty constructor } - private static TBinaryProtocol generateWriteProtocol(DataOutputStream stream) + private static TBinaryProtocol generateWriteProtocol(final OutputStream stream) throws TTransportException { - TTransport transport = new TIOStreamTransport(stream); - return new TBinaryProtocol(transport); + return new TBinaryProtocol(new TIOStreamTransport(stream)); } private static TBinaryProtocol generateWriteProtocol(ByteBuffer buffer) @@ -64,21 +65,36 @@ private static TBinaryProtocol generateWriteProtocol(ByteBuffer buffer) return new TBinaryProtocol(transport); } + private static TBinaryProtocol generateReadProtocol(final InputStream stream) + throws TTransportException { + return new TBinaryProtocol(new TIOStreamTransport(stream)); + } + private static TBinaryProtocol generateReadProtocol(ByteBuffer buffer) throws TTransportException { TTransport transport = generateTByteBuffer(buffer); return new TBinaryProtocol(transport); } - public static void serializeTEndPoint(TEndPoint endPoint, DataOutputStream stream) { + public static void serializeTEndPoint(final TEndPoint endPoint, final OutputStream stream) { try { endPoint.write(generateWriteProtocol(stream)); - } catch (TException e) { + } catch (final TException e) { throw new ThriftSerDeException("Write TEndPoint failed: ", e); } } - public static TEndPoint deserializeTEndPoint(ByteBuffer buffer) { + public static TEndPoint deserializeTEndPoint(final InputStream stream) { + final TEndPoint endPoint = new TEndPoint(); + try { + endPoint.read(generateReadProtocol(stream)); + } catch (final TException e) { + throw new ThriftSerDeException("Read TEndPoint failed: ", e); + } + return endPoint; + } + + public static TEndPoint deserializeTEndPoint(final ByteBuffer buffer) { TEndPoint endPoint = new TEndPoint(); try { endPoint.read(generateReadProtocol(buffer)); From 6f11dc4c9ed49f37193a6eb79b043cb9eedfee7b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:24:29 +0800 Subject: [PATCH 011/213] Update DeviceAttributeRemoteUpdater.java --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 43bee14d3d06b..8d62fef162376 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -153,5 +153,7 @@ private void deserialize(final InputStream inputStream) throws IOException { container.deserialize(inputStream); attributeUpdateMap.put(endPoint, container); } + + version.set(ReadWriteIOUtils.readLong(inputStream)); } } From a1c32472b81d0aff0a5bce14e245870c26206a80 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:33:48 +0800 Subject: [PATCH 012/213] partial prepare --- .../rescon/MemSchemaEngineStatistics.java | 6 ++++-- .../rescon/MemSchemaRegionStatistics.java | 4 ++-- .../update/DeviceAttributeRemoteUpdater.java | 11 +++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaEngineStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaEngineStatistics.java index 9bc847634dd68..51dc16a781875 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaEngineStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaEngineStatistics.java @@ -74,7 +74,7 @@ public long getMemoryUsage() { return memoryUsage.get(); } - public void requestMemory(long size) { + public boolean requestMemory(final long size) { memoryUsage.addAndGet(size); if (memoryUsage.get() >= memoryCapacity) { synchronized (allowToCreateNewSeriesLock) { @@ -87,10 +87,12 @@ public void requestMemory(long size) { allowToCreateNewSeries = false; } } + return false; } + return true; } - public void releaseMemory(long size) { + public void releaseMemory(final long size) { memoryUsage.addAndGet(-size); if (memoryUsage.get() < memoryCapacity) { synchronized (allowToCreateNewSeriesLock) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java index eaec23ebcdc40..19b0291a61098 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java @@ -51,9 +51,9 @@ public boolean isAllowToCreateNewSeries() { return schemaEngineStatistics.isAllowToCreateNewSeries(); } - public void requestMemory(final long size) { + public boolean requestMemory(final long size) { memoryUsage.addAndGet(size); - schemaEngineStatistics.requestMemory(size); + return schemaEngineStatistics.requestMemory(size); } public void releaseMemory(final long size) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 8d62fef162376..ece7cfc125ee3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -41,6 +41,7 @@ import java.nio.file.Files; import java.util.Collections; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -156,4 +157,14 @@ private void deserialize(final InputStream inputStream) throws IOException { version.set(ReadWriteIOUtils.readLong(inputStream)); } + + private boolean requestMemory(final long size) { + return Objects.isNull(regionStatistics) || regionStatistics.requestMemory(size); + } + + private void releaseMemory(final long size) { + if (regionStatistics != null) { + regionStatistics.releaseMemory(size); + } + } } From c9f2732e3d550d0e0bb12c65f4948708faf7d6fc Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:12:35 +0800 Subject: [PATCH 013/213] Refactor --- .../attribute/DeviceAttributeStore.java | 6 ++---- .../update/DeviceAttributeRemoteUpdater.java | 6 ++---- .../mtree/impl/mem/MemMTreeStore.java | 12 +++++------ .../mem/snapshot/MemMTreeSnapshotUtil.java | 21 +++++++++---------- .../schemaregion/tag/TagManager.java | 8 +++---- 5 files changed, 24 insertions(+), 29 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java index 7ac2618e114d4..585b76e65767b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java @@ -69,15 +69,13 @@ public synchronized boolean createSnapshot(final File targetDir) { final File snapshot = SystemFileFactory.INSTANCE.getFile(targetDir, SchemaConstant.DEVICE_ATTRIBUTE_SNAPSHOT); - try { - final FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); - final BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream); + try (final FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); + final BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream)) { try { serialize(outputStream); } finally { outputStream.flush(); fileOutputStream.getFD().sync(); - outputStream.close(); } if (snapshot.exists() && !FileUtils.deleteFileIfExist(snapshot)) { logger.error( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index ece7cfc125ee3..707f0bc46d277 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -69,15 +69,13 @@ public synchronized boolean createSnapshot(final File targetDir) { SystemFileFactory.INSTANCE.getFile( targetDir, SchemaConstant.DEVICE_ATTRIBUTE_REMOTE_UPDATER_SNAPSHOT); - try { - final FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); - final BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream); + try (final FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); + final BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream)) { try { serialize(outputStream); } finally { outputStream.flush(); fileOutputStream.getFD().sync(); - outputStream.close(); } if (snapshot.exists() && !FileUtils.deleteFileIfExist(snapshot)) { logger.error( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java index 544a0534c9fa4..54e94c8b53783 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java @@ -204,16 +204,16 @@ public void clear() { } @Override - public boolean createSnapshot(File snapshotDir) { + public boolean createSnapshot(final File snapshotDir) { return MemMTreeSnapshotUtil.createSnapshot(snapshotDir, this); } public static MemMTreeStore loadFromSnapshot( - File snapshotDir, - Consumer> measurementProcess, - Consumer> deviceProcess, - MemSchemaRegionStatistics regionStatistics, - SchemaRegionMemMetric metric) + final File snapshotDir, + final Consumer> measurementProcess, + final Consumer> deviceProcess, + final MemSchemaRegionStatistics regionStatistics, + final SchemaRegionMemMetric metric) throws IOException { return new MemMTreeStore( MemMTreeSnapshotUtil.loadSnapshot( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java index 360e4f2fde6dc..774858b46c132 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java @@ -74,29 +74,28 @@ public class MemMTreeSnapshotUtil { private static final IMNodeFactory nodeFactory = MNodeFactoryLoader.getInstance().getMemMNodeIMNodeFactory(); - public static boolean createSnapshot(File snapshotDir, MemMTreeStore store) { - File snapshotTmp = + public static boolean createSnapshot(final File snapshotDir, final MemMTreeStore store) { + final File snapshotTmp = SystemFileFactory.INSTANCE.getFile(snapshotDir, SchemaConstant.MTREE_SNAPSHOT_TMP); - File snapshot = SystemFileFactory.INSTANCE.getFile(snapshotDir, SchemaConstant.MTREE_SNAPSHOT); + final File snapshot = + SystemFileFactory.INSTANCE.getFile(snapshotDir, SchemaConstant.MTREE_SNAPSHOT); - try { - FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); - BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream); + try (final FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); + final BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream)) { try { serializeTo(store, outputStream); } finally { outputStream.flush(); fileOutputStream.getFD().sync(); - outputStream.close(); } if (snapshot.exists() && !FileUtils.deleteFileIfExist(snapshot)) { logger.error( - "Failed to delete old snapshot {} while creating mtree snapshot.", snapshot.getName()); + "Failed to delete old snapshot {} while creating mTree snapshot.", snapshot.getName()); return false; } if (!snapshotTmp.renameTo(snapshot)) { logger.error( - "Failed to rename {} to {} while creating mtree snapshot.", + "Failed to rename {} to {} while creating mTree snapshot.", snapshotTmp.getName(), snapshot.getName()); FileUtils.deleteFileIfExist(snapshot); @@ -104,8 +103,8 @@ public static boolean createSnapshot(File snapshotDir, MemMTreeStore store) { } return true; - } catch (IOException e) { - logger.error("Failed to create mtree snapshot due to {}", e.getMessage(), e); + } catch (final IOException e) { + logger.error("Failed to create mTree snapshot due to {}", e.getMessage(), e); FileUtils.deleteFileIfExist(snapshot); return false; } finally { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java index d184c4647f1d3..4bc71c7cca0ad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java @@ -86,10 +86,10 @@ public TagManager(String sgSchemaDirPath, MemSchemaRegionStatistics regionStatis this.regionStatistics = regionStatistics; } - public synchronized boolean createSnapshot(File targetDir) { - File tagLogSnapshot = + public synchronized boolean createSnapshot(final File targetDir) { + final File tagLogSnapshot = SystemFileFactory.INSTANCE.getFile(targetDir, SchemaConstant.TAG_LOG_SNAPSHOT); - File tagLogSnapshotTmp = + final File tagLogSnapshotTmp = SystemFileFactory.INSTANCE.getFile(targetDir, SchemaConstant.TAG_LOG_SNAPSHOT_TMP); try { tagLogFile.copyTo(tagLogSnapshotTmp); @@ -111,7 +111,7 @@ public synchronized boolean createSnapshot(File targetDir) { } return true; - } catch (IOException e) { + } catch (final IOException e) { logger.error("Failed to create tagManager snapshot due to {}", e.getMessage(), e); if (!FileUtils.deleteFileIfExist(tagLogSnapshot)) { logger.warn( From 5df1fcd917a2932bdf26a542ec5e03b4bbcbb6f1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:16:17 +0800 Subject: [PATCH 014/213] Create GeneralRegionAttributeSecurityService.java --- ...GeneralRegionAttributeSecurityService.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java new file mode 100644 index 0000000000000..4826ee1cae4b1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; + +public class GeneralRegionAttributeSecurityService {} From 4aaf5fd96b5fc0b5dab133ebf0948aa398c8a941 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:23:11 +0800 Subject: [PATCH 015/213] version --- .../update/DeviceAttributeRemoteUpdater.java | 10 ++++++++++ .../schemaregion/impl/SchemaRegionMemoryImpl.java | 14 ++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 707f0bc46d277..816349e5e2c5e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -61,6 +61,14 @@ public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatis this.regionStatistics = regionStatistics; } + /////////////////////////////// Service /////////////////////////////// + + public void addVersion() { + version.incrementAndGet(); + } + + /////////////////////////////// Snapshot /////////////////////////////// + public synchronized boolean createSnapshot(final File targetDir) { final File snapshotTmp = SystemFileFactory.INSTANCE.getFile( @@ -156,6 +164,8 @@ private void deserialize(final InputStream inputStream) throws IOException { version.set(ReadWriteIOUtils.readLong(inputStream)); } + /////////////////////////////// Memory /////////////////////////////// + private boolean requestMemory(final long size) { return Objects.isNull(regionStatistics) || regionStatistics.requestMemory(size); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 615f09d0fc74a..2e4ca5f40f302 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1374,14 +1374,11 @@ public void createOrUpdateTableDevice(final CreateOrUpdateTableDeviceNode node) tableName, deviceId, () -> deviceAttributeStore.createAttribute(attributeNameList, attributeValueList), - pointer -> - updateAttribute( - databaseName, - tableName, - deviceId, - pointer, - attributeNameList, - attributeValueList)); + pointer -> { + updateAttribute( + databaseName, tableName, deviceId, pointer, attributeNameList, attributeValueList); + deviceAttributeRemoteUpdater.addVersion(); + }); } writeToMLog(node); } @@ -1415,6 +1412,7 @@ public void updateTableDeviceAttribute(final TableDeviceAttributeUpdateNode upda } } writeToMLog(updateNode); + deviceAttributeRemoteUpdater.addVersion(); } private DeviceAttributeUpdater constructDevicePredicateUpdater( From 7ebcfdbbddc96c00d7fb2b8882c6a4efeae31b0f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:27:13 +0800 Subject: [PATCH 016/213] interface --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 8 ++++++++ .../schemaregion/impl/SchemaRegionMemoryImpl.java | 1 + 2 files changed, 9 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 816349e5e2c5e..1a222a8ad3cb6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -63,6 +63,14 @@ public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatis /////////////////////////////// Service /////////////////////////////// + public void update( + final String database, + final String tableName, + final String[] deviceId, + final Map attributeMap) { + // TODO + } + public void addVersion() { version.incrementAndGet(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 2e4ca5f40f302..9aa479d7b631f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1396,6 +1396,7 @@ private void updateAttribute( TableDeviceSchemaFetcher.getInstance() .getTableDeviceCache() .update(databaseName, tableName, deviceId, resultMap); + deviceAttributeRemoteUpdater.update(databaseName, tableName, deviceId, resultMap); } } From 3e67a16fbaf716b30d86abffba72d8631276bd26 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:35:41 +0800 Subject: [PATCH 017/213] Refactor --- .../rescon/MemSchemaEngineStatistics.java | 4 +--- .../rescon/MemSchemaRegionStatistics.java | 4 ++-- .../update/DeviceAttributeRemoteUpdater.java | 14 ++++++-------- .../attribute/update/UpdateClearContainer.java | 11 +++++++---- .../attribute/update/UpdateContainer.java | 4 ++-- .../attribute/update/UpdateDetailContainer.java | 7 ++++--- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 2 +- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaEngineStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaEngineStatistics.java index 51dc16a781875..faf0546df70ee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaEngineStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaEngineStatistics.java @@ -74,7 +74,7 @@ public long getMemoryUsage() { return memoryUsage.get(); } - public boolean requestMemory(final long size) { + public void requestMemory(final long size) { memoryUsage.addAndGet(size); if (memoryUsage.get() >= memoryCapacity) { synchronized (allowToCreateNewSeriesLock) { @@ -87,9 +87,7 @@ public boolean requestMemory(final long size) { allowToCreateNewSeries = false; } } - return false; } - return true; } public void releaseMemory(final long size) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java index 19b0291a61098..eaec23ebcdc40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java @@ -51,9 +51,9 @@ public boolean isAllowToCreateNewSeries() { return schemaEngineStatistics.isAllowToCreateNewSeries(); } - public boolean requestMemory(final long size) { + public void requestMemory(final long size) { memoryUsage.addAndGet(size); - return schemaEngineStatistics.requestMemory(size); + schemaEngineStatistics.requestMemory(size); } public void releaseMemory(final long size) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 1a222a8ad3cb6..ce8a0509578c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -41,7 +41,6 @@ import java.nio.file.Files; import java.util.Collections; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -64,11 +63,8 @@ public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatis /////////////////////////////// Service /////////////////////////////// public void update( - final String database, - final String tableName, - final String[] deviceId, - final Map attributeMap) { - // TODO + final String tableName, final String[] deviceId, final Map attributeMap) { + targetDataNodeLocations.forEach(pair -> {}); } public void addVersion() { @@ -174,8 +170,10 @@ private void deserialize(final InputStream inputStream) throws IOException { /////////////////////////////// Memory /////////////////////////////// - private boolean requestMemory(final long size) { - return Objects.isNull(regionStatistics) || regionStatistics.requestMemory(size); + private void requestMemory(final long size) { + if (regionStatistics != null) { + regionStatistics.requestMemory(size); + } } private void releaseMemory(final long size) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index f31d09e4e0c7b..e478420bb451e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -19,8 +19,8 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; -import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.IOException; @@ -37,9 +37,12 @@ public class UpdateClearContainer implements UpdateContainer { private final Set tableNames = Collections.newSetFromMap(new ConcurrentHashMap<>()); @Override - public int updateAttribute( - final IDeviceID deviceId, final Map updatedAttributes) { - return 0; + public long updateAttribute( + final String tableName, + final String[] deviceId, + final Map updatedAttributes) { + tableNames.add(tableName); + return RamUsageEstimator.sizeOf(tableName); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 0aa52336d7d05..29ca23b82a735 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; -import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Pair; import java.io.IOException; @@ -30,7 +29,8 @@ public interface UpdateContainer { - int updateAttribute(final IDeviceID deviceId, final Map updatedAttributes); + long updateAttribute( + final String tableName, final String[] deviceId, final Map updatedAttributes); ByteBuffer getUpdateBuffer(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 265d6c181244c..a4d2460f06a8e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; -import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Pair; import java.io.InputStream; @@ -34,8 +33,10 @@ public class UpdateDetailContainer implements UpdateContainer { new ConcurrentHashMap<>(); @Override - public int updateAttribute( - final IDeviceID deviceId, final Map updatedAttributes) { + public long updateAttribute( + final String tableName, + final String[] deviceId, + final Map updatedAttributes) { return 0; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 9aa479d7b631f..99ccdadcdd779 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1396,7 +1396,7 @@ private void updateAttribute( TableDeviceSchemaFetcher.getInstance() .getTableDeviceCache() .update(databaseName, tableName, deviceId, resultMap); - deviceAttributeRemoteUpdater.update(databaseName, tableName, deviceId, resultMap); + deviceAttributeRemoteUpdater.update(tableName, deviceId, resultMap); } } From 6241b4c04a9569afa397696d6557b3c794b2eb5b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:39:57 +0800 Subject: [PATCH 018/213] Fix --- .../update/DeviceAttributeRemoteUpdater.java | 18 +++++++++++++++++- .../attribute/update/UpdateClearContainer.java | 3 +-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index ce8a0509578c0..dc62fa84bf693 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -64,7 +64,15 @@ public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatis public void update( final String tableName, final String[] deviceId, final Map attributeMap) { - targetDataNodeLocations.forEach(pair -> {}); + targetDataNodeLocations.forEach( + pair -> { + if (attributeUpdateMap.containsKey(pair.getLeft())) { + updateMemory( + attributeUpdateMap + .get(pair.getLeft()) + .updateAttribute(tableName, deviceId, attributeMap)); + } + }); } public void addVersion() { @@ -170,6 +178,14 @@ private void deserialize(final InputStream inputStream) throws IOException { /////////////////////////////// Memory /////////////////////////////// + private void updateMemory(final long size) { + if (size > 0) { + requestMemory(size); + } else { + releaseMemory(size); + } + } + private void requestMemory(final long size) { if (regionStatistics != null) { regionStatistics.requestMemory(size); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index e478420bb451e..6c94aca5c488e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -41,8 +41,7 @@ public long updateAttribute( final String tableName, final String[] deviceId, final Map updatedAttributes) { - tableNames.add(tableName); - return RamUsageEstimator.sizeOf(tableName); + return tableNames.add(tableName) ? RamUsageEstimator.sizeOf(tableName) : 0; } @Override From d7338f50475900a7b93a68f8998ccf0f3387195f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:02:03 +0800 Subject: [PATCH 019/213] init memory calculations --- .../schemaregion/attribute/update/UpdateClearContainer.java | 6 ++++++ .../attribute/update/UpdateDetailContainer.java | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 6c94aca5c488e..c19add6dbb67a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -34,6 +34,12 @@ public class UpdateClearContainer implements UpdateContainer { + static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(UpdateClearContainer.class) + + RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class) + + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + + 2L * RamUsageEstimator.NUM_BYTES_OBJECT_REF; + private final Set tableNames = Collections.newSetFromMap(new ConcurrentHashMap<>()); @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index a4d2460f06a8e..0e80c8fe043ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.RamUsageEstimator; import java.io.InputStream; import java.io.OutputStream; @@ -29,6 +30,10 @@ public class UpdateDetailContainer implements UpdateContainer { + static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(UpdateClearContainer.class) + + RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class); + private final Map>> updateMap = new ConcurrentHashMap<>(); From aac32375086ebfcae86e536e2f1c14757859c744 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:05:41 +0800 Subject: [PATCH 020/213] Update DeviceAttributeRemoteUpdater.java --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index dc62fa84bf693..6cb5eef1e59d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -144,6 +144,12 @@ public void loadFromSnapshot(final File snapshotDir) throws IOException { final File snapshot = SystemFileFactory.INSTANCE.getFile( snapshotDir, SchemaConstant.DEVICE_ATTRIBUTE_REMOTE_UPDATER_SNAPSHOT); + if (!snapshot.exists()) { + logger.info( + "Device attribute remote updater snapshot {} not found, consider it as upgraded from the older version, will not update remote", + snapshot); + return; + } try (final BufferedInputStream inputStream = new BufferedInputStream(Files.newInputStream(snapshot.toPath()))) { deserialize(inputStream); From 2710cf5a04def71ebf429b746afd8a0c89a7608b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:10:06 +0800 Subject: [PATCH 021/213] Ram bytes --- .../schemaregion/attribute/update/UpdateClearContainer.java | 6 ++++++ .../schemaregion/attribute/update/UpdateContainer.java | 2 ++ .../attribute/update/UpdateDetailContainer.java | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index c19add6dbb67a..5af999039e0eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -75,4 +75,10 @@ public void deserialize(final InputStream fileInputStream) throws IOException { tableNames.add(ReadWriteIOUtils.readString(fileInputStream)); } } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + + tableNames.stream().mapToLong(RamUsageEstimator::sizeOf).reduce(0L, Long::sum); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 29ca23b82a735..03db29262ad40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -39,4 +39,6 @@ long updateAttribute( void serialize(final OutputStream outputstream) throws IOException; void deserialize(final InputStream fileInputStream) throws IOException; + + long ramBytesUsed(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 0e80c8fe043ca..1653746c79de9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -60,4 +60,9 @@ public void serialize(final OutputStream outputstream) {} @Override public void deserialize(final InputStream fileInputStream) {} + + @Override + public long ramBytesUsed() { + return 0; + } } From 2b40aa94121a6e1246a8a7c68746e65081ff2478 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:29:49 +0800 Subject: [PATCH 022/213] try update --- .../update/DeviceAttributeRemoteUpdater.java | 2 + .../update/UpdateDetailContainer.java | 45 +++++++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 6cb5eef1e59d6..754e10f515211 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -71,6 +71,8 @@ public void update( attributeUpdateMap .get(pair.getLeft()) .updateAttribute(tableName, deviceId, attributeMap)); + } else { + } }); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 1653746c79de9..a607ec27b4878 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -26,13 +26,15 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; public class UpdateDetailContainer implements UpdateContainer { + static final long MAP_SIZE = RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class); static final long INSTANCE_SIZE = - RamUsageEstimator.shallowSizeOfInstance(UpdateClearContainer.class) - + RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class); + RamUsageEstimator.shallowSizeOfInstance(UpdateClearContainer.class) + MAP_SIZE; private final Map>> updateMap = new ConcurrentHashMap<>(); @@ -42,6 +44,42 @@ public long updateAttribute( final String tableName, final String[] deviceId, final Map updatedAttributes) { + final AtomicLong result = new AtomicLong(0); + updateMap.compute( + tableName, + (name, value) -> { + if (Objects.isNull(value)) { + result.addAndGet( + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + + RamUsageEstimator.sizeOf(name) + + MAP_SIZE); + value = new ConcurrentHashMap<>(); + } + value.compute( + deviceId, + (device, attributes) -> { + if (Objects.isNull(attributes)) { + result.addAndGet( + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + + RamUsageEstimator.sizeOf(device) + + MAP_SIZE); + attributes = new ConcurrentHashMap<>(); + } + for (final Map.Entry updateAttribute : + updatedAttributes.entrySet()) { + attributes.compute( + updateAttribute.getKey(), + (k, v) -> { + result.addAndGet( + RamUsageEstimator.sizeOf(updateAttribute.getValue()) + - RamUsageEstimator.sizeOf(v)); + return updateAttribute.getValue(); + }); + } + return attributes; + }); + return value; + }); return 0; } @@ -63,6 +101,7 @@ public void deserialize(final InputStream fileInputStream) {} @Override public long ramBytesUsed() { - return 0; + // TODO + return INSTANCE_SIZE; } } From 6b5f97111729ed50e9b1120d2c997d4b217f6639 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:34:28 +0800 Subject: [PATCH 023/213] Update UpdateDetailContainer.java --- .../attribute/update/UpdateDetailContainer.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index a607ec27b4878..e649889751e59 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -70,11 +70,19 @@ public long updateAttribute( attributes.compute( updateAttribute.getKey(), (k, v) -> { + if (Objects.isNull(v)) { + result.addAndGet( + RamUsageEstimator.sizeOf(k) + + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY); + } result.addAndGet( RamUsageEstimator.sizeOf(updateAttribute.getValue()) - RamUsageEstimator.sizeOf(v)); return updateAttribute.getValue(); }); + if (Objects.isNull(updateAttribute.getValue())) { + attributes.put(updateAttribute.getKey(), null); + } } return attributes; }); From e1c0c8cfa7c6776c5f644758cf659b37f5f74622 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:36:07 +0800 Subject: [PATCH 024/213] Update UpdateDetailContainer.java --- .../schemaregion/attribute/update/UpdateDetailContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index e649889751e59..fe4f17c017637 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -88,7 +88,7 @@ public long updateAttribute( }); return value; }); - return 0; + return result.get(); } @Override From b741cdc0b30cb93c990d78258cb9450d63fb482d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:54:05 +0800 Subject: [PATCH 025/213] may continue --- .../update/UpdateClearContainer.java | 6 +- .../attribute/update/UpdateContainer.java | 2 +- .../update/UpdateDetailContainer.java | 60 +++++++++++++++++-- 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 5af999039e0eb..675e166c7155d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -69,10 +69,10 @@ public void serialize(final OutputStream outputstream) throws IOException { } @Override - public void deserialize(final InputStream fileInputStream) throws IOException { - final int size = ReadWriteIOUtils.readInt(fileInputStream); + public void deserialize(final InputStream inputStream) throws IOException { + final int size = ReadWriteIOUtils.readInt(inputStream); for (int i = 0; i < size; ++i) { - tableNames.add(ReadWriteIOUtils.readString(fileInputStream)); + tableNames.add(ReadWriteIOUtils.readString(inputStream)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 03db29262ad40..ac96aebd0918c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -38,7 +38,7 @@ long updateAttribute( void serialize(final OutputStream outputstream) throws IOException; - void deserialize(final InputStream fileInputStream) throws IOException; + void deserialize(final InputStream inputStream) throws IOException; long ramBytesUsed(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index fe4f17c017637..2bb49059bfc21 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -21,13 +21,16 @@ import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; public class UpdateDetailContainer implements UpdateContainer { @@ -36,8 +39,8 @@ public class UpdateDetailContainer implements UpdateContainer { static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(UpdateClearContainer.class) + MAP_SIZE; - private final Map>> updateMap = - new ConcurrentHashMap<>(); + private final ConcurrentMap>> + updateMap = new ConcurrentHashMap<>(); @Override public long updateAttribute( @@ -102,10 +105,59 @@ public Pair updateSelfByCommitBuffer(final ByteBuffer commitBu } @Override - public void serialize(final OutputStream outputstream) {} + public void serialize(final OutputStream outputstream) throws IOException { + ReadWriteIOUtils.write(updateMap.size(), outputstream); + for (final Map.Entry>> + tableEntry : updateMap.entrySet()) { + ReadWriteIOUtils.write(tableEntry.getKey(), outputstream); + for (final Map.Entry> deviceEntry : + tableEntry.getValue().entrySet()) { + ReadWriteIOUtils.write(deviceEntry.getKey().length, outputstream); + for (final String node : deviceEntry.getKey()) { + ReadWriteIOUtils.write(node, outputstream); + } + ReadWriteIOUtils.write(deviceEntry.getValue(), outputstream); + } + } + } @Override - public void deserialize(final InputStream fileInputStream) {} + public void deserialize(final InputStream inputStream) throws IOException { + final int tableSize = ReadWriteIOUtils.readInt(inputStream); + for (int i = 0; i < tableSize; ++i) { + final String tableName = ReadWriteIOUtils.readString(inputStream); + final int deviceSize = ReadWriteIOUtils.readInt(inputStream); + final ConcurrentMap> deviceMap = + new ConcurrentHashMap<>(deviceSize); + for (int j = 0; j < deviceSize; ++j) { + final int nodeSize = ReadWriteIOUtils.readInt(inputStream); + final String[] deviceId = new String[nodeSize]; + for (int k = 0; k < nodeSize; ++k) { + deviceId[k] = ReadWriteIOUtils.readString(inputStream); + } + deviceMap.put(deviceId, readConcurrentMap(inputStream)); + } + updateMap.put(tableName, deviceMap); + } + } + + private static ConcurrentMap readConcurrentMap(final InputStream inputStream) + throws IOException { + final int length = ReadWriteIOUtils.readInt(inputStream); + if (length == -1) { + return null; + } else { + final ConcurrentMap map = new ConcurrentHashMap<>(length); + + for (int i = 0; i < length; ++i) { + final String key = ReadWriteIOUtils.readString(inputStream); + final String value = ReadWriteIOUtils.readString(inputStream); + map.put(key, value); + } + + return map; + } + } @Override public long ramBytesUsed() { From 809a0b9879944f5bf10efd256cfb5c4c3cf41882 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:55:10 +0800 Subject: [PATCH 026/213] Update UpdateDetailContainer.java --- .../attribute/update/UpdateDetailContainer.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 2bb49059bfc21..1c49ead5fee98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -148,13 +148,9 @@ private static ConcurrentMap readConcurrentMap(final InputStream return null; } else { final ConcurrentMap map = new ConcurrentHashMap<>(length); - for (int i = 0; i < length; ++i) { - final String key = ReadWriteIOUtils.readString(inputStream); - final String value = ReadWriteIOUtils.readString(inputStream); - map.put(key, value); + map.put(ReadWriteIOUtils.readString(inputStream), ReadWriteIOUtils.readString(inputStream)); } - return map; } } From 046499711344dc5e73d737fad4d2659ef6138604 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:59:57 +0800 Subject: [PATCH 027/213] Degrade logic --- .../attribute/update/UpdateClearContainer.java | 8 ++++++++ .../attribute/update/UpdateDetailContainer.java | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 675e166c7155d..0335dd0c0c139 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -42,6 +42,14 @@ public class UpdateClearContainer implements UpdateContainer { private final Set tableNames = Collections.newSetFromMap(new ConcurrentHashMap<>()); + public UpdateClearContainer() { + // Public constructor + } + + UpdateClearContainer(final Set tableNames) { + this.tableNames.addAll(tableNames); + } + @Override public long updateAttribute( final String tableName, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 1c49ead5fee98..8afb1bf01a67e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -160,4 +160,8 @@ public long ramBytesUsed() { // TODO return INSTANCE_SIZE; } + + UpdateClearContainer degrade() { + return new UpdateClearContainer(updateMap.keySet()); + } } From f706080d5f5ff5049993de36db70f4fdb71110bd Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 19:37:27 +0800 Subject: [PATCH 028/213] Refactor --- .../schemaregion/attribute/update/UpdateClearContainer.java | 2 +- .../schemaregion/attribute/update/UpdateContainer.java | 2 +- .../schemaregion/attribute/update/UpdateDetailContainer.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 0335dd0c0c139..8049ce5b0527a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -59,7 +59,7 @@ public long updateAttribute( } @Override - public ByteBuffer getUpdateBuffer() { + public byte[] getUpdateContent() { return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index ac96aebd0918c..0c33f02d81c6f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -32,7 +32,7 @@ public interface UpdateContainer { long updateAttribute( final String tableName, final String[] deviceId, final Map updatedAttributes); - ByteBuffer getUpdateBuffer(); + byte[] getUpdateContent(); Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 8afb1bf01a67e..9d9dab06e5ea6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -95,7 +95,7 @@ public long updateAttribute( } @Override - public ByteBuffer getUpdateBuffer() { + public byte[] getUpdateContent() { return null; } From edfa59b4318875add644bc4d5797d57f219b7a79 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 19:40:39 +0800 Subject: [PATCH 029/213] Update UpdateClearContainer.java --- .../attribute/update/UpdateClearContainer.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 8049ce5b0527a..070d6c98083c0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -23,6 +23,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -60,7 +61,16 @@ public long updateAttribute( @Override public byte[] getUpdateContent() { - return null; + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.write(tableNames.size(), outputStream); + for (final String tableName : tableNames) { + ReadWriteIOUtils.write(tableName, outputStream); + } + } catch (final IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + return outputStream.toByteArray(); } @Override From d75f1c5df8238a231987e0d08db1d63e52ef8820 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 26 Sep 2024 20:39:52 +0800 Subject: [PATCH 030/213] GRASS --- .../GeneralRegionAttributeSecurityService.java | 12 +++++++++++- .../apache/iotdb/commons/concurrent/ThreadName.java | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 4826ee1cae4b1..b770e6462d4ef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -19,4 +19,14 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; -public class GeneralRegionAttributeSecurityService {} +import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; +import org.apache.iotdb.commons.concurrent.ThreadName; + +import java.util.concurrent.ScheduledExecutorService; + +public class GeneralRegionAttributeSecurityService { + + private static final ScheduledExecutorService SECURITY_SERVICE_EXECUTOR = + IoTDBThreadPoolFactory.newSingleThreadScheduledExecutor( + ThreadName.GENERAL_REGION_ATTRIBUTE_SECURITY_SERVICE.getName()); +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java index c82d3f12b8138..e651d4c7381ae 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java @@ -68,6 +68,7 @@ public enum ThreadName { PBTREE_RELEASE_MONITOR("PBTree-Release-Task-Monitor"), PBTREE_FLUSH_MONITOR("PBTree-Flush-Monitor"), PBTREE_WORKER_POOL("PBTree-Worker-Pool"), + GENERAL_REGION_ATTRIBUTE_SECURITY_SERVICE("General-Region-Attribute-Security-Service"), // -------------------------- ClientService -------------------------- CLIENT_RPC_SERVICE("ClientRPC-Service"), CLIENT_RPC_PROCESSOR("ClientRPC-Processor"), From 65d8234047adc6061e92cdf0852a56504f6add8e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:13:32 +0800 Subject: [PATCH 031/213] Add config --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 12 ++++++++++++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 1d369eeccd213..6770322b947cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1044,6 +1044,8 @@ public class IoTDBConfig { private int dataNodeTableCacheSemaphorePermitNum = 5; + private long generalRegionAttributeSecurityServiceIntervalSeconds = 30L; + private String readConsistencyLevel = "strong"; /** Maximum execution time of a DriverTask */ @@ -3465,6 +3467,16 @@ public void setDataNodeTableCacheSemaphorePermitNum(int dataNodeTableCacheSemaph this.dataNodeTableCacheSemaphorePermitNum = dataNodeTableCacheSemaphorePermitNum; } + public long getGeneralRegionAttributeSecurityServiceIntervalSeconds() { + return generalRegionAttributeSecurityServiceIntervalSeconds; + } + + public void setGeneralRegionAttributeSecurityServiceIntervalSeconds( + long generalRegionAttributeSecurityServiceIntervalSeconds) { + this.generalRegionAttributeSecurityServiceIntervalSeconds = + generalRegionAttributeSecurityServiceIntervalSeconds; + } + public String getReadConsistencyLevel() { return readConsistencyLevel; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index feda1c03b6f69..c830b325a776d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1072,6 +1072,12 @@ public void loadProperties(Properties properties) throws BadNodeUrlException, IO "datanode_table_cache_semaphore_permit_num", String.valueOf(conf.getDataNodeTableCacheSemaphorePermitNum())))); + conf.setGeneralRegionAttributeSecurityServiceIntervalSeconds( + Long.parseLong( + properties.getProperty( + "general_region_attribute_security_service_interval_seconds", + String.valueOf(conf.getGeneralRegionAttributeSecurityServiceIntervalSeconds())))); + loadIoTConsensusProps(properties); loadIoTConsensusV2Props(properties); } From 1907fafedddd8ed71403976ecb3618c102f492d5 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:17:59 +0800 Subject: [PATCH 032/213] Update GeneralRegionAttributeSecurityService.java --- ...GeneralRegionAttributeSecurityService.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index b770e6462d4ef..d1ec1ad5392e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -21,12 +21,52 @@ import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; +import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; public class GeneralRegionAttributeSecurityService { + private static final Logger LOGGER = + LoggerFactory.getLogger(GeneralRegionAttributeSecurityService.class); private static final ScheduledExecutorService SECURITY_SERVICE_EXECUTOR = IoTDBThreadPoolFactory.newSingleThreadScheduledExecutor( ThreadName.GENERAL_REGION_ATTRIBUTE_SECURITY_SERVICE.getName()); + + private Future executorFuture; + + public synchronized void start() { + if (executorFuture == null) { + executorFuture = + ScheduledExecutorUtil.safelyScheduleWithFixedDelay( + SECURITY_SERVICE_EXECUTOR, + this::execute, + IoTDBDescriptor.getInstance() + .getConfig() + .getGeneralRegionAttributeSecurityServiceIntervalSeconds(), + IoTDBDescriptor.getInstance() + .getConfig() + .getGeneralRegionAttributeSecurityServiceIntervalSeconds(), + TimeUnit.SECONDS); + LOGGER.info("General region attribute security service is started successfully."); + } + } + + public synchronized void stop() { + if (executorFuture != null) { + executorFuture.cancel(false); + executorFuture = null; + LOGGER.info("General region attribute security service is stopped successfully."); + } + } + + private void execute() { + // TODO + } } From bbe897fca78a8ed34fe11c377b21217fe5c3c8bf Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:26:11 +0800 Subject: [PATCH 033/213] state machine logic --- .../SchemaRegionStateMachine.java | 7 +++++ ...GeneralRegionAttributeSecurityService.java | 29 +++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java index f07cc34e5f654..8828fe0df51b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.GeneralRegionAttributeSecurityService; import org.apache.iotdb.db.tools.schema.SchemaRegionSnapshotParser; import org.apache.iotdb.rpc.TSStatusCode; @@ -65,6 +66,8 @@ public void start() { public void stop() { // Stop leader related service for schema pipe PipeDataNodeAgent.runtime().notifySchemaLeaderUnavailable(schemaRegion.getSchemaRegionId()); + GeneralRegionAttributeSecurityService.getInstance() + .stopBroadcast(schemaRegion.getSchemaRegionId()); } @Override @@ -90,6 +93,8 @@ public void notifyNotLeader() { // Shutdown leader related service for schema pipe PipeDataNodeAgent.runtime().notifySchemaLeaderUnavailable(schemaRegion.getSchemaRegionId()); + GeneralRegionAttributeSecurityService.getInstance() + .stopBroadcast(schemaRegion.getSchemaRegionId()); logger.info( "Current node [nodeId: {}] is no longer the schema region leader [regionId: {}], " @@ -107,6 +112,8 @@ public void notifyLeaderReady() { // Activate leader related service for schema pipe PipeDataNodeAgent.runtime().notifySchemaLeaderReady(schemaRegion.getSchemaRegionId()); + GeneralRegionAttributeSecurityService.getInstance() + .startBroadcast(schemaRegion.getSchemaRegionId()); logger.info( "Current node [nodeId: {}] as schema region leader [regionId: {}] is ready to work", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index d1ec1ad5392e5..ff9c6071a04bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -22,11 +22,14 @@ import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil; +import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -40,8 +43,11 @@ public class GeneralRegionAttributeSecurityService { ThreadName.GENERAL_REGION_ATTRIBUTE_SECURITY_SERVICE.getName()); private Future executorFuture; + private final Set regionLeaders = new HashSet<>(); + + public synchronized void startBroadcast(final SchemaRegionId id) { + regionLeaders.add(id); - public synchronized void start() { if (executorFuture == null) { executorFuture = ScheduledExecutorUtil.safelyScheduleWithFixedDelay( @@ -58,7 +64,9 @@ public synchronized void start() { } } - public synchronized void stop() { + public synchronized void stopBroadcast(final SchemaRegionId id) { + regionLeaders.remove(id); + if (executorFuture != null) { executorFuture.cancel(false); executorFuture = null; @@ -69,4 +77,21 @@ public synchronized void stop() { private void execute() { // TODO } + + /////////////////////////////// SingleTon /////////////////////////////// + + private GeneralRegionAttributeSecurityService() { + // Do nothing + } + + private static final class GeneralRegionAttributeSecurityServiceHolder { + private static final GeneralRegionAttributeSecurityService INSTANCE = + new GeneralRegionAttributeSecurityService(); + + private GeneralRegionAttributeSecurityServiceHolder() {} + } + + public static GeneralRegionAttributeSecurityService getInstance() { + return GeneralRegionAttributeSecurityServiceHolder.INSTANCE; + } } From 0d0b97aa91107f40327276febb7b829aeb745384 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:28:41 +0800 Subject: [PATCH 034/213] Update GeneralRegionAttributeSecurityService.java --- .../update/GeneralRegionAttributeSecurityService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index ff9c6071a04bf..6c4b96c373b47 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -46,9 +46,7 @@ public class GeneralRegionAttributeSecurityService { private final Set regionLeaders = new HashSet<>(); public synchronized void startBroadcast(final SchemaRegionId id) { - regionLeaders.add(id); - - if (executorFuture == null) { + if (regionLeaders.isEmpty()) { executorFuture = ScheduledExecutorUtil.safelyScheduleWithFixedDelay( SECURITY_SERVICE_EXECUTOR, @@ -62,12 +60,14 @@ public synchronized void startBroadcast(final SchemaRegionId id) { TimeUnit.SECONDS); LOGGER.info("General region attribute security service is started successfully."); } + + regionLeaders.add(id); } public synchronized void stopBroadcast(final SchemaRegionId id) { regionLeaders.remove(id); - if (executorFuture != null) { + if (regionLeaders.isEmpty()) { executorFuture.cancel(false); executorFuture = null; LOGGER.info("General region attribute security service is stopped successfully."); From 8d85897980779b2c80ddc9091bdd6b2d6351a81b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:31:25 +0800 Subject: [PATCH 035/213] Update GeneralRegionAttributeSecurityService.java --- .../attribute/update/GeneralRegionAttributeSecurityService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 6c4b96c373b47..39433ef2eb3bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -32,6 +32,7 @@ import java.util.Set; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; public class GeneralRegionAttributeSecurityService { @@ -44,6 +45,7 @@ public class GeneralRegionAttributeSecurityService { private Future executorFuture; private final Set regionLeaders = new HashSet<>(); + private final Semaphore serviceSemaphore = new Semaphore(1); public synchronized void startBroadcast(final SchemaRegionId id) { if (regionLeaders.isEmpty()) { From 98cb1f7ee680fa70764e188808c4a5a52af15dff Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:47:45 +0800 Subject: [PATCH 036/213] notify --- .../airgap/IoTDBAirGapReceiverAgent.java | 2 +- .../update/DeviceAttributeRemoteUpdater.java | 3 +- ...GeneralRegionAttributeSecurityService.java | 42 +++++++++---------- .../impl/SchemaRegionMemoryImpl.java | 4 +- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiverAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiverAgent.java index 6db65b00a326a..aa92af9d89d51 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiverAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiverAgent.java @@ -53,7 +53,7 @@ public void listen() { try { final Socket socket = serverSocket.accept(); new Thread(new IoTDBAirGapReceiver(socket, receiverId.incrementAndGet())).start(); - } catch (IOException e) { + } catch (final IOException e) { LOGGER.warn("Unhandled exception during pipe air gap receiver listening", e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 754e10f515211..ac78ea29cbe8c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -77,8 +77,9 @@ public void update( }); } - public void addVersion() { + public void addVersionAndNotifySend() { version.incrementAndGet(); + GeneralRegionAttributeSecurityService.getInstance().notifyAll(); } /////////////////////////////// Snapshot /////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 39433ef2eb3bf..c1519fb65bf41 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; -import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil; import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -30,43 +29,30 @@ import java.util.HashSet; import java.util.Set; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; public class GeneralRegionAttributeSecurityService { private static final Logger LOGGER = LoggerFactory.getLogger(GeneralRegionAttributeSecurityService.class); - private static final ScheduledExecutorService SECURITY_SERVICE_EXECUTOR = - IoTDBThreadPoolFactory.newSingleThreadScheduledExecutor( + private final ExecutorService securityServiceExecutor = + IoTDBThreadPoolFactory.newSingleThreadExecutor( ThreadName.GENERAL_REGION_ATTRIBUTE_SECURITY_SERVICE.getName()); private Future executorFuture; private final Set regionLeaders = new HashSet<>(); - private final Semaphore serviceSemaphore = new Semaphore(1); - public synchronized void startBroadcast(final SchemaRegionId id) { + public void startBroadcast(final SchemaRegionId id) { if (regionLeaders.isEmpty()) { - executorFuture = - ScheduledExecutorUtil.safelyScheduleWithFixedDelay( - SECURITY_SERVICE_EXECUTOR, - this::execute, - IoTDBDescriptor.getInstance() - .getConfig() - .getGeneralRegionAttributeSecurityServiceIntervalSeconds(), - IoTDBDescriptor.getInstance() - .getConfig() - .getGeneralRegionAttributeSecurityServiceIntervalSeconds(), - TimeUnit.SECONDS); + executorFuture = securityServiceExecutor.submit(this::execute); LOGGER.info("General region attribute security service is started successfully."); } regionLeaders.add(id); } - public synchronized void stopBroadcast(final SchemaRegionId id) { + public void stopBroadcast(final SchemaRegionId id) { regionLeaders.remove(id); if (regionLeaders.isEmpty()) { @@ -76,8 +62,20 @@ public synchronized void stopBroadcast(final SchemaRegionId id) { } } - private void execute() { - // TODO + private synchronized void execute() { + + try { + wait( + IoTDBDescriptor.getInstance() + .getConfig() + .getGeneralRegionAttributeSecurityServiceIntervalSeconds()); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.warn( + "Interrupted when waiting for the next attribute broadcasting: {}", e.getMessage()); + } finally { + securityServiceExecutor.submit(this::execute); + } } /////////////////////////////// SingleTon /////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 99ccdadcdd779..11691405b6fd2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1377,7 +1377,7 @@ public void createOrUpdateTableDevice(final CreateOrUpdateTableDeviceNode node) pointer -> { updateAttribute( databaseName, tableName, deviceId, pointer, attributeNameList, attributeValueList); - deviceAttributeRemoteUpdater.addVersion(); + deviceAttributeRemoteUpdater.addVersionAndNotifySend(); }); } writeToMLog(node); @@ -1413,7 +1413,7 @@ public void updateTableDeviceAttribute(final TableDeviceAttributeUpdateNode upda } } writeToMLog(updateNode); - deviceAttributeRemoteUpdater.addVersion(); + deviceAttributeRemoteUpdater.addVersionAndNotifySend(); } private DeviceAttributeUpdater constructDevicePredicateUpdater( From c83d498689197f7c761d880a77901abc1e4a447d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:50:58 +0800 Subject: [PATCH 037/213] Fix --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 2 +- .../update/GeneralRegionAttributeSecurityService.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index ac78ea29cbe8c..31d27bcc69885 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -79,7 +79,7 @@ public void update( public void addVersionAndNotifySend() { version.incrementAndGet(); - GeneralRegionAttributeSecurityService.getInstance().notifyAll(); + GeneralRegionAttributeSecurityService.getInstance().notifyBroadCast(); } /////////////////////////////// Snapshot /////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index c1519fb65bf41..63e287e29c8f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -62,6 +62,10 @@ public void stopBroadcast(final SchemaRegionId id) { } } + public synchronized void notifyBroadCast() { + notifyAll(); + } + private synchronized void execute() { try { From 64294b89368a845b5c5630a11d18e3f9a3c5f6bb Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:52:43 +0800 Subject: [PATCH 038/213] Update UpdateContainer.java --- .../schemaregion/attribute/update/UpdateContainer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 0c33f02d81c6f..9cfcdc9499616 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -32,6 +32,7 @@ public interface UpdateContainer { long updateAttribute( final String tableName, final String[] deviceId, final Map updatedAttributes); + // Only this method is not synchronize called and is called by GRASS thread byte[] getUpdateContent(); Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer); From 8a1d35592f3e35b05547e1aa105fa0a30b72c720 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 16:04:04 +0800 Subject: [PATCH 039/213] Update GeneralRegionAttributeSecurityService.java --- ...GeneralRegionAttributeSecurityService.java | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 63e287e29c8f9..1bca1dee46271 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -31,6 +31,9 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; public class GeneralRegionAttributeSecurityService { private static final Logger LOGGER = @@ -42,6 +45,9 @@ public class GeneralRegionAttributeSecurityService { private Future executorFuture; private final Set regionLeaders = new HashSet<>(); + private final ReentrantLock lock = new ReentrantLock(); + private final Condition condition = lock.newCondition(); + private volatile boolean skipNext = false; public void startBroadcast(final SchemaRegionId id) { if (regionLeaders.isEmpty()) { @@ -62,22 +68,35 @@ public void stopBroadcast(final SchemaRegionId id) { } } - public synchronized void notifyBroadCast() { - notifyAll(); + public void notifyBroadCast() { + if (lock.tryLock()) { + try { + condition.signalAll(); + } finally { + lock.unlock(); + } + } else { + skipNext = true; + } } - private synchronized void execute() { - + private void execute() { + lock.lock(); try { - wait( - IoTDBDescriptor.getInstance() - .getConfig() - .getGeneralRegionAttributeSecurityServiceIntervalSeconds()); + if (!skipNext) { + condition.await( + IoTDBDescriptor.getInstance() + .getConfig() + .getGeneralRegionAttributeSecurityServiceIntervalSeconds(), + TimeUnit.SECONDS); + } + skipNext = false; } catch (final InterruptedException e) { Thread.currentThread().interrupt(); LOGGER.warn( "Interrupted when waiting for the next attribute broadcasting: {}", e.getMessage()); } finally { + lock.unlock(); securityServiceExecutor.submit(this::execute); } } From d2db15286aea2282ef08092552ac07f63198b2bf Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 16:16:19 +0800 Subject: [PATCH 040/213] Update UpdateDetailContainer.java --- .../update/UpdateDetailContainer.java | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 9d9dab06e5ea6..cd8f5cc124dfb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -157,8 +157,31 @@ private static ConcurrentMap readConcurrentMap(final InputStream @Override public long ramBytesUsed() { - // TODO - return INSTANCE_SIZE; + final AtomicLong result = new AtomicLong(INSTANCE_SIZE); + + // Do not use "sizeOfMap" to save time and ensure that the calculated bytes is the same as + // previous calculation's sum + updateMap.forEach( + (name, value) -> { + result.addAndGet( + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + + RamUsageEstimator.sizeOf(name) + + MAP_SIZE); + value.forEach( + (device, attributes) -> { + result.addAndGet( + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + + RamUsageEstimator.sizeOf(device) + + MAP_SIZE); + attributes.forEach( + (k, v) -> + result.addAndGet( + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + + RamUsageEstimator.sizeOf(k) + + RamUsageEstimator.sizeOf(v))); + }); + }); + return result.get(); } UpdateClearContainer degrade() { From c9dcfeb1c369d865be4af711976a0701fcbe8ec1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 16:27:56 +0800 Subject: [PATCH 041/213] Update logic --- .../airgap/IoTDBAirGapReceiverAgent.java | 4 +-- .../update/DeviceAttributeRemoteUpdater.java | 25 +++++++++++++------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiverAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiverAgent.java index aa92af9d89d51..3097b0fcac85d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiverAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiverAgent.java @@ -66,7 +66,7 @@ public void listen() { public void start() throws StartupException { try { serverSocket = new ServerSocket(PipeConfig.getInstance().getPipeAirGapReceiverPort()); - } catch (IOException e) { + } catch (final IOException e) { throw new StartupException(e); } @@ -80,7 +80,7 @@ public void start() throws StartupException { public void stop() { try { serverSocket.close(); - } catch (IOException e) { + } catch (final IOException e) { LOGGER.warn("Failed to close IoTDBAirGapReceiverAgent's server socket", e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 31d27bcc69885..5ac8262cb351e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -66,17 +66,28 @@ public void update( final String tableName, final String[] deviceId, final Map attributeMap) { targetDataNodeLocations.forEach( pair -> { - if (attributeUpdateMap.containsKey(pair.getLeft())) { - updateMemory( - attributeUpdateMap - .get(pair.getLeft()) - .updateAttribute(tableName, deviceId, attributeMap)); - } else { - + if (!attributeUpdateMap.containsKey(pair.getLeft())) { + final UpdateContainer newContainer; + if (!regionStatistics.isAllowToCreateNewSeries()) { + newContainer = new UpdateClearContainer(); + requestMemory(UpdateClearContainer.INSTANCE_SIZE); + } else { + newContainer = new UpdateDetailContainer(); + requestMemory(UpdateDetailContainer.INSTANCE_SIZE); + } + attributeUpdateMap.put(pair.getLeft(), newContainer); } + updateMemory( + attributeUpdateMap + .get(pair.getLeft()) + .updateAttribute(tableName, deviceId, attributeMap)); }); } + public void addLocation(final Pair dataNodeLocation) { + targetDataNodeLocations.add(dataNodeLocation); + } + public void addVersionAndNotifySend() { version.incrementAndGet(); GeneralRegionAttributeSecurityService.getInstance().notifyBroadCast(); From 6fa59aff22a1b67f35273e973005e25413e0a086 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 16:28:38 +0800 Subject: [PATCH 042/213] Update DeviceAttributeRemoteUpdater.java --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 5ac8262cb351e..029ea2e077c28 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -84,6 +84,10 @@ public void update( }); } + public Map getSendBuffer() { + return null; + } + public void addLocation(final Pair dataNodeLocation) { targetDataNodeLocations.add(dataNodeLocation); } From 09ebec09d9f787bdb5921be0bca5613a353de53f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 17:02:48 +0800 Subject: [PATCH 043/213] Refactor --- .../update/DeviceAttributeRemoteUpdater.java | 13 +++++-- .../update/UpdateClearContainer.java | 3 ++ .../attribute/update/UpdateContainer.java | 3 ++ .../update/UpdateContainerStatistics.java | 37 +++++++++++++++++++ .../update/UpdateDetailContainer.java | 3 ++ .../impl/SchemaRegionMemoryImpl.java | 2 +- 6 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 029ea2e077c28..478697bb7df89 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -39,7 +39,8 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; -import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -49,12 +50,15 @@ public class DeviceAttributeRemoteUpdater { private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeRemoteUpdater.class); - private final Set> targetDataNodeLocations = - Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Set> targetDataNodeLocations = new HashSet<>(); private final ConcurrentMap attributeUpdateMap = new ConcurrentHashMap<>(); private final AtomicLong version = new AtomicLong(0); + + // Volatiles private final MemSchemaRegionStatistics regionStatistics; + private final Map updateContainerStatistics = + new HashMap<>(); public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatistics) { this.regionStatistics = regionStatistics; @@ -64,6 +68,9 @@ public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatis public void update( final String tableName, final String[] deviceId, final Map attributeMap) { + // Degrade + if (!regionStatistics.isAllowToCreateNewSeries()) {} + targetDataNodeLocations.forEach( pair -> { if (!attributeUpdateMap.containsKey(pair.getLeft())) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 070d6c98083c0..85523237cf340 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -23,6 +23,8 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; +import javax.annotation.concurrent.ThreadSafe; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -33,6 +35,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +@ThreadSafe public class UpdateClearContainer implements UpdateContainer { static final long INSTANCE_SIZE = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 9cfcdc9499616..bb1969cac3219 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -21,12 +21,15 @@ import org.apache.tsfile.utils.Pair; +import javax.annotation.concurrent.ThreadSafe; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Map; +@ThreadSafe public interface UpdateContainer { long updateAttribute( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java new file mode 100644 index 0000000000000..a74b52f1f9641 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; + +public class UpdateContainerStatistics { + private long lastUpdateTime = System.currentTimeMillis(); + private long size = 0; + + private void update() { + lastUpdateTime = System.currentTimeMillis(); + } + + private void addSize(final long increment) { + this.size += increment; + } + + long getDegradePriority() { + return size * (System.currentTimeMillis() - lastUpdateTime); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index cd8f5cc124dfb..872b30e7813af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -23,6 +23,8 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; +import javax.annotation.concurrent.ThreadSafe; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -33,6 +35,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; +@ThreadSafe public class UpdateDetailContainer implements UpdateContainer { static final long MAP_SIZE = RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 11691405b6fd2..10457203c4caa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1412,8 +1412,8 @@ public void updateTableDeviceAttribute(final TableDeviceAttributeUpdateNode upda mtree.updateTableDevice(pattern, batchUpdater); } } - writeToMLog(updateNode); deviceAttributeRemoteUpdater.addVersionAndNotifySend(); + writeToMLog(updateNode); } private DeviceAttributeUpdater constructDevicePredicateUpdater( From 1f8ac6386c1058d852d942b12d16f0e7372c15ec Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 17:11:23 +0800 Subject: [PATCH 044/213] Size --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++++++++++ .../java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++++++ .../attribute/update/DeviceAttributeRemoteUpdater.java | 7 +++++-- .../attribute/update/UpdateContainerStatistics.java | 9 +++++++-- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 6770322b947cf..637be158da090 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1045,6 +1045,7 @@ public class IoTDBConfig { private int dataNodeTableCacheSemaphorePermitNum = 5; private long generalRegionAttributeSecurityServiceIntervalSeconds = 30L; + private long detailContainerMinDegradeMemoryInBytes = 1024 * 1024L; private String readConsistencyLevel = "strong"; @@ -3477,6 +3478,15 @@ public void setGeneralRegionAttributeSecurityServiceIntervalSeconds( generalRegionAttributeSecurityServiceIntervalSeconds; } + public long getDetailContainerMinDegradeMemoryInBytes() { + return detailContainerMinDegradeMemoryInBytes; + } + + public void setDetailContainerMinDegradeMemoryInBytes( + long detailContainerMinDegradeMemoryInBytes) { + this.detailContainerMinDegradeMemoryInBytes = detailContainerMinDegradeMemoryInBytes; + } + public String getReadConsistencyLevel() { return readConsistencyLevel; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index c830b325a776d..2231faa0bca4f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1078,6 +1078,12 @@ public void loadProperties(Properties properties) throws BadNodeUrlException, IO "general_region_attribute_security_service_interval_seconds", String.valueOf(conf.getGeneralRegionAttributeSecurityServiceIntervalSeconds())))); + conf.setDetailContainerMinDegradeMemoryInBytes( + Long.parseLong( + properties.getProperty( + "detail_container_min_degrade_memory_in_bytes", + String.valueOf(conf.getDetailContainerMinDegradeMemoryInBytes())))); + loadIoTConsensusProps(properties); loadIoTConsensusV2Props(properties); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 478697bb7df89..6e3b28d2b309b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -83,11 +83,14 @@ public void update( requestMemory(UpdateDetailContainer.INSTANCE_SIZE); } attributeUpdateMap.put(pair.getLeft(), newContainer); + updateContainerStatistics.put(pair.getLeft(), new UpdateContainerStatistics()); } - updateMemory( + final long size = attributeUpdateMap .get(pair.getLeft()) - .updateAttribute(tableName, deviceId, attributeMap)); + .updateAttribute(tableName, deviceId, attributeMap); + updateContainerStatistics.get(pair.getLeft()).addSize(size); + updateMemory(size); }); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java index a74b52f1f9641..d429ec17cd07f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java @@ -19,7 +19,12 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; +import org.apache.iotdb.db.conf.IoTDBDescriptor; + +// For degrade public class UpdateContainerStatistics { + private static final long MIN_DEGRADE_MEMORY = + IoTDBDescriptor.getInstance().getConfig().getDetailContainerMinDegradeMemoryInBytes(); private long lastUpdateTime = System.currentTimeMillis(); private long size = 0; @@ -27,11 +32,11 @@ private void update() { lastUpdateTime = System.currentTimeMillis(); } - private void addSize(final long increment) { + void addSize(final long increment) { this.size += increment; } long getDegradePriority() { - return size * (System.currentTimeMillis() - lastUpdateTime); + return size >= MIN_DEGRADE_MEMORY ? size * (System.currentTimeMillis() - lastUpdateTime) : 0; } } From 84124e81527ac5f16dcad5bc1de7e752dace7738 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 17:40:01 +0800 Subject: [PATCH 045/213] Degrade logic --- .../update/DeviceAttributeRemoteUpdater.java | 36 +++++++++++++++++-- .../update/UpdateContainerStatistics.java | 10 +++++- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 6e3b28d2b309b..9dfd43a8779d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -39,10 +39,12 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; @@ -57,6 +59,8 @@ public class DeviceAttributeRemoteUpdater { // Volatiles private final MemSchemaRegionStatistics regionStatistics; + + // Only exist for update detail container private final Map updateContainerStatistics = new HashMap<>(); @@ -69,7 +73,28 @@ public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatis public void update( final String tableName, final String[] deviceId, final Map attributeMap) { // Degrade - if (!regionStatistics.isAllowToCreateNewSeries()) {} + if (!regionStatistics.isAllowToCreateNewSeries()) { + final TreeSet degradeSet = + new TreeSet<>( + Comparator.comparingLong(v -> updateContainerStatistics.get(v).getDegradePriority())); + updateContainerStatistics.forEach( + (k, v) -> { + if (v.needDegrade()) { + degradeSet.add(k); + } + }); + for (final TEndPoint endPoint : degradeSet) { + if (regionStatistics.isAllowToCreateNewSeries()) { + break; + } + final UpdateClearContainer newContainer = + ((UpdateDetailContainer) attributeUpdateMap.get(endPoint)).degrade(); + updateMemory( + newContainer.ramBytesUsed() - updateContainerStatistics.get(endPoint).getSize()); + attributeUpdateMap.put(endPoint, newContainer); + updateContainerStatistics.remove(endPoint); + } + } targetDataNodeLocations.forEach( pair -> { @@ -81,15 +106,20 @@ public void update( } else { newContainer = new UpdateDetailContainer(); requestMemory(UpdateDetailContainer.INSTANCE_SIZE); + updateContainerStatistics.put(pair.getLeft(), new UpdateContainerStatistics()); } attributeUpdateMap.put(pair.getLeft(), newContainer); - updateContainerStatistics.put(pair.getLeft(), new UpdateContainerStatistics()); } final long size = attributeUpdateMap .get(pair.getLeft()) .updateAttribute(tableName, deviceId, attributeMap); - updateContainerStatistics.get(pair.getLeft()).addSize(size); + updateContainerStatistics.computeIfPresent( + pair.getLeft(), + (k, v) -> { + v.addSize(size); + return v; + }); updateMemory(size); }); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java index d429ec17cd07f..12721bf985334 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java @@ -32,11 +32,19 @@ private void update() { lastUpdateTime = System.currentTimeMillis(); } + long getSize() { + return size; + } + void addSize(final long increment) { this.size += increment; } + boolean needDegrade() { + return size >= MIN_DEGRADE_MEMORY; + } + long getDegradePriority() { - return size >= MIN_DEGRADE_MEMORY ? size * (System.currentTimeMillis() - lastUpdateTime) : 0; + return size * (System.currentTimeMillis() - lastUpdateTime); } } From c85e5ac55d7bbe4f2e671738f413f41a21a5e62c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 17:47:15 +0800 Subject: [PATCH 046/213] Update DeviceAttributeRemoteUpdater.java --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 9dfd43a8779d9..f73ed5c224bdd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -76,7 +76,8 @@ public void update( if (!regionStatistics.isAllowToCreateNewSeries()) { final TreeSet degradeSet = new TreeSet<>( - Comparator.comparingLong(v -> updateContainerStatistics.get(v).getDegradePriority())); + Comparator.comparingLong(v -> updateContainerStatistics.get(v).getDegradePriority()) + .reversed()); updateContainerStatistics.forEach( (k, v) -> { if (v.needDegrade()) { From 836c51b3d25942804fe2f1910ca5c90dc7bf863f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 17:48:53 +0800 Subject: [PATCH 047/213] Refactor --- .../update/UpdateClearContainer.java | 3 +- .../attribute/update/UpdateContainer.java | 2 -- .../update/UpdateDetailContainer.java | 29 ------------------- 3 files changed, 1 insertion(+), 33 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 85523237cf340..08bbff8ac7382 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -97,8 +97,7 @@ public void deserialize(final InputStream inputStream) throws IOException { } } - @Override - public long ramBytesUsed() { + long ramBytesUsed() { return INSTANCE_SIZE + tableNames.stream().mapToLong(RamUsageEstimator::sizeOf).reduce(0L, Long::sum); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index bb1969cac3219..baa704c44a366 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -43,6 +43,4 @@ long updateAttribute( void serialize(final OutputStream outputstream) throws IOException; void deserialize(final InputStream inputStream) throws IOException; - - long ramBytesUsed(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 872b30e7813af..0d311dbab7a7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -158,35 +158,6 @@ private static ConcurrentMap readConcurrentMap(final InputStream } } - @Override - public long ramBytesUsed() { - final AtomicLong result = new AtomicLong(INSTANCE_SIZE); - - // Do not use "sizeOfMap" to save time and ensure that the calculated bytes is the same as - // previous calculation's sum - updateMap.forEach( - (name, value) -> { - result.addAndGet( - RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY - + RamUsageEstimator.sizeOf(name) - + MAP_SIZE); - value.forEach( - (device, attributes) -> { - result.addAndGet( - RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY - + RamUsageEstimator.sizeOf(device) - + MAP_SIZE); - attributes.forEach( - (k, v) -> - result.addAndGet( - RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY - + RamUsageEstimator.sizeOf(k) - + RamUsageEstimator.sizeOf(v))); - }); - }); - return result.get(); - } - UpdateClearContainer degrade() { return new UpdateClearContainer(updateMap.keySet()); } From ec7ee773c36b1d07a7a22d993a8971fbdd32c22e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 17:51:52 +0800 Subject: [PATCH 048/213] Update UpdateClearContainer.java --- .../schemaregion/attribute/update/UpdateClearContainer.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 08bbff8ac7382..a664809422ad9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -66,10 +66,7 @@ public long updateAttribute( public byte[] getUpdateContent() { final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try { - ReadWriteIOUtils.write(tableNames.size(), outputStream); - for (final String tableName : tableNames) { - ReadWriteIOUtils.write(tableName, outputStream); - } + serialize(outputStream); } catch (final IOException ignored) { // ByteArrayOutputStream won't throw IOException } From 12845365334a56f54035d1a8e66aa7a9786ba67f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 17:54:22 +0800 Subject: [PATCH 049/213] Refactor --- .../update/DeviceAttributeRemoteUpdater.java | 54 ++++++++++--------- .../impl/SchemaRegionMemoryImpl.java | 4 +- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index f73ed5c224bdd..7363fed8ecc43 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -72,31 +72,6 @@ public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatis public void update( final String tableName, final String[] deviceId, final Map attributeMap) { - // Degrade - if (!regionStatistics.isAllowToCreateNewSeries()) { - final TreeSet degradeSet = - new TreeSet<>( - Comparator.comparingLong(v -> updateContainerStatistics.get(v).getDegradePriority()) - .reversed()); - updateContainerStatistics.forEach( - (k, v) -> { - if (v.needDegrade()) { - degradeSet.add(k); - } - }); - for (final TEndPoint endPoint : degradeSet) { - if (regionStatistics.isAllowToCreateNewSeries()) { - break; - } - final UpdateClearContainer newContainer = - ((UpdateDetailContainer) attributeUpdateMap.get(endPoint)).degrade(); - updateMemory( - newContainer.ramBytesUsed() - updateContainerStatistics.get(endPoint).getSize()); - attributeUpdateMap.put(endPoint, newContainer); - updateContainerStatistics.remove(endPoint); - } - } - targetDataNodeLocations.forEach( pair -> { if (!attributeUpdateMap.containsKey(pair.getLeft())) { @@ -133,9 +108,36 @@ public void addLocation(final Pair dataNodeLocation) { targetDataNodeLocations.add(dataNodeLocation); } - public void addVersionAndNotifySend() { + public void afterUpdate() { version.incrementAndGet(); GeneralRegionAttributeSecurityService.getInstance().notifyBroadCast(); + degrade(); + } + + private void degrade() { + if (!regionStatistics.isAllowToCreateNewSeries()) { + final TreeSet degradeSet = + new TreeSet<>( + Comparator.comparingLong(v -> updateContainerStatistics.get(v).getDegradePriority()) + .reversed()); + updateContainerStatistics.forEach( + (k, v) -> { + if (v.needDegrade()) { + degradeSet.add(k); + } + }); + for (final TEndPoint endPoint : degradeSet) { + if (regionStatistics.isAllowToCreateNewSeries()) { + break; + } + final UpdateClearContainer newContainer = + ((UpdateDetailContainer) attributeUpdateMap.get(endPoint)).degrade(); + updateMemory( + newContainer.ramBytesUsed() - updateContainerStatistics.get(endPoint).getSize()); + attributeUpdateMap.put(endPoint, newContainer); + updateContainerStatistics.remove(endPoint); + } + } } /////////////////////////////// Snapshot /////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 10457203c4caa..9035147dfdef2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1377,7 +1377,7 @@ public void createOrUpdateTableDevice(final CreateOrUpdateTableDeviceNode node) pointer -> { updateAttribute( databaseName, tableName, deviceId, pointer, attributeNameList, attributeValueList); - deviceAttributeRemoteUpdater.addVersionAndNotifySend(); + deviceAttributeRemoteUpdater.afterUpdate(); }); } writeToMLog(node); @@ -1412,7 +1412,7 @@ public void updateTableDeviceAttribute(final TableDeviceAttributeUpdateNode upda mtree.updateTableDevice(pattern, batchUpdater); } } - deviceAttributeRemoteUpdater.addVersionAndNotifySend(); + deviceAttributeRemoteUpdater.afterUpdate(); writeToMLog(updateNode); } From eea2f03885e5988bf482d9f877a5ff3505837b10 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 17:54:54 +0800 Subject: [PATCH 050/213] Update DeviceAttributeRemoteUpdater.java --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 7363fed8ecc43..949994fe4ce71 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -110,8 +110,8 @@ public void addLocation(final Pair dataNodeLocation) { public void afterUpdate() { version.incrementAndGet(); - GeneralRegionAttributeSecurityService.getInstance().notifyBroadCast(); degrade(); + GeneralRegionAttributeSecurityService.getInstance().notifyBroadCast(); } private void degrade() { From 203e622fb96ede8eecdece72f86a6ef2c6474165 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 17:55:34 +0800 Subject: [PATCH 051/213] Update DeviceAttributeRemoteUpdater.java --- .../update/DeviceAttributeRemoteUpdater.java | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 949994fe4ce71..b434977a3e606 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -115,28 +115,28 @@ public void afterUpdate() { } private void degrade() { - if (!regionStatistics.isAllowToCreateNewSeries()) { - final TreeSet degradeSet = - new TreeSet<>( - Comparator.comparingLong(v -> updateContainerStatistics.get(v).getDegradePriority()) - .reversed()); - updateContainerStatistics.forEach( - (k, v) -> { - if (v.needDegrade()) { - degradeSet.add(k); - } - }); - for (final TEndPoint endPoint : degradeSet) { - if (regionStatistics.isAllowToCreateNewSeries()) { - break; - } - final UpdateClearContainer newContainer = - ((UpdateDetailContainer) attributeUpdateMap.get(endPoint)).degrade(); - updateMemory( - newContainer.ramBytesUsed() - updateContainerStatistics.get(endPoint).getSize()); - attributeUpdateMap.put(endPoint, newContainer); - updateContainerStatistics.remove(endPoint); + if (regionStatistics.isAllowToCreateNewSeries()) { + return; + } + final TreeSet degradeSet = + new TreeSet<>( + Comparator.comparingLong(v -> updateContainerStatistics.get(v).getDegradePriority()) + .reversed()); + updateContainerStatistics.forEach( + (k, v) -> { + if (v.needDegrade()) { + degradeSet.add(k); + } + }); + for (final TEndPoint endPoint : degradeSet) { + if (regionStatistics.isAllowToCreateNewSeries()) { + return; } + final UpdateClearContainer newContainer = + ((UpdateDetailContainer) attributeUpdateMap.get(endPoint)).degrade(); + updateMemory(newContainer.ramBytesUsed() - updateContainerStatistics.get(endPoint).getSize()); + attributeUpdateMap.put(endPoint, newContainer); + updateContainerStatistics.remove(endPoint); } } From f51885640ac8e1fd1d112b0c2a28f1f8da9839f9 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 27 Sep 2024 18:57:58 +0800 Subject: [PATCH 052/213] Create RewritableByteArrayOutputStream.java --- .../RewritableByteArrayOutputStream.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java new file mode 100644 index 0000000000000..8e0ef180652ce --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +public class RewritableByteArrayOutputStream extends ByteArrayOutputStream { + + private static final byte[] longPlaceHolder = new byte[4]; + + public synchronized int skipLong() throws IOException { + final int result = count; + write(longPlaceHolder); + return result; + } + + public synchronized void rewrite(final byte[] b, final int off) { + if ((off < 0) || (off > buf.length - b.length)) { + throw new IndexOutOfBoundsException(); + } + System.arraycopy(b, 0, buf, off, b.length); + } +} From 3eb768e1b251b29139c6b6206bdc06fcee83711f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 09:43:11 +0800 Subject: [PATCH 053/213] continue --- .../schemaregion/attribute/update/UpdateClearContainer.java | 2 +- .../schemaregion/attribute/update/UpdateContainer.java | 4 +++- .../schemaregion/attribute/update/UpdateDetailContainer.java | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index a664809422ad9..6268f913ae36c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -63,7 +63,7 @@ public long updateAttribute( } @Override - public byte[] getUpdateContent() { + public byte[] getUpdateContent(final long limitBytes) { final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try { serialize(outputStream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index baa704c44a366..3574d97b6b9cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -36,7 +36,9 @@ long updateAttribute( final String tableName, final String[] deviceId, final Map updatedAttributes); // Only this method is not synchronize called and is called by GRASS thread - byte[] getUpdateContent(); + // A piece of "updateContent" won't exceed "limitBytes" in order to handle + // thrift threshold and low bandwidth + byte[] getUpdateContent(final long limitBytes); Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 0d311dbab7a7c..44e77fb781078 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -98,7 +98,7 @@ public long updateAttribute( } @Override - public byte[] getUpdateContent() { + public byte[] getUpdateContent(final long limitBytes) { return null; } From 6d686ec4753a51dd90069bb8b5196e9d56ae4f1b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 09:45:46 +0800 Subject: [PATCH 054/213] Use default --- .../attribute/update/UpdateClearContainer.java | 11 ----------- .../attribute/update/UpdateContainer.java | 11 ++++++++++- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 6268f913ae36c..1837545c89047 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -62,17 +62,6 @@ public long updateAttribute( return tableNames.add(tableName) ? RamUsageEstimator.sizeOf(tableName) : 0; } - @Override - public byte[] getUpdateContent(final long limitBytes) { - final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try { - serialize(outputStream); - } catch (final IOException ignored) { - // ByteArrayOutputStream won't throw IOException - } - return outputStream.toByteArray(); - } - @Override public Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer) { return null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 3574d97b6b9cb..4e8fb1ed448f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -23,6 +23,7 @@ import javax.annotation.concurrent.ThreadSafe; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -38,7 +39,15 @@ long updateAttribute( // Only this method is not synchronize called and is called by GRASS thread // A piece of "updateContent" won't exceed "limitBytes" in order to handle // thrift threshold and low bandwidth - byte[] getUpdateContent(final long limitBytes); + default byte[] getUpdateContent(final long limitBytes) { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try { + serialize(outputStream); + } catch (final IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + return outputStream.toByteArray(); + } Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer); From 4917340fcfddc17a0af2066384e58a2f10fe412d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 10:16:53 +0800 Subject: [PATCH 055/213] Tricky --- .../RewritableByteArrayOutputStream.java | 32 ++++++++++--- .../update/UpdateClearContainer.java | 1 - .../attribute/update/UpdateContainer.java | 2 +- .../update/UpdateDetailContainer.java | 47 +++++++++++++++---- 4 files changed, 66 insertions(+), 16 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java index 8e0ef180652ce..72a09c1446fb7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java @@ -19,23 +19,43 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; +import org.apache.tsfile.utils.BytesUtils; + import java.io.ByteArrayOutputStream; import java.io.IOException; public class RewritableByteArrayOutputStream extends ByteArrayOutputStream { - private static final byte[] longPlaceHolder = new byte[4]; + private static final byte[] intPlaceHolder = new byte[4]; + private final int limit; + + public RewritableByteArrayOutputStream(final int limit) { + this.limit = limit; + } + + public synchronized boolean tryWrite(final byte[] content) throws IOException { + if (content.length + count <= limit) { + write(content); + return true; + } + return false; + } - public synchronized int skipLong() throws IOException { + public synchronized int skipInt() throws IOException { final int result = count; - write(longPlaceHolder); + write(intPlaceHolder); return result; } - public synchronized void rewrite(final byte[] b, final int off) { - if ((off < 0) || (off > buf.length - b.length)) { + public synchronized void rewrite(final int n, final int off) { + byte[] bytes = BytesUtils.intToBytes(n); + if ((off < 0) || (off > buf.length - bytes.length)) { throw new IndexOutOfBoundsException(); } - System.arraycopy(b, 0, buf, off, b.length); + System.arraycopy(bytes, 0, buf, off, bytes.length); + } + + public int getCount() { + return count; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 1837545c89047..ef02b8bb6e518 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -25,7 +25,6 @@ import javax.annotation.concurrent.ThreadSafe; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 4e8fb1ed448f1..3479feeb98e8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -39,7 +39,7 @@ long updateAttribute( // Only this method is not synchronize called and is called by GRASS thread // A piece of "updateContent" won't exceed "limitBytes" in order to handle // thrift threshold and low bandwidth - default byte[] getUpdateContent(final long limitBytes) { + default byte[] getUpdateContent(final int limitBytes) { final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try { serialize(outputStream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 44e77fb781078..c73f17e23c82d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -98,8 +98,38 @@ public long updateAttribute( } @Override - public byte[] getUpdateContent(final long limitBytes) { - return null; + public byte[] getUpdateContent(final int limitBytes) { + if (limitBytes < 0) { + return UpdateContainer.super.getUpdateContent(limitBytes); + } + final RewritableByteArrayOutputStream outputStream = + new RewritableByteArrayOutputStream(limitBytes); + try { + serializeWithLimit(outputStream); + } catch (final IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + return outputStream.toByteArray(); + } + + private void serializeWithLimit(final RewritableByteArrayOutputStream outputStream) + throws IOException { + final int mapSizeOffset = outputStream.skipInt(); + for (final Map.Entry>> + tableEntry : updateMap.entrySet()) { + ReadWriteIOUtils.write(tableEntry.getKey(), outputStream); + final int deviceSizeOffset = outputStream.skipInt(); + for (final Map.Entry> deviceEntry : + tableEntry.getValue().entrySet()) { + ReadWriteIOUtils.write(deviceEntry.getKey().length, outputStream); + for (final String node : deviceEntry.getKey()) { + ReadWriteIOUtils.write(node, outputStream); + } + ReadWriteIOUtils.write(deviceEntry.getValue(), outputStream); + } + outputStream.rewrite(tableEntry.getValue().size(), deviceSizeOffset); + } + outputStream.rewrite(updateMap.size(), mapSizeOffset); } @Override @@ -108,18 +138,19 @@ public Pair updateSelfByCommitBuffer(final ByteBuffer commitBu } @Override - public void serialize(final OutputStream outputstream) throws IOException { - ReadWriteIOUtils.write(updateMap.size(), outputstream); + public void serialize(final OutputStream outputStream) throws IOException { + ReadWriteIOUtils.write(updateMap.size(), outputStream); for (final Map.Entry>> tableEntry : updateMap.entrySet()) { - ReadWriteIOUtils.write(tableEntry.getKey(), outputstream); + ReadWriteIOUtils.write(tableEntry.getKey(), outputStream); + ReadWriteIOUtils.write(tableEntry.getValue().size(), outputStream); for (final Map.Entry> deviceEntry : tableEntry.getValue().entrySet()) { - ReadWriteIOUtils.write(deviceEntry.getKey().length, outputstream); + ReadWriteIOUtils.write(deviceEntry.getKey().length, outputStream); for (final String node : deviceEntry.getKey()) { - ReadWriteIOUtils.write(node, outputstream); + ReadWriteIOUtils.write(node, outputStream); } - ReadWriteIOUtils.write(deviceEntry.getValue(), outputstream); + ReadWriteIOUtils.write(deviceEntry.getValue(), outputStream); } } } From 58ca9735324cf414a62f1367d261fd9ed77064af Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 10:55:55 +0800 Subject: [PATCH 056/213] continue --- .../RewritableByteArrayOutputStream.java | 21 ++++++++----- .../attribute/update/UpdateContainer.java | 1 + .../update/UpdateDetailContainer.java | 30 +++++++++++++++++-- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java index 72a09c1446fb7..031b6e790cf1b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java @@ -20,10 +20,14 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import javax.annotation.concurrent.NotThreadSafe; import java.io.ByteArrayOutputStream; import java.io.IOException; +@NotThreadSafe public class RewritableByteArrayOutputStream extends ByteArrayOutputStream { private static final byte[] intPlaceHolder = new byte[4]; @@ -33,21 +37,22 @@ public RewritableByteArrayOutputStream(final int limit) { this.limit = limit; } - public synchronized boolean tryWrite(final byte[] content) throws IOException { - if (content.length + count <= limit) { - write(content); - return true; - } - return false; + public boolean checkCapacity(final int newGrowth) { + return newGrowth + count <= limit; + } + + public void writeWithLength(final byte[] bytes) throws IOException { + ReadWriteIOUtils.write(bytes.length, this); + write(bytes); } - public synchronized int skipInt() throws IOException { + public int skipInt() throws IOException { final int result = count; write(intPlaceHolder); return result; } - public synchronized void rewrite(final int n, final int off) { + public void rewrite(final int n, final int off) { byte[] bytes = BytesUtils.intToBytes(n); if ((off < 0) || (off > buf.length - bytes.length)) { throw new IndexOutOfBoundsException(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 3479feeb98e8f..112641d1d7b1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -39,6 +39,7 @@ long updateAttribute( // Only this method is not synchronize called and is called by GRASS thread // A piece of "updateContent" won't exceed "limitBytes" in order to handle // thrift threshold and low bandwidth + // The "limitBytes" shall be at least 4 for a "0" to indicate empty default byte[] getUpdateContent(final int limitBytes) { final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index c73f17e23c82d..ebbb7740f1de0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -29,6 +30,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -115,15 +117,37 @@ public byte[] getUpdateContent(final int limitBytes) { private void serializeWithLimit(final RewritableByteArrayOutputStream outputStream) throws IOException { final int mapSizeOffset = outputStream.skipInt(); + int mapEntryCount = 0; for (final Map.Entry>> tableEntry : updateMap.entrySet()) { - ReadWriteIOUtils.write(tableEntry.getKey(), outputStream); + final byte[] tableEntryBytes = tableEntry.getKey().getBytes(TSFileConfig.STRING_CHARSET); + if (!outputStream.checkCapacity(2 * Integer.BYTES + tableEntryBytes.length)) { + outputStream.rewrite(mapEntryCount, mapSizeOffset); + return; + } + ++mapEntryCount; + outputStream.writeWithLength(tableEntryBytes); final int deviceSizeOffset = outputStream.skipInt(); + int deviceEntryCount = 0; for (final Map.Entry> deviceEntry : tableEntry.getValue().entrySet()) { + final byte[][] deviceIdBytes = + Arrays.stream(deviceEntry.getKey()) + .map(str -> str.getBytes(TSFileConfig.STRING_CHARSET)) + .toArray(byte[][]::new); + if (!outputStream.checkCapacity( + (Integer.BYTES * (deviceIdBytes.length + 1)) + + Arrays.stream(deviceIdBytes) + .map(bytes -> bytes.length) + .reduce(0, Integer::sum))) { + outputStream.rewrite(mapEntryCount, mapSizeOffset); + outputStream.rewrite(deviceEntryCount, deviceSizeOffset); + return; + } + ++deviceEntryCount; ReadWriteIOUtils.write(deviceEntry.getKey().length, outputStream); - for (final String node : deviceEntry.getKey()) { - ReadWriteIOUtils.write(node, outputStream); + for (final byte[] node : deviceIdBytes) { + outputStream.writeWithLength(node); } ReadWriteIOUtils.write(deviceEntry.getValue(), outputStream); } From 6fd6ec3d2e02efe5adb04bd6fce96f5fa8fa5fdb Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 11:06:46 +0800 Subject: [PATCH 057/213] complete limit --- .../RewritableByteArrayOutputStream.java | 4 ++-- .../update/UpdateDetailContainer.java | 24 +++++++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java index 031b6e790cf1b..10aefcd978664 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java @@ -37,8 +37,8 @@ public RewritableByteArrayOutputStream(final int limit) { this.limit = limit; } - public boolean checkCapacity(final int newGrowth) { - return newGrowth + count <= limit; + public boolean exceedCapacity(final int newGrowth) { + return newGrowth + count > limit; } public void writeWithLength(final byte[] bytes) throws IOException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index ebbb7740f1de0..4bbb009644aff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -121,7 +121,7 @@ private void serializeWithLimit(final RewritableByteArrayOutputStream outputStre for (final Map.Entry>> tableEntry : updateMap.entrySet()) { final byte[] tableEntryBytes = tableEntry.getKey().getBytes(TSFileConfig.STRING_CHARSET); - if (!outputStream.checkCapacity(2 * Integer.BYTES + tableEntryBytes.length)) { + if (outputStream.exceedCapacity(2 * Integer.BYTES + tableEntryBytes.length)) { outputStream.rewrite(mapEntryCount, mapSizeOffset); return; } @@ -135,8 +135,8 @@ private void serializeWithLimit(final RewritableByteArrayOutputStream outputStre Arrays.stream(deviceEntry.getKey()) .map(str -> str.getBytes(TSFileConfig.STRING_CHARSET)) .toArray(byte[][]::new); - if (!outputStream.checkCapacity( - (Integer.BYTES * (deviceIdBytes.length + 1)) + if (outputStream.exceedCapacity( + (Integer.BYTES * (deviceIdBytes.length + 2)) + Arrays.stream(deviceIdBytes) .map(bytes -> bytes.length) .reduce(0, Integer::sum))) { @@ -149,7 +149,23 @@ private void serializeWithLimit(final RewritableByteArrayOutputStream outputStre for (final byte[] node : deviceIdBytes) { outputStream.writeWithLength(node); } - ReadWriteIOUtils.write(deviceEntry.getValue(), outputStream); + final int attributeOffset = outputStream.skipInt(); + int attributeCount = 0; + for (final Map.Entry attributeKV : deviceEntry.getValue().entrySet()) { + final byte[] keyBytes = attributeKV.getKey().getBytes(TSFileConfig.STRING_CHARSET); + final byte[] valueBytes = attributeKV.getValue().getBytes(TSFileConfig.STRING_CHARSET); + if (outputStream.exceedCapacity( + 2 * Integer.BYTES + keyBytes.length + valueBytes.length)) { + outputStream.rewrite(mapEntryCount, mapSizeOffset); + outputStream.rewrite(deviceEntryCount, deviceSizeOffset); + outputStream.rewrite(attributeCount, attributeOffset); + return; + } + outputStream.writeWithLength(keyBytes); + outputStream.writeWithLength(valueBytes); + ++attributeCount; + } + outputStream.rewrite(deviceEntry.getValue().size(), attributeOffset); } outputStream.rewrite(tableEntry.getValue().size(), deviceSizeOffset); } From 45995e3df1f89c51b7c918badb8891acce861c18 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 11:13:41 +0800 Subject: [PATCH 058/213] reuse pipe config --- .../update/DeviceAttributeRemoteUpdater.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index b434977a3e606..3b027cb061086 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.utils.FileUtils; @@ -48,6 +49,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; public class DeviceAttributeRemoteUpdater { private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeRemoteUpdater.class); @@ -101,7 +103,17 @@ public void update( } public Map getSendBuffer() { - return null; + return attributeUpdateMap.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> + entry + .getValue() + .getUpdateContent( + CommonDescriptor.getInstance() + .getConfig() + .getPipeConnectorRequestSliceThresholdBytes()))); } public void addLocation(final Pair dataNodeLocation) { From d74d449c04e9e87c4de508f765b504bd78994fb4 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 11:16:41 +0800 Subject: [PATCH 059/213] Update DeviceAttributeRemoteUpdater.java --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 3b027cb061086..84846befef535 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -102,6 +102,10 @@ public void update( }); } + public long getVersion() { + return version.get(); + } + public Map getSendBuffer() { return attributeUpdateMap.entrySet().stream() .collect( From 86bfb57af250d35f770150eab5f42e92a07308c3 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 11:21:51 +0800 Subject: [PATCH 060/213] Refactor --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 1 - .../schemaregion/attribute/update/UpdateClearContainer.java | 1 + .../schemaregion/attribute/update/UpdateDetailContainer.java | 2 ++ 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 84846befef535..10230bf67537c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -210,7 +210,6 @@ private void serialize(final OutputStream outputStream) throws IOException { ReadWriteIOUtils.write(attributeUpdateMap.size(), outputStream); for (final Map.Entry entry : attributeUpdateMap.entrySet()) { ThriftCommonsSerDeUtils.serializeTEndPoint(entry.getKey(), outputStream); - ReadWriteIOUtils.write(entry.getValue() instanceof UpdateDetailContainer, outputStream); entry.getValue().serialize(outputStream); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index ef02b8bb6e518..46129a95b20dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -68,6 +68,7 @@ public Pair updateSelfByCommitBuffer(final ByteBuffer commitBu @Override public void serialize(final OutputStream outputstream) throws IOException { + ReadWriteIOUtils.write((byte) 0, outputstream); ReadWriteIOUtils.write(tableNames.size(), outputstream); for (final String tableName : tableNames) { ReadWriteIOUtils.write(tableName, outputstream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 4bbb009644aff..371fba9e56885 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -107,6 +107,7 @@ public byte[] getUpdateContent(final int limitBytes) { final RewritableByteArrayOutputStream outputStream = new RewritableByteArrayOutputStream(limitBytes); try { + ReadWriteIOUtils.write((byte) 1, outputStream); serializeWithLimit(outputStream); } catch (final IOException ignored) { // ByteArrayOutputStream won't throw IOException @@ -179,6 +180,7 @@ public Pair updateSelfByCommitBuffer(final ByteBuffer commitBu @Override public void serialize(final OutputStream outputStream) throws IOException { + ReadWriteIOUtils.write((byte) 1, outputStream); ReadWriteIOUtils.write(updateMap.size(), outputStream); for (final Map.Entry>> tableEntry : updateMap.entrySet()) { From c9e7b944f9d69d95664403610a15cb18f2ad2488 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 11:26:46 +0800 Subject: [PATCH 061/213] Update GeneralRegionAttributeSecurityService.java --- .../update/GeneralRegionAttributeSecurityService.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 1bca1dee46271..8a3c3308240dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -30,7 +30,6 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; @@ -43,7 +42,6 @@ public class GeneralRegionAttributeSecurityService { IoTDBThreadPoolFactory.newSingleThreadExecutor( ThreadName.GENERAL_REGION_ATTRIBUTE_SECURITY_SERVICE.getName()); - private Future executorFuture; private final Set regionLeaders = new HashSet<>(); private final ReentrantLock lock = new ReentrantLock(); private final Condition condition = lock.newCondition(); @@ -51,7 +49,7 @@ public class GeneralRegionAttributeSecurityService { public void startBroadcast(final SchemaRegionId id) { if (regionLeaders.isEmpty()) { - executorFuture = securityServiceExecutor.submit(this::execute); + securityServiceExecutor.submit(this::execute); LOGGER.info("General region attribute security service is started successfully."); } @@ -62,8 +60,7 @@ public void stopBroadcast(final SchemaRegionId id) { regionLeaders.remove(id); if (regionLeaders.isEmpty()) { - executorFuture.cancel(false); - executorFuture = null; + securityServiceExecutor.shutdown(); LOGGER.info("General region attribute security service is stopped successfully."); } } From 94761dc9ac45a9679ec4d0ab68139b50367f4184 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 11:34:25 +0800 Subject: [PATCH 062/213] Refactor --- .../SchemaRegionStateMachine.java | 29 ++++++++++--------- ...GeneralRegionAttributeSecurityService.java | 5 ++++ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java index 8828fe0df51b6..22699b08f4671 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java @@ -53,7 +53,7 @@ public class SchemaRegionStateMachine extends BaseStateMachine { private static final FragmentInstanceManager QUERY_INSTANCE_MANAGER = FragmentInstanceManager.getInstance(); - public SchemaRegionStateMachine(ISchemaRegion schemaRegion) { + public SchemaRegionStateMachine(final ISchemaRegion schemaRegion) { this.schemaRegion = schemaRegion; } @@ -71,7 +71,7 @@ public void stop() { } @Override - public void notifyLeaderChanged(ConsensusGroupId groupId, int newLeaderId) { + public void notifyLeaderChanged(final ConsensusGroupId groupId, final int newLeaderId) { if (newLeaderId != IoTDBDescriptor.getInstance().getConfig().getDataNodeId()) { logger.info( "Current node [nodeId: {}] is no longer the schema region leader [regionId: {}], " @@ -84,7 +84,7 @@ public void notifyLeaderChanged(ConsensusGroupId groupId, int newLeaderId) { @Override public void notifyNotLeader() { - int dataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); + final int dataNodeId = IoTDBDescriptor.getInstance().getConfig().getDataNodeId(); logger.info( "Current node [nodeId: {}] is no longer the schema region leader [regionId: {}], " + "start cleaning up related services.", @@ -127,7 +127,7 @@ public boolean isReadOnly() { } @Override - public boolean takeSnapshot(File snapshotDir) { + public boolean takeSnapshot(final File snapshotDir) { if (schemaRegion.createSnapshot(snapshotDir) && PipeDataNodeAgent.runtime() .schemaListener(schemaRegion.getSchemaRegionId()) @@ -139,7 +139,7 @@ public boolean takeSnapshot(File snapshotDir) { } @Override - public void loadSnapshot(File latestSnapshotRootDir) { + public void loadSnapshot(final File latestSnapshotRootDir) { schemaRegion.loadSnapshot(latestSnapshotRootDir); PipeDataNodeAgent.runtime() .schemaListener(schemaRegion.getSchemaRegionId()) @@ -149,14 +149,14 @@ public void loadSnapshot(File latestSnapshotRootDir) { listen2Snapshot4PipeListener(false); } - public void listen2Snapshot4PipeListener(boolean isTmp) { - Pair snapshotPaths = + public void listen2Snapshot4PipeListener(final boolean isTmp) { + final Pair snapshotPaths = SchemaRegionSnapshotParser.getSnapshotPaths( Utils.fromConsensusGroupIdToRaftGroupId(schemaRegion.getSchemaRegionId()) .getUuid() .toString(), isTmp); - SchemaRegionListeningQueue listener = + final SchemaRegionListeningQueue listener = PipeDataNodeAgent.runtime().schemaListener(schemaRegion.getSchemaRegionId()); if (Objects.isNull(snapshotPaths) || Objects.isNull(snapshotPaths.getLeft())) { if (listener.isOpened()) { @@ -176,27 +176,28 @@ public void listen2Snapshot4PipeListener(boolean isTmp) { } @Override - public TSStatus write(IConsensusRequest request) { + public TSStatus write(final IConsensusRequest request) { try { - TSStatus result = ((PlanNode) request).accept(new SchemaExecutionVisitor(), schemaRegion); + final TSStatus result = + ((PlanNode) request).accept(new SchemaExecutionVisitor(), schemaRegion); if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { PipeDataNodeAgent.runtime() .schemaListener(schemaRegion.getSchemaRegionId()) .tryListenToNode((PlanNode) request); } return result; - } catch (IllegalArgumentException e) { + } catch (final IllegalArgumentException e) { logger.error(e.getMessage(), e); return new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode()); } } @Override - public DataSet read(IConsensusRequest request) { - FragmentInstance fragmentInstance; + public DataSet read(final IConsensusRequest request) { + final FragmentInstance fragmentInstance; try { fragmentInstance = getFragmentInstance(request); - } catch (IllegalArgumentException e) { + } catch (final IllegalArgumentException e) { logger.error(e.getMessage()); return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 8a3c3308240dc..f187a895795e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.consensus.SchemaRegionId; @@ -27,7 +28,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; @@ -80,6 +83,8 @@ public void notifyBroadCast() { private void execute() { lock.lock(); try { + final Map> attributeUpdateCommitMap = new HashMap<>(); + if (!skipNext) { condition.await( IoTDBDescriptor.getInstance() From 9e44a146c1c91e4918bec6a5b50e53c347c519e8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 11:41:04 +0800 Subject: [PATCH 063/213] Refactor --- .../SchemaRegionStateMachine.java | 9 +- .../schemaregion/ISchemaRegion.java | 86 ++++++++++--------- ...GeneralRegionAttributeSecurityService.java | 11 +-- 3 files changed, 55 insertions(+), 51 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java index 22699b08f4671..fe88c38f17472 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java @@ -66,8 +66,7 @@ public void start() { public void stop() { // Stop leader related service for schema pipe PipeDataNodeAgent.runtime().notifySchemaLeaderUnavailable(schemaRegion.getSchemaRegionId()); - GeneralRegionAttributeSecurityService.getInstance() - .stopBroadcast(schemaRegion.getSchemaRegionId()); + GeneralRegionAttributeSecurityService.getInstance().stopBroadcast(schemaRegion); } @Override @@ -93,8 +92,7 @@ public void notifyNotLeader() { // Shutdown leader related service for schema pipe PipeDataNodeAgent.runtime().notifySchemaLeaderUnavailable(schemaRegion.getSchemaRegionId()); - GeneralRegionAttributeSecurityService.getInstance() - .stopBroadcast(schemaRegion.getSchemaRegionId()); + GeneralRegionAttributeSecurityService.getInstance().stopBroadcast(schemaRegion); logger.info( "Current node [nodeId: {}] is no longer the schema region leader [regionId: {}], " @@ -112,8 +110,7 @@ public void notifyLeaderReady() { // Activate leader related service for schema pipe PipeDataNodeAgent.runtime().notifySchemaLeaderReady(schemaRegion.getSchemaRegionId()); - GeneralRegionAttributeSecurityService.getInstance() - .startBroadcast(schemaRegion.getSchemaRegionId()); + GeneralRegionAttributeSecurityService.getInstance().startBroadcast(schemaRegion); logger.info( "Current node [nodeId: {}] as schema region leader [regionId: {}] is ready to work", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 57da8f5d1d6f4..3549175fd7961 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -100,9 +100,9 @@ public interface ISchemaRegion { // delete this schemaRegion and clear all resources void deleteSchemaRegion() throws MetadataException; - boolean createSnapshot(File snapshotDir); + boolean createSnapshot(final File snapshotDir); - void loadSnapshot(File latestSnapshotRootDir); + void loadSnapshot(final File latestSnapshotRootDir); // endregion @@ -114,7 +114,8 @@ public interface ISchemaRegion { * @param offset * @throws MetadataException */ - void createTimeSeries(ICreateTimeSeriesPlan plan, long offset) throws MetadataException; + void createTimeSeries(final ICreateTimeSeriesPlan plan, final long offset) + throws MetadataException; /** * Create aligned timeseries. @@ -122,7 +123,7 @@ public interface ISchemaRegion { * @param plan a plan describes how to create the timeseries. * @throws MetadataException */ - void createAlignedTimeSeries(ICreateAlignedTimeSeriesPlan plan) throws MetadataException; + void createAlignedTimeSeries(final ICreateAlignedTimeSeriesPlan plan) throws MetadataException; /** * Check whether measurement exists. @@ -135,7 +136,9 @@ public interface ISchemaRegion { * example, a MeasurementAlreadyExistException means this measurement exists. */ Map checkMeasurementExistence( - PartialPath devicePath, List measurementList, List aliasList); + final PartialPath devicePath, + final List measurementList, + final List aliasList); /** * Check whether time series can be created. @@ -144,7 +147,7 @@ Map checkMeasurementExistence( * @param timeSeriesNum the number of time series that you want to check * @throws SchemaQuotaExceededException if the number of time series or devices exceeds the limit */ - void checkSchemaQuota(PartialPath devicePath, int timeSeriesNum) + void checkSchemaQuota(final PartialPath devicePath, final int timeSeriesNum) throws SchemaQuotaExceededException; /** @@ -165,7 +168,7 @@ Pair constructSchemaBlackList(final PathPatternTree patternTree) * @param patternTree * @throws MetadataException */ - void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException; + void rollbackSchemaBlackList(final PathPatternTree patternTree) throws MetadataException; /** * Fetch schema black list (timeseries that has been pre deleted). @@ -173,7 +176,7 @@ Pair constructSchemaBlackList(final PathPatternTree patternTree) * @param patternTree * @throws MetadataException */ - Set fetchSchemaBlackList(PathPatternTree patternTree) throws MetadataException; + Set fetchSchemaBlackList(final PathPatternTree patternTree) throws MetadataException; /** * Delete timeseries in schema black list. @@ -181,20 +184,21 @@ Pair constructSchemaBlackList(final PathPatternTree patternTree) * @param patternTree * @throws MetadataException */ - void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException; + void deleteTimeseriesInBlackList(final PathPatternTree patternTree) throws MetadataException; // endregion // region Interfaces for Logical View - void createLogicalView(ICreateLogicalViewPlan createLogicalViewPlan) throws MetadataException; + void createLogicalView(final ICreateLogicalViewPlan createLogicalViewPlan) + throws MetadataException; - long constructLogicalViewBlackList(PathPatternTree patternTree) throws MetadataException; + long constructLogicalViewBlackList(final PathPatternTree patternTree) throws MetadataException; - void rollbackLogicalViewBlackList(PathPatternTree patternTree) throws MetadataException; + void rollbackLogicalViewBlackList(final PathPatternTree patternTree) throws MetadataException; - void deleteLogicalView(PathPatternTree patternTree) throws MetadataException; + void deleteLogicalView(final PathPatternTree patternTree) throws MetadataException; - void alterLogicalView(IAlterLogicalViewPlan alterLogicalViewPlan) throws MetadataException; + void alterLogicalView(final IAlterLogicalViewPlan alterLogicalViewPlan) throws MetadataException; // endregion @@ -202,15 +206,15 @@ Pair constructSchemaBlackList(final PathPatternTree patternTree) // region Interfaces for timeSeries, measurement and schema info Query - MeasurementPath fetchMeasurementPath(PartialPath fullPath) throws MetadataException; + MeasurementPath fetchMeasurementPath(final PartialPath fullPath) throws MetadataException; ClusterSchemaTree fetchSeriesSchema( - PathPatternTree patternTree, - Map templateMap, - boolean withTags, - boolean withAttributes, - boolean withTemplate, - boolean withAliasForce) + final PathPatternTree patternTree, + final Map templateMap, + final boolean withTags, + final boolean withAttributes, + final boolean withTemplate, + final boolean withAliasForce) throws MetadataException; /** @@ -221,7 +225,8 @@ ClusterSchemaTree fetchSeriesSchema( * @param authorityScope the scope of the authority * @return pattern tree with all leaves as device nodes */ - ClusterSchemaTree fetchDeviceSchema(PathPatternTree patternTree, PathPatternTree authorityScope) + ClusterSchemaTree fetchDeviceSchema( + final PathPatternTree patternTree, final PathPatternTree authorityScope) throws MetadataException; // endregion @@ -239,10 +244,10 @@ ClusterSchemaTree fetchDeviceSchema(PathPatternTree patternTree, PathPatternTree * @param fullPath timeseries */ void upsertAliasAndTagsAndAttributes( - String alias, - Map tagsMap, - Map attributesMap, - PartialPath fullPath) + final String alias, + final Map tagsMap, + final Map attributesMap, + final PartialPath fullPath) throws MetadataException, IOException; /** @@ -252,7 +257,7 @@ void upsertAliasAndTagsAndAttributes( * @param fullPath timeseries * @throws MetadataException tagLogFile write error or attributes already exist */ - void addAttributes(Map attributesMap, PartialPath fullPath) + void addAttributes(final Map attributesMap, final PartialPath fullPath) throws MetadataException, IOException; /** @@ -262,7 +267,7 @@ void addAttributes(Map attributesMap, PartialPath fullPath) * @param fullPath timeseries * @throws MetadataException tagLogFile write error or tags already exist */ - void addTags(Map tagsMap, PartialPath fullPath) + void addTags(final Map tagsMap, final PartialPath fullPath) throws MetadataException, IOException; /** @@ -272,7 +277,7 @@ void addTags(Map tagsMap, PartialPath fullPath) * @param keySet tags key or attributes key * @param fullPath timeseries path */ - void dropTagsOrAttributes(Set keySet, PartialPath fullPath) + void dropTagsOrAttributes(final Set keySet, final PartialPath fullPath) throws MetadataException, IOException; /** @@ -282,7 +287,7 @@ void dropTagsOrAttributes(Set keySet, PartialPath fullPath) * @param fullPath timeseries * @throws MetadataException tagLogFile write error or tags/attributes do not exist */ - void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) + void setTagsOrAttributesValue(final Map alterMap, final PartialPath fullPath) throws MetadataException, IOException; /** @@ -294,22 +299,22 @@ void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath * @throws MetadataException tagLogFile write error or does not have tag/attribute or already has * a tag/attribute named newKey */ - void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) + void renameTagOrAttributeKey(final String oldKey, final String newKey, final PartialPath fullPath) throws MetadataException, IOException; // endregion // region Interfaces for Template operations - void activateSchemaTemplate(IActivateTemplateInClusterPlan plan, Template template) + void activateSchemaTemplate(final IActivateTemplateInClusterPlan plan, final Template template) throws MetadataException; - long constructSchemaBlackListWithTemplate(IPreDeactivateTemplatePlan plan) + long constructSchemaBlackListWithTemplate(final IPreDeactivateTemplatePlan plan) throws MetadataException; - void rollbackSchemaBlackListWithTemplate(IRollbackPreDeactivateTemplatePlan plan) + void rollbackSchemaBlackListWithTemplate(final IRollbackPreDeactivateTemplatePlan plan) throws MetadataException; - void deactivateTemplateInBlackList(IDeactivateTemplatePlan plan) throws MetadataException; + void deactivateTemplateInBlackList(final IDeactivateTemplatePlan plan) throws MetadataException; long countPathsUsingTemplate(int templateId, PathPatternTree patternTree) throws MetadataException; @@ -330,13 +335,13 @@ void updateTableDeviceAttribute(final TableDeviceAttributeUpdateNode updateNode) // region Interfaces for SchemaReader - ISchemaReader getDeviceReader(IShowDevicesPlan showDevicesPlan) + ISchemaReader getDeviceReader(final IShowDevicesPlan showDevicesPlan) throws MetadataException; - ISchemaReader getTimeSeriesReader(IShowTimeSeriesPlan showTimeSeriesPlan) - throws MetadataException; + ISchemaReader getTimeSeriesReader( + final IShowTimeSeriesPlan showTimeSeriesPlan) throws MetadataException; - ISchemaReader getNodeReader(IShowNodesPlan showNodesPlan) + ISchemaReader getNodeReader(final IShowNodesPlan showNodesPlan) throws MetadataException; ISchemaReader getTableDeviceReader(final PartialPath pathPattern) @@ -344,7 +349,8 @@ ISchemaReader getTableDeviceReader(final PartialPath pathPatt ISchemaReader getTableDeviceReader( final String table, final List devicePathList) throws MetadataException; - // endregion + + // region Interfaces for SchemaReader // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index f187a895795e2..cb6715753d7cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,22 +46,22 @@ public class GeneralRegionAttributeSecurityService { IoTDBThreadPoolFactory.newSingleThreadExecutor( ThreadName.GENERAL_REGION_ATTRIBUTE_SECURITY_SERVICE.getName()); - private final Set regionLeaders = new HashSet<>(); + private final Set regionLeaders = new HashSet<>(); private final ReentrantLock lock = new ReentrantLock(); private final Condition condition = lock.newCondition(); private volatile boolean skipNext = false; - public void startBroadcast(final SchemaRegionId id) { + public void startBroadcast(final ISchemaRegion schemaRegion) { if (regionLeaders.isEmpty()) { securityServiceExecutor.submit(this::execute); LOGGER.info("General region attribute security service is started successfully."); } - regionLeaders.add(id); + regionLeaders.add(schemaRegion); } - public void stopBroadcast(final SchemaRegionId id) { - regionLeaders.remove(id); + public void stopBroadcast(final ISchemaRegion schemaRegion) { + regionLeaders.remove(schemaRegion); if (regionLeaders.isEmpty()) { securityServiceExecutor.shutdown(); From 9166ec40cc6f72fa59377d83dec8f349313bfc4f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 11:45:34 +0800 Subject: [PATCH 064/213] UpdateMap --- .../schemaregion/ISchemaRegion.java | 4 ++- .../update/DeviceAttributeRemoteUpdater.java | 30 +++++++++---------- .../impl/SchemaRegionMemoryImpl.java | 6 ++++ .../impl/SchemaRegionPBTreeImpl.java | 7 +++++ 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 3549175fd7961..449e1a7f0263c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.MeasurementPath; @@ -350,7 +351,8 @@ ISchemaReader getTableDeviceReader(final PartialPath pathPatt ISchemaReader getTableDeviceReader( final String table, final List devicePathList) throws MetadataException; - // region Interfaces for SchemaReader + // region Interfaces for AttributeUpdate + Pair> getAttributeUpdateMap(); // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 10230bf67537c..9f4702265b9a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -102,22 +102,20 @@ public void update( }); } - public long getVersion() { - return version.get(); - } - - public Map getSendBuffer() { - return attributeUpdateMap.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, - entry -> - entry - .getValue() - .getUpdateContent( - CommonDescriptor.getInstance() - .getConfig() - .getPipeConnectorRequestSliceThresholdBytes()))); + public Pair> getAttributeUpdateMap(final int limit) { + return new Pair<>( + version.get(), + attributeUpdateMap.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> + entry + .getValue() + .getUpdateContent( + CommonDescriptor.getInstance() + .getConfig() + .getPipeConnectorRequestSliceThresholdBytes())))); } public void addLocation(final Pair dataNodeLocation) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 9035147dfdef2..367911bb2ae81 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.impl; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; @@ -1561,6 +1562,11 @@ public ISchemaReader getTableDeviceReader( table, devicePathList, (pointer, name) -> deviceAttributeStore.getAttribute(pointer, name)); } + @Override + public Pair> getAttributeUpdateMap() { + return deviceAttributeRemoteUpdater.getAttributeUpdateMap(0); + } + // endregion private static class RecoverOperationResult { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index cce4f0384be40..9ea704a7de190 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.impl; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.file.SystemFileFactory; @@ -1490,6 +1491,12 @@ public ISchemaReader getTableDeviceReader( throw new UnsupportedOperationException(); } + @Override + public Pair> getAttributeUpdateMap() { + // TODO + return null; + } + // endregion private static class RecoverOperationResult { From ac0e3fc5c9ac42f12cc1a9767271e293b019ebd1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 11:51:20 +0800 Subject: [PATCH 065/213] Update GeneralRegionAttributeSecurityService.java --- .../update/GeneralRegionAttributeSecurityService.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index cb6715753d7cf..19ff83cb63949 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -26,10 +26,10 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; public class GeneralRegionAttributeSecurityService { private static final Logger LOGGER = @@ -84,7 +85,13 @@ public void notifyBroadCast() { private void execute() { lock.lock(); try { - final Map> attributeUpdateCommitMap = new HashMap<>(); + final Map>> attributeUpdateCommitMap = + regionLeaders.stream() + .collect( + Collectors.toMap( + ISchemaRegion::getSchemaRegionId, ISchemaRegion::getAttributeUpdateMap)); + + // Send if (!skipNext) { condition.await( From 29861d67accdbcd2917187b4994e058a8333f54b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 12:08:35 +0800 Subject: [PATCH 066/213] Refactor --- .../db/schemaengine/schemaregion/ISchemaRegion.java | 2 +- .../update/DeviceAttributeRemoteUpdater.java | 5 ++++- .../GeneralRegionAttributeSecurityService.java | 2 +- .../attribute/update/UpdateClearContainer.java | 12 ++++++++++++ .../attribute/update/UpdateContainer.java | 11 +---------- .../attribute/update/UpdateDetailContainer.java | 5 +---- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 4 ++-- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 2 +- 8 files changed, 23 insertions(+), 20 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 449e1a7f0263c..2bc5cab231230 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -352,7 +352,7 @@ ISchemaReader getTableDeviceReader( final String table, final List devicePathList) throws MetadataException; // region Interfaces for AttributeUpdate - Pair> getAttributeUpdateMap(); + Pair> getAttributeUpdateInfo(); // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 9f4702265b9a9..2449f8eff408a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -102,7 +102,10 @@ public void update( }); } - public Pair> getAttributeUpdateMap(final int limit) { + public Pair> getAttributeUpdateInfo(final int limit) { + // Note that the "updateContainerStatistics" is unsafe to use here for whole read of detail + // container because the update map is read by GRASS thread, and the container's size may change + // during the read process return new Pair<>( version.get(), attributeUpdateMap.entrySet().stream() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 19ff83cb63949..c297ab769c816 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -89,7 +89,7 @@ private void execute() { regionLeaders.stream() .collect( Collectors.toMap( - ISchemaRegion::getSchemaRegionId, ISchemaRegion::getAttributeUpdateMap)); + ISchemaRegion::getSchemaRegionId, ISchemaRegion::getAttributeUpdateInfo)); // Send diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 46129a95b20dd..6f23f38ceba7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -25,6 +25,7 @@ import javax.annotation.concurrent.ThreadSafe; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -61,6 +62,17 @@ public long updateAttribute( return tableNames.add(tableName) ? RamUsageEstimator.sizeOf(tableName) : 0; } + @Override + public byte[] getUpdateContent(final int limitBytes) { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try { + serialize(outputStream); + } catch (final IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + return outputStream.toByteArray(); + } + @Override public Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer) { return null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 112641d1d7b1d..26177636ed31d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -23,7 +23,6 @@ import javax.annotation.concurrent.ThreadSafe; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -40,15 +39,7 @@ long updateAttribute( // A piece of "updateContent" won't exceed "limitBytes" in order to handle // thrift threshold and low bandwidth // The "limitBytes" shall be at least 4 for a "0" to indicate empty - default byte[] getUpdateContent(final int limitBytes) { - final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try { - serialize(outputStream); - } catch (final IOException ignored) { - // ByteArrayOutputStream won't throw IOException - } - return outputStream.toByteArray(); - } + byte[] getUpdateContent(final int limitBytes); Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 371fba9e56885..cb0bd805d757c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -101,13 +101,9 @@ public long updateAttribute( @Override public byte[] getUpdateContent(final int limitBytes) { - if (limitBytes < 0) { - return UpdateContainer.super.getUpdateContent(limitBytes); - } final RewritableByteArrayOutputStream outputStream = new RewritableByteArrayOutputStream(limitBytes); try { - ReadWriteIOUtils.write((byte) 1, outputStream); serializeWithLimit(outputStream); } catch (final IOException ignored) { // ByteArrayOutputStream won't throw IOException @@ -117,6 +113,7 @@ public byte[] getUpdateContent(final int limitBytes) { private void serializeWithLimit(final RewritableByteArrayOutputStream outputStream) throws IOException { + ReadWriteIOUtils.write((byte) 1, outputStream); final int mapSizeOffset = outputStream.skipInt(); int mapEntryCount = 0; for (final Map.Entry>> diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 367911bb2ae81..025d4d19c87fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1563,8 +1563,8 @@ public ISchemaReader getTableDeviceReader( } @Override - public Pair> getAttributeUpdateMap() { - return deviceAttributeRemoteUpdater.getAttributeUpdateMap(0); + public Pair> getAttributeUpdateInfo() { + return deviceAttributeRemoteUpdater.getAttributeUpdateInfo(0); } // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 9ea704a7de190..9a03bf6b6a4eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -1492,7 +1492,7 @@ public ISchemaReader getTableDeviceReader( } @Override - public Pair> getAttributeUpdateMap() { + public Pair> getAttributeUpdateInfo() { // TODO return null; } From 9157550ca427574bb01455393b87fa0dc6d88e3c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 12:19:07 +0800 Subject: [PATCH 067/213] Refactor --- .../schemaregion/ISchemaRegion.java | 3 +- .../update/DeviceAttributeRemoteUpdater.java | 13 ++------ ...GeneralRegionAttributeSecurityService.java | 10 +++++- .../RewritableByteArrayOutputStream.java | 9 ------ .../update/UpdateClearContainer.java | 4 ++- .../attribute/update/UpdateContainer.java | 4 ++- .../update/UpdateDetailContainer.java | 31 ++++++++++++------- .../impl/SchemaRegionMemoryImpl.java | 5 +-- .../impl/SchemaRegionPBTreeImpl.java | 3 +- 9 files changed, 44 insertions(+), 38 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 2bc5cab231230..3fc848639555d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -56,6 +56,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; /** * This interface defines all interfaces and behaviours that one SchemaRegion should support and @@ -352,7 +353,7 @@ ISchemaReader getTableDeviceReader( final String table, final List devicePathList) throws MetadataException; // region Interfaces for AttributeUpdate - Pair> getAttributeUpdateInfo(); + Pair> getAttributeUpdateInfo(final AtomicInteger limit); // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 2449f8eff408a..3c3e4125992a6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.utils.FileUtils; @@ -48,6 +47,7 @@ import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -102,7 +102,7 @@ public void update( }); } - public Pair> getAttributeUpdateInfo(final int limit) { + public Pair> getAttributeUpdateInfo(final AtomicInteger limit) { // Note that the "updateContainerStatistics" is unsafe to use here for whole read of detail // container because the update map is read by GRASS thread, and the container's size may change // during the read process @@ -111,14 +111,7 @@ public Pair> getAttributeUpdateInfo(final int limit attributeUpdateMap.entrySet().stream() .collect( Collectors.toMap( - Map.Entry::getKey, - entry -> - entry - .getValue() - .getUpdateContent( - CommonDescriptor.getInstance() - .getConfig() - .getPipeConnectorRequestSliceThresholdBytes())))); + Map.Entry::getKey, entry -> entry.getValue().getUpdateContent(limit)))); } public void addLocation(final Pair dataNodeLocation) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index c297ab769c816..188c4ddaad9a0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; @@ -35,6 +36,7 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @@ -85,11 +87,17 @@ public void notifyBroadCast() { private void execute() { lock.lock(); try { + final AtomicInteger limit = + new AtomicInteger( + CommonDescriptor.getInstance() + .getConfig() + .getPipeConnectorRequestSliceThresholdBytes()); final Map>> attributeUpdateCommitMap = regionLeaders.stream() .collect( Collectors.toMap( - ISchemaRegion::getSchemaRegionId, ISchemaRegion::getAttributeUpdateInfo)); + ISchemaRegion::getSchemaRegionId, + schemaRegion -> schemaRegion.getAttributeUpdateInfo(limit))); // Send diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java index 10aefcd978664..affe91ae2a2e8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java @@ -31,15 +31,6 @@ public class RewritableByteArrayOutputStream extends ByteArrayOutputStream { private static final byte[] intPlaceHolder = new byte[4]; - private final int limit; - - public RewritableByteArrayOutputStream(final int limit) { - this.limit = limit; - } - - public boolean exceedCapacity(final int newGrowth) { - return newGrowth + count > limit; - } public void writeWithLength(final byte[] bytes) throws IOException { ReadWriteIOUtils.write(bytes.length, this); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 6f23f38ceba7c..34f151f063036 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -23,6 +23,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; +import javax.annotation.Nonnull; import javax.annotation.concurrent.ThreadSafe; import java.io.ByteArrayOutputStream; @@ -34,6 +35,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; @ThreadSafe public class UpdateClearContainer implements UpdateContainer { @@ -63,7 +65,7 @@ public long updateAttribute( } @Override - public byte[] getUpdateContent(final int limitBytes) { + public byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes) { final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try { serialize(outputStream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 26177636ed31d..2f22619723716 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -21,6 +21,7 @@ import org.apache.tsfile.utils.Pair; +import javax.annotation.Nonnull; import javax.annotation.concurrent.ThreadSafe; import java.io.IOException; @@ -28,6 +29,7 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; @ThreadSafe public interface UpdateContainer { @@ -39,7 +41,7 @@ long updateAttribute( // A piece of "updateContent" won't exceed "limitBytes" in order to handle // thrift threshold and low bandwidth // The "limitBytes" shall be at least 4 for a "0" to indicate empty - byte[] getUpdateContent(final int limitBytes); + byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes); Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index cb0bd805d757c..93d080688cf6c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -24,6 +24,7 @@ import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; +import javax.annotation.Nonnull; import javax.annotation.concurrent.ThreadSafe; import java.io.IOException; @@ -35,6 +36,7 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @ThreadSafe @@ -100,29 +102,32 @@ public long updateAttribute( } @Override - public byte[] getUpdateContent(final int limitBytes) { - final RewritableByteArrayOutputStream outputStream = - new RewritableByteArrayOutputStream(limitBytes); + public byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes) { + final RewritableByteArrayOutputStream outputStream = new RewritableByteArrayOutputStream(); try { - serializeWithLimit(outputStream); + serializeWithLimit(outputStream, limitBytes); } catch (final IOException ignored) { // ByteArrayOutputStream won't throw IOException } return outputStream.toByteArray(); } - private void serializeWithLimit(final RewritableByteArrayOutputStream outputStream) + private void serializeWithLimit( + final RewritableByteArrayOutputStream outputStream, final AtomicInteger limitBytes) throws IOException { ReadWriteIOUtils.write((byte) 1, outputStream); final int mapSizeOffset = outputStream.skipInt(); int mapEntryCount = 0; + int newSize; for (final Map.Entry>> tableEntry : updateMap.entrySet()) { final byte[] tableEntryBytes = tableEntry.getKey().getBytes(TSFileConfig.STRING_CHARSET); - if (outputStream.exceedCapacity(2 * Integer.BYTES + tableEntryBytes.length)) { + newSize = 2 * Integer.BYTES + tableEntryBytes.length; + if (limitBytes.get() < newSize) { outputStream.rewrite(mapEntryCount, mapSizeOffset); return; } + limitBytes.addAndGet(-newSize); ++mapEntryCount; outputStream.writeWithLength(tableEntryBytes); final int deviceSizeOffset = outputStream.skipInt(); @@ -133,15 +138,16 @@ private void serializeWithLimit(final RewritableByteArrayOutputStream outputStre Arrays.stream(deviceEntry.getKey()) .map(str -> str.getBytes(TSFileConfig.STRING_CHARSET)) .toArray(byte[][]::new); - if (outputStream.exceedCapacity( + + newSize = (Integer.BYTES * (deviceIdBytes.length + 2)) - + Arrays.stream(deviceIdBytes) - .map(bytes -> bytes.length) - .reduce(0, Integer::sum))) { + + Arrays.stream(deviceIdBytes).map(bytes -> bytes.length).reduce(0, Integer::sum); + if (limitBytes.get() < newSize) { outputStream.rewrite(mapEntryCount, mapSizeOffset); outputStream.rewrite(deviceEntryCount, deviceSizeOffset); return; } + limitBytes.addAndGet(-newSize); ++deviceEntryCount; ReadWriteIOUtils.write(deviceEntry.getKey().length, outputStream); for (final byte[] node : deviceIdBytes) { @@ -152,13 +158,14 @@ private void serializeWithLimit(final RewritableByteArrayOutputStream outputStre for (final Map.Entry attributeKV : deviceEntry.getValue().entrySet()) { final byte[] keyBytes = attributeKV.getKey().getBytes(TSFileConfig.STRING_CHARSET); final byte[] valueBytes = attributeKV.getValue().getBytes(TSFileConfig.STRING_CHARSET); - if (outputStream.exceedCapacity( - 2 * Integer.BYTES + keyBytes.length + valueBytes.length)) { + newSize = 2 * Integer.BYTES + keyBytes.length + valueBytes.length; + if (limitBytes.get() < newSize) { outputStream.rewrite(mapEntryCount, mapSizeOffset); outputStream.rewrite(deviceEntryCount, deviceSizeOffset); outputStream.rewrite(attributeCount, attributeOffset); return; } + limitBytes.addAndGet(-newSize); outputStream.writeWithLength(keyBytes); outputStream.writeWithLength(valueBytes); ++attributeCount; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 025d4d19c87fc..2bc455cdc465d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -133,6 +133,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import static org.apache.iotdb.db.queryengine.plan.planner.TableOperatorGenerator.makeLayout; @@ -1563,8 +1564,8 @@ public ISchemaReader getTableDeviceReader( } @Override - public Pair> getAttributeUpdateInfo() { - return deviceAttributeRemoteUpdater.getAttributeUpdateInfo(0); + public Pair> getAttributeUpdateInfo(final AtomicInteger limit) { + return deviceAttributeRemoteUpdater.getAttributeUpdateInfo(limit); } // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 9a03bf6b6a4eb..93b250ce72a2d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -110,6 +110,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; @@ -1492,7 +1493,7 @@ public ISchemaReader getTableDeviceReader( } @Override - public Pair> getAttributeUpdateInfo() { + public Pair> getAttributeUpdateInfo(final AtomicInteger limit) { // TODO return null; } From 5439e0343e2f1adf37fef362921b18641089eba2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 12:22:51 +0800 Subject: [PATCH 068/213] Update GeneralRegionAttributeSecurityService.java --- .../attribute/update/GeneralRegionAttributeSecurityService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 188c4ddaad9a0..abad92fb32da7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -87,6 +87,8 @@ public void notifyBroadCast() { private void execute() { lock.lock(); try { + // All the "detailContainer"'s size will add up to at most "limit" + // UpdateClearContainer and version / TEndPoint are not calculated final AtomicInteger limit = new AtomicInteger( CommonDescriptor.getInstance() From 93e0fabd15c4122dc551a6f7560cdccad6bcd594 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 12:28:59 +0800 Subject: [PATCH 069/213] optimization --- .../update/DeviceAttributeRemoteUpdater.java | 16 +++++++++------- .../attribute/update/UpdateContainer.java | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 3c3e4125992a6..117ae26ccde48 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -49,7 +49,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; public class DeviceAttributeRemoteUpdater { private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeRemoteUpdater.class); @@ -106,12 +105,15 @@ public Pair> getAttributeUpdateInfo(final AtomicInt // Note that the "updateContainerStatistics" is unsafe to use here for whole read of detail // container because the update map is read by GRASS thread, and the container's size may change // during the read process - return new Pair<>( - version.get(), - attributeUpdateMap.entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, entry -> entry.getValue().getUpdateContent(limit)))); + final Map updateBytes = new HashMap<>(); + for (final Map.Entry entry : attributeUpdateMap.entrySet()) { + if (limit.get() < 5) { + break; + } + limit.addAndGet(-5); + updateBytes.put(entry.getKey(), entry.getValue().getUpdateContent(limit)); + } + return new Pair<>(version.get(), updateBytes); } public void addLocation(final Pair dataNodeLocation) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 2f22619723716..0b562d67507c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -40,7 +40,7 @@ long updateAttribute( // Only this method is not synchronize called and is called by GRASS thread // A piece of "updateContent" won't exceed "limitBytes" in order to handle // thrift threshold and low bandwidth - // The "limitBytes" shall be at least 4 for a "0" to indicate empty + // The "limitBytes" shall be at least 5 for a "type" and "0" to indicate empty byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes); Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer); From 919098ae53dca73f20ad1352fb1360bf4ea6abc8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 14:10:31 +0800 Subject: [PATCH 070/213] Update DeviceAttributeRemoteUpdater.java --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 117ae26ccde48..21d668240c433 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -52,6 +52,7 @@ public class DeviceAttributeRemoteUpdater { private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeRemoteUpdater.class); + private static final int UPDATE_DETAIL_CONTAINER_SEND_MIN_LIMIT_BYTES = 1024; private final Set> targetDataNodeLocations = new HashSet<>(); private final ConcurrentMap attributeUpdateMap = @@ -107,6 +108,13 @@ public Pair> getAttributeUpdateInfo(final AtomicInt // during the read process final Map updateBytes = new HashMap<>(); for (final Map.Entry entry : attributeUpdateMap.entrySet()) { + // If the remaining capacity is too low we just send clear container first + // Because they require less capacity + if (limit.get() < UPDATE_DETAIL_CONTAINER_SEND_MIN_LIMIT_BYTES + && entry.getValue() instanceof UpdateDetailContainer) { + continue; + } + // type(1) + size(4) if (limit.get() < 5) { break; } From 83b0aeb40c4af6e5b8e0a0f71fb97e477ce1b25d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 14:12:55 +0800 Subject: [PATCH 071/213] Refactor --- .../update/DeviceAttributeRemoteUpdater.java | 2 +- .../update/RewritableByteArrayOutputStream.java | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 21d668240c433..53c36d091f783 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -115,7 +115,7 @@ public Pair> getAttributeUpdateInfo(final AtomicInt continue; } // type(1) + size(4) - if (limit.get() < 5) { + if (limit.get() <= 5) { break; } limit.addAndGet(-5); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java index affe91ae2a2e8..2e70ca4781520 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java @@ -28,30 +28,26 @@ import java.io.IOException; @NotThreadSafe -public class RewritableByteArrayOutputStream extends ByteArrayOutputStream { +class RewritableByteArrayOutputStream extends ByteArrayOutputStream { private static final byte[] intPlaceHolder = new byte[4]; - public void writeWithLength(final byte[] bytes) throws IOException { + void writeWithLength(final byte[] bytes) throws IOException { ReadWriteIOUtils.write(bytes.length, this); write(bytes); } - public int skipInt() throws IOException { + int skipInt() throws IOException { final int result = count; write(intPlaceHolder); return result; } - public void rewrite(final int n, final int off) { + void rewrite(final int n, final int off) { byte[] bytes = BytesUtils.intToBytes(n); if ((off < 0) || (off > buf.length - bytes.length)) { throw new IndexOutOfBoundsException(); } System.arraycopy(bytes, 0, buf, off, bytes.length); } - - public int getCount() { - return count; - } } From addd0a753c1173cc42d28ecf1fc3954563fafc7a Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 14:17:48 +0800 Subject: [PATCH 072/213] Update GeneralRegionAttributeSecurityService.java --- ...GeneralRegionAttributeSecurityService.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index abad92fb32da7..9864771d7ea80 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -31,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -39,7 +40,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; public class GeneralRegionAttributeSecurityService { private static final Logger LOGGER = @@ -94,12 +94,19 @@ private void execute() { CommonDescriptor.getInstance() .getConfig() .getPipeConnectorRequestSliceThresholdBytes()); + final Map>> attributeUpdateCommitMap = - regionLeaders.stream() - .collect( - Collectors.toMap( - ISchemaRegion::getSchemaRegionId, - schemaRegion -> schemaRegion.getAttributeUpdateInfo(limit))); + new HashMap<>(); + for (final ISchemaRegion regionLeader : regionLeaders) { + final Pair> currentResult = + regionLeader.getAttributeUpdateInfo(limit); + if (currentResult.getRight().isEmpty()) { + break; + } + attributeUpdateCommitMap.put(regionLeader.getSchemaRegionId(), currentResult); + } + + // Send From 5ebcbc7903a9953c79d7f5cf50240c0de040bdb8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 14:23:00 +0800 Subject: [PATCH 073/213] client.thrift --- .../update/GeneralRegionAttributeSecurityService.java | 2 -- .../thrift-datanode/src/main/thrift/client.thrift | 11 +++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 9864771d7ea80..291d82cbfb984 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -106,8 +106,6 @@ private void execute() { attributeUpdateCommitMap.put(regionLeader.getSchemaRegionId(), currentResult); } - - // Send if (!skipNext) { diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index b36a70fb91229..dc06deec79cf8 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -511,6 +511,15 @@ struct TPipeSubscribeResp { 4:optional list body } +struct TAttributeUpdateReq { + 1:required map attributeUpdateMap +} + +struct TSchemaRegionAttributeInfo { + 1:required i64 version + 2:required binary body +} + struct TSBackupConfigurationResp { 1: required common.TSStatus status 2: optional bool enableOperationSync @@ -664,6 +673,8 @@ service IClientRPCService { TPipeSubscribeResp pipeSubscribe(TPipeSubscribeReq req); + common.TSStatus updateAttribute(TAttributeUpdateReq req); + TSBackupConfigurationResp getBackupConfiguration(); TSConnectionInfoResp fetchAllConnectionsInfo(); From aef9bb1dbcfe59d2207fe243ea9e2cc1bdb68a67 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 17:36:55 +0800 Subject: [PATCH 074/213] refactor --- .../update/UpdateClearContainer.java | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 34f151f063036..71a83ec375894 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -31,10 +32,12 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; @ThreadSafe @@ -66,26 +69,47 @@ public long updateAttribute( @Override public byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes) { - final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + final RewritableByteArrayOutputStream outputStream = new RewritableByteArrayOutputStream(); try { - serialize(outputStream); + serializeWithLimit(outputStream, limitBytes); } catch (final IOException ignored) { // ByteArrayOutputStream won't throw IOException } return outputStream.toByteArray(); } + private void serializeWithLimit( + final RewritableByteArrayOutputStream outputStream, final AtomicInteger limitBytes) + throws IOException { + ReadWriteIOUtils.write((byte) 0, outputStream); + final int setSizeOffset = outputStream.skipInt(); + int setEntryCount = 0; + int newSize; + for (final String tableName : tableNames) { + final byte[] tableBytes = tableName.getBytes(TSFileConfig.STRING_CHARSET); + newSize = Integer.BYTES + tableBytes.length; + if (limitBytes.get() < newSize) { + outputStream.rewrite(setEntryCount, setSizeOffset); + return; + } + limitBytes.addAndGet(-newSize); + ++setEntryCount; + outputStream.writeWithLength(tableBytes); + } + outputStream.rewrite(tableNames.size(), setSizeOffset); + } + @Override public Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer) { return null; } @Override - public void serialize(final OutputStream outputstream) throws IOException { - ReadWriteIOUtils.write((byte) 0, outputstream); - ReadWriteIOUtils.write(tableNames.size(), outputstream); + public void serialize(final OutputStream outputStream) throws IOException { + ReadWriteIOUtils.write((byte) 0, outputStream); + ReadWriteIOUtils.write(tableNames.size(), outputStream); for (final String tableName : tableNames) { - ReadWriteIOUtils.write(tableName, outputstream); + ReadWriteIOUtils.write(tableName, outputStream); } } From 8d8e2a32aa8b1cb24950e3239042be6c70456cdd Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 17:41:40 +0800 Subject: [PATCH 075/213] Update UpdateClearContainer.java --- .../attribute/update/UpdateClearContainer.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 71a83ec375894..21a5c2edc76d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -27,17 +27,14 @@ import javax.annotation.Nonnull; import javax.annotation.concurrent.ThreadSafe; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.Collections; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; @ThreadSafe @@ -79,8 +76,8 @@ public byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes) { } private void serializeWithLimit( - final RewritableByteArrayOutputStream outputStream, final AtomicInteger limitBytes) - throws IOException { + final RewritableByteArrayOutputStream outputStream, final AtomicInteger limitBytes) + throws IOException { ReadWriteIOUtils.write((byte) 0, outputStream); final int setSizeOffset = outputStream.skipInt(); int setEntryCount = 0; From d57894b8417495404de6b01ca8c8022a6ac96615 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 18:47:36 +0800 Subject: [PATCH 076/213] Move --- .../thrift-datanode/src/main/thrift/client.thrift | 11 ----------- .../thrift-datanode/src/main/thrift/datanode.thrift | 11 +++++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index dc06deec79cf8..b36a70fb91229 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -511,15 +511,6 @@ struct TPipeSubscribeResp { 4:optional list body } -struct TAttributeUpdateReq { - 1:required map attributeUpdateMap -} - -struct TSchemaRegionAttributeInfo { - 1:required i64 version - 2:required binary body -} - struct TSBackupConfigurationResp { 1: required common.TSStatus status 2: optional bool enableOperationSync @@ -673,8 +664,6 @@ service IClientRPCService { TPipeSubscribeResp pipeSubscribe(TPipeSubscribeReq req); - common.TSStatus updateAttribute(TAttributeUpdateReq req); - TSBackupConfigurationResp getBackupConfiguration(); TSConnectionInfoResp fetchAllConnectionsInfo(); diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 2cd84fdda0f9a..c4da2fffc65d8 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -343,6 +343,15 @@ struct TLoadCommandReq { 4: optional binary progressIndex } +struct TAttributeUpdateReq { + 1:required map attributeUpdateMap +} + +struct TSchemaRegionAttributeInfo { + 1:required i64 version + 2:required binary body +} + struct TLoadResp { 1: required bool accepted 2: optional string message @@ -683,6 +692,8 @@ service IDataNodeRPCService { TLoadResp sendLoadCommand(TLoadCommandReq req); + common.TSStatus updateAttribute(TAttributeUpdateReq req); + // -----------------------------------For Config Node----------------------------------------------- From 57afbe8ac2949dad858b564c6df099ad80d226ea Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 19:47:24 +0800 Subject: [PATCH 077/213] Update DataNodeInternalRPCServiceImpl.java --- .../impl/DataNodeInternalRPCServiceImpl.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index f901d556c65af..a7238dc951934 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -167,6 +167,7 @@ import org.apache.iotdb.mpp.rpc.thrift.IDataNodeRPCService; import org.apache.iotdb.mpp.rpc.thrift.TActiveTriggerInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TAlterViewReq; +import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; import org.apache.iotdb.mpp.rpc.thrift.TCancelFragmentInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TCancelPlanFragmentReq; import org.apache.iotdb.mpp.rpc.thrift.TCancelQueryReq; @@ -441,22 +442,22 @@ public TCancelResp cancelFragmentInstance(TCancelFragmentInstanceReq req) { } @Override - public TSchemaFetchResponse fetchSchema(TSchemaFetchRequest req) { + public TSchemaFetchResponse fetchSchema(final TSchemaFetchRequest req) { throw new UnsupportedOperationException(); } @Override - public TLoadResp sendTsFilePieceNode(TTsFilePieceReq req) { + public TLoadResp sendTsFilePieceNode(final TTsFilePieceReq req) { LOGGER.info("Receive load node from uuid {}.", req.uuid); - ConsensusGroupId groupId = + final ConsensusGroupId groupId = ConsensusGroupId.Factory.createFromTConsensusGroupId(req.consensusGroupId); - LoadTsFilePieceNode pieceNode = (LoadTsFilePieceNode) PlanNodeType.deserialize(req.body); + final LoadTsFilePieceNode pieceNode = (LoadTsFilePieceNode) PlanNodeType.deserialize(req.body); if (pieceNode == null) { return createTLoadResp( new TSStatus(TSStatusCode.DESERIALIZE_PIECE_OF_TSFILE_ERROR.getStatusCode())); } - TSStatus resultStatus = + final TSStatus resultStatus = StorageEngine.getInstance() .writeLoadTsFileNode((DataRegionId) groupId, pieceNode, req.uuid); @@ -464,7 +465,7 @@ public TLoadResp sendTsFilePieceNode(TTsFilePieceReq req) { } @Override - public TLoadResp sendLoadCommand(TLoadCommandReq req) { + public TLoadResp sendLoadCommand(final TLoadCommandReq req) { final ProgressIndex progressIndex; if (req.isSetProgressIndex()) { progressIndex = ProgressIndexType.deserializeFrom(ByteBuffer.wrap(req.getProgressIndex())); @@ -484,9 +485,15 @@ public TLoadResp sendLoadCommand(TLoadCommandReq req) { progressIndex)); } - private TLoadResp createTLoadResp(TSStatus resultStatus) { - boolean isAccepted = RpcUtils.SUCCESS_STATUS.equals(resultStatus); - TLoadResp loadResp = new TLoadResp(isAccepted); + @Override + public TSStatus updateAttribute(final TAttributeUpdateReq req) { + // TODO + return RpcUtils.SUCCESS_STATUS; + } + + private TLoadResp createTLoadResp(final TSStatus resultStatus) { + final boolean isAccepted = RpcUtils.SUCCESS_STATUS.equals(resultStatus); + final TLoadResp loadResp = new TLoadResp(isAccepted); if (!isAccepted) { loadResp.setMessage(resultStatus.getMessage()); loadResp.setStatus(resultStatus); @@ -495,7 +502,7 @@ private TLoadResp createTLoadResp(TSStatus resultStatus) { } @Override - public TSStatus createSchemaRegion(TCreateSchemaRegionReq req) { + public TSStatus createSchemaRegion(final TCreateSchemaRegionReq req) { return regionManager.createSchemaRegion(req.getRegionReplicaSet(), req.getStorageGroup()); } From aaf508fe7360c2a910b80035d8ebd3dba7981f68 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 19:51:03 +0800 Subject: [PATCH 078/213] little completion --- .../client/dn/DataNodeAsyncRequestRPCHandler.java | 15 ++++++++------- .../db/protocol/client/dn/DnToDnRequestType.java | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DataNodeAsyncRequestRPCHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DataNodeAsyncRequestRPCHandler.java index 76be9ac9d715a..a15e4a0b10fc4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DataNodeAsyncRequestRPCHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DataNodeAsyncRequestRPCHandler.java @@ -61,15 +61,16 @@ protected String generateFormattedTargetLocation(TDataNodeLocation dataNodeLocat } public static DataNodeAsyncRequestRPCHandler createAsyncRPCHandler( - AsyncRequestContext context, - int requestId, - TDataNodeLocation targetDataNode) { - DnToDnRequestType requestType = context.getRequestType(); - Map nodeLocationMap = context.getNodeLocationMap(); - Map responseMap = context.getResponseMap(); - CountDownLatch countDownLatch = context.getCountDownLatch(); + final AsyncRequestContext context, + final int requestId, + final TDataNodeLocation targetDataNode) { + final DnToDnRequestType requestType = context.getRequestType(); + final Map nodeLocationMap = context.getNodeLocationMap(); + final Map responseMap = context.getResponseMap(); + final CountDownLatch countDownLatch = context.getCountDownLatch(); switch (requestType) { case TEST_CONNECTION: + case UPDATE_ATTRIBUTE: return new AsyncTSStatusRPCHandler( requestType, requestId, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DnToDnRequestType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DnToDnRequestType.java index 35f5b8d93a9b4..8061d8d954273 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DnToDnRequestType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DnToDnRequestType.java @@ -22,4 +22,5 @@ /** For DataNode async call DataNode */ public enum DnToDnRequestType { TEST_CONNECTION, + UPDATE_ATTRIBUTE, } From 2d3fd7a79440d9527641a2767a3e7d69d23c3488 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sun, 29 Sep 2024 19:53:59 +0800 Subject: [PATCH 079/213] Update DnToDnInternalServiceAsyncRequestManager.java --- .../dn/DnToDnInternalServiceAsyncRequestManager.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DnToDnInternalServiceAsyncRequestManager.java index 88766458b649d..b57a10bcb4301 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DnToDnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DnToDnInternalServiceAsyncRequestManager.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.client.request.AsyncRequestContext; import org.apache.iotdb.commons.client.request.AsyncRequestRPCHandler; import org.apache.iotdb.commons.client.request.DataNodeInternalServiceRequestManager; +import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,13 +38,17 @@ protected void initActionMapBuilder() { actionMapBuilder.put( DnToDnRequestType.TEST_CONNECTION, (req, client, handler) -> client.testConnectionEmptyRPC((AsyncTSStatusRPCHandler) handler)); + actionMapBuilder.put( + DnToDnRequestType.UPDATE_ATTRIBUTE, + (req, client, handler) -> + client.updateAttribute((TAttributeUpdateReq) req, (AsyncTSStatusRPCHandler) handler)); } @Override protected AsyncRequestRPCHandler buildHandler( - AsyncRequestContext requestContext, - int requestId, - TDataNodeLocation targetNode) { + final AsyncRequestContext requestContext, + final int requestId, + final TDataNodeLocation targetNode) { return DataNodeAsyncRequestRPCHandler.createAsyncRPCHandler( requestContext, requestId, targetNode); } From 22a3482761c1dcc9a21a26f666f80a3a2dc32dc2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:47:50 +0800 Subject: [PATCH 080/213] Update DeviceAttributeRemoteUpdater.java --- .../update/DeviceAttributeRemoteUpdater.java | 77 +++++++++++-------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 53c36d091f783..aff625098d9f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; -import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.utils.FileUtils; @@ -54,8 +54,8 @@ public class DeviceAttributeRemoteUpdater { private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeRemoteUpdater.class); private static final int UPDATE_DETAIL_CONTAINER_SEND_MIN_LIMIT_BYTES = 1024; - private final Set> targetDataNodeLocations = new HashSet<>(); - private final ConcurrentMap attributeUpdateMap = + private final Set targetDataNodeLocations = new HashSet<>(); + private final ConcurrentMap attributeUpdateMap = new ConcurrentHashMap<>(); private final AtomicLong version = new AtomicLong(0); @@ -63,7 +63,7 @@ public class DeviceAttributeRemoteUpdater { private final MemSchemaRegionStatistics regionStatistics; // Only exist for update detail container - private final Map updateContainerStatistics = + private final Map updateContainerStatistics = new HashMap<>(); public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatistics) { @@ -75,8 +75,8 @@ public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatis public void update( final String tableName, final String[] deviceId, final Map attributeMap) { targetDataNodeLocations.forEach( - pair -> { - if (!attributeUpdateMap.containsKey(pair.getLeft())) { + location -> { + if (!attributeUpdateMap.containsKey(location)) { final UpdateContainer newContainer; if (!regionStatistics.isAllowToCreateNewSeries()) { newContainer = new UpdateClearContainer(); @@ -84,16 +84,14 @@ public void update( } else { newContainer = new UpdateDetailContainer(); requestMemory(UpdateDetailContainer.INSTANCE_SIZE); - updateContainerStatistics.put(pair.getLeft(), new UpdateContainerStatistics()); + updateContainerStatistics.put(location, new UpdateContainerStatistics()); } - attributeUpdateMap.put(pair.getLeft(), newContainer); + attributeUpdateMap.put(location, newContainer); } final long size = - attributeUpdateMap - .get(pair.getLeft()) - .updateAttribute(tableName, deviceId, attributeMap); + attributeUpdateMap.get(location).updateAttribute(tableName, deviceId, attributeMap); updateContainerStatistics.computeIfPresent( - pair.getLeft(), + location, (k, v) -> { v.addSize(size); return v; @@ -102,12 +100,14 @@ public void update( }); } - public Pair> getAttributeUpdateInfo(final AtomicInteger limit) { + public Pair> getAttributeUpdateInfo( + final AtomicInteger limit) { // Note that the "updateContainerStatistics" is unsafe to use here for whole read of detail // container because the update map is read by GRASS thread, and the container's size may change // during the read process - final Map updateBytes = new HashMap<>(); - for (final Map.Entry entry : attributeUpdateMap.entrySet()) { + final Map updateBytes = new HashMap<>(); + for (final Map.Entry entry : + attributeUpdateMap.entrySet()) { // If the remaining capacity is too low we just send clear container first // Because they require less capacity if (limit.get() < UPDATE_DETAIL_CONTAINER_SEND_MIN_LIMIT_BYTES @@ -124,7 +124,7 @@ public Pair> getAttributeUpdateInfo(final AtomicInt return new Pair<>(version.get(), updateBytes); } - public void addLocation(final Pair dataNodeLocation) { + public void addLocation(final TDataNodeLocation dataNodeLocation) { targetDataNodeLocations.add(dataNodeLocation); } @@ -138,7 +138,7 @@ private void degrade() { if (regionStatistics.isAllowToCreateNewSeries()) { return; } - final TreeSet degradeSet = + final TreeSet degradeSet = new TreeSet<>( Comparator.comparingLong(v -> updateContainerStatistics.get(v).getDegradePriority()) .reversed()); @@ -148,15 +148,15 @@ private void degrade() { degradeSet.add(k); } }); - for (final TEndPoint endPoint : degradeSet) { + for (final TDataNodeLocation location : degradeSet) { if (regionStatistics.isAllowToCreateNewSeries()) { return; } final UpdateClearContainer newContainer = - ((UpdateDetailContainer) attributeUpdateMap.get(endPoint)).degrade(); - updateMemory(newContainer.ramBytesUsed() - updateContainerStatistics.get(endPoint).getSize()); - attributeUpdateMap.put(endPoint, newContainer); - updateContainerStatistics.remove(endPoint); + ((UpdateDetailContainer) attributeUpdateMap.get(location)).degrade(); + updateMemory(newContainer.ramBytesUsed() - updateContainerStatistics.get(location).getSize()); + attributeUpdateMap.put(location, newContainer); + updateContainerStatistics.remove(location); } } @@ -206,14 +206,18 @@ public synchronized boolean createSnapshot(final File targetDir) { private void serialize(final OutputStream outputStream) throws IOException { ReadWriteIOUtils.write(targetDataNodeLocations.size(), outputStream); - for (final Pair targetDataNodeLocation : targetDataNodeLocations) { - ThriftCommonsSerDeUtils.serializeTEndPoint(targetDataNodeLocation.getLeft(), outputStream); - ReadWriteIOUtils.write(targetDataNodeLocation.getRight(), outputStream); + for (final TDataNodeLocation targetDataNodeLocation : targetDataNodeLocations) { + ReadWriteIOUtils.write(targetDataNodeLocation.getDataNodeId(), outputStream); + ThriftCommonsSerDeUtils.serializeTEndPoint( + targetDataNodeLocation.getInternalEndPoint(), outputStream); } ReadWriteIOUtils.write(attributeUpdateMap.size(), outputStream); - for (final Map.Entry entry : attributeUpdateMap.entrySet()) { - ThriftCommonsSerDeUtils.serializeTEndPoint(entry.getKey(), outputStream); + for (final Map.Entry entry : + attributeUpdateMap.entrySet()) { + ReadWriteIOUtils.write(entry.getKey().getDataNodeId(), outputStream); + ThriftCommonsSerDeUtils.serializeTEndPoint( + entry.getKey().getInternalEndPoint(), outputStream); entry.getValue().serialize(outputStream); } @@ -243,20 +247,31 @@ private void deserialize(final InputStream inputStream) throws IOException { int size = ReadWriteIOUtils.readInt(inputStream); for (int i = 0; i < size; i++) { targetDataNodeLocations.add( - new Pair<>( + new TDataNodeLocation( + ReadWriteIOUtils.readInt(inputStream), + null, ThriftCommonsSerDeUtils.deserializeTEndPoint(inputStream), - ReadWriteIOUtils.readInt(inputStream))); + null, + null, + null)); } size = ReadWriteIOUtils.readInt(inputStream); for (int i = 0; i < size; ++i) { - final TEndPoint endPoint = ThriftCommonsSerDeUtils.deserializeTEndPoint(inputStream); + final TDataNodeLocation location = + new TDataNodeLocation( + ReadWriteIOUtils.readInt(inputStream), + null, + ThriftCommonsSerDeUtils.deserializeTEndPoint(inputStream), + null, + null, + null); final UpdateContainer container = ReadWriteIOUtils.readBool(inputStream) ? new UpdateDetailContainer() : new UpdateClearContainer(); container.deserialize(inputStream); - attributeUpdateMap.put(endPoint, container); + attributeUpdateMap.put(location, container); } version.set(ReadWriteIOUtils.readLong(inputStream)); From 89aa30abcc744cafce6bdd37bf50948e53fb7cd1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:49:45 +0800 Subject: [PATCH 081/213] Refactor --- .../iotdb/db/schemaengine/schemaregion/ISchemaRegion.java | 4 ++-- .../update/GeneralRegionAttributeSecurityService.java | 8 ++++---- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 5 +++-- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 5 +++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 3fc848639555d..bc867bb615c1a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion; -import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.MeasurementPath; @@ -353,7 +353,7 @@ ISchemaReader getTableDeviceReader( final String table, final List devicePathList) throws MetadataException; // region Interfaces for AttributeUpdate - Pair> getAttributeUpdateInfo(final AtomicInteger limit); + Pair> getAttributeUpdateInfo(final AtomicInteger limit); // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 291d82cbfb984..b355046df65c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; -import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.conf.CommonDescriptor; @@ -95,10 +95,10 @@ private void execute() { .getConfig() .getPipeConnectorRequestSliceThresholdBytes()); - final Map>> attributeUpdateCommitMap = - new HashMap<>(); + final Map>> + attributeUpdateCommitMap = new HashMap<>(); for (final ISchemaRegion regionLeader : regionLeaders) { - final Pair> currentResult = + final Pair> currentResult = regionLeader.getAttributeUpdateInfo(limit); if (currentResult.getRight().isEmpty()) { break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 2bc455cdc465d..196c039ebdb2e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.impl; -import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; @@ -1564,7 +1564,8 @@ public ISchemaReader getTableDeviceReader( } @Override - public Pair> getAttributeUpdateInfo(final AtomicInteger limit) { + public Pair> getAttributeUpdateInfo( + final AtomicInteger limit) { return deviceAttributeRemoteUpdater.getAttributeUpdateInfo(limit); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 93b250ce72a2d..9b783a66f7e47 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.impl; -import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.commons.consensus.SchemaRegionId; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.file.SystemFileFactory; @@ -1493,7 +1493,8 @@ public ISchemaReader getTableDeviceReader( } @Override - public Pair> getAttributeUpdateInfo(final AtomicInteger limit) { + public Pair> getAttributeUpdateInfo( + final AtomicInteger limit) { // TODO return null; } From eb622fd5729dbfeb6075369ca4ca202e8ab604a7 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:19:56 +0800 Subject: [PATCH 082/213] partial --- ...GeneralRegionAttributeSecurityService.java | 36 +++++++++++++++++++ .../client/request/AsyncRequestContext.java | 10 ++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index b355046df65c9..b7554c06c1b28 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -20,17 +20,25 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.request.AsyncRequestContext; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.protocol.client.dn.DnToDnInternalServiceAsyncRequestManager; +import org.apache.iotdb.db.protocol.client.dn.DnToDnRequestType; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; +import org.apache.iotdb.mpp.rpc.thrift.TSchemaRegionAttributeInfo; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -126,6 +134,34 @@ private void execute() { } } + private Set sendUpdateRequest( + Map>> attributeUpdateCommitMap) { + final AsyncRequestContext + clientHandler = new AsyncRequestContext<>(DnToDnRequestType.UPDATE_ATTRIBUTE); + + attributeUpdateCommitMap.forEach( + (id, pair) -> + pair.getRight() + .forEach( + (location, bytes) -> { + clientHandler.putNodeLocation(location.getDataNodeId(), location); + clientHandler + .putRequestIfAbsent( + location.getDataNodeId(), new TAttributeUpdateReq(new HashMap<>())) + .getAttributeUpdateMap() + .put( + id.getId(), + new TSchemaRegionAttributeInfo( + pair.getLeft(), ByteBuffer.wrap(bytes))); + })); + + DnToDnInternalServiceAsyncRequestManager.getInstance() + .sendAsyncRequestToNodeWithRetryAndTimeoutInMs( + clientHandler, + PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3); + return clientHandler.getResponseMap(); + } + /////////////////////////////// SingleTon /////////////////////////////// private GeneralRegionAttributeSecurityService() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestContext.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestContext.java index 8b54072a4c40e..75fc648330fbd 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestContext.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestContext.java @@ -66,18 +66,22 @@ public class AsyncRequestContext { private CountDownLatch countDownLatch; /** Custom constructor. */ - public AsyncRequestContext(RequestType requestType) { + public AsyncRequestContext(final RequestType requestType) { this.requestType = requestType; this.requestMap = new ConcurrentHashMap<>(); this.nodeLocationMap = new ConcurrentHashMap<>(); this.responseMap = new ConcurrentHashMap<>(); } - public void putRequest(int requestId, Request request) { + public void putRequest(final int requestId, final Request request) { requestMap.put(requestId, request); } - public void putNodeLocation(int requestId, NodeLocation nodeLocation) { + public Request putRequestIfAbsent(final int requestId, final Request request) { + return requestMap.putIfAbsent(requestId, request); + } + + public void putNodeLocation(final int requestId, final NodeLocation nodeLocation) { nodeLocationMap.put(requestId, nodeLocation); } From fd70cee80d4008a64578de728b5260f62ffc4807 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:36:28 +0800 Subject: [PATCH 083/213] timeout config --- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 13 ++++++ .../apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +++ .../client/request/AsyncRequestManager.java | 42 ++++++++++++------- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 637be158da090..a7b9efc3b250f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1044,7 +1044,10 @@ public class IoTDBConfig { private int dataNodeTableCacheSemaphorePermitNum = 5; + /** GRASS Service */ private long generalRegionAttributeSecurityServiceIntervalSeconds = 30L; + + private long generalRegionAttributeSecurityServiceTimeoutSeconds = 20L; private long detailContainerMinDegradeMemoryInBytes = 1024 * 1024L; private String readConsistencyLevel = "strong"; @@ -3478,6 +3481,16 @@ public void setGeneralRegionAttributeSecurityServiceIntervalSeconds( generalRegionAttributeSecurityServiceIntervalSeconds; } + public long getGeneralRegionAttributeSecurityServiceTimeoutSeconds() { + return generalRegionAttributeSecurityServiceTimeoutSeconds; + } + + public void setGeneralRegionAttributeSecurityServiceTimeoutSeconds( + long generalRegionAttributeSecurityServiceTimeoutSeconds) { + this.generalRegionAttributeSecurityServiceTimeoutSeconds = + generalRegionAttributeSecurityServiceTimeoutSeconds; + } + public long getDetailContainerMinDegradeMemoryInBytes() { return detailContainerMinDegradeMemoryInBytes; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 35040e16c1e81..3f86dea6809fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1082,6 +1082,12 @@ public void loadProperties(Properties properties) throws BadNodeUrlException, IO "general_region_attribute_security_service_interval_seconds", String.valueOf(conf.getGeneralRegionAttributeSecurityServiceIntervalSeconds())))); + conf.setGeneralRegionAttributeSecurityServiceTimeoutSeconds( + Long.parseLong( + properties.getProperty( + "general_region_attribute_security_service_timeout_seconds", + String.valueOf(conf.getGeneralRegionAttributeSecurityServiceTimeoutSeconds())))); + conf.setDetailContainerMinDegradeMemoryInBytes( Long.parseLong( properties.getProperty( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestManager.java index e0a2ea0b635aa..b4d73747a8f63 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestManager.java @@ -73,7 +73,8 @@ protected AsyncRequestManager() { * @param timeoutInMs timeout in milliseconds */ public void sendAsyncRequestToNodeWithRetryAndTimeoutInMs( - AsyncRequestContext requestContext, long timeoutInMs) { + final AsyncRequestContext requestContext, + final long timeoutInMs) { sendAsyncRequest(requestContext, MAX_RETRY_NUM, timeoutInMs); } @@ -85,31 +86,44 @@ public void sendAsyncRequestToNodeWithRetryAndTimeoutInMs( * @param requestContext which will also contain the result */ public final void sendAsyncRequestWithRetry( - AsyncRequestContext requestContext) { + final AsyncRequestContext requestContext) { sendAsyncRequest(requestContext, MAX_RETRY_NUM, null); } + /** + * Send asynchronous requests to the specified Nodes with default retry num + * + *

Notice: The Nodes that failed to receive the requests will be reconnected + * + * @param requestContext which will also contain the result + */ + public final void sendAsyncRequestWithTimeoutInMs( + final AsyncRequestContext requestContext, + final long timeoutInMs) { + sendAsyncRequest(requestContext, MAX_RETRY_NUM, timeoutInMs); + } + public final void sendAsyncRequest( - AsyncRequestContext requestContext) { + final AsyncRequestContext requestContext) { sendAsyncRequest(requestContext, 1, null); } public void sendAsyncRequest( - AsyncRequestContext requestContext, - int retryNum, - Long timeoutInMs) { + final AsyncRequestContext requestContext, + final int retryNum, + final Long timeoutInMs) { if (requestContext.getRequestIndices().isEmpty()) { return; } - RequestType requestType = requestContext.getRequestType(); + final RequestType requestType = requestContext.getRequestType(); for (int retry = 0; retry < retryNum; retry++) { // Always Reset CountDownLatch first requestContext.resetCountDownLatch(); // Send requests to all targetNodes - for (int requestId : requestContext.getRequestIndices()) { - NodeLocation targetNode = requestContext.getNodeLocation(requestId); + for (final int requestId : requestContext.getRequestIndices()) { + final NodeLocation targetNode = requestContext.getNodeLocation(requestId); sendAsyncRequest(requestContext, requestId, targetNode, retry); } @@ -119,13 +133,11 @@ public void sendAsyncRequest( requestContext.getCountDownLatch().await(); } else { if (!requestContext.getCountDownLatch().await(timeoutInMs, TimeUnit.MILLISECONDS)) { - LOGGER.warn( - "Timeout during {} on ConfigNode. Retry: {}/{}", requestType, retry, retryNum); + LOGGER.warn("Timeout during {}. Retry: {}/{}", requestType, retry, retryNum); } } - } catch (InterruptedException e) { - LOGGER.error( - "Interrupted during {} on ConfigNode. Retry: {}/{}", requestType, retry, retryNum); + } catch (final InterruptedException e) { + LOGGER.error("Interrupted during {}. Retry: {}/{}", requestType, retry, retryNum); Thread.currentThread().interrupt(); } @@ -137,7 +149,7 @@ public void sendAsyncRequest( if (!requestContext.getRequestIndices().isEmpty()) { LOGGER.warn( - "Failed to {} on ConfigNode after {} retries, requestIndices: {}", + "Failed to {} after {} retries, requestIndices: {}", requestType, retryNum, requestContext.getRequestIndices()); From 267348a4d8fafe50f9aa47e782b60f635a9693c6 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:38:31 +0800 Subject: [PATCH 084/213] Update GeneralRegionAttributeSecurityService.java --- .../update/GeneralRegionAttributeSecurityService.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index b7554c06c1b28..a7f90d8a53b88 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -26,7 +26,6 @@ import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.SchemaRegionId; -import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.protocol.client.dn.DnToDnInternalServiceAsyncRequestManager; import org.apache.iotdb.db.protocol.client.dn.DnToDnRequestType; @@ -135,7 +134,8 @@ private void execute() { } private Set sendUpdateRequest( - Map>> attributeUpdateCommitMap) { + final Map>> + attributeUpdateCommitMap) { final AsyncRequestContext clientHandler = new AsyncRequestContext<>(DnToDnRequestType.UPDATE_ATTRIBUTE); @@ -156,9 +156,11 @@ private Set sendUpdateRequest( })); DnToDnInternalServiceAsyncRequestManager.getInstance() - .sendAsyncRequestToNodeWithRetryAndTimeoutInMs( + .sendAsyncRequestWithTimeoutInMs( clientHandler, - PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3); + IoTDBDescriptor.getInstance() + .getConfig() + .getGeneralRegionAttributeSecurityServiceTimeoutSeconds()); return clientHandler.getResponseMap(); } From f1fe8fa3b1fff3e44cdbf291f8f5689a4dd8bd3e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:42:34 +0800 Subject: [PATCH 085/213] Update GeneralRegionAttributeSecurityService.java --- .../update/GeneralRegionAttributeSecurityService.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index a7f90d8a53b88..4addd2aad438a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; import org.apache.iotdb.mpp.rpc.thrift.TSchemaRegionAttributeInfo; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; @@ -47,6 +48,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; public class GeneralRegionAttributeSecurityService { private static final Logger LOGGER = @@ -133,7 +135,7 @@ private void execute() { } } - private Set sendUpdateRequest( + private Set sendUpdateRequest( final Map>> attributeUpdateCommitMap) { final AsyncRequestContext @@ -161,7 +163,11 @@ private Set sendUpdateRequest( IoTDBDescriptor.getInstance() .getConfig() .getGeneralRegionAttributeSecurityServiceTimeoutSeconds()); - return clientHandler.getResponseMap(); + + return clientHandler.getResponseMap().entrySet().stream() + .filter(entry -> entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); } /////////////////////////////// SingleTon /////////////////////////////// From d6ef424427aca4596a558d93fc7dd77cffd65f2b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:56:48 +0800 Subject: [PATCH 086/213] Update GeneralRegionAttributeSecurityService.java --- ...GeneralRegionAttributeSecurityService.java | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 4addd2aad438a..11fbaa621b6d7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -41,6 +41,7 @@ import java.nio.ByteBuffer; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -116,6 +117,7 @@ private void execute() { } // Send + sendUpdateRequest(attributeUpdateCommitMap); if (!skipNext) { condition.await( @@ -135,7 +137,7 @@ private void execute() { } } - private Set sendUpdateRequest( + private void sendUpdateRequest( final Map>> attributeUpdateCommitMap) { final AsyncRequestContext @@ -164,10 +166,24 @@ private Set sendUpdateRequest( .getConfig() .getGeneralRegionAttributeSecurityServiceTimeoutSeconds()); - return clientHandler.getResponseMap().entrySet().stream() - .filter(entry -> entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) - .map(Map.Entry::getKey) - .collect(Collectors.toSet()); + final Set failedDataNodes = + clientHandler.getResponseMap().entrySet().stream() + .filter( + entry -> entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + for (final Iterator>>> it = + attributeUpdateCommitMap.entrySet().iterator(); + it.hasNext(); ) { + final Map dataNodeLocationMap = it.next().getValue().getRight(); + dataNodeLocationMap + .entrySet() + .removeIf( + locationEntry -> failedDataNodes.contains(locationEntry.getKey().getDataNodeId())); + if (dataNodeLocationMap.isEmpty()) { + it.remove(); + } + } } /////////////////////////////// SingleTon /////////////////////////////// From 88a76a043687239b1b8e485c6aead6bdb47e512b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:12:00 +0800 Subject: [PATCH 087/213] Create UpdateAttributeCommitNode.java --- .../write/UpdateAttributeCommitNode.java | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/UpdateAttributeCommitNode.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/UpdateAttributeCommitNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/UpdateAttributeCommitNode.java new file mode 100644 index 0000000000000..bcb6bea4db941 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/UpdateAttributeCommitNode.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Set; + +public class UpdateAttributeCommitNode extends PlanNode { + + private final long version; + private final byte[] commitBuffer; + private final Set shrunkNodes; + + protected UpdateAttributeCommitNode( + final PlanNodeId id, + final long version, + final byte[] commitBuffer, + final Set shrunkNodes) { + super(id); + this.version = version; + this.commitBuffer = commitBuffer; + this.shrunkNodes = shrunkNodes; + } + + public long getVersion() { + return version; + } + + public byte[] getCommitBuffer() { + return commitBuffer; + } + + public Set getShrunkNodes() { + return shrunkNodes; + } + + @Override + public List getChildren() { + return null; + } + + @Override + public void addChild(final PlanNode child) { + // Do nothing + } + + @Override + public PlanNode clone() { + return new UpdateAttributeCommitNode(id, version, commitBuffer, shrunkNodes); + } + + @Override + public int allowedChildCount() { + return 0; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + protected void serializeAttributes(final ByteBuffer byteBuffer) {} + + @Override + protected void serializeAttributes(final DataOutputStream stream) throws IOException {} +} From 512d7c25842695551ba07cf20d846c7ceb0f5de2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:24:21 +0800 Subject: [PATCH 088/213] Refactor --- .../schemaregion/SchemaExecutionVisitor.java | 4 ++-- .../plan/planner/TableOperatorGenerator.java | 6 +++--- .../plan/planner/plan/node/PlanNodeType.java | 14 +++++++++----- .../plan/planner/plan/node/PlanVisitor.java | 10 +++++----- .../read/AbstractTableDeviceQueryNode.java | 2 ++ .../planner/TableLogicalPlanner.java | 10 +++++----- .../planner/distribute/AddExchangeNodes.java | 6 +++--- .../TableDistributedPlanGenerator.java | 6 +++--- .../TableModelTypeProviderExtractor.java | 4 ++-- .../CreateOrUpdateTableDeviceNode.java | 2 +- .../TableDeviceAttributeCommitNode.java} | 18 ++++++++++++++---- .../TableDeviceAttributeUpdateNode.java | 6 +++--- .../node/schema}/TableDeviceFetchNode.java | 3 ++- .../schema}/TableDeviceQueryCountNode.java | 3 ++- .../node/schema}/TableDeviceQueryScanNode.java | 3 ++- .../schemaregion/ISchemaRegion.java | 4 ++-- .../schemaregion/SchemaRegionPlanVisitor.java | 4 ++-- .../impl/SchemaRegionMemoryImpl.java | 4 ++-- .../impl/SchemaRegionPBTreeImpl.java | 4 ++-- .../visitor/SchemaRegionPlanDeserializer.java | 4 ++-- .../visitor/SchemaRegionPlanSerializer.java | 4 ++-- .../req/SchemaRegionWritePlanFactory.java | 4 ++-- .../SchemaRegionSimpleRecoverTest.java | 2 +- .../schemaRegion/SchemaRegionTestUtil.java | 2 +- 24 files changed, 74 insertions(+), 55 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/{ => schema}/CreateOrUpdateTableDeviceNode.java (99%) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/{planner/plan/node/metadata/write/UpdateAttributeCommitNode.java => relational/planner/node/schema/TableDeviceAttributeCommitNode.java} (80%) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/{planner/plan/node/metadata/read => relational/planner/node/schema}/TableDeviceAttributeUpdateNode.java (98%) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/{planner/plan/node/metadata/read => relational/planner/node/schema}/TableDeviceFetchNode.java (97%) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/{planner/plan/node/metadata/read => relational/planner/node/schema}/TableDeviceQueryCountNode.java (94%) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/{planner/plan/node/metadata/read => relational/planner/node/schema}/TableDeviceQueryScanNode.java (95%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index ebfa62888926b..93ff084804919 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -33,7 +33,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ActivateTemplateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterTimeSeriesNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.BatchActivateTemplateNode; @@ -58,7 +57,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedNonWritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedWritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeOperateSchemaQueueNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.ICreateAlignedTimeSeriesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.ICreateTimeSeriesPlan; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 7a5cc0db04532..a67e0c1f324d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -61,9 +61,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.CountSchemaMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceFetchNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryCountNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.InputLocation; @@ -87,6 +84,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.leaf.LeafColumnTransformer; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 26d0d4c82ea2d..2b17f3dc1e64e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -35,10 +35,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.SeriesSchemaFetchScanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceAttributeUpdateNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceFetchNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryCountNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ActivateTemplateNode; @@ -118,7 +114,12 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -238,6 +239,7 @@ public enum PlanNodeType { DELETE_TABLE_DEVICE((short) 905), TABLE_DEVICE_QUERY_COUNT((short) 906), TABLE_DEVICE_ATTRIBUTE_UPDATE((short) 907), + TABLE_DEVICE_ATTRIBUTE_COMMIT((short) 908), TABLE_SCAN_NODE((short) 1000), TABLE_FILTER_NODE((short) 1001), @@ -531,6 +533,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return TableDeviceQueryCountNode.deserialize(buffer); case 907: return TableDeviceAttributeUpdateNode.deserialize(buffer); + case 908: + return TableDeviceAttributeCommitNode.deserialize(buffer); case 1000: return org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode .deserialize(buffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index fd284fe251485..0bc4341f7c083 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -32,10 +32,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.SchemaQueryScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.SeriesSchemaFetchScanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceAttributeUpdateNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceFetchNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryCountNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ActivateTemplateNode; @@ -123,8 +119,12 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.GroupReference; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; @SuppressWarnings("java:S6539") // suppress "Monster class" warning public abstract class PlanVisitor { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/AbstractTableDeviceQueryNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/AbstractTableDeviceQueryNode.java index 295d274403297..ac4e6f50e3c52 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/AbstractTableDeviceQueryNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/AbstractTableDeviceQueryNode.java @@ -24,6 +24,8 @@ import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java index cb83d60007be4..5ca62420b2bab 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java @@ -26,18 +26,18 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.CountSchemaMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceAttributeUpdateNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceFetchNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryCountNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Field; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.RelationType; import org.apache.iotdb.db.queryengine.plan.relational.execution.querystats.PlanOptimizersStatsCollector; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.LogicalOptimizeFactory; import org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.PlanOptimizer; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AbstractQueryDeviceWithCache; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java index a9a27d12b316e..622c93d74bdf7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java @@ -25,12 +25,12 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceFetchNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryCountNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; import static org.apache.iotdb.db.queryengine.plan.planner.distribution.NodeDistributionType.SAME_WITH_ALL_CHILDREN; import static org.apache.iotdb.db.queryengine.plan.planner.distribution.NodeDistributionType.SAME_WITH_SOME_CHILD; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 5c2efcafc31e2..f7be87bf644c2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -26,9 +26,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.AbstractTableDeviceQueryNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceFetchNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryCountNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme; @@ -49,6 +46,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelTypeProviderExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelTypeProviderExtractor.java index a78f8695b0700..336bd78b7b585 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelTypeProviderExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelTypeProviderExtractor.java @@ -19,8 +19,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.SimplePlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.AbstractTableDeviceQueryNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.CountSchemaMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryCountNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; @@ -37,6 +35,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.tsfile.enums.TSDataType; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateOrUpdateTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/CreateOrUpdateTableDeviceNode.java similarity index 99% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateOrUpdateTableDeviceNode.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/CreateOrUpdateTableDeviceNode.java index edb8b2d9c08d2..8a07aa279539b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateOrUpdateTableDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/CreateOrUpdateTableDeviceNode.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.queryengine.plan.relational.planner.node; +package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/UpdateAttributeCommitNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java similarity index 80% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/UpdateAttributeCommitNode.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java index bcb6bea4db941..e7b99da9d2943 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/UpdateAttributeCommitNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java @@ -17,10 +17,11 @@ * under the License. */ -package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write; +package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import java.io.DataOutputStream; import java.io.IOException; @@ -28,13 +29,13 @@ import java.util.List; import java.util.Set; -public class UpdateAttributeCommitNode extends PlanNode { +public class TableDeviceAttributeCommitNode extends PlanNode { private final long version; private final byte[] commitBuffer; private final Set shrunkNodes; - protected UpdateAttributeCommitNode( + protected TableDeviceAttributeCommitNode( final PlanNodeId id, final long version, final byte[] commitBuffer, @@ -67,9 +68,14 @@ public void addChild(final PlanNode child) { // Do nothing } + @Override + public PlanNodeType getType() { + return PlanNodeType.TABLE_DEVICE_ATTRIBUTE_COMMIT; + } + @Override public PlanNode clone() { - return new UpdateAttributeCommitNode(id, version, commitBuffer, shrunkNodes); + return new TableDeviceAttributeCommitNode(id, version, commitBuffer, shrunkNodes); } @Override @@ -87,4 +93,8 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) {} @Override protected void serializeAttributes(final DataOutputStream stream) throws IOException {} + + public static PlanNode deserialize(final ByteBuffer buffer) { + return null; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceAttributeUpdateNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeUpdateNode.java similarity index 98% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceAttributeUpdateNode.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeUpdateNode.java index c4163818ec5f6..8d19341092894 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceAttributeUpdateNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeUpdateNode.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read; +package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.schema.filter.SchemaFilter; @@ -303,9 +303,9 @@ public String toString() { + ", tableName='" + tableName + '\'' - + ", idDeterminedFilterList=" + + ", idDeterminedPredicateList=" + idDeterminedPredicateList - + ", idFuzzyFilter=" + + ", idFuzzyPredicate=" + idFuzzyPredicate + ", columnHeaderList=" + columnHeaderList diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceFetchNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java similarity index 97% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceFetchNode.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java index d598a351cb9be..2916b5f6ad404 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceFetchNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read; +package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; @@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceQueryCountNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java similarity index 94% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceQueryCountNode.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java index b09996bf5bf5e..a01d517e9e94a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceQueryCountNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read; +package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.schema.filter.SchemaFilter; @@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.AbstractTableDeviceQueryNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import java.nio.ByteBuffer; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceQueryScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java similarity index 95% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceQueryScanNode.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java index 2b81c5292286c..de1488efa2760 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceQueryScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read; +package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.schema.filter.SchemaFilter; @@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.AbstractTableDeviceQueryNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index bc867bb615c1a..ef60e693096c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -28,8 +28,8 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceAttributeUpdateNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.metric.ISchemaRegionMetric; import org.apache.iotdb.db.schemaengine.rescon.ISchemaRegionStatistics; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowDevicesPlan; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java index fd93ab37f6d4e..303d2a35b1afc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java @@ -19,8 +19,8 @@ package org.apache.iotdb.db.schemaengine.schemaregion; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceAttributeUpdateNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.IActivateTemplateInClusterPlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.IAutoCreateDeviceMNodePlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.IChangeAliasPlan; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 196c039ebdb2e..60bb0d57d19f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -47,12 +47,12 @@ import org.apache.iotdb.db.queryengine.execution.relational.ColumnTransformerBuilder; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.InputLocation; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 9b783a66f7e47..b57fd872b126d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -39,8 +39,8 @@ import org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException; import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceAttributeUpdateNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.metric.ISchemaRegionMetric; import org.apache.iotdb.db.schemaengine.metric.SchemaRegionCachedMetric; import org.apache.iotdb.db.schemaengine.rescon.CachedSchemaRegionStatistics; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java index c31a3d3041787..c5c06c0903d95 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java @@ -25,8 +25,8 @@ import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceAttributeUpdateNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java index 5d2bd8165a2f8..918369e431bf2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java @@ -21,8 +21,8 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceAttributeUpdateNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.ISerializer; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java index 46940bd72c6c8..3dbdfc66b5d1a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java @@ -22,8 +22,8 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceAttributeUpdateNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.ActivateTemplateInClusterPlanImpl; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java index 2d731b0f15617..cfe294cb37849 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java @@ -32,7 +32,7 @@ import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceAttributeUpdateNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.UpdateAssignment; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTestUtil.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTestUtil.java index 9444e8646d992..5a700d74d7731 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTestUtil.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTestUtil.java @@ -27,7 +27,7 @@ import org.apache.iotdb.commons.schema.filter.SchemaFilterFactory; import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterUtil; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.SchemaRegionReadPlanFactory; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchemaInfo; From 278a938e8559c8a93c33e50229b462c1fd5d2043 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:36:13 +0800 Subject: [PATCH 089/213] Partial --- .../TableDistributedPlanGenerator.java | 2 +- .../TableModelTypeProviderExtractor.java | 2 +- .../schema}/AbstractTableDeviceQueryNode.java | 5 ++-- .../TableDeviceAttributeCommitNode.java | 24 +++++++++++++++++-- .../schema/TableDeviceQueryCountNode.java | 1 - .../node/schema/TableDeviceQueryScanNode.java | 1 - 6 files changed, 26 insertions(+), 9 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/{planner/plan/node/metadata/read => relational/planner/node/schema}/AbstractTableDeviceQueryNode.java (96%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index f7be87bf644c2..8c1d9c9b3bfee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -25,7 +25,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.AbstractTableDeviceQueryNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme; @@ -46,6 +45,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.AbstractTableDeviceQueryNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelTypeProviderExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelTypeProviderExtractor.java index 336bd78b7b585..7fa0a4d888d21 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelTypeProviderExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelTypeProviderExtractor.java @@ -17,7 +17,6 @@ import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.SimplePlanVisitor; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.AbstractTableDeviceQueryNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.CountSchemaMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode; @@ -35,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.AbstractTableDeviceQueryNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/AbstractTableDeviceQueryNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java similarity index 96% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/AbstractTableDeviceQueryNode.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java index ac4e6f50e3c52..e76941ee53ba8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/AbstractTableDeviceQueryNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java @@ -17,15 +17,14 @@ * under the License. */ -package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read; +package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java index e7b99da9d2943..be683e15680a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java @@ -23,6 +23,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.tsfile.utils.ReadWriteIOUtils; + import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; @@ -89,10 +91,28 @@ public List getOutputColumnNames() { } @Override - protected void serializeAttributes(final ByteBuffer byteBuffer) {} + protected void serializeAttributes(final ByteBuffer byteBuffer) { + getType().serialize(byteBuffer); + ReadWriteIOUtils.write(version, byteBuffer); + ReadWriteIOUtils.write(commitBuffer.length, byteBuffer); + byteBuffer.put(commitBuffer); + ReadWriteIOUtils.write(shrunkNodes.size(), byteBuffer); + for (final Integer nodeId : shrunkNodes) { + ReadWriteIOUtils.write(nodeId, byteBuffer); + } + } @Override - protected void serializeAttributes(final DataOutputStream stream) throws IOException {} + protected void serializeAttributes(final DataOutputStream stream) throws IOException { + getType().serialize(stream); + ReadWriteIOUtils.write(version, stream); + ReadWriteIOUtils.write(commitBuffer.length, stream); + stream.write(commitBuffer); + ReadWriteIOUtils.write(shrunkNodes.size(), stream); + for (final Integer nodeId : shrunkNodes) { + ReadWriteIOUtils.write(nodeId, stream); + } + } public static PlanNode deserialize(final ByteBuffer buffer) { return null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java index a01d517e9e94a..b611c1c0095f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java @@ -26,7 +26,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.AbstractTableDeviceQueryNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import java.nio.ByteBuffer; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java index de1488efa2760..a78ef0992875e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java @@ -26,7 +26,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.AbstractTableDeviceQueryNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.tsfile.utils.ReadWriteIOUtils; From 49bda898d100b131ecccfb1ac8173f1dc45a163c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:50:35 +0800 Subject: [PATCH 090/213] Update TableDeviceAttributeCommitNode.java --- .../node/schema/TableDeviceAttributeCommitNode.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java index be683e15680a2..c2062000278d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java @@ -28,6 +28,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -115,6 +116,15 @@ protected void serializeAttributes(final DataOutputStream stream) throws IOExcep } public static PlanNode deserialize(final ByteBuffer buffer) { - return null; + final long version = ReadWriteIOUtils.readLong(buffer); + final byte[] commitBuffer = new byte[ReadWriteIOUtils.readInt(buffer)]; + buffer.get(commitBuffer); + final int size = ReadWriteIOUtils.readInt(buffer); + final Set shrunkNodes = new HashSet<>(); + for (int i = 0; i < size; ++i) { + shrunkNodes.add(ReadWriteIOUtils.readInt(buffer)); + } + final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); + return new TableDeviceAttributeCommitNode(planNodeId, version, commitBuffer, shrunkNodes); } } From 6179b0b85e004cfae3dd41d4cefe269a9848e50c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:27:24 +0800 Subject: [PATCH 091/213] Add write --- .../TableDeviceAttributeCommitNode.java | 57 ++++++++++++++----- .../update/DeviceAttributeRemoteUpdater.java | 2 + ...GeneralRegionAttributeSecurityService.java | 21 +++++++ .../utils/ThriftCommonsSerDeUtils.java | 12 +++- 4 files changed, 75 insertions(+), 17 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java index c2062000278d6..0fe5bcec26538 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -28,24 +30,26 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; public class TableDeviceAttributeCommitNode extends PlanNode { private final long version; - private final byte[] commitBuffer; + private final Map commitMap; private final Set shrunkNodes; - protected TableDeviceAttributeCommitNode( + public TableDeviceAttributeCommitNode( final PlanNodeId id, final long version, - final byte[] commitBuffer, + final Map commitMap, final Set shrunkNodes) { super(id); this.version = version; - this.commitBuffer = commitBuffer; + this.commitMap = commitMap; this.shrunkNodes = shrunkNodes; } @@ -53,8 +57,8 @@ public long getVersion() { return version; } - public byte[] getCommitBuffer() { - return commitBuffer; + public Map getCommitMap() { + return commitMap; } public Set getShrunkNodes() { @@ -78,7 +82,7 @@ public PlanNodeType getType() { @Override public PlanNode clone() { - return new TableDeviceAttributeCommitNode(id, version, commitBuffer, shrunkNodes); + return new TableDeviceAttributeCommitNode(id, version, commitMap, shrunkNodes); } @Override @@ -95,8 +99,13 @@ public List getOutputColumnNames() { protected void serializeAttributes(final ByteBuffer byteBuffer) { getType().serialize(byteBuffer); ReadWriteIOUtils.write(version, byteBuffer); - ReadWriteIOUtils.write(commitBuffer.length, byteBuffer); - byteBuffer.put(commitBuffer); + ReadWriteIOUtils.write(commitMap.size(), byteBuffer); + for (final Map.Entry entry : commitMap.entrySet()) { + ReadWriteIOUtils.write(entry.getKey().getDataNodeId(), byteBuffer); + ThriftCommonsSerDeUtils.serializeTEndPoint(entry.getKey().getInternalEndPoint(), byteBuffer); + ReadWriteIOUtils.write(entry.getValue().length, byteBuffer); + byteBuffer.put(entry.getValue()); + } ReadWriteIOUtils.write(shrunkNodes.size(), byteBuffer); for (final Integer nodeId : shrunkNodes) { ReadWriteIOUtils.write(nodeId, byteBuffer); @@ -107,8 +116,13 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { protected void serializeAttributes(final DataOutputStream stream) throws IOException { getType().serialize(stream); ReadWriteIOUtils.write(version, stream); - ReadWriteIOUtils.write(commitBuffer.length, stream); - stream.write(commitBuffer); + ReadWriteIOUtils.write(commitMap.size(), stream); + for (final Map.Entry entry : commitMap.entrySet()) { + ReadWriteIOUtils.write(entry.getKey().getDataNodeId(), stream); + ThriftCommonsSerDeUtils.serializeTEndPoint(entry.getKey().getInternalEndPoint(), stream); + ReadWriteIOUtils.write(entry.getValue().length, stream); + stream.write(entry.getValue()); + } ReadWriteIOUtils.write(shrunkNodes.size(), stream); for (final Integer nodeId : shrunkNodes) { ReadWriteIOUtils.write(nodeId, stream); @@ -117,14 +131,27 @@ protected void serializeAttributes(final DataOutputStream stream) throws IOExcep public static PlanNode deserialize(final ByteBuffer buffer) { final long version = ReadWriteIOUtils.readLong(buffer); - final byte[] commitBuffer = new byte[ReadWriteIOUtils.readInt(buffer)]; - buffer.get(commitBuffer); - final int size = ReadWriteIOUtils.readInt(buffer); + int size = ReadWriteIOUtils.readInt(buffer); + final Map commitMap = new HashMap<>(size); + for (int i = 0; i < size; ++i) { + final TDataNodeLocation location = + new TDataNodeLocation( + ReadWriteIOUtils.readInt(buffer), + null, + ThriftCommonsSerDeUtils.deserializeTEndPoint(buffer), + null, + null, + null); + final byte[] commitBuffer = new byte[ReadWriteIOUtils.readInt(buffer)]; + buffer.get(commitBuffer); + commitMap.put(location, commitBuffer); + } + size = ReadWriteIOUtils.readInt(buffer); final Set shrunkNodes = new HashSet<>(); for (int i = 0; i < size; ++i) { shrunkNodes.add(ReadWriteIOUtils.readInt(buffer)); } final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); - return new TableDeviceAttributeCommitNode(planNodeId, version, commitBuffer, shrunkNodes); + return new TableDeviceAttributeCommitNode(planNodeId, version, commitMap, shrunkNodes); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index aff625098d9f1..c03cc97da3e35 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -54,6 +54,8 @@ public class DeviceAttributeRemoteUpdater { private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeRemoteUpdater.class); private static final int UPDATE_DETAIL_CONTAINER_SEND_MIN_LIMIT_BYTES = 1024; + // All the data node locations shall only have internal endpoint with all the other endpoints set + // to null private final Set targetDataNodeLocations = new HashSet<>(); private final ConcurrentMap attributeUpdateMap = new ConcurrentHashMap<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 11fbaa621b6d7..f528cb4cc38fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -29,6 +29,9 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.protocol.client.dn.DnToDnInternalServiceAsyncRequestManager; import org.apache.iotdb.db.protocol.client.dn.DnToDnRequestType; +import org.apache.iotdb.db.queryengine.execution.executor.RegionWriteExecutor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; import org.apache.iotdb.mpp.rpc.thrift.TSchemaRegionAttributeInfo; @@ -39,6 +42,7 @@ import org.slf4j.LoggerFactory; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -119,6 +123,23 @@ private void execute() { // Send sendUpdateRequest(attributeUpdateCommitMap); + // Commit + // TODO: Execute in parallel by thread pool + attributeUpdateCommitMap.forEach( + (schemaRegionId, pair) -> { + if (!new RegionWriteExecutor() + .execute( + schemaRegionId, + new TableDeviceAttributeCommitNode( + new PlanNodeId(""), + pair.getLeft(), + pair.getRight(), + Collections.emptySet())) + .isAccepted()) { + LOGGER.warn("Failed to write attribute commit message to region {}.", schemaRegionId); + } + }); + if (!skipNext) { condition.await( IoTDBDescriptor.getInstance() diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/ThriftCommonsSerDeUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/ThriftCommonsSerDeUtils.java index aa6a8d09d8290..ee5e0677f0ba3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/ThriftCommonsSerDeUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/ThriftCommonsSerDeUtils.java @@ -59,7 +59,7 @@ private static TBinaryProtocol generateWriteProtocol(final OutputStream stream) return new TBinaryProtocol(new TIOStreamTransport(stream)); } - private static TBinaryProtocol generateWriteProtocol(ByteBuffer buffer) + private static TBinaryProtocol generateWriteProtocol(final ByteBuffer buffer) throws TTransportException { TTransport transport = generateTByteBuffer(buffer); return new TBinaryProtocol(transport); @@ -70,7 +70,7 @@ private static TBinaryProtocol generateReadProtocol(final InputStream stream) return new TBinaryProtocol(new TIOStreamTransport(stream)); } - private static TBinaryProtocol generateReadProtocol(ByteBuffer buffer) + private static TBinaryProtocol generateReadProtocol(final ByteBuffer buffer) throws TTransportException { TTransport transport = generateTByteBuffer(buffer); return new TBinaryProtocol(transport); @@ -84,6 +84,14 @@ public static void serializeTEndPoint(final TEndPoint endPoint, final OutputStre } } + public static void serializeTEndPoint(final TEndPoint endPoint, final ByteBuffer buffer) { + try { + endPoint.write(generateWriteProtocol(buffer)); + } catch (final TException e) { + throw new ThriftSerDeException("Write TEndPoint failed: ", e); + } + } + public static TEndPoint deserializeTEndPoint(final InputStream stream) { final TEndPoint endPoint = new TEndPoint(); try { From d0f55e474a2229077c561c9575396e1a4b6de482 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:33:47 +0800 Subject: [PATCH 092/213] Partial --- .../schemaregion/SchemaExecutionVisitor.java | 10 +++++++++- .../plan/planner/plan/node/PlanVisitor.java | 9 ++++++++- .../node/schema/CreateOrUpdateTableDeviceNode.java | 2 +- .../db/schemaengine/schemaregion/ISchemaRegion.java | 3 +++ .../attribute/update/DeviceAttributeRemoteUpdater.java | 5 +++++ .../schemaregion/impl/SchemaRegionMemoryImpl.java | 6 ++++++ .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 9 +++++++-- 7 files changed, 39 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index 93ff084804919..bf92f3a34f79b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -58,6 +58,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedWritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeOperateSchemaQueueNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.ICreateAlignedTimeSeriesPlan; @@ -580,7 +581,7 @@ public TSStatus visitDeleteLogicalView(DeleteLogicalViewNode node, ISchemaRegion } @Override - public TSStatus visitCreateTableDevice( + public TSStatus visitCreateOrUpdateTableDevice( final CreateOrUpdateTableDeviceNode node, final ISchemaRegion schemaRegion) { try { // todo implement storage for device of diverse data types @@ -604,6 +605,13 @@ public TSStatus visitTableDeviceAttributeUpdate( } } + @Override + public TSStatus visitTableDeviceAttributeCommit( + final TableDeviceAttributeCommitNode node, final ISchemaRegion schemaRegion) { + schemaRegion.commitUpdateAttribute(node); + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } + @Override public TSStatus visitPipeEnrichedWritePlanNode( final PipeEnrichedWritePlanNode node, final ISchemaRegion schemaRegion) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 0bc4341f7c083..cc726078da14f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -121,6 +121,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.GroupReference; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; @@ -501,7 +502,8 @@ public R visitAlterLogicalView(AlterLogicalViewNode node, C context) { return visitPlan(node, context); } - public R visitCreateTableDevice(final CreateOrUpdateTableDeviceNode node, final C context) { + public R visitCreateOrUpdateTableDevice( + final CreateOrUpdateTableDeviceNode node, final C context) { return visitPlan(node, context); } @@ -522,6 +524,11 @@ public R visitTableDeviceQueryCount(final TableDeviceQueryCountNode node, final return visitPlan(node, context); } + public R visitTableDeviceAttributeCommit( + final TableDeviceAttributeCommitNode node, final C context) { + return visitPlan(node, context); + } + ///////////////////////////////////////////////////////////////////////////////////////////////// // Data Write Node ///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/CreateOrUpdateTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/CreateOrUpdateTableDeviceNode.java index 8a07aa279539b..bba380559cdfb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/CreateOrUpdateTableDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/CreateOrUpdateTableDeviceNode.java @@ -286,7 +286,7 @@ public List splitByPartition(final IAnalysis analysis) { @Override public R accept(final PlanVisitor visitor, C context) { - return visitor.visitCreateTableDevice(this, context); + return visitor.visitCreateOrUpdateTableDevice(this, context); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index ef60e693096c3..15366b6b07783 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.metric.ISchemaRegionMetric; import org.apache.iotdb.db.schemaengine.rescon.ISchemaRegionStatistics; @@ -355,5 +356,7 @@ ISchemaReader getTableDeviceReader( // region Interfaces for AttributeUpdate Pair> getAttributeUpdateInfo(final AtomicInteger limit); + void commitUpdateAttribute(final TableDeviceAttributeCommitNode node); + // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index c03cc97da3e35..307121d5424a5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; import org.apache.tsfile.utils.Pair; @@ -126,6 +127,10 @@ public Pair> getAttributeUpdateInfo( return new Pair<>(version.get(), updateBytes); } + public void commit(final TableDeviceAttributeCommitNode node) { + // TODO + } + public void addLocation(final TDataNodeLocation dataNodeLocation) { targetDataNodeLocations.add(dataNodeLocation); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 60bb0d57d19f8..cc2a8f17cf154 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -52,6 +52,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; @@ -1569,6 +1570,11 @@ public Pair> getAttributeUpdateInfo( return deviceAttributeRemoteUpdater.getAttributeUpdateInfo(limit); } + @Override + public void commitUpdateAttribute(final TableDeviceAttributeCommitNode node) { + deviceAttributeRemoteUpdater.commit(node); + } + // endregion private static class RecoverOperationResult { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index b57fd872b126d..8b71d23f9269f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.metric.ISchemaRegionMetric; import org.apache.iotdb.db.schemaengine.metric.SchemaRegionCachedMetric; @@ -1495,8 +1496,12 @@ public ISchemaReader getTableDeviceReader( @Override public Pair> getAttributeUpdateInfo( final AtomicInteger limit) { - // TODO - return null; + throw new UnsupportedOperationException(); + } + + @Override + public void commitUpdateAttribute(final TableDeviceAttributeCommitNode node) { + throw new UnsupportedOperationException(); } // endregion From ea25db72138f14312fbb80a6d93c475b61af2bd0 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:04:14 +0800 Subject: [PATCH 093/213] Add mlog --- .../schemaregion/SchemaExecutionVisitor.java | 13 ++++++---- .../plan/planner/plan/node/PlanNodeType.java | 4 ++-- .../plan/planner/plan/node/PlanVisitor.java | 4 ++-- ...TableDeviceAttributeCommitUpdateNode.java} | 24 +++++++++++++++---- .../schemaregion/ISchemaRegion.java | 5 ++-- .../schemaregion/SchemaRegionPlanType.java | 1 + .../schemaregion/SchemaRegionPlanVisitor.java | 7 ++++++ .../update/DeviceAttributeRemoteUpdater.java | 4 ++-- ...GeneralRegionAttributeSecurityService.java | 4 ++-- .../impl/SchemaRegionMemoryImpl.java | 18 ++++++++++++-- .../impl/SchemaRegionPBTreeImpl.java | 4 ++-- .../visitor/SchemaRegionPlanDeserializer.java | 8 +++++++ .../visitor/SchemaRegionPlanSerializer.java | 13 ++++++++++ .../req/SchemaRegionWritePlanFactory.java | 3 +++ 14 files changed, 90 insertions(+), 22 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/{TableDeviceAttributeCommitNode.java => TableDeviceAttributeCommitUpdateNode.java} (83%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index bf92f3a34f79b..00b6744ab5049 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -58,7 +58,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedWritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeOperateSchemaQueueNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.ICreateAlignedTimeSeriesPlan; @@ -607,9 +607,14 @@ public TSStatus visitTableDeviceAttributeUpdate( @Override public TSStatus visitTableDeviceAttributeCommit( - final TableDeviceAttributeCommitNode node, final ISchemaRegion schemaRegion) { - schemaRegion.commitUpdateAttribute(node); - return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + final TableDeviceAttributeCommitUpdateNode node, final ISchemaRegion schemaRegion) { + try { + schemaRegion.commitUpdateAttribute(node); + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } catch (final MetadataException e) { + logger.error(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 2b17f3dc1e64e..5a6354cb4a1d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -115,7 +115,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; @@ -534,7 +534,7 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { case 907: return TableDeviceAttributeUpdateNode.deserialize(buffer); case 908: - return TableDeviceAttributeCommitNode.deserialize(buffer); + return TableDeviceAttributeCommitUpdateNode.deserialize(buffer); case 1000: return org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode .deserialize(buffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index cc726078da14f..e1f2b635e2ae8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -121,7 +121,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.GroupReference; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; @@ -525,7 +525,7 @@ public R visitTableDeviceQueryCount(final TableDeviceQueryCountNode node, final } public R visitTableDeviceAttributeCommit( - final TableDeviceAttributeCommitNode node, final C context) { + final TableDeviceAttributeCommitUpdateNode node, final C context) { return visitPlan(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java similarity index 83% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java index 0fe5bcec26538..80cb93c4d94ba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java @@ -24,6 +24,9 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; +import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; +import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -36,13 +39,16 @@ import java.util.Map; import java.util.Set; -public class TableDeviceAttributeCommitNode extends PlanNode { +public class TableDeviceAttributeCommitUpdateNode extends PlanNode implements ISchemaRegionPlan { private final long version; private final Map commitMap; private final Set shrunkNodes; - public TableDeviceAttributeCommitNode( + public static final TableDeviceAttributeCommitUpdateNode MOCK_INSTANCE = + new TableDeviceAttributeCommitUpdateNode(new PlanNodeId(""), 0L, null, null); + + public TableDeviceAttributeCommitUpdateNode( final PlanNodeId id, final long version, final Map commitMap, @@ -82,7 +88,7 @@ public PlanNodeType getType() { @Override public PlanNode clone() { - return new TableDeviceAttributeCommitNode(id, version, commitMap, shrunkNodes); + return new TableDeviceAttributeCommitUpdateNode(id, version, commitMap, shrunkNodes); } @Override @@ -152,6 +158,16 @@ public static PlanNode deserialize(final ByteBuffer buffer) { shrunkNodes.add(ReadWriteIOUtils.readInt(buffer)); } final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); - return new TableDeviceAttributeCommitNode(planNodeId, version, commitMap, shrunkNodes); + return new TableDeviceAttributeCommitUpdateNode(planNodeId, version, commitMap, shrunkNodes); + } + + @Override + public SchemaRegionPlanType getPlanType() { + return SchemaRegionPlanType.COMMIT_UPDATE_TABLE_DEVICE_ATTRIBUTE; + } + + @Override + public R accept(final SchemaRegionPlanVisitor visitor, final C context) { + return visitor.visitCommitUpdateTableDeviceAttribute(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 15366b6b07783..6613b9cf5db7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -29,7 +29,7 @@ import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.metric.ISchemaRegionMetric; import org.apache.iotdb.db.schemaengine.rescon.ISchemaRegionStatistics; @@ -356,7 +356,8 @@ ISchemaReader getTableDeviceReader( // region Interfaces for AttributeUpdate Pair> getAttributeUpdateInfo(final AtomicInteger limit); - void commitUpdateAttribute(final TableDeviceAttributeCommitNode node); + void commitUpdateAttribute(final TableDeviceAttributeCommitUpdateNode node) + throws MetadataException; // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java index f4480e9a39eca..7e318f660f1b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java @@ -54,6 +54,7 @@ public enum SchemaRegionPlanType { // Table CREATE_TABLE_DEVICE((byte) 100), UPDATE_TABLE_DEVICE_ATTRIBUTE((byte) 101), + COMMIT_UPDATE_TABLE_DEVICE_ATTRIBUTE((byte) 102), // query plan doesn't need any ser/deSer, thus use one type to represent all READ_SCHEMA(Byte.MAX_VALUE); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java index 303d2a35b1afc..009d9f730fade 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.IActivateTemplateInClusterPlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.IAutoCreateDeviceMNodePlan; @@ -135,4 +136,10 @@ public R visitUpdateTableDeviceAttribute( final TableDeviceAttributeUpdateNode updateTableDeviceAttributePlan, final C context) { return visitSchemaRegionPlan(updateTableDeviceAttributePlan, context); } + + public R visitCommitUpdateTableDeviceAttribute( + final TableDeviceAttributeCommitUpdateNode commitUpdateTableDeviceAttributePlan, + final C context) { + return visitSchemaRegionPlan(commitUpdateTableDeviceAttributePlan, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 307121d5424a5..e3588c1910625 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; import org.apache.tsfile.utils.Pair; @@ -127,7 +127,7 @@ public Pair> getAttributeUpdateInfo( return new Pair<>(version.get(), updateBytes); } - public void commit(final TableDeviceAttributeCommitNode node) { + public void commit(final TableDeviceAttributeCommitUpdateNode node) { // TODO } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index f528cb4cc38fa..832e94afd66ef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -31,7 +31,7 @@ import org.apache.iotdb.db.protocol.client.dn.DnToDnRequestType; import org.apache.iotdb.db.queryengine.execution.executor.RegionWriteExecutor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; import org.apache.iotdb.mpp.rpc.thrift.TSchemaRegionAttributeInfo; @@ -130,7 +130,7 @@ private void execute() { if (!new RegionWriteExecutor() .execute( schemaRegionId, - new TableDeviceAttributeCommitNode( + new TableDeviceAttributeCommitUpdateNode( new PlanNodeId(""), pair.getLeft(), pair.getRight(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index cc2a8f17cf154..e736441d03260 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -52,7 +52,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; @@ -1571,8 +1571,10 @@ public Pair> getAttributeUpdateInfo( } @Override - public void commitUpdateAttribute(final TableDeviceAttributeCommitNode node) { + public void commitUpdateAttribute(final TableDeviceAttributeCommitUpdateNode node) + throws MetadataException { deviceAttributeRemoteUpdater.commit(node); + writeToMLog(node); } // endregion @@ -1814,5 +1816,17 @@ public RecoverOperationResult visitUpdateTableDeviceAttribute( return new RecoverOperationResult(e); } } + + @Override + public RecoverOperationResult visitCommitUpdateTableDeviceAttribute( + final TableDeviceAttributeCommitUpdateNode commitUpdateTableDeviceAttributePlan, + final SchemaRegionMemoryImpl context) { + try { + commitUpdateAttribute(commitUpdateTableDeviceAttributePlan); + return RecoverOperationResult.SUCCESS; + } catch (final MetadataException e) { + return new RecoverOperationResult(e); + } + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 8b71d23f9269f..a00b99aaec9d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -40,7 +40,7 @@ import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.metric.ISchemaRegionMetric; import org.apache.iotdb.db.schemaengine.metric.SchemaRegionCachedMetric; @@ -1500,7 +1500,7 @@ public Pair> getAttributeUpdateInfo( } @Override - public void commitUpdateAttribute(final TableDeviceAttributeCommitNode node) { + public void commitUpdateAttribute(final TableDeviceAttributeCommitUpdateNode node) { throw new UnsupportedOperationException(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java index c5c06c0903d95..eda82a67cee98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; @@ -422,5 +423,12 @@ public ISchemaRegionPlan visitUpdateTableDeviceAttribute( final ByteBuffer buffer) { return (TableDeviceAttributeUpdateNode) PlanNodeType.deserialize(buffer); } + + @Override + public ISchemaRegionPlan visitCommitUpdateTableDeviceAttribute( + final TableDeviceAttributeCommitUpdateNode commitUpdateTableDeviceAttributePlan, + final ByteBuffer buffer) { + return (TableDeviceAttributeCommitUpdateNode) PlanNodeType.deserialize(buffer); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java index 918369e431bf2..51f1c9c43002d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; @@ -500,6 +501,18 @@ public SchemaRegionPlanSerializationResult visitUpdateTableDeviceAttribute( } } + @Override + public SchemaRegionPlanSerializationResult visitCommitUpdateTableDeviceAttribute( + final TableDeviceAttributeCommitUpdateNode commitUpdateTableDeviceAttributePlan, + final DataOutputStream outputStream) { + try { + commitUpdateTableDeviceAttributePlan.serialize(outputStream); + return SchemaRegionPlanSerializationResult.SUCCESS; + } catch (final IOException e) { + return new SchemaRegionPlanSerializationResult(e); + } + } + @Override public SchemaRegionPlanSerializationResult visitAlterLogicalView( final IAlterLogicalViewPlan alterLogicalViewPlan, final DataOutputStream stream) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java index 3dbdfc66b5d1a..75ffea30b77cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; @@ -100,6 +101,8 @@ public static ISchemaRegionPlan getEmptyPlan(final SchemaRegionPlanType planType return CreateOrUpdateTableDeviceNode.MOCK_INSTANCE; case UPDATE_TABLE_DEVICE_ATTRIBUTE: return TableDeviceAttributeUpdateNode.MOCK_INSTANCE; + case COMMIT_UPDATE_TABLE_DEVICE_ATTRIBUTE: + return TableDeviceAttributeCommitUpdateNode.MOCK_INSTANCE; default: throw new UnsupportedOperationException( String.format( From 8e55ed29f57b0997a1de540453cb55edb14b1614 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:15:04 +0800 Subject: [PATCH 094/213] non-complete commit logic --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 10 +++++++++- .../attribute/update/UpdateClearContainer.java | 3 +-- .../schemaregion/attribute/update/UpdateContainer.java | 3 +-- .../attribute/update/UpdateDetailContainer.java | 3 +-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index e3588c1910625..e0551120f282d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -128,7 +128,15 @@ public Pair> getAttributeUpdateInfo( } public void commit(final TableDeviceAttributeCommitUpdateNode node) { - // TODO + node.getCommitMap() + .forEach( + ((location, bytes) -> + attributeUpdateMap.computeIfPresent( + location, + (dataNode, container) -> { + container.updateSelfByCommitBuffer(bytes); + return container; + }))); } public void addLocation(final TDataNodeLocation dataNodeLocation) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 21a5c2edc76d2..557f424bceb1a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -97,7 +96,7 @@ private void serializeWithLimit( } @Override - public Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer) { + public Pair updateSelfByCommitBuffer(final byte[] commitBuffer) { return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index 0b562d67507c4..bb49040929577 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -27,7 +27,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -43,7 +42,7 @@ long updateAttribute( // The "limitBytes" shall be at least 5 for a "type" and "0" to indicate empty byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes); - Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer); + Pair updateSelfByCommitBuffer(final byte[] commitBuffer); void serialize(final OutputStream outputstream) throws IOException; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 93d080688cf6c..1b208773a9308 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Map; import java.util.Objects; @@ -178,7 +177,7 @@ private void serializeWithLimit( } @Override - public Pair updateSelfByCommitBuffer(final ByteBuffer commitBuffer) { + public Pair updateSelfByCommitBuffer(final byte[] commitBuffer) { return null; } From 0566fba5de57d5436d95d0734f9bebf71f060d19 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:53:24 +0800 Subject: [PATCH 095/213] partial commit --- .../update/DeviceAttributeRemoteUpdater.java | 33 ++++++++++++++++++- .../update/UpdateClearContainer.java | 2 +- .../attribute/update/UpdateContainer.java | 3 +- .../update/UpdateContainerStatistics.java | 9 ++--- .../update/UpdateDetailContainer.java | 2 +- 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index e0551120f282d..fb9cc8d06d2a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -34,6 +34,7 @@ import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -134,11 +135,41 @@ public void commit(final TableDeviceAttributeCommitUpdateNode node) { attributeUpdateMap.computeIfPresent( location, (dataNode, container) -> { - container.updateSelfByCommitBuffer(bytes); + if (container instanceof UpdateDetailContainer + || version.get() == node.getVersion()) { + final Pair result = + container.updateSelfByCommitContainer(getContainer(bytes)); + releaseMemory(result.getLeft()); + // isEmpty + if (result.getRight()) { + releaseMemory( + container instanceof UpdateDetailContainer + ? UpdateDetailContainer.INSTANCE_SIZE + : UpdateClearContainer.INSTANCE_SIZE); + updateContainerStatistics.remove(dataNode); + } else if (updateContainerStatistics.containsKey(dataNode)) { + updateContainerStatistics.get(dataNode).decreaseSize(result.getLeft()); + } + } return container; }))); } + private UpdateContainer getContainer(final byte[] bytes) { + final ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes); + UpdateContainer result = null; + try { + result = + ReadWriteIOUtils.readBool(inputStream) + ? new UpdateDetailContainer() + : new UpdateClearContainer(); + result.deserialize(inputStream); + } catch (final IOException ignore) { + // ByteArrayInputStream won't throw IOException + } + return result; + } + public void addLocation(final TDataNodeLocation dataNodeLocation) { targetDataNodeLocations.add(dataNodeLocation); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 557f424bceb1a..617f96c23bc82 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -96,7 +96,7 @@ private void serializeWithLimit( } @Override - public Pair updateSelfByCommitBuffer(final byte[] commitBuffer) { + public Pair updateSelfByCommitContainer(final UpdateContainer commitBuffer) { return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index bb49040929577..f38fc8f85f4cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -39,10 +39,11 @@ long updateAttribute( // Only this method is not synchronize called and is called by GRASS thread // A piece of "updateContent" won't exceed "limitBytes" in order to handle // thrift threshold and low bandwidth + // Note that a "piece" returned is also a complete "updateContainer"'s serialized bytes // The "limitBytes" shall be at least 5 for a "type" and "0" to indicate empty byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes); - Pair updateSelfByCommitBuffer(final byte[] commitBuffer); + Pair updateSelfByCommitContainer(final UpdateContainer commitBuffer); void serialize(final OutputStream outputstream) throws IOException; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java index 12721bf985334..ee20c99f92a35 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java @@ -28,10 +28,6 @@ public class UpdateContainerStatistics { private long lastUpdateTime = System.currentTimeMillis(); private long size = 0; - private void update() { - lastUpdateTime = System.currentTimeMillis(); - } - long getSize() { return size; } @@ -40,6 +36,11 @@ void addSize(final long increment) { this.size += increment; } + void decreaseSize(final long decrement) { + this.size -= decrement; + lastUpdateTime = System.currentTimeMillis(); + } + boolean needDegrade() { return size >= MIN_DEGRADE_MEMORY; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 1b208773a9308..919ac30983dbb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -177,7 +177,7 @@ private void serializeWithLimit( } @Override - public Pair updateSelfByCommitBuffer(final byte[] commitBuffer) { + public Pair updateSelfByCommitContainer(final UpdateContainer commitBuffer) { return null; } From 6f36622f3e503d6ed5f6b5a30feaf40d36ade0fd Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:55:02 +0800 Subject: [PATCH 096/213] Update DeviceAttributeRemoteUpdater.java --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index fb9cc8d06d2a8..4f441de187db9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -147,6 +147,7 @@ public void commit(final TableDeviceAttributeCommitUpdateNode node) { ? UpdateDetailContainer.INSTANCE_SIZE : UpdateClearContainer.INSTANCE_SIZE); updateContainerStatistics.remove(dataNode); + return null; } else if (updateContainerStatistics.containsKey(dataNode)) { updateContainerStatistics.get(dataNode).decreaseSize(result.getLeft()); } From 40907c6f94890bd8874601d18e9163e945797451 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:07:37 +0800 Subject: [PATCH 097/213] Refactor --- .../update/DeviceAttributeRemoteUpdater.java | 2 +- .../update/UpdateClearContainer.java | 22 +++++++++++++++++-- .../attribute/update/UpdateContainer.java | 2 +- .../update/UpdateDetailContainer.java | 2 +- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 4f441de187db9..7fd0bc3fcc404 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -137,7 +137,7 @@ public void commit(final TableDeviceAttributeCommitUpdateNode node) { (dataNode, container) -> { if (container instanceof UpdateDetailContainer || version.get() == node.getVersion()) { - final Pair result = + final Pair result = container.updateSelfByCommitContainer(getContainer(bytes)); releaseMemory(result.getLeft()); // isEmpty diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 617f96c23bc82..0a316c8106b81 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -35,6 +35,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; @ThreadSafe public class UpdateClearContainer implements UpdateContainer { @@ -96,8 +97,21 @@ private void serializeWithLimit( } @Override - public Pair updateSelfByCommitContainer(final UpdateContainer commitBuffer) { - return null; + public Pair updateSelfByCommitContainer(final UpdateContainer commitContainer) { + if (!(commitContainer instanceof UpdateClearContainer)) { + return new Pair<>(0L, false); + } + final AtomicLong reducedBytes = new AtomicLong(0); + ((UpdateClearContainer) commitContainer) + .getTableNames() + .forEach( + tableName -> { + if (tableNames.contains(tableName)) { + tableNames.remove(tableName); + reducedBytes.addAndGet(RamUsageEstimator.sizeOf(tableName)); + } + }); + return new Pair<>(reducedBytes.get(), tableNames.isEmpty()); } @Override @@ -117,6 +131,10 @@ public void deserialize(final InputStream inputStream) throws IOException { } } + Set getTableNames() { + return tableNames; + } + long ramBytesUsed() { return INSTANCE_SIZE + tableNames.stream().mapToLong(RamUsageEstimator::sizeOf).reduce(0L, Long::sum); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index f38fc8f85f4cb..ab6b00b47ee02 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -43,7 +43,7 @@ long updateAttribute( // The "limitBytes" shall be at least 5 for a "type" and "0" to indicate empty byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes); - Pair updateSelfByCommitContainer(final UpdateContainer commitBuffer); + Pair updateSelfByCommitContainer(final UpdateContainer commitContainer); void serialize(final OutputStream outputstream) throws IOException; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 919ac30983dbb..09cde67f4ec34 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -177,7 +177,7 @@ private void serializeWithLimit( } @Override - public Pair updateSelfByCommitContainer(final UpdateContainer commitBuffer) { + public Pair updateSelfByCommitContainer(final UpdateContainer commitContainer) { return null; } From 0003106909013ed41af606bfc531407116247b5c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:34:46 +0800 Subject: [PATCH 098/213] partial --- .../RewritableByteArrayOutputStream.java | 4 +++ .../update/UpdateDetailContainer.java | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java index 2e70ca4781520..2f26eabb192b1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java @@ -32,6 +32,10 @@ class RewritableByteArrayOutputStream extends ByteArrayOutputStream { private static final byte[] intPlaceHolder = new byte[4]; + void writeNull() throws IOException { + ReadWriteIOUtils.write(ReadWriteIOUtils.NO_BYTE_TO_READ, this); + } + void writeWithLength(final byte[] bytes) throws IOException { ReadWriteIOUtils.write(bytes.length, this); write(bytes); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 09cde67f4ec34..c6e78c2feec8a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -45,6 +45,8 @@ public class UpdateDetailContainer implements UpdateContainer { static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(UpdateClearContainer.class) + MAP_SIZE; + // <@Nonnull TableName, >> private final ConcurrentMap>> updateMap = new ConcurrentHashMap<>(); @@ -178,6 +180,39 @@ private void serializeWithLimit( @Override public Pair updateSelfByCommitContainer(final UpdateContainer commitContainer) { + final AtomicLong result = new AtomicLong(0); + if (commitContainer instanceof UpdateDetailContainer) { + ((UpdateDetailContainer) commitContainer) + .updateMap.forEach( + (table, commitMap) -> { + if (!this.updateMap.containsKey(table)) { + return; + } + final ConcurrentMap> thisDeviceMap = + this.updateMap.get(table); + commitMap.forEach( + (device, attributes) -> { + if (!thisDeviceMap.containsKey(device)) { + return; + } + final Map thisAttributes = thisDeviceMap.get(device); + attributes.forEach( + (k, v) -> { + if (!thisAttributes.containsKey(k)) { + return; + } + final String thisV = thisAttributes.get(k); + if (Objects.equals(thisV, v)) { + result.addAndGet( + RamUsageEstimator.sizeOf(k) + + RamUsageEstimator.sizeOf(thisV) + + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY); + thisAttributes.remove(k); + } + }); + }); + }); + } return null; } From 84698e9be6fc4208d4ed6e3a3f69eb307f55647b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:50:29 +0800 Subject: [PATCH 099/213] Update UpdateDetailContainer.java --- .../attribute/update/UpdateDetailContainer.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index c6e78c2feec8a..4250f59085064 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -210,7 +210,21 @@ public Pair updateSelfByCommitContainer(final UpdateContainer com thisAttributes.remove(k); } }); + if (thisAttributes.isEmpty()) { + result.addAndGet( + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + + RamUsageEstimator.sizeOf(device) + + MAP_SIZE); + thisDeviceMap.remove(device); + } }); + if (thisDeviceMap.isEmpty()) { + result.addAndGet( + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + + RamUsageEstimator.sizeOf(table) + + MAP_SIZE); + this.updateMap.remove(table); + } }); } return null; From 4286c6fb3e8ec6bdb4908a55077fd179ccf66d34 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:51:17 +0800 Subject: [PATCH 100/213] Update UpdateDetailContainer.java --- .../schemaregion/attribute/update/UpdateDetailContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 4250f59085064..986b12c94573a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -227,7 +227,7 @@ public Pair updateSelfByCommitContainer(final UpdateContainer com } }); } - return null; + return new Pair<>(result.get(), updateMap.isEmpty()); } @Override From e4a51b43681b0e0785bff9e52517204fff0b5b45 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 17:50:46 +0800 Subject: [PATCH 101/213] Handle null --- .../update/RewritableByteArrayOutputStream.java | 9 +++++---- .../attribute/update/UpdateDetailContainer.java | 16 ++++++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java index 2f26eabb192b1..a5a32ebe2164c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/RewritableByteArrayOutputStream.java @@ -26,17 +26,18 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Objects; @NotThreadSafe class RewritableByteArrayOutputStream extends ByteArrayOutputStream { private static final byte[] intPlaceHolder = new byte[4]; - void writeNull() throws IOException { - ReadWriteIOUtils.write(ReadWriteIOUtils.NO_BYTE_TO_READ, this); - } - void writeWithLength(final byte[] bytes) throws IOException { + if (Objects.isNull(bytes)) { + ReadWriteIOUtils.write(ReadWriteIOUtils.NO_BYTE_TO_READ, this); + return; + } ReadWriteIOUtils.write(bytes.length, this); write(bytes); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 986b12c94573a..375e17a1029d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -137,12 +137,14 @@ private void serializeWithLimit( tableEntry.getValue().entrySet()) { final byte[][] deviceIdBytes = Arrays.stream(deviceEntry.getKey()) - .map(str -> str.getBytes(TSFileConfig.STRING_CHARSET)) + .map(str -> Objects.nonNull(str) ? str.getBytes(TSFileConfig.STRING_CHARSET) : null) .toArray(byte[][]::new); newSize = (Integer.BYTES * (deviceIdBytes.length + 2)) - + Arrays.stream(deviceIdBytes).map(bytes -> bytes.length).reduce(0, Integer::sum); + + Arrays.stream(deviceIdBytes) + .map(bytes -> Objects.nonNull(bytes) ? bytes.length : 0) + .reduce(0, Integer::sum); if (limitBytes.get() < newSize) { outputStream.rewrite(mapEntryCount, mapSizeOffset); outputStream.rewrite(deviceEntryCount, deviceSizeOffset); @@ -158,8 +160,14 @@ private void serializeWithLimit( int attributeCount = 0; for (final Map.Entry attributeKV : deviceEntry.getValue().entrySet()) { final byte[] keyBytes = attributeKV.getKey().getBytes(TSFileConfig.STRING_CHARSET); - final byte[] valueBytes = attributeKV.getValue().getBytes(TSFileConfig.STRING_CHARSET); - newSize = 2 * Integer.BYTES + keyBytes.length + valueBytes.length; + final byte[] valueBytes = + Objects.nonNull(attributeKV.getValue()) + ? attributeKV.getValue().getBytes(TSFileConfig.STRING_CHARSET) + : null; + newSize = + 2 * Integer.BYTES + + keyBytes.length + + (Objects.nonNull(valueBytes) ? valueBytes.length : 0); if (limitBytes.get() < newSize) { outputStream.rewrite(mapEntryCount, mapSizeOffset); outputStream.rewrite(deviceEntryCount, deviceSizeOffset); From f5686a7ce4a7409815edc724676f9d631f652b94 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 18:01:34 +0800 Subject: [PATCH 102/213] Update UpdateDetailContainer.java --- .../update/UpdateDetailContainer.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 375e17a1029d3..762097a099890 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -234,6 +234,33 @@ public Pair updateSelfByCommitContainer(final UpdateContainer com this.updateMap.remove(table); } }); + } else { + ((UpdateClearContainer) commitContainer) + .getTableNames() + .forEach( + tableName -> { + final ConcurrentMap> deviceMap = + updateMap.remove(tableName); + if (Objects.nonNull(deviceMap)) { + result.addAndGet( + deviceMap.size() + * (RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + MAP_SIZE) + + deviceMap.entrySet().stream() + .mapToLong( + entry -> + RamUsageEstimator.sizeOf(entry.getKey()) + + entry.getValue().size() + * RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + + entry.getValue().entrySet().stream() + .mapToLong( + attributeKV -> + RamUsageEstimator.sizeOf(attributeKV.getKey()) + + RamUsageEstimator.sizeOf( + attributeKV.getValue())) + .reduce(0, Long::sum)) + .reduce(0, Long::sum)); + } + }); } return new Pair<>(result.get(), updateMap.isEmpty()); } From 0b3763c80914c6d9f2682036b275b4cb6dd82bf1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 30 Sep 2024 18:04:55 +0800 Subject: [PATCH 103/213] Update DeviceAttributeRemoteUpdater.java --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 7fd0bc3fcc404..be0ccf4047250 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -141,7 +141,7 @@ public void commit(final TableDeviceAttributeCommitUpdateNode node) { container.updateSelfByCommitContainer(getContainer(bytes)); releaseMemory(result.getLeft()); // isEmpty - if (result.getRight()) { + if (Boolean.TRUE.equals(result.getRight())) { releaseMemory( container instanceof UpdateDetailContainer ? UpdateDetailContainer.INSTANCE_SIZE From 66c51e28f6bcb8b06aa3798285a9bd14e290834a Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 11:55:22 +0800 Subject: [PATCH 104/213] Update GeneralRegionAttributeSecurityService.java --- ...GeneralRegionAttributeSecurityService.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 832e94afd66ef..8075f0583c98c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -21,12 +21,17 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.client.request.AsyncRequestContext; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.protocol.client.ConfigNodeClient; +import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; +import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.client.dn.DnToDnInternalServiceAsyncRequestManager; import org.apache.iotdb.db.protocol.client.dn.DnToDnRequestType; import org.apache.iotdb.db.queryengine.execution.executor.RegionWriteExecutor; @@ -37,6 +42,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TSchemaRegionAttributeInfo; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.thrift.TException; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -158,6 +164,42 @@ private void execute() { } } + private Map> detectNodeShrinkage( + final Map>> + attributeUpdateCommitMap) { + final TShowClusterResp showClusterResp; + try (final ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + showClusterResp = client.showCluster(); + } catch (final ClientManagerException | TException e) { + LOGGER.warn("Failed to fetch dataNodeLocations, will retry."); + return null; + } + final Set dataNodeLocations = new HashSet<>(); + showClusterResp + .getDataNodeList() + .forEach( + location -> { + location.setDataRegionConsensusEndPoint(null); + location.setMPPDataExchangeEndPoint(null); + location.setSchemaRegionConsensusEndPoint(null); + location.setClientRpcEndPoint(null); + dataNodeLocations.add(location); + }); + return attributeUpdateCommitMap.entrySet().stream() + .filter( + entry -> { + entry + .getValue() + .getRight() + .keySet() + .removeIf(location -> !dataNodeLocations.contains(location)); + return !entry.getValue().getRight().isEmpty(); + }) + .collect( + Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getRight().keySet())); + } + private void sendUpdateRequest( final Map>> attributeUpdateCommitMap) { From e9baf261fb1760fc1878963e7e04557aeb6acad8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:01:37 +0800 Subject: [PATCH 105/213] Update GeneralRegionAttributeSecurityService.java --- .../update/GeneralRegionAttributeSecurityService.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 8075f0583c98c..5ca9014b443e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -164,7 +164,7 @@ private void execute() { } } - private Map> detectNodeShrinkage( + private Map> detectNodeShrinkage( final Map>> attributeUpdateCommitMap) { final TShowClusterResp showClusterResp; @@ -197,7 +197,12 @@ private Map> detectNodeShrinkage( return !entry.getValue().getRight().isEmpty(); }) .collect( - Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getRight().keySet())); + Collectors.toMap( + Map.Entry::getKey, + entry -> + entry.getValue().getRight().keySet().stream() + .map(TDataNodeLocation::getDataNodeId) + .collect(Collectors.toSet()))); } private void sendUpdateRequest( From ba8677c0c4c4b933757f1b22be3f170627e12f81 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:13:07 +0800 Subject: [PATCH 106/213] Added property --- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 22 +++++++++++++++++++ .../apache/iotdb/db/conf/IoTDBDescriptor.java | 15 +++++++++++++ ...GeneralRegionAttributeSecurityService.java | 6 ++++- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index d3e3753406aaf..0a3a5a902a2fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1048,6 +1048,8 @@ public class IoTDBConfig { private long generalRegionAttributeSecurityServiceIntervalSeconds = 30L; private long generalRegionAttributeSecurityServiceTimeoutSeconds = 20L; + private long generalRegionAttributeSecurityServiceFailureDurationSecondsToFetch = 600L; + private int generalRegionAttributeSecurityServiceFailureTimesToFetch = 20; private long detailContainerMinDegradeMemoryInBytes = 1024 * 1024L; private String readConsistencyLevel = "strong"; @@ -3491,6 +3493,26 @@ public void setGeneralRegionAttributeSecurityServiceTimeoutSeconds( generalRegionAttributeSecurityServiceTimeoutSeconds; } + public long getGeneralRegionAttributeSecurityServiceFailureDurationSecondsToFetch() { + return generalRegionAttributeSecurityServiceFailureDurationSecondsToFetch; + } + + public void setGeneralRegionAttributeSecurityServiceFailureDurationSecondsToFetch( + long generalRegionAttributeSecurityServiceFailureDurationSecondsToFetch) { + this.generalRegionAttributeSecurityServiceFailureDurationSecondsToFetch = + generalRegionAttributeSecurityServiceFailureDurationSecondsToFetch; + } + + public int getGeneralRegionAttributeSecurityServiceFailureTimesToFetch() { + return generalRegionAttributeSecurityServiceFailureTimesToFetch; + } + + public void setGeneralRegionAttributeSecurityServiceFailureTimesToFetch( + int generalRegionAttributeSecurityServiceFailureTimesToFetch) { + this.generalRegionAttributeSecurityServiceFailureTimesToFetch = + generalRegionAttributeSecurityServiceFailureTimesToFetch; + } + public long getDetailContainerMinDegradeMemoryInBytes() { return detailContainerMinDegradeMemoryInBytes; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 3f86dea6809fc..eb84d313cbee8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1088,6 +1088,21 @@ public void loadProperties(Properties properties) throws BadNodeUrlException, IO "general_region_attribute_security_service_timeout_seconds", String.valueOf(conf.getGeneralRegionAttributeSecurityServiceTimeoutSeconds())))); + conf.setGeneralRegionAttributeSecurityServiceFailureDurationSecondsToFetch( + Long.parseLong( + properties.getProperty( + "general_region_attribute_security_service_failure_duration_seconds_to_fetch", + String.valueOf( + conf + .getGeneralRegionAttributeSecurityServiceFailureDurationSecondsToFetch())))); + + conf.setGeneralRegionAttributeSecurityServiceFailureTimesToFetch( + Integer.parseInt( + properties.getProperty( + "general_region_attribute_security_service_failure_times_to_fetch", + String.valueOf( + conf.getGeneralRegionAttributeSecurityServiceFailureTimesToFetch())))); + conf.setDetailContainerMinDegradeMemoryInBytes( Long.parseLong( properties.getProperty( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 5ca9014b443e7..3565e925d964c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -69,6 +69,8 @@ public class GeneralRegionAttributeSecurityService { IoTDBThreadPoolFactory.newSingleThreadExecutor( ThreadName.GENERAL_REGION_ATTRIBUTE_SECURITY_SERVICE.getName()); + private final Map> dataNodeId2FailureDurationAndTimesMap = + new HashMap<>(); private final Set regionLeaders = new HashSet<>(); private final ReentrantLock lock = new ReentrantLock(); private final Condition condition = lock.newCondition(); @@ -129,6 +131,8 @@ private void execute() { // Send sendUpdateRequest(attributeUpdateCommitMap); + // May shrink + // Commit // TODO: Execute in parallel by thread pool attributeUpdateCommitMap.forEach( @@ -173,7 +177,7 @@ private Map> detectNodeShrinkage( showClusterResp = client.showCluster(); } catch (final ClientManagerException | TException e) { LOGGER.warn("Failed to fetch dataNodeLocations, will retry."); - return null; + return Collections.emptyMap(); } final Set dataNodeLocations = new HashSet<>(); showClusterResp From cba17a4582631c0cf20e6f405d1dc23450cc064f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:19:25 +0800 Subject: [PATCH 107/213] Update GeneralRegionAttributeSecurityService.java --- ...GeneralRegionAttributeSecurityService.java | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 3565e925d964c..c29c64b56b12f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -53,9 +53,11 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; @@ -209,7 +211,7 @@ private Map> detectNodeShrinkage( .collect(Collectors.toSet()))); } - private void sendUpdateRequest( + private boolean sendUpdateRequest( final Map>> attributeUpdateCommitMap) { final AsyncRequestContext @@ -238,10 +240,29 @@ private void sendUpdateRequest( .getConfig() .getGeneralRegionAttributeSecurityServiceTimeoutSeconds()); + final AtomicBoolean needFetch = new AtomicBoolean(false); + final Set failedDataNodes = clientHandler.getResponseMap().entrySet().stream() .filter( - entry -> entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) + entry -> { + final boolean failed = + entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode(); + if (failed) { + dataNodeId2FailureDurationAndTimesMap.compute( + entry.getKey(), + (k, v) -> { + if (Objects.isNull(v)) { + return new Pair<>(System.currentTimeMillis(), 1); + } + v.setRight(v.getRight() + 1); + return v; + }); + } else { + dataNodeId2FailureDurationAndTimesMap.remove(entry.getKey()); + } + return failed; + }) .map(Map.Entry::getKey) .collect(Collectors.toSet()); for (final Iterator>>> it = @@ -256,6 +277,7 @@ private void sendUpdateRequest( it.remove(); } } + return needFetch.get(); } /////////////////////////////// SingleTon /////////////////////////////// From 7b71e0c828c61d324cfcab907a8974d027c53e07 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:25:24 +0800 Subject: [PATCH 108/213] Update GeneralRegionAttributeSecurityService.java --- ...GeneralRegionAttributeSecurityService.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index c29c64b56b12f..64309a2f0b605 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -47,6 +47,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; + import java.nio.ByteBuffer; import java.util.Collections; import java.util.HashMap; @@ -130,10 +132,11 @@ private void execute() { attributeUpdateCommitMap.put(regionLeader.getSchemaRegionId(), currentResult); } - // Send - sendUpdateRequest(attributeUpdateCommitMap); - - // May shrink + // Send & may shrink + final Map> shrinkMap = + sendUpdateRequest(attributeUpdateCommitMap) + ? detectNodeShrinkage(attributeUpdateCommitMap) + : Collections.emptyMap(); // Commit // TODO: Execute in parallel by thread pool @@ -146,7 +149,7 @@ private void execute() { new PlanNodeId(""), pair.getLeft(), pair.getRight(), - Collections.emptySet())) + shrinkMap.getOrDefault(schemaRegionId, Collections.emptySet()))) .isAccepted()) { LOGGER.warn("Failed to write attribute commit message to region {}.", schemaRegionId); } @@ -170,7 +173,7 @@ private void execute() { } } - private Map> detectNodeShrinkage( + private @Nonnull Map> detectNodeShrinkage( final Map>> attributeUpdateCommitMap) { final TShowClusterResp showClusterResp; @@ -181,6 +184,7 @@ private Map> detectNodeShrinkage( LOGGER.warn("Failed to fetch dataNodeLocations, will retry."); return Collections.emptyMap(); } + dataNodeId2FailureDurationAndTimesMap.clear(); final Set dataNodeLocations = new HashSet<>(); showClusterResp .getDataNodeList() @@ -256,6 +260,16 @@ private boolean sendUpdateRequest( return new Pair<>(System.currentTimeMillis(), 1); } v.setRight(v.getRight() + 1); + if (System.currentTimeMillis() - v.getLeft() + >= IoTDBDescriptor.getInstance() + .getConfig() + .getGeneralRegionAttributeSecurityServiceFailureDurationSecondsToFetch() + || v.getRight() + >= IoTDBDescriptor.getInstance() + .getConfig() + .getGeneralRegionAttributeSecurityServiceFailureTimesToFetch()) { + needFetch.set(true); + } return v; }); } else { From 7b0b0e4aaded0b13077340e9b429614c26164796 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:37:11 +0800 Subject: [PATCH 109/213] complete shrink logic --- .../TableDeviceAttributeCommitUpdateNode.java | 27 ++++++++++++------- .../update/DeviceAttributeRemoteUpdater.java | 5 ++++ ...GeneralRegionAttributeSecurityService.java | 11 +++----- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java index 80cb93c4d94ba..21234fb98e4b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java @@ -43,7 +43,7 @@ public class TableDeviceAttributeCommitUpdateNode extends PlanNode implements IS private final long version; private final Map commitMap; - private final Set shrunkNodes; + private final Set shrunkNodes; public static final TableDeviceAttributeCommitUpdateNode MOCK_INSTANCE = new TableDeviceAttributeCommitUpdateNode(new PlanNodeId(""), 0L, null, null); @@ -52,7 +52,7 @@ public TableDeviceAttributeCommitUpdateNode( final PlanNodeId id, final long version, final Map commitMap, - final Set shrunkNodes) { + final Set shrunkNodes) { super(id); this.version = version; this.commitMap = commitMap; @@ -67,7 +67,7 @@ public Map getCommitMap() { return commitMap; } - public Set getShrunkNodes() { + public Set getShrunkNodes() { return shrunkNodes; } @@ -113,8 +113,9 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { byteBuffer.put(entry.getValue()); } ReadWriteIOUtils.write(shrunkNodes.size(), byteBuffer); - for (final Integer nodeId : shrunkNodes) { - ReadWriteIOUtils.write(nodeId, byteBuffer); + for (final TDataNodeLocation location : shrunkNodes) { + ReadWriteIOUtils.write(location.getDataNodeId(), byteBuffer); + ThriftCommonsSerDeUtils.serializeTEndPoint(location.getInternalEndPoint(), byteBuffer); } } @@ -130,8 +131,9 @@ protected void serializeAttributes(final DataOutputStream stream) throws IOExcep stream.write(entry.getValue()); } ReadWriteIOUtils.write(shrunkNodes.size(), stream); - for (final Integer nodeId : shrunkNodes) { - ReadWriteIOUtils.write(nodeId, stream); + for (final TDataNodeLocation location : shrunkNodes) { + ReadWriteIOUtils.write(location.getDataNodeId(), stream); + ThriftCommonsSerDeUtils.serializeTEndPoint(location.getInternalEndPoint(), stream); } } @@ -153,9 +155,16 @@ public static PlanNode deserialize(final ByteBuffer buffer) { commitMap.put(location, commitBuffer); } size = ReadWriteIOUtils.readInt(buffer); - final Set shrunkNodes = new HashSet<>(); + final Set shrunkNodes = new HashSet<>(); for (int i = 0; i < size; ++i) { - shrunkNodes.add(ReadWriteIOUtils.readInt(buffer)); + shrunkNodes.add( + new TDataNodeLocation( + ReadWriteIOUtils.readInt(buffer), + null, + ThriftCommonsSerDeUtils.deserializeTEndPoint(buffer), + null, + null, + null)); } final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); return new TableDeviceAttributeCommitUpdateNode(planNodeId, version, commitMap, shrunkNodes); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index be0ccf4047250..3cd891b09f6a0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -129,6 +129,11 @@ public Pair> getAttributeUpdateInfo( } public void commit(final TableDeviceAttributeCommitUpdateNode node) { + final Set shrunkNodes = node.getShrunkNodes(); + targetDataNodeLocations.removeAll(shrunkNodes); + attributeUpdateMap.keySet().removeAll(shrunkNodes); + updateContainerStatistics.keySet().removeAll(shrunkNodes); + node.getCommitMap() .forEach( ((location, bytes) -> diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 64309a2f0b605..2e91548c5b249 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -133,7 +133,7 @@ private void execute() { } // Send & may shrink - final Map> shrinkMap = + final Map> shrinkMap = sendUpdateRequest(attributeUpdateCommitMap) ? detectNodeShrinkage(attributeUpdateCommitMap) : Collections.emptyMap(); @@ -173,7 +173,7 @@ private void execute() { } } - private @Nonnull Map> detectNodeShrinkage( + private @Nonnull Map> detectNodeShrinkage( final Map>> attributeUpdateCommitMap) { final TShowClusterResp showClusterResp; @@ -207,12 +207,7 @@ private void execute() { return !entry.getValue().getRight().isEmpty(); }) .collect( - Collectors.toMap( - Map.Entry::getKey, - entry -> - entry.getValue().getRight().keySet().stream() - .map(TDataNodeLocation::getDataNodeId) - .collect(Collectors.toSet()))); + Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getRight().keySet())); } private boolean sendUpdateRequest( From b3e948fdc973bfedf530edf607092e2de8975350 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 15:11:21 +0800 Subject: [PATCH 110/213] Update ThreadName.java --- .../java/org/apache/iotdb/commons/concurrent/ThreadName.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java index e651d4c7381ae..b56a104c4d1e7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java @@ -236,7 +236,8 @@ public enum ThreadName { PBTREE_RELEASE_MONITOR, SCHEMA_FORCE_MLOG, PBTREE_FLUSH_MONITOR, - PBTREE_WORKER_POOL)); + PBTREE_WORKER_POOL, + GENERAL_REGION_ATTRIBUTE_SECURITY_SERVICE)); private static final Set clientServiceThreadNames = new HashSet<>(Arrays.asList(CLIENT_RPC_SERVICE, CLIENT_RPC_PROCESSOR)); From b6d39e9ef07266739544b488aa03509b6bdd5f24 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 15:36:36 +0800 Subject: [PATCH 111/213] Update GeneralRegionAttributeSecurityService.java --- .../attribute/update/GeneralRegionAttributeSecurityService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 2e91548c5b249..0718d39427626 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -139,7 +139,6 @@ private void execute() { : Collections.emptyMap(); // Commit - // TODO: Execute in parallel by thread pool attributeUpdateCommitMap.forEach( (schemaRegionId, pair) -> { if (!new RegionWriteExecutor() From 989fdb8e1ae1967a8d2abedb2e45f1a4a96862a7 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 17:18:54 +0800 Subject: [PATCH 112/213] start write in query --- .../impl/DataNodeInternalRPCServiceImpl.java | 16 +++--- .../schema/TableSchemaQueryWriteVisitor.java | 55 +++++++++++++++++++ 2 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 610a66c0b34b0..5390f0904699d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -330,7 +330,7 @@ public DataNodeInternalRPCServiceImpl() { } @Override - public TSendFragmentInstanceResp sendFragmentInstance(TSendFragmentInstanceReq req) { + public TSendFragmentInstanceResp sendFragmentInstance(final TSendFragmentInstanceReq req) { LOGGER.debug("receive FragmentInstance to group[{}]", req.getConsensusGroupId()); // Deserialize ConsensusGroupId @@ -338,9 +338,9 @@ public TSendFragmentInstanceResp sendFragmentInstance(TSendFragmentInstanceReq r if (req.consensusGroupId != null) { try { groupId = ConsensusGroupId.Factory.createFromTConsensusGroupId(req.getConsensusGroupId()); - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn("Deserialize ConsensusGroupId failed. ", e); - TSendFragmentInstanceResp resp = new TSendFragmentInstanceResp(false); + final TSendFragmentInstanceResp resp = new TSendFragmentInstanceResp(false); resp.setMessage("Deserialize ConsensusGroupId failed: " + e.getMessage()); return resp; } @@ -348,22 +348,22 @@ public TSendFragmentInstanceResp sendFragmentInstance(TSendFragmentInstanceReq r // We deserialize here instead of the underlying state machine because parallelism is possible // here but not at the underlying state machine - FragmentInstance fragmentInstance; + final FragmentInstance fragmentInstance; try { fragmentInstance = FragmentInstance.deserializeFrom(req.fragmentInstance.body); - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn("Deserialize FragmentInstance failed.", e); TSendFragmentInstanceResp resp = new TSendFragmentInstanceResp(false); resp.setMessage("Deserialize FragmentInstance failed: " + e.getMessage()); return resp; } - RegionReadExecutor executor = new RegionReadExecutor(); - RegionExecutionResult executionResult = + final RegionReadExecutor executor = new RegionReadExecutor(); + final RegionExecutionResult executionResult = groupId == null ? executor.execute(fragmentInstance) : executor.execute(groupId, fragmentInstance); - TSendFragmentInstanceResp resp = new TSendFragmentInstanceResp(); + final TSendFragmentInstanceResp resp = new TSendFragmentInstanceResp(); resp.setAccepted(executionResult.isAccepted()); resp.setMessage(executionResult.getMessage()); resp.setNeedRetry(executionResult.isReadNeedRetry()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java new file mode 100644 index 0000000000000..cc7d6fce15e96 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.utils.StatusUtils; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.FilterNode; + +public class TableSchemaQueryWriteVisitor extends PlanVisitor { + + @Override + public TSStatus visitPlan(final PlanNode node, final Void context) { + return StatusUtils.OK; + } + + @Override + public TSStatus visitFilter(final FilterNode node, final Void context) { + return node.getChild().accept(this, context); + } + + @Override + public TSStatus visitTableDeviceFetch(final TableDeviceFetchNode node, final Void context) { + return visitTableDeviceSourceNode(node); + } + + @Override + public TSStatus visitTableDeviceQueryScan( + final TableDeviceQueryScanNode node, final Void context) { + return visitTableDeviceSourceNode(node); + } + + private TSStatus visitTableDeviceSourceNode(final TableDeviceSourceNode node) { + return StatusUtils.OK; + } +} From a7233659d5d2f243c825d9167881fbc4242c6e65 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 17:42:43 +0800 Subject: [PATCH 113/213] completing --- .../metadata/read/TableDeviceSourceNode.java | 23 ++++++++++++ .../TableDeviceCacheAttributeGuard.java | 35 +++++++++++++++++++ .../fetcher/TableDeviceSchemaFetcher.java | 7 ++++ 3 files changed, 65 insertions(+) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceSourceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceSourceNode.java index 1329028a97dd5..82ea2bc656288 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceSourceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceSourceNode.java @@ -19,11 +19,16 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SourceNode; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import java.util.Collections; @@ -32,6 +37,7 @@ import java.util.stream.Collectors; public abstract class TableDeviceSourceNode extends SourceNode { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); protected String database; @@ -40,6 +46,7 @@ public abstract class TableDeviceSourceNode extends SourceNode { protected List columnHeaderList; protected TRegionReplicaSet schemaRegionReplicaSet; + private TDataNodeLocation senderLocation; protected TableDeviceSourceNode( final PlanNodeId id, @@ -66,6 +73,10 @@ public List getColumnHeaderList() { return columnHeaderList; } + public TDataNodeLocation getSenderLocation() { + return senderLocation; + } + @Override public List getOutputColumnNames() { return columnHeaderList.stream().map(ColumnHeader::getColumnName).collect(Collectors.toList()); @@ -79,6 +90,18 @@ public TRegionReplicaSet getRegionReplicaSet() { @Override public void setRegionReplicaSet(final TRegionReplicaSet regionReplicaSet) { this.schemaRegionReplicaSet = regionReplicaSet; + if (!TableDeviceSchemaFetcher.getInstance() + .getAttributeGuard() + .isRegionFetched(regionReplicaSet.getRegionId().getId())) { + this.senderLocation = + new TDataNodeLocation( + config.getDataNodeId(), + null, + new TEndPoint(config.getInternalAddress(), config.getInternalPort()), + null, + null, + null); + } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java new file mode 100644 index 0000000000000..6227705bd402f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher; + +import java.util.HashSet; +import java.util.Set; + +public class TableDeviceCacheAttributeGuard { + final Set fetchedSchemaRegionIds = new HashSet<>(); + + public boolean isRegionFetched(final Integer schemaRegionId) { + return fetchedSchemaRegionIds.contains(schemaRegionId); + } + + public void addFetchedRegions(final Set schemaRegionIds) { + fetchedSchemaRegionIds.addAll(schemaRegionIds); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index 9222aa1f1025a..9883e7418c1db 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -79,6 +79,9 @@ public class TableDeviceSchemaFetcher { private final TableDeviceSchemaCache cache = new TableDeviceSchemaCache(); + private final TableDeviceCacheAttributeGuard attributeGuard = + new TableDeviceCacheAttributeGuard(); + private TableDeviceSchemaFetcher() { // do nothing } @@ -95,6 +98,10 @@ public TableDeviceSchemaCache getTableDeviceCache() { return cache; } + public TableDeviceCacheAttributeGuard getAttributeGuard() { + return attributeGuard; + } + Map> fetchMissingDeviceSchemaForDataInsertion( final FetchDevice statement, final MPPQueryContext context) { final long queryId = SessionManager.getInstance().requestQueryId(); From afb674e3ac117eed826cfaffd8b092eabcb8cf4f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 17:44:26 +0800 Subject: [PATCH 114/213] hak --- .../planner/node/schema/TableDeviceQueryCountNode.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java index b611c1c0095f5..166eada7ff684 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java @@ -69,6 +69,11 @@ public PlanNodeType getType() { return PlanNodeType.TABLE_DEVICE_QUERY_COUNT; } + @Override + public void setRegionReplicaSet(final TRegionReplicaSet regionReplicaSet) { + this.schemaRegionReplicaSet = regionReplicaSet; + } + @Override public PlanNode clone() { return new TableDeviceQueryCountNode( From dcf32fc886b0e5507bd7ffd7711434faaa199be1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:00:46 +0800 Subject: [PATCH 115/213] enrich --- .../metadata/read/TableDeviceSourceNode.java | 2 +- .../planner/TableLogicalPlanner.java | 1 + .../TableDistributedPlanGenerator.java | 14 +++---- .../schema/AbstractTableDeviceQueryNode.java | 17 +++++++++ .../node/schema/TableDeviceFetchNode.java | 38 +++++++++++++++++-- 5 files changed, 60 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceSourceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceSourceNode.java index 82ea2bc656288..57fd4a309a69a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceSourceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceSourceNode.java @@ -46,7 +46,7 @@ public abstract class TableDeviceSourceNode extends SourceNode { protected List columnHeaderList; protected TRegionReplicaSet schemaRegionReplicaSet; - private TDataNodeLocation senderLocation; + protected TDataNodeLocation senderLocation; protected TableDeviceSourceNode( final PlanNodeId id, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java index 5ca62420b2bab..56cd627f1161d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java @@ -282,6 +282,7 @@ private PlanNode planFetchDevice(final FetchDevice statement, final Analysis ana statement.getTableName(), statement.getDeviceIdList(), columnHeaderList, + null, null); final SchemaPartition schemaPartition = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 8c1d9c9b3bfee..295fab8880f34 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -775,14 +775,12 @@ public List visitTableDeviceFetch( tableDeviceFetchMap .computeIfAbsent( regionReplicaSet, - k -> - new TableDeviceFetchNode( - queryId.genPlanNodeId(), - node.getDatabase(), - node.getTableName(), - new ArrayList<>(), - node.getColumnHeaderList(), - regionReplicaSet)) + k -> { + final TableDeviceFetchNode clonedNode = (TableDeviceFetchNode) node.clone(); + clonedNode.setPlanNodeId(queryId.genPlanNodeId()); + clonedNode.setRegionReplicaSet(regionReplicaSet); + return clonedNode; + }) .addDeviceId(deviceIdArray); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java index e76941ee53ba8..f34ec510565b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; @@ -95,6 +96,14 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { for (final ColumnHeader columnHeader : columnHeaderList) { columnHeader.serialize(byteBuffer); } + + if (Objects.nonNull(senderLocation)) { + ReadWriteIOUtils.write(true, byteBuffer); + ReadWriteIOUtils.write(senderLocation.getDataNodeId(), byteBuffer); + ThriftCommonsSerDeUtils.serializeTEndPoint(senderLocation.getInternalEndPoint(), byteBuffer); + } else { + ReadWriteIOUtils.write(false, byteBuffer); + } } @Override @@ -120,6 +129,14 @@ protected void serializeAttributes(final DataOutputStream stream) throws IOExcep for (final ColumnHeader columnHeader : columnHeaderList) { columnHeader.serialize(stream); } + + if (Objects.nonNull(senderLocation)) { + ReadWriteIOUtils.write(true, stream); + ReadWriteIOUtils.write(senderLocation.getDataNodeId(), stream); + ThriftCommonsSerDeUtils.serializeTEndPoint(senderLocation.getInternalEndPoint(), stream); + } else { + ReadWriteIOUtils.write(false, stream); + } } protected static PlanNode deserialize(final ByteBuffer buffer, final boolean isScan) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java index 2916b5f6ad404..e1365c5ab5571 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java @@ -19,7 +19,9 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; @@ -46,7 +48,8 @@ public TableDeviceFetchNode( final String tableName, final List deviceIdList, final List columnHeaderList, - final TRegionReplicaSet regionReplicaSet) { + final TRegionReplicaSet regionReplicaSet, + final TDataNodeLocation senderLocation) { super(id, database, tableName, columnHeaderList, regionReplicaSet); this.deviceIdList = deviceIdList; } @@ -72,7 +75,8 @@ public PlanNode clone() { tableName, deviceIdList, columnHeaderList, - schemaRegionReplicaSet); + schemaRegionReplicaSet, + senderLocation); } @Override @@ -93,6 +97,14 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { for (final ColumnHeader columnHeader : columnHeaderList) { columnHeader.serialize(byteBuffer); } + + if (Objects.nonNull(senderLocation)) { + ReadWriteIOUtils.write(true, byteBuffer); + ReadWriteIOUtils.write(senderLocation.getDataNodeId(), byteBuffer); + ThriftCommonsSerDeUtils.serializeTEndPoint(senderLocation.getInternalEndPoint(), byteBuffer); + } else { + ReadWriteIOUtils.write(false, byteBuffer); + } } @Override @@ -113,6 +125,14 @@ protected void serializeAttributes(final DataOutputStream stream) throws IOExcep for (final ColumnHeader columnHeader : columnHeaderList) { columnHeader.serialize(stream); } + + if (Objects.nonNull(senderLocation)) { + ReadWriteIOUtils.write(true, stream); + ReadWriteIOUtils.write(senderLocation.getDataNodeId(), stream); + ThriftCommonsSerDeUtils.serializeTEndPoint(senderLocation.getInternalEndPoint(), stream); + } else { + ReadWriteIOUtils.write(false, stream); + } } public static TableDeviceFetchNode deserialize(final ByteBuffer buffer) { @@ -136,9 +156,21 @@ public static TableDeviceFetchNode deserialize(final ByteBuffer buffer) { columnHeaderList.add(ColumnHeader.deserialize(buffer)); } + TDataNodeLocation senderLocation = null; + if (ReadWriteIOUtils.readBool(buffer)) { + senderLocation = + new TDataNodeLocation( + ReadWriteIOUtils.readInt(buffer), + null, + ThriftCommonsSerDeUtils.deserializeTEndPoint(buffer), + null, + null, + null); + } + final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); return new TableDeviceFetchNode( - planNodeId, database, tableName, deviceIdList, columnHeaderList, null); + planNodeId, database, tableName, deviceIdList, columnHeaderList, null, senderLocation); } @Override From 694cef3302b4aae3ad70d93d56cdb55b57fefab5 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:08:15 +0800 Subject: [PATCH 116/213] Refactor --- .../metadata/read/TableDeviceSourceNode.java | 4 ++-- .../planner/TableLogicalPlanner.java | 3 +-- .../schema/AbstractTableDeviceQueryNode.java | 23 ++++++++++++++----- .../node/schema/TableDeviceFetchNode.java | 2 +- .../schema/TableDeviceQueryCountNode.java | 5 ++-- .../node/schema/TableDeviceQueryScanNode.java | 6 ++--- 6 files changed, 26 insertions(+), 17 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceSourceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceSourceNode.java index 57fd4a309a69a..46aa165ed5d8a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceSourceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/read/TableDeviceSourceNode.java @@ -53,12 +53,12 @@ protected TableDeviceSourceNode( final String database, final String tableName, final List columnHeaderList, - final TRegionReplicaSet schemaRegionReplicaSet) { + final TDataNodeLocation senderLocation) { super(id); this.database = database; this.tableName = tableName; this.columnHeaderList = columnHeaderList; - this.schemaRegionReplicaSet = schemaRegionReplicaSet; + this.senderLocation = senderLocation; } public String getDatabase() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java index 56cd627f1161d..4204dab4b72d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java @@ -328,8 +328,7 @@ private PlanNode planCountDevice(final CountDevice statement, final Analysis ana statement.getTableName(), statement.getIdDeterminedFilterList(), statement.getIdFuzzyPredicate(), - statement.getColumnHeaderList(), - null); + statement.getColumnHeaderList()); final CountSchemaMergeNode countMergeNode = new CountSchemaMergeNode(queryContext.getQueryId().genPlanNodeId()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java index f34ec510565b2..269b779dff2c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; -import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; @@ -59,8 +59,8 @@ protected AbstractTableDeviceQueryNode( final List> idDeterminedPredicateList, final Expression idFuzzyPredicate, final List columnHeaderList, - final TRegionReplicaSet schemaRegionReplicaSet) { - super(planNodeId, database, tableName, columnHeaderList, schemaRegionReplicaSet); + final TDataNodeLocation senderLocation) { + super(planNodeId, database, tableName, columnHeaderList, senderLocation); this.idDeterminedPredicateList = idDeterminedPredicateList; this.idFuzzyPredicate = idFuzzyPredicate; } @@ -164,6 +164,18 @@ protected static PlanNode deserialize(final ByteBuffer buffer, final boolean isS columnHeaderList.add(ColumnHeader.deserialize(buffer)); } + TDataNodeLocation senderLocation = null; + if (ReadWriteIOUtils.readBool(buffer)) { + senderLocation = + new TDataNodeLocation( + ReadWriteIOUtils.readInt(buffer), + null, + ThriftCommonsSerDeUtils.deserializeTEndPoint(buffer), + null, + null, + null); + } + final long offset = isScan ? ReadWriteIOUtils.readLong(buffer) : 0; final long limit = isScan ? ReadWriteIOUtils.readLong(buffer) : 0; @@ -176,7 +188,7 @@ protected static PlanNode deserialize(final ByteBuffer buffer, final boolean isS idDeterminedFilterList, idFuzzyFilter, columnHeaderList, - null, + senderLocation, offset, limit) : new TableDeviceQueryCountNode( @@ -185,8 +197,7 @@ protected static PlanNode deserialize(final ByteBuffer buffer, final boolean isS tableName, idDeterminedFilterList, idFuzzyFilter, - columnHeaderList, - null); + columnHeaderList); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java index e1365c5ab5571..c13b35930ef04 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java @@ -50,7 +50,7 @@ public TableDeviceFetchNode( final List columnHeaderList, final TRegionReplicaSet regionReplicaSet, final TDataNodeLocation senderLocation) { - super(id, database, tableName, columnHeaderList, regionReplicaSet); + super(id, database, tableName, columnHeaderList, regionReplicaSet, senderLocation); this.deviceIdList = deviceIdList; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java index 166eada7ff684..03a0a8ebdb618 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java @@ -42,8 +42,7 @@ public TableDeviceQueryCountNode( final String tableName, final List> idDeterminedPredicateList, final Expression idFuzzyPredicate, - final List columnHeaderList, - final TRegionReplicaSet schemaRegionReplicaSet) { + final List columnHeaderList) { super( planNodeId, database, @@ -51,7 +50,7 @@ public TableDeviceQueryCountNode( idDeterminedPredicateList, idFuzzyPredicate, columnHeaderList, - schemaRegionReplicaSet); + null); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java index a78ef0992875e..61ce4b6851204 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; -import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; @@ -48,7 +48,7 @@ public TableDeviceQueryScanNode( final List> idDeterminedPredicateList, final Expression idFuzzyPredicate, final List columnHeaderList, - final TRegionReplicaSet schemaRegionReplicaSet, + final TDataNodeLocation senderLocation, final long offset, final long limit) { super( @@ -58,7 +58,7 @@ public TableDeviceQueryScanNode( idDeterminedPredicateList, idFuzzyPredicate, columnHeaderList, - schemaRegionReplicaSet); + senderLocation); this.offset = offset; this.limit = limit; } From 85617771ea576ad01e2fdc27717db67ba82b926e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:09:45 +0800 Subject: [PATCH 117/213] Refactor --- .../planner/node/schema/TableDeviceQueryCountNode.java | 3 +-- .../planner/node/schema/TableDeviceQueryScanNode.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java index 03a0a8ebdb618..07b4169c543b5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryCountNode.java @@ -81,8 +81,7 @@ public PlanNode clone() { tableName, idDeterminedPredicateList, idFuzzyPredicate, - columnHeaderList, - schemaRegionReplicaSet); + columnHeaderList); } public static PlanNode deserialize(final ByteBuffer buffer) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java index 61ce4b6851204..25c7a87be962e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceQueryScanNode.java @@ -90,7 +90,7 @@ public PlanNode clone() { idDeterminedPredicateList, idFuzzyPredicate, columnHeaderList, - schemaRegionReplicaSet, + senderLocation, offset, limit); } From 1e640f99ad186722339299dbe993cb3ba2246a15 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:24:26 +0800 Subject: [PATCH 118/213] Prepare --- .../planner/TableLogicalPlanner.java | 1 - .../node/schema/TableDeviceFetchNode.java | 14 +--- .../node/schema/TableNodeLocationAddNode.java | 79 +++++++++++++++++++ .../schema/TableSchemaQueryWriteVisitor.java | 32 +++++--- 4 files changed, 101 insertions(+), 25 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java index 4204dab4b72d6..d9afec350e31d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java @@ -282,7 +282,6 @@ private PlanNode planFetchDevice(final FetchDevice statement, final Analysis ana statement.getTableName(), statement.getDeviceIdList(), columnHeaderList, - null, null); final SchemaPartition schemaPartition = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java index c13b35930ef04..12d546445d86c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; -import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; @@ -48,9 +47,8 @@ public TableDeviceFetchNode( final String tableName, final List deviceIdList, final List columnHeaderList, - final TRegionReplicaSet regionReplicaSet, final TDataNodeLocation senderLocation) { - super(id, database, tableName, columnHeaderList, regionReplicaSet, senderLocation); + super(id, database, tableName, columnHeaderList, senderLocation); this.deviceIdList = deviceIdList; } @@ -70,13 +68,7 @@ public PlanNodeType getType() { @Override public PlanNode clone() { return new TableDeviceFetchNode( - getPlanNodeId(), - database, - tableName, - deviceIdList, - columnHeaderList, - schemaRegionReplicaSet, - senderLocation); + getPlanNodeId(), database, tableName, deviceIdList, columnHeaderList, senderLocation); } @Override @@ -170,7 +162,7 @@ public static TableDeviceFetchNode deserialize(final ByteBuffer buffer) { final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); return new TableDeviceFetchNode( - planNodeId, database, tableName, deviceIdList, columnHeaderList, null, senderLocation); + planNodeId, database, tableName, deviceIdList, columnHeaderList, senderLocation); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java new file mode 100644 index 0000000000000..e2cc2ba5d3794 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; +import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; +import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +public class TableNodeLocationAddNode extends PlanNode implements ISchemaRegionPlan { + + protected TableNodeLocationAddNode(final PlanNodeId id) { + super(id); + } + + @Override + public List getChildren() { + return null; + } + + @Override + public void addChild(final PlanNode child) { + // Do nothing + } + + @Override + public PlanNode clone() { + return null; + } + + @Override + public int allowedChildCount() { + return 0; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + protected void serializeAttributes(final ByteBuffer byteBuffer) {} + + @Override + protected void serializeAttributes(final DataOutputStream stream) throws IOException {} + + @Override + public SchemaRegionPlanType getPlanType() { + return null; + } + + @Override + public R accept(final SchemaRegionPlanVisitor visitor, C context) { + return null; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java index cc7d6fce15e96..056ab87601b06 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java @@ -19,37 +19,43 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.utils.StatusUtils; +import org.apache.iotdb.commons.consensus.ConsensusGroupId; +import org.apache.iotdb.db.queryengine.execution.executor.RegionExecutionResult; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.FilterNode; -public class TableSchemaQueryWriteVisitor extends PlanVisitor { +import java.util.Objects; +public class TableSchemaQueryWriteVisitor + extends PlanVisitor { @Override - public TSStatus visitPlan(final PlanNode node, final Void context) { - return StatusUtils.OK; + public RegionExecutionResult visitPlan(final PlanNode node, final ConsensusGroupId context) { + return null; } @Override - public TSStatus visitFilter(final FilterNode node, final Void context) { + public RegionExecutionResult visitFilter(final FilterNode node, final ConsensusGroupId context) { return node.getChild().accept(this, context); } @Override - public TSStatus visitTableDeviceFetch(final TableDeviceFetchNode node, final Void context) { - return visitTableDeviceSourceNode(node); + public RegionExecutionResult visitTableDeviceFetch( + final TableDeviceFetchNode node, final ConsensusGroupId context) { + return visitTableDeviceSourceNode(node, context); } @Override - public TSStatus visitTableDeviceQueryScan( - final TableDeviceQueryScanNode node, final Void context) { - return visitTableDeviceSourceNode(node); + public RegionExecutionResult visitTableDeviceQueryScan( + final TableDeviceQueryScanNode node, final ConsensusGroupId context) { + return visitTableDeviceSourceNode(node, context); } - private TSStatus visitTableDeviceSourceNode(final TableDeviceSourceNode node) { - return StatusUtils.OK; + private RegionExecutionResult visitTableDeviceSourceNode( + final TableDeviceSourceNode node, final ConsensusGroupId context) { + if (Objects.nonNull(node.getSenderLocation())) { + return null; + } } } From cfbc20f9b75c3f1fddd544a0ff13be57e9b7f5c9 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:36:01 +0800 Subject: [PATCH 119/213] small completion --- .../plan/planner/plan/node/PlanNodeType.java | 4 ++ .../node/schema/TableNodeLocationAddNode.java | 43 ++++++++++++++++--- .../schema/TableSchemaQueryWriteVisitor.java | 1 + 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 9e988977770a3..764833153c4bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -120,6 +120,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -240,6 +241,7 @@ public enum PlanNodeType { TABLE_DEVICE_QUERY_COUNT((short) 906), TABLE_DEVICE_ATTRIBUTE_UPDATE((short) 907), TABLE_DEVICE_ATTRIBUTE_COMMIT((short) 908), + TABLE_DEVICE_LOCATION_ADD((short) 909), TABLE_SCAN_NODE((short) 1000), TABLE_FILTER_NODE((short) 1001), @@ -537,6 +539,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return TableDeviceAttributeUpdateNode.deserialize(buffer); case 908: return TableDeviceAttributeCommitUpdateNode.deserialize(buffer); + case 909: + return TableNodeLocationAddNode.deserialize(buffer); case 1000: return org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode .deserialize(buffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java index e2cc2ba5d3794..296f07e39c4fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java @@ -19,21 +19,28 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; +import org.apache.tsfile.utils.ReadWriteIOUtils; + import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; public class TableNodeLocationAddNode extends PlanNode implements ISchemaRegionPlan { + private final TDataNodeLocation location; - protected TableNodeLocationAddNode(final PlanNodeId id) { + public TableNodeLocationAddNode(final PlanNodeId id, final TDataNodeLocation location) { super(id); + this.location = location; } @Override @@ -48,7 +55,7 @@ public void addChild(final PlanNode child) { @Override public PlanNode clone() { - return null; + return new TableNodeLocationAddNode(id, location); } @Override @@ -62,14 +69,40 @@ public List getOutputColumnNames() { } @Override - protected void serializeAttributes(final ByteBuffer byteBuffer) {} + protected void serializeAttributes(final ByteBuffer byteBuffer) { + getType().serialize(byteBuffer); + ReadWriteIOUtils.write(location.getDataNodeId(), byteBuffer); + ThriftCommonsSerDeUtils.serializeTEndPoint(location.getInternalEndPoint(), byteBuffer); + } + + @Override + protected void serializeAttributes(final DataOutputStream stream) throws IOException { + getType().serialize(stream); + ReadWriteIOUtils.write(location.getDataNodeId(), stream); + ThriftCommonsSerDeUtils.serializeTEndPoint(location.getInternalEndPoint(), stream); + } + + public static PlanNode deserialize(final ByteBuffer buffer) { + final TDataNodeLocation location = + new TDataNodeLocation( + ReadWriteIOUtils.readInt(buffer), + null, + ThriftCommonsSerDeUtils.deserializeTEndPoint(buffer), + null, + null, + null); + final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); + return new TableNodeLocationAddNode(planNodeId, location); + } @Override - protected void serializeAttributes(final DataOutputStream stream) throws IOException {} + public PlanNodeType getType() { + return PlanNodeType.TABLE_DEVICE_LOCATION_ADD; + } @Override public SchemaRegionPlanType getPlanType() { - return null; + return PlanNodeType.TABLE_DEVICE_LOCATION_ADD; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java index 056ab87601b06..78a1198c8a0af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java @@ -57,5 +57,6 @@ private RegionExecutionResult visitTableDeviceSourceNode( if (Objects.nonNull(node.getSenderLocation())) { return null; } + return null; } } From 78bfbe95509e5b8bfbbb6d45780f76b57a2fe518 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:38:55 +0800 Subject: [PATCH 120/213] small progress --- .../planner/node/schema/TableNodeLocationAddNode.java | 5 ++++- .../db/schemaengine/schemaregion/SchemaRegionPlanType.java | 1 + .../schemaregion/write/req/SchemaRegionWritePlanFactory.java | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java index 296f07e39c4fd..791d676d67253 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java @@ -38,6 +38,9 @@ public class TableNodeLocationAddNode extends PlanNode implements ISchemaRegionPlan { private final TDataNodeLocation location; + public static final TableNodeLocationAddNode MOCK_INSTANCE = + new TableNodeLocationAddNode(new PlanNodeId(""), null); + public TableNodeLocationAddNode(final PlanNodeId id, final TDataNodeLocation location) { super(id); this.location = location; @@ -102,7 +105,7 @@ public PlanNodeType getType() { @Override public SchemaRegionPlanType getPlanType() { - return PlanNodeType.TABLE_DEVICE_LOCATION_ADD; + return SchemaRegionPlanType.ADD_NODE_LOCATION; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java index 7e318f660f1b3..0553e3bf23bd3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java @@ -55,6 +55,7 @@ public enum SchemaRegionPlanType { CREATE_TABLE_DEVICE((byte) 100), UPDATE_TABLE_DEVICE_ATTRIBUTE((byte) 101), COMMIT_UPDATE_TABLE_DEVICE_ATTRIBUTE((byte) 102), + ADD_NODE_LOCATION((byte) 103), // query plan doesn't need any ser/deSer, thus use one type to represent all READ_SCHEMA(Byte.MAX_VALUE); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java index 75ffea30b77cc..5c057be3247ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.ActivateTemplateInClusterPlanImpl; @@ -103,6 +104,8 @@ public static ISchemaRegionPlan getEmptyPlan(final SchemaRegionPlanType planType return TableDeviceAttributeUpdateNode.MOCK_INSTANCE; case COMMIT_UPDATE_TABLE_DEVICE_ATTRIBUTE: return TableDeviceAttributeCommitUpdateNode.MOCK_INSTANCE; + case ADD_NODE_LOCATION: + return TableNodeLocationAddNode.MOCK_INSTANCE; default: throw new UnsupportedOperationException( String.format( From ed79b3fbf04653d131ecd0b635fa70e9884a3cb1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:43:24 +0800 Subject: [PATCH 121/213] little --- .../schemaengine/schemaregion/SchemaRegionPlanVisitor.java | 6 ++++++ .../logfile/visitor/SchemaRegionPlanDeserializer.java | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java index 009d9f730fade..1b108bd99b82b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.IActivateTemplateInClusterPlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.IAutoCreateDeviceMNodePlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.IChangeAliasPlan; @@ -142,4 +143,9 @@ public R visitCommitUpdateTableDeviceAttribute( final C context) { return visitSchemaRegionPlan(commitUpdateTableDeviceAttributePlan, context); } + + public R visitAddNodeLocation( + final TableNodeLocationAddNode addNodeLocationPlan, final C context) { + return visitSchemaRegionPlan(addNodeLocationPlan, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java index eda82a67cee98..0d9e44a8c4f6e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; @@ -430,5 +431,11 @@ public ISchemaRegionPlan visitCommitUpdateTableDeviceAttribute( final ByteBuffer buffer) { return (TableDeviceAttributeCommitUpdateNode) PlanNodeType.deserialize(buffer); } + + @Override + public ISchemaRegionPlan visitAddNodeLocation( + final TableNodeLocationAddNode addNodeLocationPlan, final ByteBuffer buffer) { + return (TableNodeLocationAddNode) PlanNodeType.deserialize(buffer); + } } } From f5513b2686c7f4222beda45f6284a4345ce90b72 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:45:39 +0800 Subject: [PATCH 122/213] some --- .../node/schema/TableNodeLocationAddNode.java | 2 +- .../logfile/visitor/SchemaRegionPlanSerializer.java | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java index 791d676d67253..12c140f5dc8d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java @@ -110,6 +110,6 @@ public SchemaRegionPlanType getPlanType() { @Override public R accept(final SchemaRegionPlanVisitor visitor, C context) { - return null; + return visitor.visitAddNodeLocation(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java index 51f1c9c43002d..7f5ee7c61e93f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.ISerializer; @@ -513,6 +514,17 @@ public SchemaRegionPlanSerializationResult visitCommitUpdateTableDeviceAttribute } } + @Override + public SchemaRegionPlanSerializationResult visitAddNodeLocation( + final TableNodeLocationAddNode addNodeLocationPlan, final DataOutputStream outputStream) { + try { + addNodeLocationPlan.serialize(outputStream); + return SchemaRegionPlanSerializationResult.SUCCESS; + } catch (final IOException e) { + return new SchemaRegionPlanSerializationResult(e); + } + } + @Override public SchemaRegionPlanSerializationResult visitAlterLogicalView( final IAlterLogicalViewPlan alterLogicalViewPlan, final DataOutputStream stream) { From 1f718670c378c773840f22594b14baf04a99103a Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 20:32:30 +0800 Subject: [PATCH 123/213] Completion --- .../schemaregion/SchemaExecutionVisitor.java | 13 +++++++++++++ .../plan/planner/plan/node/PlanVisitor.java | 5 +++++ .../TableDeviceAttributeCommitUpdateNode.java | 6 ++++++ .../node/schema/TableNodeLocationAddNode.java | 6 ++++++ .../db/schemaengine/schemaregion/ISchemaRegion.java | 3 +++ .../schemaregion/impl/SchemaRegionMemoryImpl.java | 4 ++++ .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 6 ++++++ 7 files changed, 43 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index 00b6744ab5049..cdf7c92a928fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -60,6 +60,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.ICreateAlignedTimeSeriesPlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.ICreateTimeSeriesPlan; @@ -617,6 +618,18 @@ public TSStatus visitTableDeviceAttributeCommit( } } + @Override + public TSStatus visitTableDeviceAttributeCommit( + final TableNodeLocationAddNode node, final ISchemaRegion schemaRegion) { + try { + schemaRegion.addNodeLocation(node); + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } catch (final MetadataException e) { + logger.error(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } + } + @Override public TSStatus visitPipeEnrichedWritePlanNode( final PipeEnrichedWritePlanNode node, final ISchemaRegion schemaRegion) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index e1f2b635e2ae8..cf4d047251ff8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -126,6 +126,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; @SuppressWarnings("java:S6539") // suppress "Monster class" warning public abstract class PlanVisitor { @@ -529,6 +530,10 @@ public R visitTableDeviceAttributeCommit( return visitPlan(node, context); } + public R visitTableNodeLocationAdd(final TableNodeLocationAddNode node, final C context) { + return visitPlan(node, context); + } + ///////////////////////////////////////////////////////////////////////////////////////////////// // Data Write Node ///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java index 21234fb98e4b8..0dcc7779c87f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; @@ -101,6 +102,11 @@ public List getOutputColumnNames() { return null; } + @Override + public R accept(final PlanVisitor visitor, final C context) { + return visitor.visitTableDeviceAttributeCommit(this, context); + } + @Override protected void serializeAttributes(final ByteBuffer byteBuffer) { getType().serialize(byteBuffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java index 12c140f5dc8d2..a3f34e36e57c2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; @@ -71,6 +72,11 @@ public List getOutputColumnNames() { return null; } + @Override + public R accept(final PlanVisitor visitor, final C context) { + return visitor.visitTableNodeLocationAdd(this, context); + } + @Override protected void serializeAttributes(final ByteBuffer byteBuffer) { getType().serialize(byteBuffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 6613b9cf5db7d..e72b924453d55 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; import org.apache.iotdb.db.schemaengine.metric.ISchemaRegionMetric; import org.apache.iotdb.db.schemaengine.rescon.ISchemaRegionStatistics; import org.apache.iotdb.db.schemaengine.schemaregion.read.req.IShowDevicesPlan; @@ -359,5 +360,7 @@ ISchemaReader getTableDeviceReader( void commitUpdateAttribute(final TableDeviceAttributeCommitUpdateNode node) throws MetadataException; + void addNodeLocation(final TableNodeLocationAddNode node) throws MetadataException; + // endregion } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index e736441d03260..cd82893c4a75d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -54,6 +54,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; @@ -1577,6 +1578,9 @@ public void commitUpdateAttribute(final TableDeviceAttributeCommitUpdateNode nod writeToMLog(node); } + @Override + public void addNodeLocation(TableNodeLocationAddNode node) throws MetadataException {} + // endregion private static class RecoverOperationResult { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index a00b99aaec9d5..39113d8c865f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -42,6 +42,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; import org.apache.iotdb.db.schemaengine.metric.ISchemaRegionMetric; import org.apache.iotdb.db.schemaengine.metric.SchemaRegionCachedMetric; import org.apache.iotdb.db.schemaengine.rescon.CachedSchemaRegionStatistics; @@ -1504,6 +1505,11 @@ public void commitUpdateAttribute(final TableDeviceAttributeCommitUpdateNode nod throw new UnsupportedOperationException(); } + @Override + public void addNodeLocation(final TableNodeLocationAddNode node) { + throw new UnsupportedOperationException(); + } + // endregion private static class RecoverOperationResult { From 91f23a1d88bb52d15f68e6651ec610ed2ce54a68 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 20:35:21 +0800 Subject: [PATCH 124/213] Small completion --- .../schemaregion/SchemaExecutionVisitor.java | 2 +- .../node/schema/TableNodeLocationAddNode.java | 4 ++++ .../impl/SchemaRegionMemoryImpl.java | 16 +++++++++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index cdf7c92a928fe..0d6fd015ba92e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -619,7 +619,7 @@ public TSStatus visitTableDeviceAttributeCommit( } @Override - public TSStatus visitTableDeviceAttributeCommit( + public TSStatus visitTableNodeLocationAdd( final TableNodeLocationAddNode node, final ISchemaRegion schemaRegion) { try { schemaRegion.addNodeLocation(node); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java index a3f34e36e57c2..206aa00781265 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java @@ -47,6 +47,10 @@ public TableNodeLocationAddNode(final PlanNodeId id, final TDataNodeLocation loc this.location = location; } + public TDataNodeLocation getLocation() { + return location; + } + @Override public List getChildren() { return null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index cd82893c4a75d..32d680e1c1641 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1579,7 +1579,10 @@ public void commitUpdateAttribute(final TableDeviceAttributeCommitUpdateNode nod } @Override - public void addNodeLocation(TableNodeLocationAddNode node) throws MetadataException {} + public void addNodeLocation(final TableNodeLocationAddNode node) throws MetadataException { + deviceAttributeRemoteUpdater.addLocation(node.getLocation()); + writeToMLog(node); + } // endregion @@ -1832,5 +1835,16 @@ public RecoverOperationResult visitCommitUpdateTableDeviceAttribute( return new RecoverOperationResult(e); } } + + @Override + public RecoverOperationResult visitAddNodeLocation( + final TableNodeLocationAddNode addNodeLocationPlan, final SchemaRegionMemoryImpl context) { + try { + addNodeLocation(addNodeLocationPlan); + return RecoverOperationResult.SUCCESS; + } catch (final MetadataException e) { + return new RecoverOperationResult(e); + } + } } } From 3396b94c7a64e7dc9ab0a443629ecc8efcc3e340 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 20:39:32 +0800 Subject: [PATCH 125/213] Update TableSchemaQueryWriteVisitor.java --- .../node/schema/TableSchemaQueryWriteVisitor.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java index 78a1198c8a0af..b631112880f80 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java @@ -21,7 +21,9 @@ import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.db.queryengine.execution.executor.RegionExecutionResult; +import org.apache.iotdb.db.queryengine.execution.executor.RegionWriteExecutor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.FilterNode; @@ -55,7 +57,12 @@ public RegionExecutionResult visitTableDeviceQueryScan( private RegionExecutionResult visitTableDeviceSourceNode( final TableDeviceSourceNode node, final ConsensusGroupId context) { if (Objects.nonNull(node.getSenderLocation())) { - return null; + final RegionExecutionResult result = + new RegionWriteExecutor() + .execute( + context, + new TableNodeLocationAddNode(new PlanNodeId(""), node.getSenderLocation())); + return !result.isAccepted() ? result : null; } return null; } From 8efdca6afcaaeaf37d896fae1690811f216df69f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 20:46:14 +0800 Subject: [PATCH 126/213] Update DataNodeInternalRPCServiceImpl.java --- .../impl/DataNodeInternalRPCServiceImpl.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 5390f0904699d..bc11b26261d9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -138,6 +138,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedNonWritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableSchemaQueryWriteVisitor; import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; @@ -279,6 +280,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -358,11 +360,22 @@ public TSendFragmentInstanceResp sendFragmentInstance(final TSendFragmentInstanc return resp; } - final RegionReadExecutor executor = new RegionReadExecutor(); - final RegionExecutionResult executionResult = - groupId == null - ? executor.execute(fragmentInstance) - : executor.execute(groupId, fragmentInstance); + RegionExecutionResult executionResult = null; + if (Objects.nonNull(groupId)) { + // For schema query, there may be a "write" before "read" + // the result is not null iff the "write" has failed + executionResult = + new TableSchemaQueryWriteVisitor() + .visitPlan(fragmentInstance.getFragment().getPlanNodeTree(), groupId); + } + if (Objects.isNull(executionResult)) { + final RegionReadExecutor executor = new RegionReadExecutor(); + executionResult = + groupId == null + ? executor.execute(fragmentInstance) + : executor.execute(groupId, fragmentInstance); + } + final TSendFragmentInstanceResp resp = new TSendFragmentInstanceResp(); resp.setAccepted(executionResult.isAccepted()); resp.setMessage(executionResult.getMessage()); From f977f518e5902e40d69b788d262af7b533c9c213 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 9 Oct 2024 20:54:16 +0800 Subject: [PATCH 127/213] Update GeneralRegionAttributeSecurityService.java --- .../attribute/update/GeneralRegionAttributeSecurityService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 0718d39427626..974e032f663e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -139,6 +139,7 @@ private void execute() { : Collections.emptyMap(); // Commit + // TODO: correct shrinkage logic attributeUpdateCommitMap.forEach( (schemaRegionId, pair) -> { if (!new RegionWriteExecutor() From a60085e79f3e23627c8b99ee0d7a84a001aa8889 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 10:21:48 +0800 Subject: [PATCH 128/213] Update GeneralRegionAttributeSecurityService.java --- ...GeneralRegionAttributeSecurityService.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 974e032f663e2..71cb5bb67ef3e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -134,9 +134,7 @@ private void execute() { // Send & may shrink final Map> shrinkMap = - sendUpdateRequest(attributeUpdateCommitMap) - ? detectNodeShrinkage(attributeUpdateCommitMap) - : Collections.emptyMap(); + sendUpdateRequestAndMayShrink(attributeUpdateCommitMap); // Commit // TODO: correct shrinkage logic @@ -173,6 +171,7 @@ private void execute() { } } + // Shrink the locations which do not exist in cluster dataNodes private @Nonnull Map> detectNodeShrinkage( final Map>> attributeUpdateCommitMap) { @@ -185,7 +184,7 @@ private void execute() { return Collections.emptyMap(); } dataNodeId2FailureDurationAndTimesMap.clear(); - final Set dataNodeLocations = new HashSet<>(); + final Set realDataNodeLocations = new HashSet<>(); showClusterResp .getDataNodeList() .forEach( @@ -194,23 +193,19 @@ private void execute() { location.setMPPDataExchangeEndPoint(null); location.setSchemaRegionConsensusEndPoint(null); location.setClientRpcEndPoint(null); - dataNodeLocations.add(location); + realDataNodeLocations.add(location); }); return attributeUpdateCommitMap.entrySet().stream() .filter( entry -> { - entry - .getValue() - .getRight() - .keySet() - .removeIf(location -> !dataNodeLocations.contains(location)); + entry.getValue().getRight().keySet().removeIf(realDataNodeLocations::contains); return !entry.getValue().getRight().isEmpty(); }) .collect( Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getRight().keySet())); } - private boolean sendUpdateRequest( + private @Nonnull Map> sendUpdateRequestAndMayShrink( final Map>> attributeUpdateCommitMap) { final AsyncRequestContext @@ -274,6 +269,11 @@ private boolean sendUpdateRequest( }) .map(Map.Entry::getKey) .collect(Collectors.toSet()); + + // Compute node shrinkage before failure removal in commit + final Map> result = + needFetch.get() ? detectNodeShrinkage(attributeUpdateCommitMap) : Collections.emptyMap(); + for (final Iterator>>> it = attributeUpdateCommitMap.entrySet().iterator(); it.hasNext(); ) { @@ -286,7 +286,7 @@ private boolean sendUpdateRequest( it.remove(); } } - return needFetch.get(); + return result; } /////////////////////////////// SingleTon /////////////////////////////// From 11d63fa29935970c245da68c5c786e61673915f7 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 10:27:00 +0800 Subject: [PATCH 129/213] Update GeneralRegionAttributeSecurityService.java --- .../update/GeneralRegionAttributeSecurityService.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 71cb5bb67ef3e..7fc89f7efae72 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -277,12 +277,15 @@ private void execute() { for (final Iterator>>> it = attributeUpdateCommitMap.entrySet().iterator(); it.hasNext(); ) { - final Map dataNodeLocationMap = it.next().getValue().getRight(); + final Map.Entry>> currentEntry = + it.next(); + final Map dataNodeLocationMap = currentEntry.getValue().getRight(); dataNodeLocationMap .entrySet() .removeIf( locationEntry -> failedDataNodes.contains(locationEntry.getKey().getDataNodeId())); - if (dataNodeLocationMap.isEmpty()) { + // Reserve the schemaIds with shrinkage for commit convenience + if (dataNodeLocationMap.isEmpty() && !result.containsKey(currentEntry.getKey())) { it.remove(); } } From 580ef9d26d80dd35ab19307afc79f0b6b763986d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 10:29:31 +0800 Subject: [PATCH 130/213] Update GeneralRegionAttributeSecurityService.java --- .../attribute/update/GeneralRegionAttributeSecurityService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 7fc89f7efae72..269c93cd88ebb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -137,7 +137,6 @@ private void execute() { sendUpdateRequestAndMayShrink(attributeUpdateCommitMap); // Commit - // TODO: correct shrinkage logic attributeUpdateCommitMap.forEach( (schemaRegionId, pair) -> { if (!new RegionWriteExecutor() From c8108957f3eb4a307fc5a113caaa1eebdafbea2d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 11:04:57 +0800 Subject: [PATCH 131/213] Update TableLogicalPlanner.java --- .../plan/relational/planner/TableLogicalPlanner.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java index d9afec350e31d..d787fb7de1248 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java @@ -163,7 +163,7 @@ public LogicalQueryPlan plan(Analysis analysis) { private PlanNode planStatement(final Analysis analysis, final Statement statement) { if (statement instanceof CreateOrUpdateDevice) { - return planCreateDevice((CreateOrUpdateDevice) statement, analysis); + return planCreateOrUpdateDevice((CreateOrUpdateDevice) statement, analysis); } if (statement instanceof FetchDevice) { return planFetchDevice((FetchDevice) statement, analysis); @@ -248,7 +248,8 @@ private RelationPlanner getRelationPlanner(Analysis analysis) { analysis, symbolAllocator, queryContext, Optional.empty(), sessionInfo, ImmutableMap.of()); } - private PlanNode planCreateDevice(final CreateOrUpdateDevice statement, final Analysis analysis) { + private PlanNode planCreateOrUpdateDevice( + final CreateOrUpdateDevice statement, final Analysis analysis) { final CreateOrUpdateTableDeviceNode node = new CreateOrUpdateTableDeviceNode( queryContext.getQueryId().genPlanNodeId(), From 97f0f251cca583c8e1e62681d98197e285177dfb Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 11:40:58 +0800 Subject: [PATCH 132/213] Update DeviceAttributeRemoteUpdater.java --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 3cd891b09f6a0..e010986d3c172 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -58,6 +58,7 @@ public class DeviceAttributeRemoteUpdater { // All the data node locations shall only have internal endpoint with all the other endpoints set // to null + // Note that the locations in SR is also placed here in case the regions are migrated private final Set targetDataNodeLocations = new HashSet<>(); private final ConcurrentMap attributeUpdateMap = new ConcurrentHashMap<>(); From 1564e596be82d2680b26a340d9f35fddf6bf9a9e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 12:30:47 +0800 Subject: [PATCH 133/213] comments --- .../protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java | 3 +++ .../iotdb/commons/pipe/datastructure/pattern/TablePattern.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index bc11b26261d9a..63f3938bf3ca5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -364,6 +364,9 @@ public TSendFragmentInstanceResp sendFragmentInstance(final TSendFragmentInstanc if (Objects.nonNull(groupId)) { // For schema query, there may be a "write" before "read" // the result is not null iff the "write" has failed + // Note that we do not handle (either write or record as fetched) schema regions + // written on local because currently a successful local dispatch MUST be executed + // on a local schema region (either leader or follower) executionResult = new TableSchemaQueryWriteVisitor() .visitPlan(fragmentInstance.getFragment().getPlanNodeTree(), groupId); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java index 61a851b7f03e8..82f67a05e4071 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java @@ -67,7 +67,7 @@ public String getTablePattern() { /** * Interpret from source parameters and get a pipe pattern. * - * @return The interpreted {@link TablePattern} which is not null. + * @return The interpreted {@link TablePattern} which is not {@code null}. */ public static TablePattern parsePipePatternFromSourceParameters( final PipeParameters sourceParameters) { From d2de995a959858fea81b1dab851efe256a00179d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:11:34 +0800 Subject: [PATCH 134/213] Added local --- .../impl/DataNodeInternalRPCServiceImpl.java | 3 -- ...leSchemaQueryAttributeSecurityVisitor.java | 22 ++++++++++ .../FragmentInstanceDispatcherImpl.java | 43 ++++++++++++------- 3 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 63f3938bf3ca5..bc11b26261d9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -364,9 +364,6 @@ public TSendFragmentInstanceResp sendFragmentInstance(final TSendFragmentInstanc if (Objects.nonNull(groupId)) { // For schema query, there may be a "write" before "read" // the result is not null iff the "write" has failed - // Note that we do not handle (either write or record as fetched) schema regions - // written on local because currently a successful local dispatch MUST be executed - // on a local schema region (either leader or follower) executionResult = new TableSchemaQueryWriteVisitor() .visitPlan(fragmentInstance.getFragment().getPlanNodeTree(), groupId); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java new file mode 100644 index 0000000000000..e5cc679e3e242 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; + +public abstract class AbstractTableSchemaQueryAttributeSecurityVisitor {} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java index 6f695b41ac84e..911ee8bb2cdae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java @@ -41,6 +41,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableSchemaQueryWriteVisitor; import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstance; import org.apache.iotdb.mpp.rpc.thrift.TPlanNode; @@ -60,6 +61,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -450,26 +452,37 @@ private void dispatchLocally(final FragmentInstance instance) switch (instance.getType()) { case READ: - final RegionReadExecutor readExecutor = new RegionReadExecutor(); - final RegionExecutionResult readResult = - groupId == null - ? readExecutor.execute(instance) - : readExecutor.execute(groupId, instance); - if (!readResult.isAccepted()) { - LOGGER.warn(readResult.getMessage()); - if (readResult.isReadNeedRetry()) { - if (readResult.getStatus() != null - && readResult.getStatus().getCode() + RegionExecutionResult executionResult = null; + if (Objects.nonNull(groupId)) { + // For schema query, there may be a "write" before "read" + // the result is not null iff the "write" has failed + executionResult = + new TableSchemaQueryWriteVisitor() + .visitPlan(instance.getFragment().getPlanNodeTree(), groupId); + } + if (Objects.isNull(executionResult)) { + final RegionReadExecutor readExecutor = new RegionReadExecutor(); + executionResult = + groupId == null + ? readExecutor.execute(instance) + : readExecutor.execute(groupId, instance); + } + if (!executionResult.isAccepted()) { + LOGGER.warn(executionResult.getMessage()); + if (executionResult.isReadNeedRetry()) { + if (executionResult.getStatus() != null + && executionResult.getStatus().getCode() == TSStatusCode.TOO_MANY_CONCURRENT_QUERIES_ERROR.getStatusCode()) { - throw new FragmentInstanceDispatchException(readResult.getStatus()); + throw new FragmentInstanceDispatchException(executionResult.getStatus()); } throw new FragmentInstanceDispatchException( - RpcUtils.getStatus(TSStatusCode.DISPATCH_ERROR, readResult.getMessage())); - } else if (readResult.getStatus() != null) { - throw new FragmentInstanceDispatchException(readResult.getStatus()); + RpcUtils.getStatus(TSStatusCode.DISPATCH_ERROR, executionResult.getMessage())); + } else if (executionResult.getStatus() != null) { + throw new FragmentInstanceDispatchException(executionResult.getStatus()); } else { throw new FragmentInstanceDispatchException( - RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, readResult.getMessage())); + RpcUtils.getStatus( + TSStatusCode.EXECUTE_STATEMENT_ERROR, executionResult.getMessage())); } } break; From e63c80bd821261e4db1b8b72f9be739c807b10a4 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:21:51 +0800 Subject: [PATCH 135/213] Refactor --- .../impl/DataNodeInternalRPCServiceImpl.java | 3 +- ...leSchemaQueryAttributeSecurityVisitor.java | 42 ++++++++++++++++++- .../schema/TableSchemaQueryWriteVisitor.java | 27 +----------- .../FragmentInstanceDispatcherImpl.java | 4 +- 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index bc11b26261d9a..68a5c79204e72 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -365,8 +365,7 @@ public TSendFragmentInstanceResp sendFragmentInstance(final TSendFragmentInstanc // For schema query, there may be a "write" before "read" // the result is not null iff the "write" has failed executionResult = - new TableSchemaQueryWriteVisitor() - .visitPlan(fragmentInstance.getFragment().getPlanNodeTree(), groupId); + new TableSchemaQueryWriteVisitor().processFragment(fragmentInstance, groupId); } if (Objects.isNull(executionResult)) { final RegionReadExecutor executor = new RegionReadExecutor(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java index e5cc679e3e242..99733708a55c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java @@ -19,4 +19,44 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; -public abstract class AbstractTableSchemaQueryAttributeSecurityVisitor {} +import org.apache.iotdb.commons.consensus.ConsensusGroupId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.FilterNode; + +import javax.annotation.Nonnull; + +public abstract class AbstractTableSchemaQueryAttributeSecurityVisitor + extends PlanVisitor { + + public R processFragment( + final @Nonnull FragmentInstance instance, final ConsensusGroupId groupId) { + return instance.getFragment().getPlanNodeTree().accept(this, groupId); + } + + @Override + public R visitPlan(final PlanNode node, final ConsensusGroupId context) { + return null; + } + + @Override + public R visitFilter(final FilterNode node, final ConsensusGroupId context) { + return node.getChild().accept(this, context); + } + + @Override + public R visitTableDeviceFetch(final TableDeviceFetchNode node, final ConsensusGroupId context) { + return visitTableDeviceSourceNode(node, context); + } + + @Override + public R visitTableDeviceQueryScan( + final TableDeviceQueryScanNode node, final ConsensusGroupId context) { + return visitTableDeviceSourceNode(node, context); + } + + protected abstract R visitTableDeviceSourceNode( + final TableDeviceSourceNode node, final ConsensusGroupId context); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java index b631112880f80..c2ff3820cbad3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQueryWriteVisitor.java @@ -22,39 +22,16 @@ import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.db.queryengine.execution.executor.RegionExecutionResult; import org.apache.iotdb.db.queryengine.execution.executor.RegionWriteExecutor; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.FilterNode; import java.util.Objects; public class TableSchemaQueryWriteVisitor - extends PlanVisitor { - @Override - public RegionExecutionResult visitPlan(final PlanNode node, final ConsensusGroupId context) { - return null; - } - - @Override - public RegionExecutionResult visitFilter(final FilterNode node, final ConsensusGroupId context) { - return node.getChild().accept(this, context); - } + extends AbstractTableSchemaQueryAttributeSecurityVisitor { @Override - public RegionExecutionResult visitTableDeviceFetch( - final TableDeviceFetchNode node, final ConsensusGroupId context) { - return visitTableDeviceSourceNode(node, context); - } - - @Override - public RegionExecutionResult visitTableDeviceQueryScan( - final TableDeviceQueryScanNode node, final ConsensusGroupId context) { - return visitTableDeviceSourceNode(node, context); - } - - private RegionExecutionResult visitTableDeviceSourceNode( + protected RegionExecutionResult visitTableDeviceSourceNode( final TableDeviceSourceNode node, final ConsensusGroupId context) { if (Objects.nonNull(node.getSenderLocation())) { final RegionExecutionResult result = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java index 911ee8bb2cdae..a9b7fc5b43910 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java @@ -456,9 +456,7 @@ private void dispatchLocally(final FragmentInstance instance) if (Objects.nonNull(groupId)) { // For schema query, there may be a "write" before "read" // the result is not null iff the "write" has failed - executionResult = - new TableSchemaQueryWriteVisitor() - .visitPlan(instance.getFragment().getPlanNodeTree(), groupId); + executionResult = new TableSchemaQueryWriteVisitor().processFragment(instance, groupId); } if (Objects.isNull(executionResult)) { final RegionReadExecutor readExecutor = new RegionReadExecutor(); From 2620662fae4785dca2411b8547ea56bee78f310a Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:00:22 +0800 Subject: [PATCH 136/213] Successful callback --- ...stractTableSchemaQueryAttributeSecurityVisitor.java | 2 +- .../plan/scheduler/FragmentInstanceDispatcherImpl.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java index 99733708a55c1..8a16897be580d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java @@ -32,7 +32,7 @@ public abstract class AbstractTableSchemaQueryAttributeSecurityVisitor extends PlanVisitor { public R processFragment( - final @Nonnull FragmentInstance instance, final ConsensusGroupId groupId) { + final @Nonnull FragmentInstance instance, final @Nonnull ConsensusGroupId groupId) { return instance.getFragment().getPlanNodeTree().accept(this, groupId); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java index a9b7fc5b43910..7bcaa3cd07a7e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java @@ -346,6 +346,13 @@ private void dispatchRemoteHelper(final FragmentInstance instance, final TEndPoi throw new FragmentInstanceDispatchException( RpcUtils.getStatus( TSStatusCode.EXECUTE_STATEMENT_ERROR, sendFragmentInstanceResp.message)); + } else if (Objects.nonNull(sendFragmentInstanceReq.getConsensusGroupId())) { + // Assume the consensus group casting will not generate any exceptions + new TableSchemaQueryWriteVisitor() + .processFragment( + instance, + ConsensusGroupId.Factory.createFromTConsensusGroupId( + sendFragmentInstanceReq.getConsensusGroupId())); } break; case WRITE: @@ -482,6 +489,9 @@ private void dispatchLocally(final FragmentInstance instance) RpcUtils.getStatus( TSStatusCode.EXECUTE_STATEMENT_ERROR, executionResult.getMessage())); } + } else if (Objects.nonNull(groupId)) { + // Assume the consensus group casting will not generate any exceptions + new TableSchemaQueryWriteVisitor().processFragment(instance, groupId); } break; case WRITE: From 8add8bf245b079d27ec840ab5f099522d2b6ffb1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:04:08 +0800 Subject: [PATCH 137/213] Add SR id --- .../TableDeviceCacheAttributeGuard.java | 4 +-- ...eSchemaQuerySuccessfulCallbackVisitor.java | 34 +++++++++++++++++++ .../FragmentInstanceDispatcherImpl.java | 5 +-- 3 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQuerySuccessfulCallbackVisitor.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index 6227705bd402f..4b3e874a06108 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -29,7 +29,7 @@ public boolean isRegionFetched(final Integer schemaRegionId) { return fetchedSchemaRegionIds.contains(schemaRegionId); } - public void addFetchedRegions(final Set schemaRegionIds) { - fetchedSchemaRegionIds.addAll(schemaRegionIds); + public void addFetchedRegion(final Integer schemaRegionId) { + fetchedSchemaRegionIds.add(schemaRegionId); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQuerySuccessfulCallbackVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQuerySuccessfulCallbackVisitor.java new file mode 100644 index 0000000000000..f4eb0114b6911 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableSchemaQuerySuccessfulCallbackVisitor.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; + +import org.apache.iotdb.commons.consensus.ConsensusGroupId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher; + +public class TableSchemaQuerySuccessfulCallbackVisitor + extends AbstractTableSchemaQueryAttributeSecurityVisitor { + @Override + protected Void visitTableDeviceSourceNode( + final TableDeviceSourceNode node, final ConsensusGroupId context) { + TableDeviceSchemaFetcher.getInstance().getAttributeGuard().addFetchedRegion(context.getId()); + return null; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java index 7bcaa3cd07a7e..6bdc5c20f641d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java @@ -41,6 +41,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableSchemaQuerySuccessfulCallbackVisitor; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableSchemaQueryWriteVisitor; import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstance; @@ -348,7 +349,7 @@ private void dispatchRemoteHelper(final FragmentInstance instance, final TEndPoi TSStatusCode.EXECUTE_STATEMENT_ERROR, sendFragmentInstanceResp.message)); } else if (Objects.nonNull(sendFragmentInstanceReq.getConsensusGroupId())) { // Assume the consensus group casting will not generate any exceptions - new TableSchemaQueryWriteVisitor() + new TableSchemaQuerySuccessfulCallbackVisitor() .processFragment( instance, ConsensusGroupId.Factory.createFromTConsensusGroupId( @@ -491,7 +492,7 @@ private void dispatchLocally(final FragmentInstance instance) } } else if (Objects.nonNull(groupId)) { // Assume the consensus group casting will not generate any exceptions - new TableSchemaQueryWriteVisitor().processFragment(instance, groupId); + new TableSchemaQuerySuccessfulCallbackVisitor().processFragment(instance, groupId); } break; case WRITE: From f9fb0912e65cdc8839268b45314fafd3eb0bcd80 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:23:03 +0800 Subject: [PATCH 138/213] Update GeneralRegionAttributeSecurityService.java --- ...GeneralRegionAttributeSecurityService.java | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 269c93cd88ebb..22f86d997ed2c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -132,25 +132,28 @@ private void execute() { attributeUpdateCommitMap.put(regionLeader.getSchemaRegionId(), currentResult); } - // Send & may shrink - final Map> shrinkMap = - sendUpdateRequestAndMayShrink(attributeUpdateCommitMap); + if (!attributeUpdateCommitMap.isEmpty()) { + // Send & may shrink + final Map> shrinkMap = + sendUpdateRequestAndMayShrink(attributeUpdateCommitMap); - // Commit - attributeUpdateCommitMap.forEach( - (schemaRegionId, pair) -> { - if (!new RegionWriteExecutor() - .execute( - schemaRegionId, - new TableDeviceAttributeCommitUpdateNode( - new PlanNodeId(""), - pair.getLeft(), - pair.getRight(), - shrinkMap.getOrDefault(schemaRegionId, Collections.emptySet()))) - .isAccepted()) { - LOGGER.warn("Failed to write attribute commit message to region {}.", schemaRegionId); - } - }); + // Commit + attributeUpdateCommitMap.forEach( + (schemaRegionId, pair) -> { + if (!new RegionWriteExecutor() + .execute( + schemaRegionId, + new TableDeviceAttributeCommitUpdateNode( + new PlanNodeId(""), + pair.getLeft(), + pair.getRight(), + shrinkMap.getOrDefault(schemaRegionId, Collections.emptySet()))) + .isAccepted()) { + LOGGER.warn( + "Failed to write attribute commit message to region {}.", schemaRegionId); + } + }); + } if (!skipNext) { condition.await( From bffa976c3f2232e4dfbf87097494a4a933574ec1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:33:38 +0800 Subject: [PATCH 139/213] Update GeneralRegionAttributeSecurityService.java --- ...GeneralRegionAttributeSecurityService.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 22f86d997ed2c..8c90fb3c6e3cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -27,6 +27,9 @@ import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.StartupException; +import org.apache.iotdb.commons.service.IService; +import org.apache.iotdb.commons.service.ServiceType; import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; @@ -57,6 +60,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -65,7 +69,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; -public class GeneralRegionAttributeSecurityService { +public class GeneralRegionAttributeSecurityService implements IService { private static final Logger LOGGER = LoggerFactory.getLogger(GeneralRegionAttributeSecurityService.class); @@ -75,7 +79,8 @@ public class GeneralRegionAttributeSecurityService { private final Map> dataNodeId2FailureDurationAndTimesMap = new HashMap<>(); - private final Set regionLeaders = new HashSet<>(); + private final Set regionLeaders = + Collections.newSetFromMap(new ConcurrentHashMap<>()); private final ReentrantLock lock = new ReentrantLock(); private final Condition condition = lock.newCondition(); private volatile boolean skipNext = false; @@ -294,6 +299,19 @@ private void execute() { return result; } + /////////////////////////////// IService /////////////////////////////// + + @Override + public void start() throws StartupException {} + + @Override + public void stop() {} + + @Override + public ServiceType getID() { + return null; + } + /////////////////////////////// SingleTon /////////////////////////////// private GeneralRegionAttributeSecurityService() { From 6df8af7d820ada8f5c59185e29e9111890b2c7da Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:41:36 +0800 Subject: [PATCH 140/213] IService --- ...GeneralRegionAttributeSecurityService.java | 32 +++++++++++-------- .../org/apache/iotdb/db/service/DataNode.java | 4 +++ .../iotdb/commons/service/ServiceType.java | 2 ++ 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 8c90fb3c6e3cf..76d0b104932f0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -84,23 +84,14 @@ public class GeneralRegionAttributeSecurityService implements IService { private final ReentrantLock lock = new ReentrantLock(); private final Condition condition = lock.newCondition(); private volatile boolean skipNext = false; + private volatile boolean allowSubmitListen = false; public void startBroadcast(final ISchemaRegion schemaRegion) { - if (regionLeaders.isEmpty()) { - securityServiceExecutor.submit(this::execute); - LOGGER.info("General region attribute security service is started successfully."); - } - regionLeaders.add(schemaRegion); } public void stopBroadcast(final ISchemaRegion schemaRegion) { regionLeaders.remove(schemaRegion); - - if (regionLeaders.isEmpty()) { - securityServiceExecutor.shutdown(); - LOGGER.info("General region attribute security service is stopped successfully."); - } } public void notifyBroadCast() { @@ -174,7 +165,10 @@ private void execute() { "Interrupted when waiting for the next attribute broadcasting: {}", e.getMessage()); } finally { lock.unlock(); - securityServiceExecutor.submit(this::execute); + + if (allowSubmitListen) { + securityServiceExecutor.submit(this::execute); + } } } @@ -302,14 +296,24 @@ private void execute() { /////////////////////////////// IService /////////////////////////////// @Override - public void start() throws StartupException {} + public void start() throws StartupException { + allowSubmitListen = true; + securityServiceExecutor.submit(this::execute); + + LOGGER.info("General region attribute security service is started successfully."); + } @Override - public void stop() {} + public void stop() { + allowSubmitListen = false; + securityServiceExecutor.shutdown(); + + LOGGER.info("General region attribute security service is stopped successfully."); + } @Override public ServiceType getID() { - return null; + return ServiceType.GENERAL_REGION_ATTRIBUTE_SECURITY_SERVICE; } /////////////////////////////// SingleTon /////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index d85ac18989e6b..9d3754764f803 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -90,6 +90,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.distribution.SourceRewriter; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; import org.apache.iotdb.db.schemaengine.SchemaEngine; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.GeneralRegionAttributeSecurityService; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.db.service.metrics.DataNodeMetricsHelper; @@ -813,6 +814,9 @@ private void setUp() throws StartupException { // Register subscription agent before pipe agent registerManager.register(SubscriptionAgent.runtime()); registerManager.register(PipeDataNodeAgent.runtime()); + + // Start GRASS Service + registerManager.register(GeneralRegionAttributeSecurityService.getInstance()); } /** Set up RPC and protocols after DataNode is available */ diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ServiceType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ServiceType.java index 2898e14d987c6..205cbc4f6a436 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ServiceType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ServiceType.java @@ -83,6 +83,8 @@ public enum ServiceType { PIPE_RUNTIME_DATA_NODE_AGENT("Pipe Runtime Data Node Agent", "PipeRuntimeDataNodeAgent"), PIPE_RUNTIME_CONFIG_NODE_AGENT("Pipe Runtime Config Node Agent", "PipeRuntimeConfigNodeAgent"), SUBSCRIPTION_RUNTIME_AGENT("Subscription Runtime Agent", "SubscriptionRuntimeAgent"), + GENERAL_REGION_ATTRIBUTE_SECURITY_SERVICE( + "General Region Attribute Security Service", "GeneralRegionAttributeSecurityService"), SESSION_MANAGER("Session Manager", "RpcSession"), CONFIG_NODE("Config Node", "ConfigNode"), From 41a27b228cfb926388a5e33805e64849e1400b77 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 16:55:20 +0800 Subject: [PATCH 141/213] Update DeviceAttributeRemoteUpdater.java --- .../attribute/update/DeviceAttributeRemoteUpdater.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index e010986d3c172..308ca2cc91b11 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; @@ -81,6 +82,11 @@ public void update( final String tableName, final String[] deviceId, final Map attributeMap) { targetDataNodeLocations.forEach( location -> { + // Skip update on local + if (location.getDataNodeId() + == IoTDBDescriptor.getInstance().getConfig().getDataNodeId()) { + return; + } if (!attributeUpdateMap.containsKey(location)) { final UpdateContainer newContainer; if (!regionStatistics.isAllowToCreateNewSeries()) { From d4f5f24588e3c35864b86c847dfe4c8885c9eb33 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 16:59:53 +0800 Subject: [PATCH 142/213] Update GeneralRegionAttributeSecurityService.java --- .../attribute/update/GeneralRegionAttributeSecurityService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 76d0b104932f0..6da37b7b749ff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -145,6 +145,8 @@ private void execute() { pair.getRight(), shrinkMap.getOrDefault(schemaRegionId, Collections.emptySet()))) .isAccepted()) { + // May fail due to region shutdown, migration or other reasons + // Just ignore LOGGER.warn( "Failed to write attribute commit message to region {}.", schemaRegionId); } From 0514249ffef2d0fcb34f4bdf4fa7a5526c776e17 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:10:05 +0800 Subject: [PATCH 143/213] Refactor --- .../update/DeviceAttributeRemoteUpdater.java | 17 ++++++++++++++--- ...va => UpdateDetailContainerStatistics.java} | 18 ++++++++++-------- 2 files changed, 24 insertions(+), 11 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/{UpdateContainerStatistics.java => UpdateDetailContainerStatistics.java} (78%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 308ca2cc91b11..5d6b648109529 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -69,7 +69,7 @@ public class DeviceAttributeRemoteUpdater { private final MemSchemaRegionStatistics regionStatistics; // Only exist for update detail container - private final Map updateContainerStatistics = + private final Map updateContainerStatistics = new HashMap<>(); public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatistics) { @@ -95,7 +95,7 @@ public void update( } else { newContainer = new UpdateDetailContainer(); requestMemory(UpdateDetailContainer.INSTANCE_SIZE); - updateContainerStatistics.put(location, new UpdateContainerStatistics()); + updateContainerStatistics.put(location, new UpdateDetailContainerStatistics()); } attributeUpdateMap.put(location, newContainer); } @@ -139,6 +139,16 @@ public void commit(final TableDeviceAttributeCommitUpdateNode node) { final Set shrunkNodes = node.getShrunkNodes(); targetDataNodeLocations.removeAll(shrunkNodes); attributeUpdateMap.keySet().removeAll(shrunkNodes); + attributeUpdateMap + .keySet() + .removeIf( + location -> { + if (shrunkNodes.contains(location)) { + releaseMemory(1L); + return true; + } + return false; + }); updateContainerStatistics.keySet().removeAll(shrunkNodes); node.getCommitMap() @@ -213,7 +223,8 @@ private void degrade() { } final UpdateClearContainer newContainer = ((UpdateDetailContainer) attributeUpdateMap.get(location)).degrade(); - updateMemory(newContainer.ramBytesUsed() - updateContainerStatistics.get(location).getSize()); + updateMemory( + newContainer.ramBytesUsed() - updateContainerStatistics.get(location).getContainerSize()); attributeUpdateMap.put(location, newContainer); updateContainerStatistics.remove(location); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainerStatistics.java similarity index 78% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainerStatistics.java index ee20c99f92a35..d39c19aa8abca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainerStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainerStatistics.java @@ -22,30 +22,32 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; // For degrade -public class UpdateContainerStatistics { +public class UpdateDetailContainerStatistics { private static final long MIN_DEGRADE_MEMORY = IoTDBDescriptor.getInstance().getConfig().getDetailContainerMinDegradeMemoryInBytes(); private long lastUpdateTime = System.currentTimeMillis(); - private long size = 0; - long getSize() { - return size; + // Not include instance size + private long entrySize = 0; + + long getContainerSize() { + return entrySize + UpdateDetailContainer.INSTANCE_SIZE; } void addSize(final long increment) { - this.size += increment; + this.entrySize += increment; } void decreaseSize(final long decrement) { - this.size -= decrement; + this.entrySize -= decrement; lastUpdateTime = System.currentTimeMillis(); } boolean needDegrade() { - return size >= MIN_DEGRADE_MEMORY; + return entrySize >= MIN_DEGRADE_MEMORY; } long getDegradePriority() { - return size * (System.currentTimeMillis() - lastUpdateTime); + return entrySize * (System.currentTimeMillis() - lastUpdateTime); } } From 43da7c07ff880376d85a20757537586727c78556 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:14:15 +0800 Subject: [PATCH 144/213] bug fix --- .../update/DeviceAttributeRemoteUpdater.java | 18 +++++++++++------- .../UpdateDetailContainerStatistics.java | 5 ++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 5d6b648109529..81076b5092559 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -104,7 +104,7 @@ public void update( updateContainerStatistics.computeIfPresent( location, (k, v) -> { - v.addSize(size); + v.addEntrySize(size); return v; }); updateMemory(size); @@ -138,13 +138,15 @@ public Pair> getAttributeUpdateInfo( public void commit(final TableDeviceAttributeCommitUpdateNode node) { final Set shrunkNodes = node.getShrunkNodes(); targetDataNodeLocations.removeAll(shrunkNodes); - attributeUpdateMap.keySet().removeAll(shrunkNodes); attributeUpdateMap - .keySet() + .entrySet() .removeIf( - location -> { - if (shrunkNodes.contains(location)) { - releaseMemory(1L); + entry -> { + if (shrunkNodes.contains(entry.getKey())) { + releaseMemory( + updateContainerStatistics.containsKey(entry.getKey()) + ? updateContainerStatistics.get(entry.getKey()).getContainerSize() + : ((UpdateClearContainer) entry.getValue()).ramBytesUsed()); return true; } return false; @@ -171,7 +173,9 @@ public void commit(final TableDeviceAttributeCommitUpdateNode node) { updateContainerStatistics.remove(dataNode); return null; } else if (updateContainerStatistics.containsKey(dataNode)) { - updateContainerStatistics.get(dataNode).decreaseSize(result.getLeft()); + updateContainerStatistics + .get(dataNode) + .decreaseEntrySize(result.getLeft()); } } return container; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainerStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainerStatistics.java index d39c19aa8abca..adf4a66ed5d6c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainerStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainerStatistics.java @@ -27,18 +27,17 @@ public class UpdateDetailContainerStatistics { IoTDBDescriptor.getInstance().getConfig().getDetailContainerMinDegradeMemoryInBytes(); private long lastUpdateTime = System.currentTimeMillis(); - // Not include instance size private long entrySize = 0; long getContainerSize() { return entrySize + UpdateDetailContainer.INSTANCE_SIZE; } - void addSize(final long increment) { + void addEntrySize(final long increment) { this.entrySize += increment; } - void decreaseSize(final long decrement) { + void decreaseEntrySize(final long decrement) { this.entrySize -= decrement; lastUpdateTime = System.currentTimeMillis(); } From 4abe00115a8f2a14490b25b5d606c092ac9f8d98 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:33:47 +0800 Subject: [PATCH 145/213] Partial refactor --- .../TableDeviceAttributeCommitUpdateNode.java | 57 ++++++++++--------- .../update/DeviceAttributeRemoteUpdater.java | 53 +++++++++++------ 2 files changed, 67 insertions(+), 43 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java index 0dcc7779c87f3..913e8e0569b2a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; -import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -28,6 +27,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeRemoteUpdater; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -45,19 +45,22 @@ public class TableDeviceAttributeCommitUpdateNode extends PlanNode implements IS private final long version; private final Map commitMap; private final Set shrunkNodes; + private final TDataNodeLocation localLocation; public static final TableDeviceAttributeCommitUpdateNode MOCK_INSTANCE = - new TableDeviceAttributeCommitUpdateNode(new PlanNodeId(""), 0L, null, null); + new TableDeviceAttributeCommitUpdateNode(new PlanNodeId(""), 0L, null, null, null); public TableDeviceAttributeCommitUpdateNode( final PlanNodeId id, final long version, final Map commitMap, - final Set shrunkNodes) { + final Set shrunkNodes, + final TDataNodeLocation localLocation) { super(id); this.version = version; this.commitMap = commitMap; this.shrunkNodes = shrunkNodes; + this.localLocation = localLocation; } public long getVersion() { @@ -89,7 +92,8 @@ public PlanNodeType getType() { @Override public PlanNode clone() { - return new TableDeviceAttributeCommitUpdateNode(id, version, commitMap, shrunkNodes); + return new TableDeviceAttributeCommitUpdateNode( + id, version, commitMap, shrunkNodes, localLocation); } @Override @@ -112,17 +116,20 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { getType().serialize(byteBuffer); ReadWriteIOUtils.write(version, byteBuffer); ReadWriteIOUtils.write(commitMap.size(), byteBuffer); + for (final Map.Entry entry : commitMap.entrySet()) { - ReadWriteIOUtils.write(entry.getKey().getDataNodeId(), byteBuffer); - ThriftCommonsSerDeUtils.serializeTEndPoint(entry.getKey().getInternalEndPoint(), byteBuffer); + DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate( + entry.getKey(), byteBuffer); ReadWriteIOUtils.write(entry.getValue().length, byteBuffer); byteBuffer.put(entry.getValue()); } + ReadWriteIOUtils.write(shrunkNodes.size(), byteBuffer); for (final TDataNodeLocation location : shrunkNodes) { - ReadWriteIOUtils.write(location.getDataNodeId(), byteBuffer); - ThriftCommonsSerDeUtils.serializeTEndPoint(location.getInternalEndPoint(), byteBuffer); + DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(location, byteBuffer); } + + DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(localLocation, byteBuffer); } @Override @@ -130,50 +137,46 @@ protected void serializeAttributes(final DataOutputStream stream) throws IOExcep getType().serialize(stream); ReadWriteIOUtils.write(version, stream); ReadWriteIOUtils.write(commitMap.size(), stream); + for (final Map.Entry entry : commitMap.entrySet()) { - ReadWriteIOUtils.write(entry.getKey().getDataNodeId(), stream); - ThriftCommonsSerDeUtils.serializeTEndPoint(entry.getKey().getInternalEndPoint(), stream); + DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(entry.getKey(), stream); ReadWriteIOUtils.write(entry.getValue().length, stream); stream.write(entry.getValue()); } + ReadWriteIOUtils.write(shrunkNodes.size(), stream); for (final TDataNodeLocation location : shrunkNodes) { - ReadWriteIOUtils.write(location.getDataNodeId(), stream); - ThriftCommonsSerDeUtils.serializeTEndPoint(location.getInternalEndPoint(), stream); + DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(location, stream); } + + DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(localLocation, stream); } public static PlanNode deserialize(final ByteBuffer buffer) { final long version = ReadWriteIOUtils.readLong(buffer); + int size = ReadWriteIOUtils.readInt(buffer); final Map commitMap = new HashMap<>(size); for (int i = 0; i < size; ++i) { final TDataNodeLocation location = - new TDataNodeLocation( - ReadWriteIOUtils.readInt(buffer), - null, - ThriftCommonsSerDeUtils.deserializeTEndPoint(buffer), - null, - null, - null); + DeviceAttributeRemoteUpdater.deserializeNodeLocationForAttributeUpdate(buffer); final byte[] commitBuffer = new byte[ReadWriteIOUtils.readInt(buffer)]; buffer.get(commitBuffer); commitMap.put(location, commitBuffer); } + size = ReadWriteIOUtils.readInt(buffer); final Set shrunkNodes = new HashSet<>(); for (int i = 0; i < size; ++i) { shrunkNodes.add( - new TDataNodeLocation( - ReadWriteIOUtils.readInt(buffer), - null, - ThriftCommonsSerDeUtils.deserializeTEndPoint(buffer), - null, - null, - null)); + DeviceAttributeRemoteUpdater.deserializeNodeLocationForAttributeUpdate(buffer)); } + + final TDataNodeLocation localLocation = + DeviceAttributeRemoteUpdater.deserializeNodeLocationForAttributeUpdate(buffer); final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); - return new TableDeviceAttributeCommitUpdateNode(planNodeId, version, commitMap, shrunkNodes); + return new TableDeviceAttributeCommitUpdateNode( + planNodeId, version, commitMap, shrunkNodes, localLocation); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 81076b5092559..40ad19b5b1aad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -41,6 +41,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.Comparator; import java.util.HashMap; @@ -320,26 +321,12 @@ public void loadFromSnapshot(final File snapshotDir) throws IOException { private void deserialize(final InputStream inputStream) throws IOException { int size = ReadWriteIOUtils.readInt(inputStream); for (int i = 0; i < size; i++) { - targetDataNodeLocations.add( - new TDataNodeLocation( - ReadWriteIOUtils.readInt(inputStream), - null, - ThriftCommonsSerDeUtils.deserializeTEndPoint(inputStream), - null, - null, - null)); + targetDataNodeLocations.add(deserializeNodeLocationForAttributeUpdate(inputStream)); } size = ReadWriteIOUtils.readInt(inputStream); for (int i = 0; i < size; ++i) { - final TDataNodeLocation location = - new TDataNodeLocation( - ReadWriteIOUtils.readInt(inputStream), - null, - ThriftCommonsSerDeUtils.deserializeTEndPoint(inputStream), - null, - null, - null); + final TDataNodeLocation location = deserializeNodeLocationForAttributeUpdate(inputStream); final UpdateContainer container = ReadWriteIOUtils.readBool(inputStream) ? new UpdateDetailContainer() @@ -351,6 +338,40 @@ private void deserialize(final InputStream inputStream) throws IOException { version.set(ReadWriteIOUtils.readLong(inputStream)); } + public static void serializeNodeLocation4AttributeUpdate( + final TDataNodeLocation location, final ByteBuffer buffer) { + ReadWriteIOUtils.write(location.getDataNodeId(), buffer); + ThriftCommonsSerDeUtils.serializeTEndPoint(location.getInternalEndPoint(), buffer); + } + + public static void serializeNodeLocation4AttributeUpdate( + final TDataNodeLocation location, final OutputStream stream) throws IOException { + ReadWriteIOUtils.write(location.getDataNodeId(), stream); + ThriftCommonsSerDeUtils.serializeTEndPoint(location.getInternalEndPoint(), stream); + } + + public static TDataNodeLocation deserializeNodeLocationForAttributeUpdate( + final ByteBuffer buffer) { + return new TDataNodeLocation( + ReadWriteIOUtils.readInt(buffer), + null, + ThriftCommonsSerDeUtils.deserializeTEndPoint(buffer), + null, + null, + null); + } + + public static TDataNodeLocation deserializeNodeLocationForAttributeUpdate( + final InputStream inputStream) throws IOException { + return new TDataNodeLocation( + ReadWriteIOUtils.readInt(inputStream), + null, + ThriftCommonsSerDeUtils.deserializeTEndPoint(inputStream), + null, + null, + null); + } + /////////////////////////////// Memory /////////////////////////////// private void updateMemory(final long size) { From 91413e324cf6d1ef649e01510d5447f3fd0ea896 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:36:16 +0800 Subject: [PATCH 146/213] More --- .../node/schema/TableNodeLocationAddNode.java | 18 ++++-------------- .../update/DeviceAttributeRemoteUpdater.java | 8 ++------ 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java index 206aa00781265..09c180fc7ea10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; -import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -28,8 +27,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; - -import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeRemoteUpdater; import java.io.DataOutputStream; import java.io.IOException; @@ -84,26 +82,18 @@ public R accept(final PlanVisitor visitor, final C context) { @Override protected void serializeAttributes(final ByteBuffer byteBuffer) { getType().serialize(byteBuffer); - ReadWriteIOUtils.write(location.getDataNodeId(), byteBuffer); - ThriftCommonsSerDeUtils.serializeTEndPoint(location.getInternalEndPoint(), byteBuffer); + DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(location, byteBuffer); } @Override protected void serializeAttributes(final DataOutputStream stream) throws IOException { getType().serialize(stream); - ReadWriteIOUtils.write(location.getDataNodeId(), stream); - ThriftCommonsSerDeUtils.serializeTEndPoint(location.getInternalEndPoint(), stream); + DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(location, stream); } public static PlanNode deserialize(final ByteBuffer buffer) { final TDataNodeLocation location = - new TDataNodeLocation( - ReadWriteIOUtils.readInt(buffer), - null, - ThriftCommonsSerDeUtils.deserializeTEndPoint(buffer), - null, - null, - null); + DeviceAttributeRemoteUpdater.deserializeNodeLocationForAttributeUpdate(buffer); final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); return new TableNodeLocationAddNode(planNodeId, location); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java index 40ad19b5b1aad..8c318fce94c49 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java @@ -282,17 +282,13 @@ public synchronized boolean createSnapshot(final File targetDir) { private void serialize(final OutputStream outputStream) throws IOException { ReadWriteIOUtils.write(targetDataNodeLocations.size(), outputStream); for (final TDataNodeLocation targetDataNodeLocation : targetDataNodeLocations) { - ReadWriteIOUtils.write(targetDataNodeLocation.getDataNodeId(), outputStream); - ThriftCommonsSerDeUtils.serializeTEndPoint( - targetDataNodeLocation.getInternalEndPoint(), outputStream); + serializeNodeLocation4AttributeUpdate(targetDataNodeLocation, outputStream); } ReadWriteIOUtils.write(attributeUpdateMap.size(), outputStream); for (final Map.Entry entry : attributeUpdateMap.entrySet()) { - ReadWriteIOUtils.write(entry.getKey().getDataNodeId(), outputStream); - ThriftCommonsSerDeUtils.serializeTEndPoint( - entry.getKey().getInternalEndPoint(), outputStream); + serializeNodeLocation4AttributeUpdate(entry.getKey(), outputStream); entry.getValue().serialize(outputStream); } From c95c41abcfef3a3c0360331c94eb631ed783f886 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:38:34 +0800 Subject: [PATCH 147/213] Update AbstractTableDeviceQueryNode.java --- .../schema/AbstractTableDeviceQueryNode.java | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java index 269b779dff2c3..f273cf6e64c30 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java @@ -21,12 +21,12 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.commons.schema.filter.SchemaFilter; -import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeRemoteUpdater; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -99,8 +99,8 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { if (Objects.nonNull(senderLocation)) { ReadWriteIOUtils.write(true, byteBuffer); - ReadWriteIOUtils.write(senderLocation.getDataNodeId(), byteBuffer); - ThriftCommonsSerDeUtils.serializeTEndPoint(senderLocation.getInternalEndPoint(), byteBuffer); + DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate( + senderLocation, byteBuffer); } else { ReadWriteIOUtils.write(false, byteBuffer); } @@ -132,8 +132,7 @@ protected void serializeAttributes(final DataOutputStream stream) throws IOExcep if (Objects.nonNull(senderLocation)) { ReadWriteIOUtils.write(true, stream); - ReadWriteIOUtils.write(senderLocation.getDataNodeId(), stream); - ThriftCommonsSerDeUtils.serializeTEndPoint(senderLocation.getInternalEndPoint(), stream); + DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(senderLocation, stream); } else { ReadWriteIOUtils.write(false, stream); } @@ -167,13 +166,7 @@ protected static PlanNode deserialize(final ByteBuffer buffer, final boolean isS TDataNodeLocation senderLocation = null; if (ReadWriteIOUtils.readBool(buffer)) { senderLocation = - new TDataNodeLocation( - ReadWriteIOUtils.readInt(buffer), - null, - ThriftCommonsSerDeUtils.deserializeTEndPoint(buffer), - null, - null, - null); + DeviceAttributeRemoteUpdater.deserializeNodeLocationForAttributeUpdate(buffer); } final long offset = isScan ? ReadWriteIOUtils.readLong(buffer) : 0; From 67f5ace9d7bdc0e8e09d4a39f86285155aafc3c5 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:39:30 +0800 Subject: [PATCH 148/213] Update TableDeviceFetchNode.java --- .../node/schema/TableDeviceFetchNode.java | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java index 12d546445d86c..fbf11f469a272 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java @@ -20,13 +20,13 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; -import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeRemoteUpdater; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -92,8 +92,8 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { if (Objects.nonNull(senderLocation)) { ReadWriteIOUtils.write(true, byteBuffer); - ReadWriteIOUtils.write(senderLocation.getDataNodeId(), byteBuffer); - ThriftCommonsSerDeUtils.serializeTEndPoint(senderLocation.getInternalEndPoint(), byteBuffer); + DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate( + senderLocation, byteBuffer); } else { ReadWriteIOUtils.write(false, byteBuffer); } @@ -120,8 +120,7 @@ protected void serializeAttributes(final DataOutputStream stream) throws IOExcep if (Objects.nonNull(senderLocation)) { ReadWriteIOUtils.write(true, stream); - ReadWriteIOUtils.write(senderLocation.getDataNodeId(), stream); - ThriftCommonsSerDeUtils.serializeTEndPoint(senderLocation.getInternalEndPoint(), stream); + DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(senderLocation, stream); } else { ReadWriteIOUtils.write(false, stream); } @@ -151,13 +150,7 @@ public static TableDeviceFetchNode deserialize(final ByteBuffer buffer) { TDataNodeLocation senderLocation = null; if (ReadWriteIOUtils.readBool(buffer)) { senderLocation = - new TDataNodeLocation( - ReadWriteIOUtils.readInt(buffer), - null, - ThriftCommonsSerDeUtils.deserializeTEndPoint(buffer), - null, - null, - null); + DeviceAttributeRemoteUpdater.deserializeNodeLocationForAttributeUpdate(buffer); } final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); From a3dce1f73c77f8ef5e32ec7d0b4ae9510212319b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:42:58 +0800 Subject: [PATCH 149/213] Update GeneralRegionAttributeSecurityService.java --- ...GeneralRegionAttributeSecurityService.java | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 6da37b7b749ff..b83d8ecaf19a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.attribute.update; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.client.request.AsyncRequestContext; @@ -31,6 +32,7 @@ import org.apache.iotdb.commons.service.IService; import org.apache.iotdb.commons.service.ServiceType; import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp; +import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; @@ -73,6 +75,8 @@ public class GeneralRegionAttributeSecurityService implements IService { private static final Logger LOGGER = LoggerFactory.getLogger(GeneralRegionAttributeSecurityService.class); + private static final IoTDBConfig iotdbConfig = IoTDBDescriptor.getInstance().getConfig(); + private final ExecutorService securityServiceExecutor = IoTDBThreadPoolFactory.newSingleThreadExecutor( ThreadName.GENERAL_REGION_ATTRIBUTE_SECURITY_SERVICE.getName()); @@ -143,7 +147,15 @@ private void execute() { new PlanNodeId(""), pair.getLeft(), pair.getRight(), - shrinkMap.getOrDefault(schemaRegionId, Collections.emptySet()))) + shrinkMap.getOrDefault(schemaRegionId, Collections.emptySet()), + new TDataNodeLocation( + iotdbConfig.getDataNodeId(), + null, + new TEndPoint( + iotdbConfig.getInternalAddress(), iotdbConfig.getInternalPort()), + null, + null, + null))) .isAccepted()) { // May fail due to region shutdown, migration or other reasons // Just ignore @@ -155,9 +167,7 @@ private void execute() { if (!skipNext) { condition.await( - IoTDBDescriptor.getInstance() - .getConfig() - .getGeneralRegionAttributeSecurityServiceIntervalSeconds(), + iotdbConfig.getGeneralRegionAttributeSecurityServiceIntervalSeconds(), TimeUnit.SECONDS); } skipNext = false; @@ -254,12 +264,10 @@ private void execute() { } v.setRight(v.getRight() + 1); if (System.currentTimeMillis() - v.getLeft() - >= IoTDBDescriptor.getInstance() - .getConfig() + >= iotdbConfig .getGeneralRegionAttributeSecurityServiceFailureDurationSecondsToFetch() || v.getRight() - >= IoTDBDescriptor.getInstance() - .getConfig() + >= iotdbConfig .getGeneralRegionAttributeSecurityServiceFailureTimesToFetch()) { needFetch.set(true); } From d15fc973ff1ab37b098b46c4c6c2017987b6067e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 18:07:57 +0800 Subject: [PATCH 150/213] Rename --- .../schema/AbstractTableDeviceQueryNode.java | 8 +++---- .../TableDeviceAttributeCommitUpdateNode.java | 20 ++++++++-------- .../node/schema/TableDeviceFetchNode.java | 8 +++---- .../node/schema/TableNodeLocationAddNode.java | 8 +++---- ....java => DeviceAttributeCacheUpdater.java} | 6 ++--- .../impl/SchemaRegionMemoryImpl.java | 24 +++++++++---------- 6 files changed, 37 insertions(+), 37 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/{DeviceAttributeRemoteUpdater.java => DeviceAttributeCacheUpdater.java} (98%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java index f273cf6e64c30..00a512eed62a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java @@ -26,7 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeRemoteUpdater; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeCacheUpdater; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -99,7 +99,7 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { if (Objects.nonNull(senderLocation)) { ReadWriteIOUtils.write(true, byteBuffer); - DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate( + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate( senderLocation, byteBuffer); } else { ReadWriteIOUtils.write(false, byteBuffer); @@ -132,7 +132,7 @@ protected void serializeAttributes(final DataOutputStream stream) throws IOExcep if (Objects.nonNull(senderLocation)) { ReadWriteIOUtils.write(true, stream); - DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(senderLocation, stream); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(senderLocation, stream); } else { ReadWriteIOUtils.write(false, stream); } @@ -166,7 +166,7 @@ protected static PlanNode deserialize(final ByteBuffer buffer, final boolean isS TDataNodeLocation senderLocation = null; if (ReadWriteIOUtils.readBool(buffer)) { senderLocation = - DeviceAttributeRemoteUpdater.deserializeNodeLocationForAttributeUpdate(buffer); + DeviceAttributeCacheUpdater.deserializeNodeLocationForAttributeUpdate(buffer); } final long offset = isScan ? ReadWriteIOUtils.readLong(buffer) : 0; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java index 913e8e0569b2a..801a541a1662f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java @@ -27,7 +27,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; -import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeRemoteUpdater; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeCacheUpdater; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -118,7 +118,7 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { ReadWriteIOUtils.write(commitMap.size(), byteBuffer); for (final Map.Entry entry : commitMap.entrySet()) { - DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate( + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate( entry.getKey(), byteBuffer); ReadWriteIOUtils.write(entry.getValue().length, byteBuffer); byteBuffer.put(entry.getValue()); @@ -126,10 +126,10 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { ReadWriteIOUtils.write(shrunkNodes.size(), byteBuffer); for (final TDataNodeLocation location : shrunkNodes) { - DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(location, byteBuffer); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(location, byteBuffer); } - DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(localLocation, byteBuffer); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(localLocation, byteBuffer); } @Override @@ -139,17 +139,17 @@ protected void serializeAttributes(final DataOutputStream stream) throws IOExcep ReadWriteIOUtils.write(commitMap.size(), stream); for (final Map.Entry entry : commitMap.entrySet()) { - DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(entry.getKey(), stream); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(entry.getKey(), stream); ReadWriteIOUtils.write(entry.getValue().length, stream); stream.write(entry.getValue()); } ReadWriteIOUtils.write(shrunkNodes.size(), stream); for (final TDataNodeLocation location : shrunkNodes) { - DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(location, stream); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(location, stream); } - DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(localLocation, stream); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(localLocation, stream); } public static PlanNode deserialize(final ByteBuffer buffer) { @@ -159,7 +159,7 @@ public static PlanNode deserialize(final ByteBuffer buffer) { final Map commitMap = new HashMap<>(size); for (int i = 0; i < size; ++i) { final TDataNodeLocation location = - DeviceAttributeRemoteUpdater.deserializeNodeLocationForAttributeUpdate(buffer); + DeviceAttributeCacheUpdater.deserializeNodeLocationForAttributeUpdate(buffer); final byte[] commitBuffer = new byte[ReadWriteIOUtils.readInt(buffer)]; buffer.get(commitBuffer); commitMap.put(location, commitBuffer); @@ -169,11 +169,11 @@ public static PlanNode deserialize(final ByteBuffer buffer) { final Set shrunkNodes = new HashSet<>(); for (int i = 0; i < size; ++i) { shrunkNodes.add( - DeviceAttributeRemoteUpdater.deserializeNodeLocationForAttributeUpdate(buffer)); + DeviceAttributeCacheUpdater.deserializeNodeLocationForAttributeUpdate(buffer)); } final TDataNodeLocation localLocation = - DeviceAttributeRemoteUpdater.deserializeNodeLocationForAttributeUpdate(buffer); + DeviceAttributeCacheUpdater.deserializeNodeLocationForAttributeUpdate(buffer); final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); return new TableDeviceAttributeCommitUpdateNode( planNodeId, version, commitMap, shrunkNodes, localLocation); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java index fbf11f469a272..07fe92b32c0ac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java @@ -26,7 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; -import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeRemoteUpdater; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeCacheUpdater; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -92,7 +92,7 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { if (Objects.nonNull(senderLocation)) { ReadWriteIOUtils.write(true, byteBuffer); - DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate( + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate( senderLocation, byteBuffer); } else { ReadWriteIOUtils.write(false, byteBuffer); @@ -120,7 +120,7 @@ protected void serializeAttributes(final DataOutputStream stream) throws IOExcep if (Objects.nonNull(senderLocation)) { ReadWriteIOUtils.write(true, stream); - DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(senderLocation, stream); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(senderLocation, stream); } else { ReadWriteIOUtils.write(false, stream); } @@ -150,7 +150,7 @@ public static TableDeviceFetchNode deserialize(final ByteBuffer buffer) { TDataNodeLocation senderLocation = null; if (ReadWriteIOUtils.readBool(buffer)) { senderLocation = - DeviceAttributeRemoteUpdater.deserializeNodeLocationForAttributeUpdate(buffer); + DeviceAttributeCacheUpdater.deserializeNodeLocationForAttributeUpdate(buffer); } final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java index 09c180fc7ea10..23481d498661d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableNodeLocationAddNode.java @@ -27,7 +27,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; -import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeRemoteUpdater; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeCacheUpdater; import java.io.DataOutputStream; import java.io.IOException; @@ -82,18 +82,18 @@ public R accept(final PlanVisitor visitor, final C context) { @Override protected void serializeAttributes(final ByteBuffer byteBuffer) { getType().serialize(byteBuffer); - DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(location, byteBuffer); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(location, byteBuffer); } @Override protected void serializeAttributes(final DataOutputStream stream) throws IOException { getType().serialize(stream); - DeviceAttributeRemoteUpdater.serializeNodeLocation4AttributeUpdate(location, stream); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(location, stream); } public static PlanNode deserialize(final ByteBuffer buffer) { final TDataNodeLocation location = - DeviceAttributeRemoteUpdater.deserializeNodeLocationForAttributeUpdate(buffer); + DeviceAttributeCacheUpdater.deserializeNodeLocationForAttributeUpdate(buffer); final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); return new TableNodeLocationAddNode(planNodeId, location); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java similarity index 98% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index 8c318fce94c49..e5155f9995269 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeRemoteUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -54,8 +54,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -public class DeviceAttributeRemoteUpdater { - private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeRemoteUpdater.class); +public class DeviceAttributeCacheUpdater { + private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeCacheUpdater.class); private static final int UPDATE_DETAIL_CONTAINER_SEND_MIN_LIMIT_BYTES = 1024; // All the data node locations shall only have internal endpoint with all the other endpoints set @@ -73,7 +73,7 @@ public class DeviceAttributeRemoteUpdater { private final Map updateContainerStatistics = new HashMap<>(); - public DeviceAttributeRemoteUpdater(final MemSchemaRegionStatistics regionStatistics) { + public DeviceAttributeCacheUpdater(final MemSchemaRegionStatistics regionStatistics) { this.regionStatistics = regionStatistics; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 32d680e1c1641..a8a1e731713d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -71,7 +71,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionUtils; import org.apache.iotdb.db.schemaengine.schemaregion.attribute.DeviceAttributeStore; import org.apache.iotdb.db.schemaengine.schemaregion.attribute.IDeviceAttributeStore; -import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeRemoteUpdater; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeCacheUpdater; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.FakeCRC32Deserializer; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.FakeCRC32Serializer; import org.apache.iotdb.db.schemaengine.schemaregion.logfile.SchemaLogReader; @@ -196,7 +196,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { private MTreeBelowSGMemoryImpl mtree; private TagManager tagManager; private IDeviceAttributeStore deviceAttributeStore; - private DeviceAttributeRemoteUpdater deviceAttributeRemoteUpdater; + private DeviceAttributeCacheUpdater deviceAttributeCacheUpdater; // region Interfaces and Implementation of initialization、snapshot、recover and clear public SchemaRegionMemoryImpl(ISchemaRegionParams schemaRegionParams) throws MetadataException { @@ -247,7 +247,7 @@ public synchronized void init() throws MetadataException { isRecovering = true; deviceAttributeStore = new DeviceAttributeStore(regionStatistics); - deviceAttributeRemoteUpdater = new DeviceAttributeRemoteUpdater(regionStatistics); + deviceAttributeCacheUpdater = new DeviceAttributeCacheUpdater(regionStatistics); tagManager = new TagManager(schemaRegionDirPath, regionStatistics); mtree = new MTreeBelowSGMemoryImpl( @@ -507,7 +507,7 @@ public synchronized boolean createSnapshot(final File snapshotDir) { currentResult); snapshotStartTime = System.currentTimeMillis(); - currentResult = deviceAttributeRemoteUpdater.createSnapshot(snapshotDir); + currentResult = deviceAttributeCacheUpdater.createSnapshot(snapshotDir); isSuccess = isSuccess && currentResult; logger.info( "Device attribute remote updater snapshot creation of schemaRegion {} costs {}ms. Status: {}", @@ -549,8 +549,8 @@ public void loadSnapshot(final File latestSnapshotRootDir) { System.currentTimeMillis() - snapshotStartTime); snapshotStartTime = System.currentTimeMillis(); - deviceAttributeRemoteUpdater = new DeviceAttributeRemoteUpdater(regionStatistics); - deviceAttributeRemoteUpdater.loadFromSnapshot(latestSnapshotRootDir); + deviceAttributeCacheUpdater = new DeviceAttributeCacheUpdater(regionStatistics); + deviceAttributeCacheUpdater.loadFromSnapshot(latestSnapshotRootDir); logger.info( "Device attribute remote updater snapshot loading of schemaRegion {} costs {}ms.", schemaRegionId, @@ -1381,7 +1381,7 @@ public void createOrUpdateTableDevice(final CreateOrUpdateTableDeviceNode node) pointer -> { updateAttribute( databaseName, tableName, deviceId, pointer, attributeNameList, attributeValueList); - deviceAttributeRemoteUpdater.afterUpdate(); + deviceAttributeCacheUpdater.afterUpdate(); }); } writeToMLog(node); @@ -1400,7 +1400,7 @@ private void updateAttribute( TableDeviceSchemaFetcher.getInstance() .getTableDeviceCache() .update(databaseName, tableName, deviceId, resultMap); - deviceAttributeRemoteUpdater.update(tableName, deviceId, resultMap); + deviceAttributeCacheUpdater.update(tableName, deviceId, resultMap); } } @@ -1416,7 +1416,7 @@ public void updateTableDeviceAttribute(final TableDeviceAttributeUpdateNode upda mtree.updateTableDevice(pattern, batchUpdater); } } - deviceAttributeRemoteUpdater.afterUpdate(); + deviceAttributeCacheUpdater.afterUpdate(); writeToMLog(updateNode); } @@ -1568,19 +1568,19 @@ public ISchemaReader getTableDeviceReader( @Override public Pair> getAttributeUpdateInfo( final AtomicInteger limit) { - return deviceAttributeRemoteUpdater.getAttributeUpdateInfo(limit); + return deviceAttributeCacheUpdater.getAttributeUpdateInfo(limit); } @Override public void commitUpdateAttribute(final TableDeviceAttributeCommitUpdateNode node) throws MetadataException { - deviceAttributeRemoteUpdater.commit(node); + deviceAttributeCacheUpdater.commit(node); writeToMLog(node); } @Override public void addNodeLocation(final TableNodeLocationAddNode node) throws MetadataException { - deviceAttributeRemoteUpdater.addLocation(node.getLocation()); + deviceAttributeCacheUpdater.addLocation(node.getLocation()); writeToMLog(node); } From 7251ecd81a63fb055eed00e70d85518bd6d81724 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 18:17:32 +0800 Subject: [PATCH 151/213] handle leader location --- .../schema/AbstractTableDeviceQueryNode.java | 3 +-- .../TableDeviceAttributeCommitUpdateNode.java | 19 +++++++++++-------- .../node/schema/TableDeviceFetchNode.java | 3 +-- .../update/DeviceAttributeCacheUpdater.java | 9 +++++++++ 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java index 00a512eed62a7..ea391c1390ab4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableDeviceQueryNode.java @@ -99,8 +99,7 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { if (Objects.nonNull(senderLocation)) { ReadWriteIOUtils.write(true, byteBuffer); - DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate( - senderLocation, byteBuffer); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(senderLocation, byteBuffer); } else { ReadWriteIOUtils.write(false, byteBuffer); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java index 801a541a1662f..430d2a99695b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceAttributeCommitUpdateNode.java @@ -45,7 +45,7 @@ public class TableDeviceAttributeCommitUpdateNode extends PlanNode implements IS private final long version; private final Map commitMap; private final Set shrunkNodes; - private final TDataNodeLocation localLocation; + private final TDataNodeLocation leaderLocation; public static final TableDeviceAttributeCommitUpdateNode MOCK_INSTANCE = new TableDeviceAttributeCommitUpdateNode(new PlanNodeId(""), 0L, null, null, null); @@ -55,12 +55,12 @@ public TableDeviceAttributeCommitUpdateNode( final long version, final Map commitMap, final Set shrunkNodes, - final TDataNodeLocation localLocation) { + final TDataNodeLocation leaderLocation) { super(id); this.version = version; this.commitMap = commitMap; this.shrunkNodes = shrunkNodes; - this.localLocation = localLocation; + this.leaderLocation = leaderLocation; } public long getVersion() { @@ -75,6 +75,10 @@ public Set getShrunkNodes() { return shrunkNodes; } + public TDataNodeLocation getLeaderLocation() { + return leaderLocation; + } + @Override public List getChildren() { return null; @@ -93,7 +97,7 @@ public PlanNodeType getType() { @Override public PlanNode clone() { return new TableDeviceAttributeCommitUpdateNode( - id, version, commitMap, shrunkNodes, localLocation); + id, version, commitMap, shrunkNodes, leaderLocation); } @Override @@ -118,8 +122,7 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { ReadWriteIOUtils.write(commitMap.size(), byteBuffer); for (final Map.Entry entry : commitMap.entrySet()) { - DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate( - entry.getKey(), byteBuffer); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(entry.getKey(), byteBuffer); ReadWriteIOUtils.write(entry.getValue().length, byteBuffer); byteBuffer.put(entry.getValue()); } @@ -129,7 +132,7 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(location, byteBuffer); } - DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(localLocation, byteBuffer); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(leaderLocation, byteBuffer); } @Override @@ -149,7 +152,7 @@ protected void serializeAttributes(final DataOutputStream stream) throws IOExcep DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(location, stream); } - DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(localLocation, stream); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(leaderLocation, stream); } public static PlanNode deserialize(final ByteBuffer buffer) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java index 07fe92b32c0ac..7fcfbc7c7dcc4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/TableDeviceFetchNode.java @@ -92,8 +92,7 @@ protected void serializeAttributes(final ByteBuffer byteBuffer) { if (Objects.nonNull(senderLocation)) { ReadWriteIOUtils.write(true, byteBuffer); - DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate( - senderLocation, byteBuffer); + DeviceAttributeCacheUpdater.serializeNodeLocation4AttributeUpdate(senderLocation, byteBuffer); } else { ReadWriteIOUtils.write(false, byteBuffer); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index e5155f9995269..d9745c4704952 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -154,6 +154,15 @@ public void commit(final TableDeviceAttributeCommitUpdateNode node) { }); updateContainerStatistics.keySet().removeAll(shrunkNodes); + final TDataNodeLocation leaderLocation = node.getLeaderLocation(); + if (version.get() == node.getVersion() && attributeUpdateMap.containsKey(leaderLocation)) { + releaseMemory( + updateContainerStatistics.containsKey(leaderLocation) + ? updateContainerStatistics.get(leaderLocation).getContainerSize() + : ((UpdateClearContainer) attributeUpdateMap.get(leaderLocation)).ramBytesUsed()); + attributeUpdateMap.remove(leaderLocation); + } + node.getCommitMap() .forEach( ((location, bytes) -> From ebf51a1693596f17112ee8f0aea6ac1aede64d3c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 18:19:34 +0800 Subject: [PATCH 152/213] Update DeviceAttributeCacheUpdater.java --- .../update/DeviceAttributeCacheUpdater.java | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index d9745c4704952..e7d234d8930d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -139,19 +139,16 @@ public Pair> getAttributeUpdateInfo( public void commit(final TableDeviceAttributeCommitUpdateNode node) { final Set shrunkNodes = node.getShrunkNodes(); targetDataNodeLocations.removeAll(shrunkNodes); - attributeUpdateMap - .entrySet() - .removeIf( - entry -> { - if (shrunkNodes.contains(entry.getKey())) { - releaseMemory( - updateContainerStatistics.containsKey(entry.getKey()) - ? updateContainerStatistics.get(entry.getKey()).getContainerSize() - : ((UpdateClearContainer) entry.getValue()).ramBytesUsed()); - return true; - } - return false; - }); + shrunkNodes.forEach( + location -> { + if (attributeUpdateMap.containsKey(location)) { + releaseMemory( + updateContainerStatistics.containsKey(location) + ? updateContainerStatistics.get(location).getContainerSize() + : ((UpdateClearContainer) attributeUpdateMap.get(location)).ramBytesUsed()); + attributeUpdateMap.remove(location); + } + }); updateContainerStatistics.keySet().removeAll(shrunkNodes); final TDataNodeLocation leaderLocation = node.getLeaderLocation(); From 1de5403454237d096bf5a8247f25a60dd5aee203 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 18:22:25 +0800 Subject: [PATCH 153/213] Update DeviceAttributeCacheUpdater.java --- .../attribute/update/DeviceAttributeCacheUpdater.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index e7d234d8930d6..0a6cac0bf17cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -286,6 +286,8 @@ public synchronized boolean createSnapshot(final File targetDir) { } private void serialize(final OutputStream outputStream) throws IOException { + ReadWriteIOUtils.write(version.get(), outputStream); + ReadWriteIOUtils.write(targetDataNodeLocations.size(), outputStream); for (final TDataNodeLocation targetDataNodeLocation : targetDataNodeLocations) { serializeNodeLocation4AttributeUpdate(targetDataNodeLocation, outputStream); @@ -297,8 +299,6 @@ private void serialize(final OutputStream outputStream) throws IOException { serializeNodeLocation4AttributeUpdate(entry.getKey(), outputStream); entry.getValue().serialize(outputStream); } - - ReadWriteIOUtils.write(version.get(), outputStream); } public void loadFromSnapshot(final File snapshotDir) throws IOException { @@ -321,6 +321,8 @@ public void loadFromSnapshot(final File snapshotDir) throws IOException { } private void deserialize(final InputStream inputStream) throws IOException { + version.set(ReadWriteIOUtils.readLong(inputStream)); + int size = ReadWriteIOUtils.readInt(inputStream); for (int i = 0; i < size; i++) { targetDataNodeLocations.add(deserializeNodeLocationForAttributeUpdate(inputStream)); @@ -336,8 +338,6 @@ private void deserialize(final InputStream inputStream) throws IOException { container.deserialize(inputStream); attributeUpdateMap.put(location, container); } - - version.set(ReadWriteIOUtils.readLong(inputStream)); } public static void serializeNodeLocation4AttributeUpdate( From b6188528eeb5c2aec6b93aa479033a875b840ee5 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 18:24:26 +0800 Subject: [PATCH 154/213] Update DeviceAttributeCacheUpdater.java --- .../attribute/update/DeviceAttributeCacheUpdater.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index 0a6cac0bf17cc..679ab7140011c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -336,6 +336,8 @@ private void deserialize(final InputStream inputStream) throws IOException { ? new UpdateDetailContainer() : new UpdateClearContainer(); container.deserialize(inputStream); + + // TODO: Update local cache for newly migrated region attributeUpdateMap.put(location, container); } } From 3a63917d5f434b2af8df4d96dcc7cafeba10d4d1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 18:35:09 +0800 Subject: [PATCH 155/213] Update GeneralRegionAttributeSecurityService.java --- .../update/GeneralRegionAttributeSecurityService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index b83d8ecaf19a2..9c2a050304097 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -87,7 +87,7 @@ public class GeneralRegionAttributeSecurityService implements IService { Collections.newSetFromMap(new ConcurrentHashMap<>()); private final ReentrantLock lock = new ReentrantLock(); private final Condition condition = lock.newCondition(); - private volatile boolean skipNext = false; + private volatile boolean skipNextSleep = false; private volatile boolean allowSubmitListen = false; public void startBroadcast(final ISchemaRegion schemaRegion) { @@ -106,7 +106,7 @@ public void notifyBroadCast() { lock.unlock(); } } else { - skipNext = true; + skipNextSleep = true; } } @@ -165,12 +165,12 @@ private void execute() { }); } - if (!skipNext) { + if (!skipNextSleep) { condition.await( iotdbConfig.getGeneralRegionAttributeSecurityServiceIntervalSeconds(), TimeUnit.SECONDS); } - skipNext = false; + skipNextSleep = false; } catch (final InterruptedException e) { Thread.currentThread().interrupt(); LOGGER.warn( From 9cb20921d4108d30d7d608430f9113cacf64f31a Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 20:31:03 +0800 Subject: [PATCH 156/213] Update TableDeviceCacheAttributeGuard.java --- .../TableDeviceCacheAttributeGuard.java | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index 4b3e874a06108..e2aaa5ea66b6c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -19,17 +19,33 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher; -import java.util.HashSet; -import java.util.Set; +import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; public class TableDeviceCacheAttributeGuard { - final Set fetchedSchemaRegionIds = new HashSet<>(); + final Map fetchedSchemaRegionIds2LargestVersionMap = new ConcurrentHashMap<>(); public boolean isRegionFetched(final Integer schemaRegionId) { - return fetchedSchemaRegionIds.contains(schemaRegionId); + return fetchedSchemaRegionIds2LargestVersionMap.containsKey(schemaRegionId); } public void addFetchedRegion(final Integer schemaRegionId) { - fetchedSchemaRegionIds.add(schemaRegionId); + fetchedSchemaRegionIds2LargestVersionMap.put(schemaRegionId, Long.MIN_VALUE); + } + + public void handleAttributeUpdate(final TAttributeUpdateReq updateReq) { + // Trim the schema regions with lower or equal version + // or with this node restarted before fetching + updateReq + .getAttributeUpdateMap() + .entrySet() + .removeIf( + entry -> + !fetchedSchemaRegionIds2LargestVersionMap.containsKey(entry.getKey()) + || entry.getValue().getVersion() + <= fetchedSchemaRegionIds2LargestVersionMap.get(entry.getKey())); + } } From ba5fe3f0660291dde1666df2907883c8cf523226 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 20:38:24 +0800 Subject: [PATCH 157/213] Update TableDeviceCacheAttributeGuard.java --- .../fetcher/TableDeviceCacheAttributeGuard.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index e2aaa5ea66b6c..32b2f70e59883 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -42,10 +42,15 @@ public void handleAttributeUpdate(final TAttributeUpdateReq updateReq) { .getAttributeUpdateMap() .entrySet() .removeIf( - entry -> - !fetchedSchemaRegionIds2LargestVersionMap.containsKey(entry.getKey()) - || entry.getValue().getVersion() - <= fetchedSchemaRegionIds2LargestVersionMap.get(entry.getKey())); - + entry -> { + if (!fetchedSchemaRegionIds2LargestVersionMap.containsKey(entry.getKey()) + || entry.getValue().getVersion() + <= fetchedSchemaRegionIds2LargestVersionMap.get(entry.getKey())) { + return true; + } + fetchedSchemaRegionIds2LargestVersionMap.put( + entry.getKey(), entry.getValue().getVersion()); + return false; + }); } } From b0645dbaf92efadb9a7e26b92da5b4310e0477ef Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 10 Oct 2024 20:59:51 +0800 Subject: [PATCH 158/213] some --- .../impl/DataNodeInternalRPCServiceImpl.java | 2 +- .../TableDeviceCacheAttributeGuard.java | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 68a5c79204e72..df836ad4cb040 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -499,7 +499,7 @@ public TLoadResp sendLoadCommand(final TLoadCommandReq req) { @Override public TSStatus updateAttribute(final TAttributeUpdateReq req) { - // TODO + TableDeviceSchemaFetcher.getInstance().getAttributeGuard().handleAttributeUpdate(req); return RpcUtils.SUCCESS_STATUS; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index 32b2f70e59883..cce004ac5e6b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -20,11 +20,19 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher; import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; +import org.apache.iotdb.mpp.rpc.thrift.TSchemaRegionAttributeInfo; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.stream.Collectors; public class TableDeviceCacheAttributeGuard { + + // Unbounded queue + final LinkedBlockingDeque> applyQueue = new LinkedBlockingDeque<>(); final Map fetchedSchemaRegionIds2LargestVersionMap = new ConcurrentHashMap<>(); public boolean isRegionFetched(final Integer schemaRegionId) { @@ -52,5 +60,29 @@ public void handleAttributeUpdate(final TAttributeUpdateReq updateReq) { entry.getKey(), entry.getValue().getVersion()); return false; }); + applyQueue.add( + updateReq.getAttributeUpdateMap().values().stream() + .map(TSchemaRegionAttributeInfo::getBody) + .collect(Collectors.toSet())); + tryUpdateCache(); + } + + public synchronized void tryUpdateCache() { + while (!applyQueue.isEmpty()) { + final Set firstElement = applyQueue.peek(); + if (firstElement instanceof HashSet) { + applyUpdateMessage(firstElement); + } else if (firstElement.isEmpty()) { + applyQueue.removeFirst(); + } else { + break; + } + } + } + + private void applyUpdateMessage(final Set updateContainerBytes) { + for (final Object element : updateContainerBytes) { + final byte[] containerByte = (byte[]) element; + } } } From 369a3a1552c3cad39a211196ea764f2d82e0bd3b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 09:58:42 +0800 Subject: [PATCH 159/213] Update TableDeviceCacheAttributeGuard.java --- .../TableDeviceCacheAttributeGuard.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index cce004ac5e6b8..0aadc034bce15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -22,8 +22,10 @@ import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; import org.apache.iotdb.mpp.rpc.thrift.TSchemaRegionAttributeInfo; +import java.util.Collections; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingDeque; @@ -43,6 +45,21 @@ public void addFetchedRegion(final Integer schemaRegionId) { fetchedSchemaRegionIds2LargestVersionMap.put(schemaRegionId, Long.MIN_VALUE); } + @SuppressWarnings("unchecked") + public Set addFetchQueryId(final long queryId) { + final Set lastSet = applyQueue.peekLast(); + final Set longSet; + if (Objects.isNull(lastSet) || lastSet instanceof HashSet) { + longSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); + applyQueue.add(longSet); + } else { + // The elements must be Set or HashSet + longSet = (Set) lastSet; + } + longSet.add(queryId); + return longSet; + } + public void handleAttributeUpdate(final TAttributeUpdateReq updateReq) { // Trim the schema regions with lower or equal version // or with this node restarted before fetching @@ -72,6 +89,7 @@ public synchronized void tryUpdateCache() { final Set firstElement = applyQueue.peek(); if (firstElement instanceof HashSet) { applyUpdateMessage(firstElement); + applyQueue.removeFirst(); } else if (firstElement.isEmpty()) { applyQueue.removeFirst(); } else { From ac2a8a420fef02d35b6ff0978984e7fa8cc1d5fc Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 10:05:32 +0800 Subject: [PATCH 160/213] Better --- .../TableDeviceCacheAttributeGuard.java | 4 +- .../fetcher/TableDeviceSchemaFetcher.java | 38 ++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index 0aadc034bce15..d861f0918ecfc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -22,6 +22,8 @@ import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; import org.apache.iotdb.mpp.rpc.thrift.TSchemaRegionAttributeInfo; +import javax.annotation.Nonnull; + import java.util.Collections; import java.util.HashSet; import java.util.Map; @@ -46,7 +48,7 @@ public void addFetchedRegion(final Integer schemaRegionId) { } @SuppressWarnings("unchecked") - public Set addFetchQueryId(final long queryId) { + public @Nonnull Set addFetchQueryId(final long queryId) { final Set lastSet = applyQueue.peekLast(); final Set longSet; if (Objects.isNull(lastSet) || lastSet instanceof HashSet) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index 9883e7418c1db..31b136485ff21 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -111,6 +111,8 @@ Map> fetchMissingDeviceSchemaForDat final String table = statement.getTableName(); final TsTable tableInstance = DataNodeTableCache.getInstance().getTable(database, table); + // For the correctness of attribute remote update + final Set queryIdSet = attributeGuard.addFetchQueryId(queryId); final ExecutionResult executionResult = coordinator.executeForTableModel( statement, @@ -122,6 +124,8 @@ Map> fetchMissingDeviceSchemaForDat "Fetch Device for insert", LocalExecutionPlanner.getInstance().metadata, config.getQueryTimeoutThreshold()); + queryIdSet.remove(queryId); + attributeGuard.tryUpdateCache(); if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new RuntimeException( new IoTDBException( @@ -379,19 +383,27 @@ private void fetchMissingDeviceSchemaForQuery( final String table = tableInstance.getTableName(); final long queryId = SessionManager.getInstance().requestQueryId(); - final ExecutionResult executionResult = - coordinator.executeForTableModel( - statement, - relationSqlParser, - SessionManager.getInstance().getCurrSession(), - queryId, - SessionManager.getInstance() - .getSessionInfo(SessionManager.getInstance().getCurrSession()), - String.format( - "fetch device for query %s : %s", - mppQueryContext.getQueryId(), mppQueryContext.getSql()), - LocalExecutionPlanner.getInstance().metadata, - config.getQueryTimeoutThreshold()); + // For the correctness of attribute remote update + final Set queryIdSet = attributeGuard.addFetchQueryId(queryId); + final ExecutionResult executionResult; + try { + executionResult = + coordinator.executeForTableModel( + statement, + relationSqlParser, + SessionManager.getInstance().getCurrSession(), + queryId, + SessionManager.getInstance() + .getSessionInfo(SessionManager.getInstance().getCurrSession()), + String.format( + "fetch device for query %s : %s", + mppQueryContext.getQueryId(), mppQueryContext.getSql()), + LocalExecutionPlanner.getInstance().metadata, + config.getQueryTimeoutThreshold()); + } finally { + queryIdSet.remove(queryId); + attributeGuard.tryUpdateCache(); + } if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new RuntimeException( new IoTDBException( From ae49b8a01ddf9e4be5f92fd7ce466aead44e847a Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 10:07:30 +0800 Subject: [PATCH 161/213] Update TableDeviceSchemaFetcher.java --- .../fetcher/TableDeviceSchemaFetcher.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index 31b136485ff21..3221bcb675db7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -113,19 +113,23 @@ Map> fetchMissingDeviceSchemaForDat // For the correctness of attribute remote update final Set queryIdSet = attributeGuard.addFetchQueryId(queryId); - final ExecutionResult executionResult = - coordinator.executeForTableModel( - statement, - relationSqlParser, - SessionManager.getInstance().getCurrSession(), - queryId, - SessionManager.getInstance() - .getSessionInfo(SessionManager.getInstance().getCurrSession()), - "Fetch Device for insert", - LocalExecutionPlanner.getInstance().metadata, - config.getQueryTimeoutThreshold()); - queryIdSet.remove(queryId); - attributeGuard.tryUpdateCache(); + final ExecutionResult executionResult; + try { + executionResult = + coordinator.executeForTableModel( + statement, + relationSqlParser, + SessionManager.getInstance().getCurrSession(), + queryId, + SessionManager.getInstance() + .getSessionInfo(SessionManager.getInstance().getCurrSession()), + "Fetch Device for insert", + LocalExecutionPlanner.getInstance().metadata, + config.getQueryTimeoutThreshold()); + } finally { + queryIdSet.remove(queryId); + attributeGuard.tryUpdateCache(); + } if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new RuntimeException( new IoTDBException( From ff66f4d39395635fc3e687bf2316d92def70e9b0 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 10:10:24 +0800 Subject: [PATCH 162/213] Update TableDeviceSchemaFetcher.java --- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index 3221bcb675db7..7185051b0abe1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -388,7 +388,10 @@ private void fetchMissingDeviceSchemaForQuery( final long queryId = SessionManager.getInstance().requestQueryId(); // For the correctness of attribute remote update - final Set queryIdSet = attributeGuard.addFetchQueryId(queryId); + Set queryIdSet = null; + if (Objects.nonNull(statement.getPartitionKeyList())) { + queryIdSet = attributeGuard.addFetchQueryId(queryId); + } final ExecutionResult executionResult; try { executionResult = @@ -405,8 +408,10 @@ private void fetchMissingDeviceSchemaForQuery( LocalExecutionPlanner.getInstance().metadata, config.getQueryTimeoutThreshold()); } finally { - queryIdSet.remove(queryId); - attributeGuard.tryUpdateCache(); + if (Objects.nonNull(queryIdSet)) { + queryIdSet.remove(queryId); + attributeGuard.tryUpdateCache(); + } } if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new RuntimeException( From 5b0fe5089c64956e2efa91e1ff070daf8972e993 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 10:10:55 +0800 Subject: [PATCH 163/213] Update TableDeviceCacheAttributeGuard.java --- .../metadata/fetcher/TableDeviceCacheAttributeGuard.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index d861f0918ecfc..49bf52dfc01e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -55,7 +55,7 @@ public void addFetchedRegion(final Integer schemaRegionId) { longSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); applyQueue.add(longSet); } else { - // The elements must be Set or HashSet + // The elements must be SetFromMap or HashSet longSet = (Set) lastSet; } longSet.add(queryId); From 4f4c0e1898040e7d045fc60b73545967ed21f3ec Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 10:37:07 +0800 Subject: [PATCH 164/213] remaining improvement --- .../db/schemaengine/schemaregion/ISchemaRegion.java | 4 +++- .../GeneralRegionAttributeSecurityService.java | 8 +++++++- .../attribute/update/UpdateClearContainer.java | 11 ++++++++--- .../attribute/update/UpdateContainer.java | 4 +++- .../attribute/update/UpdateDetailContainer.java | 13 ++++++++++--- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 4 ++-- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 2 +- 7 files changed, 34 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index e72b924453d55..cc36504722999 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -58,6 +58,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; /** @@ -355,7 +356,8 @@ ISchemaReader getTableDeviceReader( final String table, final List devicePathList) throws MetadataException; // region Interfaces for AttributeUpdate - Pair> getAttributeUpdateInfo(final AtomicInteger limit); + Pair> getAttributeUpdateInfo( + final AtomicInteger limit, final AtomicBoolean hasRemaining); void commitUpdateAttribute(final TableDeviceAttributeCommitUpdateNode node) throws MetadataException; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 9c2a050304097..d86ff26a777e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -121,17 +121,22 @@ private void execute() { .getConfig() .getPipeConnectorRequestSliceThresholdBytes()); + final AtomicBoolean hasRemaining = new AtomicBoolean(false); final Map>> attributeUpdateCommitMap = new HashMap<>(); for (final ISchemaRegion regionLeader : regionLeaders) { final Pair> currentResult = - regionLeader.getAttributeUpdateInfo(limit); + regionLeader.getAttributeUpdateInfo(limit, hasRemaining); if (currentResult.getRight().isEmpty()) { break; } attributeUpdateCommitMap.put(regionLeader.getSchemaRegionId(), currentResult); } + if (hasRemaining.get()) { + skipNextSleep = true; + } + if (!attributeUpdateCommitMap.isEmpty()) { // Send & may shrink final Map> shrinkMap = @@ -159,6 +164,7 @@ private void execute() { .isAccepted()) { // May fail due to region shutdown, migration or other reasons // Just ignore + skipNextSleep = false; LOGGER.warn( "Failed to write attribute commit message to region {}.", schemaRegionId); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index 0a316c8106b81..e9bf6c591fa41 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -65,10 +66,11 @@ public long updateAttribute( } @Override - public byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes) { + public byte[] getUpdateContent( + final @Nonnull AtomicInteger limitBytes, final @Nonnull AtomicBoolean hasRemaining) { final RewritableByteArrayOutputStream outputStream = new RewritableByteArrayOutputStream(); try { - serializeWithLimit(outputStream, limitBytes); + serializeWithLimit(outputStream, limitBytes, hasRemaining); } catch (final IOException ignored) { // ByteArrayOutputStream won't throw IOException } @@ -76,7 +78,9 @@ public byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes) { } private void serializeWithLimit( - final RewritableByteArrayOutputStream outputStream, final AtomicInteger limitBytes) + final RewritableByteArrayOutputStream outputStream, + final AtomicInteger limitBytes, + final AtomicBoolean hasRemaining) throws IOException { ReadWriteIOUtils.write((byte) 0, outputStream); final int setSizeOffset = outputStream.skipInt(); @@ -87,6 +91,7 @@ private void serializeWithLimit( newSize = Integer.BYTES + tableBytes.length; if (limitBytes.get() < newSize) { outputStream.rewrite(setEntryCount, setSizeOffset); + hasRemaining.set(true); return; } limitBytes.addAndGet(-newSize); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java index ab6b00b47ee02..74bd000751be6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateContainer.java @@ -28,6 +28,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @ThreadSafe @@ -41,7 +42,8 @@ long updateAttribute( // thrift threshold and low bandwidth // Note that a "piece" returned is also a complete "updateContainer"'s serialized bytes // The "limitBytes" shall be at least 5 for a "type" and "0" to indicate empty - byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes); + byte[] getUpdateContent( + final @Nonnull AtomicInteger limitBytes, final @Nonnull AtomicBoolean hasRemaining); Pair updateSelfByCommitContainer(final UpdateContainer commitContainer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 762097a099890..07d4dd954a057 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -35,6 +35,7 @@ import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -103,10 +104,11 @@ public long updateAttribute( } @Override - public byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes) { + public byte[] getUpdateContent( + final @Nonnull AtomicInteger limitBytes, final @Nonnull AtomicBoolean hasRemaining) { final RewritableByteArrayOutputStream outputStream = new RewritableByteArrayOutputStream(); try { - serializeWithLimit(outputStream, limitBytes); + serializeWithLimit(outputStream, limitBytes, hasRemaining); } catch (final IOException ignored) { // ByteArrayOutputStream won't throw IOException } @@ -114,7 +116,9 @@ public byte[] getUpdateContent(final @Nonnull AtomicInteger limitBytes) { } private void serializeWithLimit( - final RewritableByteArrayOutputStream outputStream, final AtomicInteger limitBytes) + final RewritableByteArrayOutputStream outputStream, + final AtomicInteger limitBytes, + final AtomicBoolean hasRemaining) throws IOException { ReadWriteIOUtils.write((byte) 1, outputStream); final int mapSizeOffset = outputStream.skipInt(); @@ -126,6 +130,7 @@ private void serializeWithLimit( newSize = 2 * Integer.BYTES + tableEntryBytes.length; if (limitBytes.get() < newSize) { outputStream.rewrite(mapEntryCount, mapSizeOffset); + hasRemaining.set(true); return; } limitBytes.addAndGet(-newSize); @@ -148,6 +153,7 @@ private void serializeWithLimit( if (limitBytes.get() < newSize) { outputStream.rewrite(mapEntryCount, mapSizeOffset); outputStream.rewrite(deviceEntryCount, deviceSizeOffset); + hasRemaining.set(true); return; } limitBytes.addAndGet(-newSize); @@ -172,6 +178,7 @@ private void serializeWithLimit( outputStream.rewrite(mapEntryCount, mapSizeOffset); outputStream.rewrite(deviceEntryCount, deviceSizeOffset); outputStream.rewrite(attributeCount, attributeOffset); + hasRemaining.set(true); return; } limitBytes.addAndGet(-newSize); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index a8a1e731713d2..8b149f2e6097e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1567,8 +1567,8 @@ public ISchemaReader getTableDeviceReader( @Override public Pair> getAttributeUpdateInfo( - final AtomicInteger limit) { - return deviceAttributeCacheUpdater.getAttributeUpdateInfo(limit); + final AtomicInteger limit, final AtomicBoolean hasRemaining) { + return deviceAttributeCacheUpdater.getAttributeUpdateInfo(limit, hasRemaining); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 39113d8c865f5..11c2461a326fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -1496,7 +1496,7 @@ public ISchemaReader getTableDeviceReader( @Override public Pair> getAttributeUpdateInfo( - final AtomicInteger limit) { + final AtomicInteger limit, final AtomicBoolean hasRemaining) { throw new UnsupportedOperationException(); } From 36f42e49ee452607f87dff1c83f9042b185c4659 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:00:27 +0800 Subject: [PATCH 165/213] Update TableDeviceCacheAttributeGuard.java --- .../fetcher/TableDeviceCacheAttributeGuard.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index 49bf52dfc01e2..971bcc5772ff7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeCacheUpdater; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.UpdateContainer; import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; import org.apache.iotdb.mpp.rpc.thrift.TSchemaRegionAttributeInfo; @@ -90,7 +92,9 @@ public synchronized void tryUpdateCache() { while (!applyQueue.isEmpty()) { final Set firstElement = applyQueue.peek(); if (firstElement instanceof HashSet) { - applyUpdateMessage(firstElement); + for (final Object element : firstElement) { + handleContainer(DeviceAttributeCacheUpdater.getContainer((byte[]) element)); + } applyQueue.removeFirst(); } else if (firstElement.isEmpty()) { applyQueue.removeFirst(); @@ -100,9 +104,7 @@ public synchronized void tryUpdateCache() { } } - private void applyUpdateMessage(final Set updateContainerBytes) { - for (final Object element : updateContainerBytes) { - final byte[] containerByte = (byte[]) element; - } + public void handleContainer(final UpdateContainer container) { + // TODO: } } From df8d852d01083ac450ba9677585269e33c93b3ae Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:01:03 +0800 Subject: [PATCH 166/213] Update DeviceAttributeCacheUpdater.java --- .../update/DeviceAttributeCacheUpdater.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index 679ab7140011c..f02a10ed45df7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -24,7 +24,9 @@ import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; +import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; @@ -33,6 +35,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; + import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; @@ -51,12 +55,14 @@ import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; public class DeviceAttributeCacheUpdater { private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeCacheUpdater.class); - private static final int UPDATE_DETAIL_CONTAINER_SEND_MIN_LIMIT_BYTES = 1024; + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + public static final int UPDATE_DETAIL_CONTAINER_SEND_MIN_LIMIT_BYTES = 1024; // All the data node locations shall only have internal endpoint with all the other endpoints set // to null @@ -84,8 +90,7 @@ public void update( targetDataNodeLocations.forEach( location -> { // Skip update on local - if (location.getDataNodeId() - == IoTDBDescriptor.getInstance().getConfig().getDataNodeId()) { + if (location.getDataNodeId() == config.getDataNodeId()) { return; } if (!attributeUpdateMap.containsKey(location)) { @@ -113,7 +118,7 @@ public void update( } public Pair> getAttributeUpdateInfo( - final AtomicInteger limit) { + final @Nonnull AtomicInteger limit, final @Nonnull AtomicBoolean hasRemaining) { // Note that the "updateContainerStatistics" is unsafe to use here for whole read of detail // container because the update map is read by GRASS thread, and the container's size may change // during the read process @@ -124,14 +129,16 @@ public Pair> getAttributeUpdateInfo( // Because they require less capacity if (limit.get() < UPDATE_DETAIL_CONTAINER_SEND_MIN_LIMIT_BYTES && entry.getValue() instanceof UpdateDetailContainer) { + hasRemaining.set(true); continue; } // type(1) + size(4) if (limit.get() <= 5) { + hasRemaining.set(true); break; } limit.addAndGet(-5); - updateBytes.put(entry.getKey(), entry.getValue().getUpdateContent(limit)); + updateBytes.put(entry.getKey(), entry.getValue().getUpdateContent(limit, hasRemaining)); } return new Pair<>(version.get(), updateBytes); } @@ -189,7 +196,7 @@ public void commit(final TableDeviceAttributeCommitUpdateNode node) { }))); } - private UpdateContainer getContainer(final byte[] bytes) { + public static UpdateContainer getContainer(final byte[] bytes) { final ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes); UpdateContainer result = null; try { @@ -337,7 +344,13 @@ private void deserialize(final InputStream inputStream) throws IOException { : new UpdateClearContainer(); container.deserialize(inputStream); - // TODO: Update local cache for newly migrated region + // Update local cache for region migration + if (config.getDataNodeId() == location.getDataNodeId() + && config.getInternalAddress().equals(location.getInternalEndPoint().getIp()) + && config.getInternalPort() == location.getInternalEndPoint().getPort()) { + TableDeviceSchemaFetcher.getInstance().getAttributeGuard().handleContainer(container); + } + attributeUpdateMap.put(location, container); } } From e4f3cc498b77b9fc320d74aa9b5b6bfa3d27042b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:06:37 +0800 Subject: [PATCH 167/213] Update DeviceAttributeCacheUpdater.java --- .../update/DeviceAttributeCacheUpdater.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index f02a10ed45df7..18d6cf262df4a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -149,22 +149,14 @@ public void commit(final TableDeviceAttributeCommitUpdateNode node) { shrunkNodes.forEach( location -> { if (attributeUpdateMap.containsKey(location)) { - releaseMemory( - updateContainerStatistics.containsKey(location) - ? updateContainerStatistics.get(location).getContainerSize() - : ((UpdateClearContainer) attributeUpdateMap.get(location)).ramBytesUsed()); - attributeUpdateMap.remove(location); + removeLocation(location); } }); updateContainerStatistics.keySet().removeAll(shrunkNodes); final TDataNodeLocation leaderLocation = node.getLeaderLocation(); if (version.get() == node.getVersion() && attributeUpdateMap.containsKey(leaderLocation)) { - releaseMemory( - updateContainerStatistics.containsKey(leaderLocation) - ? updateContainerStatistics.get(leaderLocation).getContainerSize() - : ((UpdateClearContainer) attributeUpdateMap.get(leaderLocation)).ramBytesUsed()); - attributeUpdateMap.remove(leaderLocation); + removeLocation(leaderLocation); } node.getCommitMap() @@ -196,6 +188,14 @@ public void commit(final TableDeviceAttributeCommitUpdateNode node) { }))); } + private void removeLocation(final TDataNodeLocation location) { + releaseMemory( + updateContainerStatistics.containsKey(location) + ? updateContainerStatistics.get(location).getContainerSize() + : ((UpdateClearContainer) attributeUpdateMap.get(location)).ramBytesUsed()); + attributeUpdateMap.remove(location); + } + public static UpdateContainer getContainer(final byte[] bytes) { final ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes); UpdateContainer result = null; From 51ce35fdf4e65b6998ffbe8f8b5eafd811202bae Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:09:45 +0800 Subject: [PATCH 168/213] Update DeviceAttributeCacheUpdater.java --- .../attribute/update/DeviceAttributeCacheUpdater.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index 18d6cf262df4a..64540377f0a2c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -91,6 +91,8 @@ public void update( location -> { // Skip update on local if (location.getDataNodeId() == config.getDataNodeId()) { + // Remove potential entries put when the dataNodeId == -1 + removeLocation(location); return; } if (!attributeUpdateMap.containsKey(location)) { From 5b2124874f1880188ed12ddd802609e896c11372 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:26:40 +0800 Subject: [PATCH 169/213] Region id security --- .../metadata/fetcher/TableDeviceCacheAttributeGuard.java | 9 ++++++++- .../attribute/update/DeviceAttributeCacheUpdater.java | 6 +++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index 971bcc5772ff7..ad1d7e5a97bb5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -64,7 +64,7 @@ public void addFetchedRegion(final Integer schemaRegionId) { return longSet; } - public void handleAttributeUpdate(final TAttributeUpdateReq updateReq) { + public synchronized void handleAttributeUpdate(final TAttributeUpdateReq updateReq) { // Trim the schema regions with lower or equal version // or with this node restarted before fetching updateReq @@ -104,6 +104,13 @@ public synchronized void tryUpdateCache() { } } + // This must be synchronized with "handleUpdate" to avoid unordered execution + // with region migration's "handleContainer" and stale "handleUpdate" logic + public synchronized void setVersion(final int schemaRegionId, final long newVersion) { + fetchedSchemaRegionIds2LargestVersionMap.computeIfPresent( + schemaRegionId, (id, version) -> Math.max(version, newVersion)); + } + public void handleContainer(final UpdateContainer container) { // TODO: } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index 64540377f0a2c..6b4b214e8ecd5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceCacheAttributeGuard; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; @@ -350,7 +351,10 @@ private void deserialize(final InputStream inputStream) throws IOException { if (config.getDataNodeId() == location.getDataNodeId() && config.getInternalAddress().equals(location.getInternalEndPoint().getIp()) && config.getInternalPort() == location.getInternalEndPoint().getPort()) { - TableDeviceSchemaFetcher.getInstance().getAttributeGuard().handleContainer(container); + final TableDeviceCacheAttributeGuard guard = + TableDeviceSchemaFetcher.getInstance().getAttributeGuard(); + guard.setVersion(regionStatistics.getSchemaRegionId(), version.get()); + guard.handleContainer(container); } attributeUpdateMap.put(location, container); From 0738f53fde4354216aeb31383739c2a8250b6323 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:29:37 +0800 Subject: [PATCH 170/213] Update IoTDBConfig.java --- .../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 0a3a5a902a2fc..1c8f650b2d4c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1046,7 +1046,6 @@ public class IoTDBConfig { /** GRASS Service */ private long generalRegionAttributeSecurityServiceIntervalSeconds = 30L; - private long generalRegionAttributeSecurityServiceTimeoutSeconds = 20L; private long generalRegionAttributeSecurityServiceFailureDurationSecondsToFetch = 600L; private int generalRegionAttributeSecurityServiceFailureTimesToFetch = 20; From 59fab2ea31faf9a1d653451e32de8333d12571bf Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:44:58 +0800 Subject: [PATCH 171/213] Update IoTDBConfig.java --- .../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 1c8f650b2d4c4..0a3a5a902a2fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1046,6 +1046,7 @@ public class IoTDBConfig { /** GRASS Service */ private long generalRegionAttributeSecurityServiceIntervalSeconds = 30L; + private long generalRegionAttributeSecurityServiceTimeoutSeconds = 20L; private long generalRegionAttributeSecurityServiceFailureDurationSecondsToFetch = 600L; private int generalRegionAttributeSecurityServiceFailureTimesToFetch = 20; From fdb3ea67de1da50729ba9706d2a005e7bf882a25 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:33:34 +0800 Subject: [PATCH 172/213] Snapshot Fix --- .../schemaregion/attribute/DeviceAttributeStore.java | 6 ++++-- .../attribute/update/DeviceAttributeCacheUpdater.java | 6 ++++-- .../mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java index 585b76e65767b..7ac2618e114d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java @@ -69,13 +69,15 @@ public synchronized boolean createSnapshot(final File targetDir) { final File snapshot = SystemFileFactory.INSTANCE.getFile(targetDir, SchemaConstant.DEVICE_ATTRIBUTE_SNAPSHOT); - try (final FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); - final BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream)) { + try { + final FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); + final BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream); try { serialize(outputStream); } finally { outputStream.flush(); fileOutputStream.getFD().sync(); + outputStream.close(); } if (snapshot.exists() && !FileUtils.deleteFileIfExist(snapshot)) { logger.error( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index 6b4b214e8ecd5..56c99eddf0b85 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -261,13 +261,15 @@ public synchronized boolean createSnapshot(final File targetDir) { SystemFileFactory.INSTANCE.getFile( targetDir, SchemaConstant.DEVICE_ATTRIBUTE_REMOTE_UPDATER_SNAPSHOT); - try (final FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); - final BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream)) { + try { + final FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); + final BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream); try { serialize(outputStream); } finally { outputStream.flush(); fileOutputStream.getFD().sync(); + outputStream.close(); } if (snapshot.exists() && !FileUtils.deleteFileIfExist(snapshot)) { logger.error( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java index 774858b46c132..5c1b66781c27c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java @@ -80,13 +80,15 @@ public static boolean createSnapshot(final File snapshotDir, final MemMTreeStore final File snapshot = SystemFileFactory.INSTANCE.getFile(snapshotDir, SchemaConstant.MTREE_SNAPSHOT); - try (final FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); - final BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream)) { + try { + final FileOutputStream fileOutputStream = new FileOutputStream(snapshotTmp); + final BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream); try { serializeTo(store, outputStream); } finally { outputStream.flush(); fileOutputStream.getFD().sync(); + outputStream.close(); } if (snapshot.exists() && !FileUtils.deleteFileIfExist(snapshot)) { logger.error( From b761d286d61533c2a23ff3081183a9cd52b35cf4 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:46:30 +0800 Subject: [PATCH 173/213] spotless --- .../queryengine/plan/planner/TableOperatorGenerator.java | 2 +- .../queryengine/plan/planner/plan/node/PlanNodeType.java | 8 ++++---- .../queryengine/plan/planner/plan/node/PlanVisitor.java | 2 +- .../planner/distribute/TableDistributedPlanGenerator.java | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index f76d6372cf085..78cc4c4cecd29 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -109,10 +109,10 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index c5536099ecfce..c8793f7881633 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -114,6 +114,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; @@ -121,10 +125,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.tsfile.utils.ReadWriteIOUtils; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index a81b209bf8768..26e5c008661ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -123,6 +123,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; @@ -130,7 +131,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; @SuppressWarnings("java:S6539") // suppress "Monster class" warning public abstract class PlanVisitor { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 371433722c564..66a70d82d9fde 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -46,11 +46,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.AbstractTableDeviceQueryNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; From 4cf2fc18d56e777e1d146cbf4e2d033d2dd31c6d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 17:03:24 +0800 Subject: [PATCH 174/213] fix --- .../db/queryengine/plan/planner/plan/node/PlanNodeType.java | 2 +- .../db/queryengine/plan/planner/plan/node/PlanVisitor.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index c8793f7881633..599ed1d140388 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.queryengine.plan.planner.plan.node; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; @@ -114,7 +115,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 26e5c008661ec..a1c2f94d5892e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -119,7 +119,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.GroupReference; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; From 452b7bb2f54cfb4ff6f1fcbc24e83ae304865276 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 17:05:16 +0800 Subject: [PATCH 175/213] Update DataNodeTableCache.java --- .../apache/iotdb/db/schemaengine/table/DataNodeTableCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 13aeff55c8797..e99df7ccd3653 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -203,7 +203,7 @@ public TsTable getTable(String database, final String tableName) { database = PathUtils.unQualifyDatabaseName(database); final Map> preUpdateTables = mayGetTableInPreUpdateMap(database, tableName); - if (Objects.nonNull(preUpdateTables)) { + if (Objects.nonNull(preUpdateTables) && !preUpdateTables.isEmpty()) { updateTable(getTablesInConfigNode(preUpdateTables), preUpdateTables); } return getTableInCache(database, tableName); From c789d0e03a67c8793f93c9ee98fe9f9456961611 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 17:23:27 +0800 Subject: [PATCH 176/213] Update DeviceAttributeCacheUpdater.java --- .../attribute/update/DeviceAttributeCacheUpdater.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index 56c99eddf0b85..5a0f4a277e703 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -357,9 +357,9 @@ private void deserialize(final InputStream inputStream) throws IOException { TableDeviceSchemaFetcher.getInstance().getAttributeGuard(); guard.setVersion(regionStatistics.getSchemaRegionId(), version.get()); guard.handleContainer(container); + } else { + attributeUpdateMap.put(location, container); } - - attributeUpdateMap.put(location, container); } } From 73a12199e656f98305c5f919382752a7ac78c4f9 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 17:50:17 +0800 Subject: [PATCH 177/213] Update DeviceAttributeCacheUpdater.java --- .../update/DeviceAttributeCacheUpdater.java | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index 5a0f4a277e703..e5cab45990dfa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -51,6 +51,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -126,12 +127,26 @@ public Pair> getAttributeUpdateInfo( // container because the update map is read by GRASS thread, and the container's size may change // during the read process final Map updateBytes = new HashMap<>(); - for (final Map.Entry entry : - attributeUpdateMap.entrySet()) { + for (final Iterator> it = + attributeUpdateMap.entrySet().iterator(); + it.hasNext(); ) { + final Map.Entry entry = it.next(); + final TDataNodeLocation location = entry.getKey(); + final UpdateContainer container = entry.getValue(); + + if (location.getDataNodeId() == config.getDataNodeId()) { + // Remove potential entries put when the dataNodeId == -1 + releaseMemory( + updateContainerStatistics.containsKey(location) + ? updateContainerStatistics.get(location).getContainerSize() + : ((UpdateClearContainer) container).ramBytesUsed()); + it.remove(); + continue; + } // If the remaining capacity is too low we just send clear container first // Because they require less capacity if (limit.get() < UPDATE_DETAIL_CONTAINER_SEND_MIN_LIMIT_BYTES - && entry.getValue() instanceof UpdateDetailContainer) { + && container instanceof UpdateDetailContainer) { hasRemaining.set(true); continue; } @@ -141,7 +156,7 @@ public Pair> getAttributeUpdateInfo( break; } limit.addAndGet(-5); - updateBytes.put(entry.getKey(), entry.getValue().getUpdateContent(limit, hasRemaining)); + updateBytes.put(location, container.getUpdateContent(limit, hasRemaining)); } return new Pair<>(version.get(), updateBytes); } From 436e1855c199ed1a6ace6adfb0684cbdfc0e46d6 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 17:59:51 +0800 Subject: [PATCH 178/213] Update GeneralRegionAttributeSecurityService.java --- .../update/GeneralRegionAttributeSecurityService.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index d86ff26a777e2..269debf5ead96 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -113,6 +113,15 @@ public void notifyBroadCast() { private void execute() { lock.lock(); try { + // Wait for the dataNodeId to get ready in order to clear the local location in + // updateMap after dataNode restart + if (iotdbConfig.getDataNodeId() == -1) { + condition.await( + iotdbConfig.getGeneralRegionAttributeSecurityServiceIntervalSeconds(), + TimeUnit.SECONDS); + return; + } + // All the "detailContainer"'s size will add up to at most "limit" // UpdateClearContainer and version / TEndPoint are not calculated final AtomicInteger limit = From eb93f8774e3d5c6458d4848b1f6019eb0b2db9f0 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:27:58 +0800 Subject: [PATCH 179/213] small fix --- .../attribute/update/DeviceAttributeCacheUpdater.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index e5cab45990dfa..c05d41ab4390c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -135,12 +135,7 @@ public Pair> getAttributeUpdateInfo( final UpdateContainer container = entry.getValue(); if (location.getDataNodeId() == config.getDataNodeId()) { - // Remove potential entries put when the dataNodeId == -1 - releaseMemory( - updateContainerStatistics.containsKey(location) - ? updateContainerStatistics.get(location).getContainerSize() - : ((UpdateClearContainer) container).ramBytesUsed()); - it.remove(); + // Remove when commit continue; } // If the remaining capacity is too low we just send clear container first From 345c25b99a4db84464665a1026beaedd09a9dad7 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:29:08 +0800 Subject: [PATCH 180/213] Update DeviceAttributeCacheUpdater.java --- .../attribute/update/DeviceAttributeCacheUpdater.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index c05d41ab4390c..33ba8615ef1f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -51,7 +51,6 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -127,10 +126,8 @@ public Pair> getAttributeUpdateInfo( // container because the update map is read by GRASS thread, and the container's size may change // during the read process final Map updateBytes = new HashMap<>(); - for (final Iterator> it = - attributeUpdateMap.entrySet().iterator(); - it.hasNext(); ) { - final Map.Entry entry = it.next(); + for (final Map.Entry entry : + attributeUpdateMap.entrySet()) { final TDataNodeLocation location = entry.getKey(); final UpdateContainer container = entry.getValue(); From 1a7725325f5b608564b219862859f63cdefcda7e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:30:58 +0800 Subject: [PATCH 181/213] Update GeneralRegionAttributeSecurityService.java --- .../update/GeneralRegionAttributeSecurityService.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 269debf5ead96..d86ff26a777e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -113,15 +113,6 @@ public void notifyBroadCast() { private void execute() { lock.lock(); try { - // Wait for the dataNodeId to get ready in order to clear the local location in - // updateMap after dataNode restart - if (iotdbConfig.getDataNodeId() == -1) { - condition.await( - iotdbConfig.getGeneralRegionAttributeSecurityServiceIntervalSeconds(), - TimeUnit.SECONDS); - return; - } - // All the "detailContainer"'s size will add up to at most "limit" // UpdateClearContainer and version / TEndPoint are not calculated final AtomicInteger limit = From e1d2e401bd1eb70be5593d4f1d41b374cea66182 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sat, 12 Oct 2024 19:17:38 +0800 Subject: [PATCH 182/213] Bug fix --- .../fetcher/TableDeviceSchemaFetcher.java | 64 +++++++++---------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index 7185051b0abe1..9b5b60fb44253 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -113,9 +113,8 @@ Map> fetchMissingDeviceSchemaForDat // For the correctness of attribute remote update final Set queryIdSet = attributeGuard.addFetchQueryId(queryId); - final ExecutionResult executionResult; try { - executionResult = + final ExecutionResult executionResult = coordinator.executeForTableModel( statement, relationSqlParser, @@ -126,22 +125,18 @@ Map> fetchMissingDeviceSchemaForDat "Fetch Device for insert", LocalExecutionPlanner.getInstance().metadata, config.getQueryTimeoutThreshold()); - } finally { - queryIdSet.remove(queryId); - attributeGuard.tryUpdateCache(); - } - if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new RuntimeException( - new IoTDBException( - executionResult.status.getMessage(), executionResult.status.getCode())); - } - final List columnHeaderList = - coordinator.getQueryExecution(queryId).getDatasetHeader().getColumnHeaders(); - final int idLength = DataNodeTableCache.getInstance().getTable(database, table).getIdNums(); - final Map> fetchedDeviceSchema = new HashMap<>(); + if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new RuntimeException( + new IoTDBException( + executionResult.status.getMessage(), executionResult.status.getCode())); + } + + final List columnHeaderList = + coordinator.getQueryExecution(queryId).getDatasetHeader().getColumnHeaders(); + final int idLength = DataNodeTableCache.getInstance().getTable(database, table).getIdNums(); + final Map> fetchedDeviceSchema = new HashMap<>(); - try { while (coordinator.getQueryExecution(queryId).hasNextResult()) { final Optional tsBlock; try { @@ -165,13 +160,15 @@ Map> fetchMissingDeviceSchemaForDat new TableDeviceId((String[]) truncateTailingNull(nodes)), attributeMap); } } + return fetchedDeviceSchema; } catch (final Throwable throwable) { t = throwable; throw throwable; } finally { + queryIdSet.remove(queryId); + attributeGuard.tryUpdateCache(); coordinator.cleanupQueryExecution(queryId, null, t); } - return fetchedDeviceSchema; } public List fetchDeviceSchemaForDataQuery( @@ -392,9 +389,9 @@ private void fetchMissingDeviceSchemaForQuery( if (Objects.nonNull(statement.getPartitionKeyList())) { queryIdSet = attributeGuard.addFetchQueryId(queryId); } - final ExecutionResult executionResult; + try { - executionResult = + final ExecutionResult executionResult = coordinator.executeForTableModel( statement, relationSqlParser, @@ -407,23 +404,18 @@ private void fetchMissingDeviceSchemaForQuery( mppQueryContext.getQueryId(), mppQueryContext.getSql()), LocalExecutionPlanner.getInstance().metadata, config.getQueryTimeoutThreshold()); - } finally { - if (Objects.nonNull(queryIdSet)) { - queryIdSet.remove(queryId); - attributeGuard.tryUpdateCache(); + + if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new RuntimeException( + new IoTDBException( + executionResult.status.getMessage(), executionResult.status.getCode())); } - } - if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new RuntimeException( - new IoTDBException( - executionResult.status.getMessage(), executionResult.status.getCode())); - } - final List columnHeaderList = - coordinator.getQueryExecution(queryId).getDatasetHeader().getColumnHeaders(); - final int idLength = DataNodeTableCache.getInstance().getTable(database, table).getIdNums(); - Throwable t = null; - try { + final List columnHeaderList = + coordinator.getQueryExecution(queryId).getDatasetHeader().getColumnHeaders(); + final int idLength = DataNodeTableCache.getInstance().getTable(database, table).getIdNums(); + Throwable t = null; + while (coordinator.getQueryExecution(queryId).hasNextResult()) { final Optional tsBlock; try { @@ -461,6 +453,10 @@ private void fetchMissingDeviceSchemaForQuery( t = throwable; throw throwable; } finally { + if (Objects.nonNull(queryIdSet)) { + queryIdSet.remove(queryId); + attributeGuard.tryUpdateCache(); + } coordinator.cleanupQueryExecution(queryId, null, t); } } From 7cbe4249d32aa07a14df2426d376713c73028827 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sat, 12 Oct 2024 19:18:27 +0800 Subject: [PATCH 183/213] Update TableDeviceSchemaFetcher.java --- .../relational/metadata/fetcher/TableDeviceSchemaFetcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index 9b5b60fb44253..c31461ded1371 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -382,6 +382,7 @@ private void fetchMissingDeviceSchemaForQuery( final MPPQueryContext mppQueryContext) { final String table = tableInstance.getTableName(); + Throwable t = null; final long queryId = SessionManager.getInstance().requestQueryId(); // For the correctness of attribute remote update @@ -414,7 +415,6 @@ private void fetchMissingDeviceSchemaForQuery( final List columnHeaderList = coordinator.getQueryExecution(queryId).getDatasetHeader().getColumnHeaders(); final int idLength = DataNodeTableCache.getInstance().getTable(database, table).getIdNums(); - Throwable t = null; while (coordinator.getQueryExecution(queryId).hasNextResult()) { final Optional tsBlock; From 9a18d567d343b4f77eb8864dbce5f1bd13919bd2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sat, 12 Oct 2024 20:21:02 +0800 Subject: [PATCH 184/213] improvement --- .../attribute/update/DeviceAttributeCacheUpdater.java | 4 ++-- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index 33ba8615ef1f5..54e4e651edd0f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -221,8 +221,8 @@ public static UpdateContainer getContainer(final byte[] bytes) { return result; } - public void addLocation(final TDataNodeLocation dataNodeLocation) { - targetDataNodeLocations.add(dataNodeLocation); + public boolean addLocation(final TDataNodeLocation dataNodeLocation) { + return targetDataNodeLocations.add(dataNodeLocation); } public void afterUpdate() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 8b149f2e6097e..eed5780e0766f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1580,8 +1580,9 @@ public void commitUpdateAttribute(final TableDeviceAttributeCommitUpdateNode nod @Override public void addNodeLocation(final TableNodeLocationAddNode node) throws MetadataException { - deviceAttributeCacheUpdater.addLocation(node.getLocation()); - writeToMLog(node); + if (deviceAttributeCacheUpdater.addLocation(node.getLocation())) { + writeToMLog(node); + } } // endregion From 6a4b929b5c1905fbd05b7d278ac1b900860933c5 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sat, 12 Oct 2024 20:25:25 +0800 Subject: [PATCH 185/213] bug fix --- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 4 ++++ .../metadata/fetcher/TableDeviceSchemaValidator.java | 10 ---------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index c31461ded1371..c27b237b55bd1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -160,6 +160,10 @@ Map> fetchMissingDeviceSchemaForDat new TableDeviceId((String[]) truncateTailingNull(nodes)), attributeMap); } } + + fetchedDeviceSchema.forEach( + (key, value) -> cache.put(database, table, key.getIdValues(), value)); + return fetchedDeviceSchema; } catch (final Throwable throwable) { t = throwable; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java index 0ade6d20e97a1..25d258c14b9e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java @@ -128,16 +128,6 @@ private ValidateResult fetchAndValidateDeviceSchema( .collect(Collectors.toList())), context); - fetchedDeviceSchema.forEach( - (key, value) -> - fetcher - .getTableDeviceCache() - .put( - schemaValidation.getDatabase(), - schemaValidation.getTableName(), - key.getIdValues(), - value)); - final List attributeKeyList = schemaValidation.getAttributeColumnNameList(); final List attributeValueList = schemaValidation.getAttributeValueList(); From f21119fbe1765af330af332d12e84fb6566f9d79 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 20:58:53 +0800 Subject: [PATCH 186/213] Update TableDeviceSchemaFetcher.java --- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java index abc20c99473fe..4d31458f7cced 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java @@ -88,15 +88,10 @@ public static TableDeviceSchemaFetcher getInstance() { return TableDeviceSchemaFetcherHolder.INSTANCE; } - private TableDeviceSchemaCache getTableDeviceCache() { - return cache; - } - public TableDeviceCacheAttributeGuard getAttributeGuard() { return attributeGuard; } - Map> fetchMissingDeviceSchemaForDataInsertion( Map> fetchMissingDeviceSchemaForDataInsertion( final FetchDevice statement, final MPPQueryContext context) { final long queryId = SessionManager.getInstance().requestQueryId(); @@ -154,8 +149,7 @@ Map> fetchMissingDeviceSchemaForDataInsertion( } } - fetchedDeviceSchema.forEach( - (key, value) -> cache.put(database, table, key.getIdValues(), value)); + fetchedDeviceSchema.forEach((key, value) -> cache.putAttributes(database, key, value)); return fetchedDeviceSchema; } catch (final Throwable throwable) { From 94d12c7101fd88078b93035a156a981aa415df36 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 21:10:19 +0800 Subject: [PATCH 187/213] Fix --- .../thrift/impl/DataNodeInternalRPCServiceImpl.java | 1 + .../metadata/fetcher/TableDeviceCacheAttributeGuard.java | 9 ++++++--- .../attribute/update/UpdateDetailContainer.java | 5 +++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index a5581c8c605c1..941a64c74dce4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -136,6 +136,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedNonWritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableSchemaQueryWriteVisitor; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index ad1d7e5a97bb5..7a013788e6a17 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeCacheUpdater; import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.UpdateContainer; import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; @@ -38,8 +39,10 @@ public class TableDeviceCacheAttributeGuard { // Unbounded queue - final LinkedBlockingDeque> applyQueue = new LinkedBlockingDeque<>(); - final Map fetchedSchemaRegionIds2LargestVersionMap = new ConcurrentHashMap<>(); + private final LinkedBlockingDeque> applyQueue = new LinkedBlockingDeque<>(); + private final Map fetchedSchemaRegionIds2LargestVersionMap = + new ConcurrentHashMap<>(); + private final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); public boolean isRegionFetched(final Integer schemaRegionId) { return fetchedSchemaRegionIds2LargestVersionMap.containsKey(schemaRegionId); @@ -112,6 +115,6 @@ public synchronized void setVersion(final int schemaRegionId, final long newVers } public void handleContainer(final UpdateContainer container) { - // TODO: + // TODO } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 07d4dd954a057..63a61242681cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -51,6 +51,11 @@ public class UpdateDetailContainer implements UpdateContainer { private final ConcurrentMap>> updateMap = new ConcurrentHashMap<>(); + public ConcurrentMap>> + getUpdateMap() { + return updateMap; + } + @Override public long updateAttribute( final String tableName, From 46ead7a1e9544149e4609baf3fe86fafeb2134d1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 09:44:12 +0800 Subject: [PATCH 188/213] bug fix --- .../db/queryengine/plan/planner/plan/node/PlanNodeType.java | 1 - .../db/queryengine/plan/planner/plan/node/PlanVisitor.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 3397b38e1956e..d59543f827a82 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -115,7 +115,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 75d3e9b44456c..f0f877d30db3e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.queryengine.plan.planner.plan.node; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.CountSchemaMergeNode; @@ -119,7 +120,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.GroupReference; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; From a2885396973d34250ca1b1bfc4ff6579edc4cf58 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:16:31 +0800 Subject: [PATCH 189/213] Partially add database --- .../update/GeneralRegionAttributeSecurityService.java | 9 ++++----- .../thrift-datanode/src/main/thrift/datanode.thrift | 7 ++++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index d86ff26a777e2..a741726f175a6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -83,19 +83,18 @@ public class GeneralRegionAttributeSecurityService implements IService { private final Map> dataNodeId2FailureDurationAndTimesMap = new HashMap<>(); - private final Set regionLeaders = - Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Map regionLeaders2DatabaseMap = new ConcurrentHashMap<>(); private final ReentrantLock lock = new ReentrantLock(); private final Condition condition = lock.newCondition(); private volatile boolean skipNextSleep = false; private volatile boolean allowSubmitListen = false; public void startBroadcast(final ISchemaRegion schemaRegion) { - regionLeaders.add(schemaRegion); + regionLeaders2DatabaseMap.put(schemaRegion, schemaRegion.getDatabaseFullPath().substring(5)); } public void stopBroadcast(final ISchemaRegion schemaRegion) { - regionLeaders.remove(schemaRegion); + regionLeaders2DatabaseMap.remove(schemaRegion); } public void notifyBroadCast() { @@ -124,7 +123,7 @@ private void execute() { final AtomicBoolean hasRemaining = new AtomicBoolean(false); final Map>> attributeUpdateCommitMap = new HashMap<>(); - for (final ISchemaRegion regionLeader : regionLeaders) { + for (final ISchemaRegion regionLeader : regionLeaders2DatabaseMap.keySet()) { final Pair> currentResult = regionLeader.getAttributeUpdateInfo(limit, hasRemaining); if (currentResult.getRight().isEmpty()) { diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 9e34c5515ce99..cab252ac971c7 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -344,12 +344,13 @@ struct TLoadCommandReq { } struct TAttributeUpdateReq { - 1:required map attributeUpdateMap + 1: required map attributeUpdateMap } struct TSchemaRegionAttributeInfo { - 1:required i64 version - 2:required binary body + 1: required i64 version + 2: required string database + 3: required binary body } struct TLoadResp { From e6a7af80c016ff1724554841778a6a76aaaa008e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:36:42 +0800 Subject: [PATCH 190/213] partial --- .../TableDeviceCacheAttributeGuard.java | 24 ++++++++++++------- ...GeneralRegionAttributeSecurityService.java | 17 +++++++++---- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index 7a013788e6a17..bd05053bdb0b0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -25,6 +25,8 @@ import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; import org.apache.iotdb.mpp.rpc.thrift.TSchemaRegionAttributeInfo; +import org.apache.tsfile.utils.Pair; + import javax.annotation.Nonnull; import java.util.Collections; @@ -40,16 +42,17 @@ public class TableDeviceCacheAttributeGuard { // Unbounded queue private final LinkedBlockingDeque> applyQueue = new LinkedBlockingDeque<>(); - private final Map fetchedSchemaRegionIds2LargestVersionMap = - new ConcurrentHashMap<>(); + private final Map> + fetchedSchemaRegionIds2LargestVersionAndDatabaseMap = new ConcurrentHashMap<>(); private final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); public boolean isRegionFetched(final Integer schemaRegionId) { - return fetchedSchemaRegionIds2LargestVersionMap.containsKey(schemaRegionId); + return fetchedSchemaRegionIds2LargestVersionAndDatabaseMap.containsKey(schemaRegionId); } public void addFetchedRegion(final Integer schemaRegionId) { - fetchedSchemaRegionIds2LargestVersionMap.put(schemaRegionId, Long.MIN_VALUE); + fetchedSchemaRegionIds2LargestVersionAndDatabaseMap.put( + schemaRegionId, new Pair<>(Long.MIN_VALUE, null)); } @SuppressWarnings("unchecked") @@ -75,13 +78,16 @@ public synchronized void handleAttributeUpdate(final TAttributeUpdateReq updateR .entrySet() .removeIf( entry -> { - if (!fetchedSchemaRegionIds2LargestVersionMap.containsKey(entry.getKey()) + if (!fetchedSchemaRegionIds2LargestVersionAndDatabaseMap.containsKey(entry.getKey()) || entry.getValue().getVersion() - <= fetchedSchemaRegionIds2LargestVersionMap.get(entry.getKey())) { + <= fetchedSchemaRegionIds2LargestVersionAndDatabaseMap + .get(entry.getKey()) + .getLeft()) { return true; } - fetchedSchemaRegionIds2LargestVersionMap.put( - entry.getKey(), entry.getValue().getVersion()); + fetchedSchemaRegionIds2LargestVersionAndDatabaseMap.put( + entry.getKey(), + new Pair<>(entry.getValue().getVersion(), entry.getValue().getDatabase())); return false; }); applyQueue.add( @@ -110,7 +116,7 @@ public synchronized void tryUpdateCache() { // This must be synchronized with "handleUpdate" to avoid unordered execution // with region migration's "handleContainer" and stale "handleUpdate" logic public synchronized void setVersion(final int schemaRegionId, final long newVersion) { - fetchedSchemaRegionIds2LargestVersionMap.computeIfPresent( + fetchedSchemaRegionIds2LargestVersionAndDatabaseMap.computeIfPresent( schemaRegionId, (id, version) -> Math.max(version, newVersion)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index a741726f175a6..18d9c190c8928 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -83,18 +83,23 @@ public class GeneralRegionAttributeSecurityService implements IService { private final Map> dataNodeId2FailureDurationAndTimesMap = new HashMap<>(); - private final Map regionLeaders2DatabaseMap = new ConcurrentHashMap<>(); + private final Set regionLeaders = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Map regionId2DatabaseMap = new ConcurrentHashMap<>(); private final ReentrantLock lock = new ReentrantLock(); private final Condition condition = lock.newCondition(); private volatile boolean skipNextSleep = false; private volatile boolean allowSubmitListen = false; public void startBroadcast(final ISchemaRegion schemaRegion) { - regionLeaders2DatabaseMap.put(schemaRegion, schemaRegion.getDatabaseFullPath().substring(5)); + regionLeaders.add(schemaRegion); + regionId2DatabaseMap.put( + schemaRegion.getSchemaRegionId(), schemaRegion.getDatabaseFullPath().substring(5)); } public void stopBroadcast(final ISchemaRegion schemaRegion) { - regionLeaders2DatabaseMap.remove(schemaRegion); + regionLeaders.remove(schemaRegion); + regionId2DatabaseMap.remove(schemaRegion.getSchemaRegionId()); } public void notifyBroadCast() { @@ -123,7 +128,7 @@ private void execute() { final AtomicBoolean hasRemaining = new AtomicBoolean(false); final Map>> attributeUpdateCommitMap = new HashMap<>(); - for (final ISchemaRegion regionLeader : regionLeaders2DatabaseMap.keySet()) { + for (final ISchemaRegion regionLeader : regionLeaders) { final Pair> currentResult = regionLeader.getAttributeUpdateInfo(limit, hasRemaining); if (currentResult.getRight().isEmpty()) { @@ -242,7 +247,9 @@ private void execute() { .put( id.getId(), new TSchemaRegionAttributeInfo( - pair.getLeft(), ByteBuffer.wrap(bytes))); + pair.getLeft(), + regionId2DatabaseMap.get(id), + ByteBuffer.wrap(bytes))); })); DnToDnInternalServiceAsyncRequestManager.getInstance() From 778171665a0691aa4c9a44eb0639d8d145286a88 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:39:08 +0800 Subject: [PATCH 191/213] Update TableDeviceCacheAttributeGuard.java --- .../metadata/fetcher/TableDeviceCacheAttributeGuard.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index bd05053bdb0b0..c21e9ea68e3b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -117,7 +117,13 @@ public synchronized void tryUpdateCache() { // with region migration's "handleContainer" and stale "handleUpdate" logic public synchronized void setVersion(final int schemaRegionId, final long newVersion) { fetchedSchemaRegionIds2LargestVersionAndDatabaseMap.computeIfPresent( - schemaRegionId, (id, version) -> Math.max(version, newVersion)); + schemaRegionId, + (id, pair) -> { + if (newVersion > pair.getLeft()) { + pair.setLeft(newVersion); + } + return pair; + }); } public void handleContainer(final UpdateContainer container) { From 24f44f20ee3f41ae7ecf700c50ba68b159e8dc67 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:02:46 +0800 Subject: [PATCH 192/213] wait for tables --- .../TableDeviceCacheAttributeGuard.java | 19 ++++++++++++++++++- .../update/UpdateClearContainer.java | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index c21e9ea68e3b2..ccd1c0ae882da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -21,7 +21,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.DeviceAttributeCacheUpdater; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.UpdateClearContainer; import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.UpdateContainer; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.UpdateDetailContainer; import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; import org.apache.iotdb.mpp.rpc.thrift.TSchemaRegionAttributeInfo; @@ -38,6 +40,8 @@ import java.util.concurrent.LinkedBlockingDeque; import java.util.stream.Collectors; +import static org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher.convertIdValuesToDeviceID; + public class TableDeviceCacheAttributeGuard { // Unbounded queue @@ -127,6 +131,19 @@ public synchronized void setVersion(final int schemaRegionId, final long newVers } public void handleContainer(final UpdateContainer container) { - // TODO + if (container instanceof UpdateDetailContainer) { + ((UpdateDetailContainer) container) + .getUpdateMap() + .forEach( + (table, deviceNodesMap) -> + deviceNodesMap.forEach( + (nodes, attributes) -> + cache.updateAttributes( + null, convertIdValuesToDeviceID(table, nodes), attributes))); + } else { + ((UpdateClearContainer) container) + .getTableNames() + .forEach(table -> cache.invalidateAttributes(null, table)); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java index e9bf6c591fa41..9a07cc3265128 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateClearContainer.java @@ -136,7 +136,7 @@ public void deserialize(final InputStream inputStream) throws IOException { } } - Set getTableNames() { + public Set getTableNames() { return tableNames; } From 3531f56dcc67a68b34e9f7e525c95dbc32fb6aa8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:13:14 +0800 Subject: [PATCH 193/213] Add db name --- .../TableDeviceCacheAttributeGuard.java | 21 ++++++++++++------- .../update/DeviceAttributeCacheUpdater.java | 7 +++++-- .../impl/SchemaRegionMemoryImpl.java | 6 ++++-- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index ccd1c0ae882da..0797c40a9b89a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -25,7 +25,6 @@ import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.UpdateContainer; import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.UpdateDetailContainer; import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq; -import org.apache.iotdb.mpp.rpc.thrift.TSchemaRegionAttributeInfo; import org.apache.tsfile.utils.Pair; @@ -95,18 +94,26 @@ public synchronized void handleAttributeUpdate(final TAttributeUpdateReq updateR return false; }); applyQueue.add( - updateReq.getAttributeUpdateMap().values().stream() - .map(TSchemaRegionAttributeInfo::getBody) + updateReq.getAttributeUpdateMap().entrySet().stream() + .map(entry -> new Pair<>(entry.getKey(), entry.getValue().getBody())) .collect(Collectors.toSet())); tryUpdateCache(); } + @SuppressWarnings("unchecked") public synchronized void tryUpdateCache() { while (!applyQueue.isEmpty()) { final Set firstElement = applyQueue.peek(); if (firstElement instanceof HashSet) { for (final Object element : firstElement) { - handleContainer(DeviceAttributeCacheUpdater.getContainer((byte[]) element)); + final Pair schemaRegionIdContainerBytesPair = + (Pair) element; + handleContainer( + fetchedSchemaRegionIds2LargestVersionAndDatabaseMap + .get(schemaRegionIdContainerBytesPair.getLeft()) + .getRight(), + DeviceAttributeCacheUpdater.getContainer( + schemaRegionIdContainerBytesPair.getRight())); } applyQueue.removeFirst(); } else if (firstElement.isEmpty()) { @@ -130,7 +137,7 @@ public synchronized void setVersion(final int schemaRegionId, final long newVers }); } - public void handleContainer(final UpdateContainer container) { + public void handleContainer(final String database, final UpdateContainer container) { if (container instanceof UpdateDetailContainer) { ((UpdateDetailContainer) container) .getUpdateMap() @@ -139,11 +146,11 @@ public void handleContainer(final UpdateContainer container) { deviceNodesMap.forEach( (nodes, attributes) -> cache.updateAttributes( - null, convertIdValuesToDeviceID(table, nodes), attributes))); + database, convertIdValuesToDeviceID(table, nodes), attributes))); } else { ((UpdateClearContainer) container) .getTableNames() - .forEach(table -> cache.invalidateAttributes(null, table)); + .forEach(table -> cache.invalidateAttributes(database, table)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index 54e4e651edd0f..9fdae96748c58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -75,13 +75,16 @@ public class DeviceAttributeCacheUpdater { // Volatiles private final MemSchemaRegionStatistics regionStatistics; + private final String databaseName; // Only exist for update detail container private final Map updateContainerStatistics = new HashMap<>(); - public DeviceAttributeCacheUpdater(final MemSchemaRegionStatistics regionStatistics) { + public DeviceAttributeCacheUpdater( + final MemSchemaRegionStatistics regionStatistics, final String databaseName) { this.regionStatistics = regionStatistics; + this.databaseName = databaseName; } /////////////////////////////// Service /////////////////////////////// @@ -363,7 +366,7 @@ private void deserialize(final InputStream inputStream) throws IOException { final TableDeviceCacheAttributeGuard guard = TableDeviceSchemaFetcher.getInstance().getAttributeGuard(); guard.setVersion(regionStatistics.getSchemaRegionId(), version.get()); - guard.handleContainer(container); + guard.handleContainer(databaseName, container); } else { attributeUpdateMap.put(location, container); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index f5504caca89ad..69762019b6831 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -248,7 +248,8 @@ public synchronized void init() throws MetadataException { isRecovering = true; deviceAttributeStore = new DeviceAttributeStore(regionStatistics); - deviceAttributeCacheUpdater = new DeviceAttributeCacheUpdater(regionStatistics); + deviceAttributeCacheUpdater = + new DeviceAttributeCacheUpdater(regionStatistics, storageGroupFullPath.substring(5)); tagManager = new TagManager(schemaRegionDirPath, regionStatistics); mtree = new MTreeBelowSGMemoryImpl( @@ -550,7 +551,8 @@ public void loadSnapshot(final File latestSnapshotRootDir) { System.currentTimeMillis() - snapshotStartTime); snapshotStartTime = System.currentTimeMillis(); - deviceAttributeCacheUpdater = new DeviceAttributeCacheUpdater(regionStatistics); + deviceAttributeCacheUpdater = + new DeviceAttributeCacheUpdater(regionStatistics, storageGroupFullPath.substring(5)); deviceAttributeCacheUpdater.loadFromSnapshot(latestSnapshotRootDir); logger.info( "Device attribute remote updater snapshot loading of schemaRegion {} costs {}ms.", From ac20ee8c1c73f5c359c3ffc6cfb9d18c8b218933 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:20:15 +0800 Subject: [PATCH 194/213] Update SchemaRegionMemoryImpl.java --- .../schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 69762019b6831..89685a6e442e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1403,8 +1403,8 @@ private void updateAttribute( TableDeviceSchemaCache.getInstance() .updateAttributes( databaseName, convertIdValuesToDeviceID(tableName, deviceId), resultMap); - deviceAttributeCacheUpdater.update(tableName, deviceId, resultMap); } + deviceAttributeCacheUpdater.update(tableName, deviceId, resultMap); } @Override From 722af6fa9df5e7c1d54a6b053d0c629f9c122787 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:31:33 +0800 Subject: [PATCH 195/213] Update AbstractTableSchemaQueryAttributeSecurityVisitor.java --- .../AbstractTableSchemaQueryAttributeSecurityVisitor.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java index 8a16897be580d..71be86f04fa3c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/AbstractTableSchemaQueryAttributeSecurityVisitor.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.FilterNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode; import javax.annotation.Nonnull; @@ -41,6 +42,12 @@ public R visitPlan(final PlanNode node, final ConsensusGroupId context) { return null; } + @Override + public R visitIdentitySink(final IdentitySinkNode node, final ConsensusGroupId context) { + // TODO: apply multi regions? + return node.getChildren().get(0).accept(this, context); + } + @Override public R visitFilter(final FilterNode node, final ConsensusGroupId context) { return node.getChild().accept(this, context); From e03be549c1dabe44abae06a15ae9b89386e20288 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:40:39 +0800 Subject: [PATCH 196/213] Update GeneralRegionAttributeSecurityService.java --- .../update/GeneralRegionAttributeSecurityService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 18d9c190c8928..07f075a7dd19e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -240,9 +240,10 @@ private void execute() { .forEach( (location, bytes) -> { clientHandler.putNodeLocation(location.getDataNodeId(), location); + clientHandler.putRequestIfAbsent( + location.getDataNodeId(), new TAttributeUpdateReq(new HashMap<>())); clientHandler - .putRequestIfAbsent( - location.getDataNodeId(), new TAttributeUpdateReq(new HashMap<>())) + .getRequest(location.getDataNodeId()) .getAttributeUpdateMap() .put( id.getId(), From 4262432689d941bc4d16eefdaf917d2470576f24 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:11:21 +0800 Subject: [PATCH 197/213] Use list --- ...GeneralRegionAttributeSecurityService.java | 1 + .../update/UpdateDetailContainer.java | 43 +++++++++++-------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index 07f075a7dd19e..db8aaca503044 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -146,6 +146,7 @@ private void execute() { final Map> shrinkMap = sendUpdateRequestAndMayShrink(attributeUpdateCommitMap); + System.out.println(attributeUpdateCommitMap); // Commit attributeUpdateCommitMap.forEach( (schemaRegionId, pair) -> { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index 63a61242681cb..b5b018ccec18d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -30,7 +30,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -43,15 +45,16 @@ public class UpdateDetailContainer implements UpdateContainer { static final long MAP_SIZE = RamUsageEstimator.shallowSizeOfInstance(ConcurrentHashMap.class); + static final long LIST_SIZE = RamUsageEstimator.shallowSizeOfInstance(ArrayList.class); static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(UpdateClearContainer.class) + MAP_SIZE; // <@Nonnull TableName, >> - private final ConcurrentMap>> + private final ConcurrentMap, ConcurrentMap>> updateMap = new ConcurrentHashMap<>(); - public ConcurrentMap>> + public ConcurrentMap, ConcurrentMap>> getUpdateMap() { return updateMap; } @@ -73,12 +76,12 @@ public long updateAttribute( value = new ConcurrentHashMap<>(); } value.compute( - deviceId, + Arrays.asList(deviceId), (device, attributes) -> { if (Objects.isNull(attributes)) { result.addAndGet( RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY - + RamUsageEstimator.sizeOf(device) + + sizeOfList(device) + MAP_SIZE); attributes = new ConcurrentHashMap<>(); } @@ -129,7 +132,7 @@ private void serializeWithLimit( final int mapSizeOffset = outputStream.skipInt(); int mapEntryCount = 0; int newSize; - for (final Map.Entry>> + for (final Map.Entry, ConcurrentMap>> tableEntry : updateMap.entrySet()) { final byte[] tableEntryBytes = tableEntry.getKey().getBytes(TSFileConfig.STRING_CHARSET); newSize = 2 * Integer.BYTES + tableEntryBytes.length; @@ -143,10 +146,10 @@ private void serializeWithLimit( outputStream.writeWithLength(tableEntryBytes); final int deviceSizeOffset = outputStream.skipInt(); int deviceEntryCount = 0; - for (final Map.Entry> deviceEntry : + for (final Map.Entry, ConcurrentMap> deviceEntry : tableEntry.getValue().entrySet()) { final byte[][] deviceIdBytes = - Arrays.stream(deviceEntry.getKey()) + deviceEntry.getKey().stream() .map(str -> Objects.nonNull(str) ? str.getBytes(TSFileConfig.STRING_CHARSET) : null) .toArray(byte[][]::new); @@ -163,7 +166,7 @@ private void serializeWithLimit( } limitBytes.addAndGet(-newSize); ++deviceEntryCount; - ReadWriteIOUtils.write(deviceEntry.getKey().length, outputStream); + ReadWriteIOUtils.write(deviceEntry.getKey().size(), outputStream); for (final byte[] node : deviceIdBytes) { outputStream.writeWithLength(node); } @@ -208,7 +211,7 @@ public Pair updateSelfByCommitContainer(final UpdateContainer com if (!this.updateMap.containsKey(table)) { return; } - final ConcurrentMap> thisDeviceMap = + final ConcurrentMap, ConcurrentMap> thisDeviceMap = this.updateMap.get(table); commitMap.forEach( (device, attributes) -> { @@ -233,7 +236,7 @@ public Pair updateSelfByCommitContainer(final UpdateContainer com if (thisAttributes.isEmpty()) { result.addAndGet( RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY - + RamUsageEstimator.sizeOf(device) + + sizeOfList(device) + MAP_SIZE); thisDeviceMap.remove(device); } @@ -251,7 +254,7 @@ public Pair updateSelfByCommitContainer(final UpdateContainer com .getTableNames() .forEach( tableName -> { - final ConcurrentMap> deviceMap = + final ConcurrentMap, ConcurrentMap> deviceMap = updateMap.remove(tableName); if (Objects.nonNull(deviceMap)) { result.addAndGet( @@ -260,7 +263,7 @@ public Pair updateSelfByCommitContainer(final UpdateContainer com + deviceMap.entrySet().stream() .mapToLong( entry -> - RamUsageEstimator.sizeOf(entry.getKey()) + sizeOfList(entry.getKey()) + entry.getValue().size() * RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + entry.getValue().entrySet().stream() @@ -277,17 +280,21 @@ public Pair updateSelfByCommitContainer(final UpdateContainer com return new Pair<>(result.get(), updateMap.isEmpty()); } + private static long sizeOfList(final List input) { + return input.stream().map(RamUsageEstimator::sizeOf).reduce(LIST_SIZE, Long::sum); + } + @Override public void serialize(final OutputStream outputStream) throws IOException { ReadWriteIOUtils.write((byte) 1, outputStream); ReadWriteIOUtils.write(updateMap.size(), outputStream); - for (final Map.Entry>> + for (final Map.Entry, ConcurrentMap>> tableEntry : updateMap.entrySet()) { ReadWriteIOUtils.write(tableEntry.getKey(), outputStream); ReadWriteIOUtils.write(tableEntry.getValue().size(), outputStream); - for (final Map.Entry> deviceEntry : + for (final Map.Entry, ConcurrentMap> deviceEntry : tableEntry.getValue().entrySet()) { - ReadWriteIOUtils.write(deviceEntry.getKey().length, outputStream); + ReadWriteIOUtils.write(deviceEntry.getKey().size(), outputStream); for (final String node : deviceEntry.getKey()) { ReadWriteIOUtils.write(node, outputStream); } @@ -302,13 +309,13 @@ public void deserialize(final InputStream inputStream) throws IOException { for (int i = 0; i < tableSize; ++i) { final String tableName = ReadWriteIOUtils.readString(inputStream); final int deviceSize = ReadWriteIOUtils.readInt(inputStream); - final ConcurrentMap> deviceMap = + final ConcurrentMap, ConcurrentMap> deviceMap = new ConcurrentHashMap<>(deviceSize); for (int j = 0; j < deviceSize; ++j) { final int nodeSize = ReadWriteIOUtils.readInt(inputStream); - final String[] deviceId = new String[nodeSize]; + final List deviceId = new ArrayList<>(nodeSize); for (int k = 0; k < nodeSize; ++k) { - deviceId[k] = ReadWriteIOUtils.readString(inputStream); + deviceId.add(ReadWriteIOUtils.readString(inputStream)); } deviceMap.put(deviceId, readConcurrentMap(inputStream)); } From b745968515eda2ecf5e97e57f87f0007081f6c45 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:13:36 +0800 Subject: [PATCH 198/213] Update TableDeviceCacheAttributeGuard.java --- .../metadata/fetcher/TableDeviceCacheAttributeGuard.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java index 0797c40a9b89a..45aee85fda362 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceCacheAttributeGuard.java @@ -146,7 +146,9 @@ public void handleContainer(final String database, final UpdateContainer contain deviceNodesMap.forEach( (nodes, attributes) -> cache.updateAttributes( - database, convertIdValuesToDeviceID(table, nodes), attributes))); + database, + convertIdValuesToDeviceID(table, nodes.toArray(new String[0])), + attributes))); } else { ((UpdateClearContainer) container) .getTableNames() From 4c094d4a970cb41df37afbc91c9c6f11bb7dcc08 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:20:35 +0800 Subject: [PATCH 199/213] Update GeneralRegionAttributeSecurityService.java --- .../attribute/update/GeneralRegionAttributeSecurityService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java index db8aaca503044..07f075a7dd19e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/GeneralRegionAttributeSecurityService.java @@ -146,7 +146,6 @@ private void execute() { final Map> shrinkMap = sendUpdateRequestAndMayShrink(attributeUpdateCommitMap); - System.out.println(attributeUpdateCommitMap); // Commit attributeUpdateCommitMap.forEach( (schemaRegionId, pair) -> { From 283056da83b350cd327b3120b22df8bd7d2377ba Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:22:39 +0800 Subject: [PATCH 200/213] Fix --- .../update/DeviceAttributeCacheUpdater.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java index 9fdae96748c58..35c9ff4054bc9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/DeviceAttributeCacheUpdater.java @@ -202,11 +202,13 @@ public void commit(final TableDeviceAttributeCommitUpdateNode node) { } private void removeLocation(final TDataNodeLocation location) { - releaseMemory( - updateContainerStatistics.containsKey(location) - ? updateContainerStatistics.get(location).getContainerSize() - : ((UpdateClearContainer) attributeUpdateMap.get(location)).ramBytesUsed()); - attributeUpdateMap.remove(location); + if (attributeUpdateMap.containsKey(location)) { + releaseMemory( + updateContainerStatistics.containsKey(location) + ? updateContainerStatistics.get(location).getContainerSize() + : ((UpdateClearContainer) attributeUpdateMap.get(location)).ramBytesUsed()); + attributeUpdateMap.remove(location); + } } public static UpdateContainer getContainer(final byte[] bytes) { From 84d89855b0387d1ae85cfc41a0218951728cd881 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 18:03:12 +0800 Subject: [PATCH 201/213] Support clear cache --- .../plan/relational/sql/ast/ClearCache.java | 65 +++++++++++++++++++ .../relational/sql/parser/AstBuilder.java | 2 +- .../relational/grammar/sql/RelationalSql.g4 | 2 +- 3 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ClearCache.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ClearCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ClearCache.java new file mode 100644 index 0000000000000..e9d9fd631a2bf --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ClearCache.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; + +public class ClearCache extends Statement { + + private final boolean onCluster; + + public ClearCache(final boolean onCluster) { + super(null); + this.onCluster = onCluster; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public int hashCode() { + return Objects.hash(onCluster); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null || (getClass() != obj.getClass())) { + return false; + } + + final ClearCache other = (ClearCache) obj; + return this.onCluster == other.onCluster; + } + + @Override + public String toString() { + return toStringHelper(this).toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 800fe7ea7ca96..529518baeb0ff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -893,7 +893,7 @@ public Node visitFlushStatement(RelationalSqlParser.FlushStatementContext ctx) { } @Override - public Node visitClearCacheStatement(RelationalSqlParser.ClearCacheStatementContext ctx) { + public Node visitClearCacheStatement(final RelationalSqlParser.ClearCacheStatementContext ctx) { return super.visitClearCacheStatement(ctx); } diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index a3cd45ad6530a..4aef90641ca30 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -433,7 +433,7 @@ flushStatement ; clearCacheStatement - : CLEAR CACHE (localOrClusterMode)? + : CLEAR (ATTRIBUTE)? CACHE (localOrClusterMode)? ; repairDataStatement From 08e8f632eaae58ea1a5fedc395bd33bb138648e9 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 19:01:07 +0800 Subject: [PATCH 202/213] Doing clear cache --- .../impl/DataNodeInternalRPCServiceImpl.java | 2 +- .../schema/ClusterSchemaQuotaLevel.java | 1 + .../schema/cache/CacheClearOptions.java | 27 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/cache/CacheClearOptions.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index a0a0636f9e32f..9a5e77989fa7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -1914,7 +1914,7 @@ public TSStatus flush(TFlushReq req) throws TException { public TSStatus clearCache() throws TException { try { storageEngine.clearCache(); - } catch (Exception e) { + } catch (final Exception e) { return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); } return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ClusterSchemaQuotaLevel.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ClusterSchemaQuotaLevel.java index 996b08ebfb90c..3c10920a1b81d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ClusterSchemaQuotaLevel.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ClusterSchemaQuotaLevel.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.commons.schema; public enum ClusterSchemaQuotaLevel { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/cache/CacheClearOptions.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/cache/CacheClearOptions.java new file mode 100644 index 0000000000000..95bb36b913c4b --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/cache/CacheClearOptions.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.schema.cache; + +public enum CacheClearOptions { + STORAGE, + TABLE_ATTRIBUTE, + LAST_VALUE, + TREE_SCHEMA, +} From 3224f34303032d59d8ef4886d58ac7cb73691053 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 19:03:03 +0800 Subject: [PATCH 203/213] Update ClearCache.java --- .../plan/relational/sql/ast/ClearCache.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ClearCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ClearCache.java index e9d9fd631a2bf..ce07528f129fb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ClearCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ClearCache.java @@ -19,20 +19,33 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; +import org.apache.iotdb.commons.schema.cache.CacheClearOptions; + import com.google.common.collect.ImmutableList; import java.util.List; import java.util.Objects; +import java.util.Set; import static com.google.common.base.MoreObjects.toStringHelper; public class ClearCache extends Statement { private final boolean onCluster; + private final Set options; - public ClearCache(final boolean onCluster) { + public ClearCache(final boolean onCluster, final Set options) { super(null); this.onCluster = onCluster; + this.options = options; + } + + public boolean isOnCluster() { + return onCluster; + } + + public Set getOptions() { + return options; } @Override From a24e14999949a91d19ba816f79b6b8fb7a04a79f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 20:28:20 +0800 Subject: [PATCH 204/213] Partially clear cache --- .../relational/ShowDataNodesTask.java | 12 ++++----- .../relational/sql/parser/AstBuilder.java | 26 ++++++++++++++++++- .../schema/cache/CacheClearOptions.java | 4 +-- .../relational/grammar/sql/RelationalSql.g4 | 8 +++++- 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDataNodesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDataNodesTask.java index 744fe2018dff9..c54e1a27571c2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDataNodesTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDataNodesTask.java @@ -42,20 +42,20 @@ public class ShowDataNodesTask implements IConfigTask { @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.showDataNodes(); } public static void buildTSBlock( - TShowDataNodesResp showDataNodesResp, SettableFuture future) { - List outputDataTypes = + final TShowDataNodesResp showDataNodesResp, final SettableFuture future) { + final List outputDataTypes = ColumnHeaderConstant.showDataNodesColumnHeaders.stream() .map(ColumnHeader::getColumnType) .collect(Collectors.toList()); - TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); + final TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); if (showDataNodesResp.getDataNodesInfoList() != null) { - for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) { + for (final TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) { builder.getTimeColumnBuilder().writeLong(0L); builder.getColumnBuilder(0).writeInt(dataNodeInfo.getDataNodeId()); builder @@ -72,7 +72,7 @@ public static void buildTSBlock( builder.declarePosition(); } } - DatasetHeader datasetHeader = DatasetHeaderFactory.getShowDataNodesHeader(); + final DatasetHeader datasetHeader = DatasetHeaderFactory.getShowDataNodesHeader(); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 529518baeb0ff..0530327bf8ce4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.cache.CacheClearOptions; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.PathUtils; @@ -38,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BinaryLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Cast; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ClearCache; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CoalesceExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ColumnDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; @@ -182,12 +184,14 @@ import java.util.Collections; import java.util.Deque; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -894,7 +898,27 @@ public Node visitFlushStatement(RelationalSqlParser.FlushStatementContext ctx) { @Override public Node visitClearCacheStatement(final RelationalSqlParser.ClearCacheStatementContext ctx) { - return super.visitClearCacheStatement(ctx); + final Set options; + final RelationalSqlParser.ClearCacheOptionsContext context = ctx.clearCacheOptions(); + + if (Objects.isNull(context)) { + options = Collections.singleton(CacheClearOptions.DEFAULT); + } else if (context.ATTRIBUTE() != null) { + options = Collections.singleton(CacheClearOptions.TABLE_ATTRIBUTE); + } else if (context.QUERY() != null) { + options = Collections.singleton(CacheClearOptions.QUERY); + } else { + options = + new HashSet<>( + Arrays.asList( + CacheClearOptions.TABLE_ATTRIBUTE, + CacheClearOptions.TREE_SCHEMA, + CacheClearOptions.QUERY)); + } + return new ClearCache( + Objects.isNull(ctx.localOrClusterMode()) + || Objects.nonNull(ctx.localOrClusterMode().CLUSTER()), + options); } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/cache/CacheClearOptions.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/cache/CacheClearOptions.java index 95bb36b913c4b..74b39dfd52917 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/cache/CacheClearOptions.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/cache/CacheClearOptions.java @@ -20,8 +20,8 @@ package org.apache.iotdb.commons.schema.cache; public enum CacheClearOptions { - STORAGE, TABLE_ATTRIBUTE, - LAST_VALUE, TREE_SCHEMA, + QUERY, + DEFAULT, } diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 4aef90641ca30..4c04f5e2ef6aa 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -433,7 +433,7 @@ flushStatement ; clearCacheStatement - : CLEAR (ATTRIBUTE)? CACHE (localOrClusterMode)? + : CLEAR clearCacheOptions? CACHE localOrClusterMode? ; repairDataStatement @@ -469,6 +469,12 @@ setConfigurationStatement : SET CONFIGURATION propertyAssignments (ON INTEGER_VALUE)? ; +clearCacheOptions + : ATTRIBUTE + | QUERY + | ALL + ; + localOrClusterMode : (ON (LOCAL | CLUSTER)) ; From 941474bb4aa7c65f4d1f3a0620c1d5558fa5f6e1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 20:41:56 +0800 Subject: [PATCH 205/213] Clear cache --- .../config/TableConfigTaskVisitor.java | 9 ++++ .../executor/ClusterConfigTaskExecutor.java | 12 +++--- .../config/executor/IConfigTaskExecutor.java | 3 +- .../metadata/relational/ClearCacheTask.java | 42 +++++++++++++++++++ .../execution/config/sys/ClearCacheTask.java | 4 +- .../plan/relational/sql/ast/AstVisitor.java | 4 ++ .../plan/relational/sql/ast/ClearCache.java | 5 +++ .../src/main/thrift/confignode.thrift | 2 +- .../src/main/thrift/datanode.thrift | 2 +- 9 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ClearCacheTask.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index c26780185a315..9c8857b405901 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableAddColumnTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableSetPropertiesTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ClearCacheTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DescribeTableTask; @@ -59,6 +60,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AddColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ClearCache; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ColumnDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipe; @@ -263,6 +265,13 @@ protected IConfigTask visitShowAINodes( return new ShowAINodesTask(); } + @Override + protected IConfigTask visitClearCache( + final ClearCache clearCacheStatement, final MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + return new ClearCacheTask(clearCacheStatement); + } + @Override protected IConfigTask visitCreateTable(final CreateTable node, final MPPQueryContext context) { context.setQueryType(QueryType.WRITE); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index b233a2fc6c566..3fed682eb0de4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -47,6 +47,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.service.PipePluginExecutableManager; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.pipe.connector.payload.airgap.AirGapPseudoTPipeTransferRequest; +import org.apache.iotdb.commons.schema.cache.CacheClearOptions; import org.apache.iotdb.commons.schema.table.AlterTableOperationType; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; @@ -1061,22 +1062,23 @@ public SettableFuture flush(TFlushReq tFlushReq, boolean onClu } @Override - public SettableFuture clearCache(boolean onCluster) { - SettableFuture future = SettableFuture.create(); + public SettableFuture clearCache( + final boolean onCluster, final Set options) { + final SettableFuture future = SettableFuture.create(); TSStatus tsStatus = new TSStatus(); if (onCluster) { - try (ConfigNodeClient client = + try (final ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { // Send request to some API server tsStatus = client.clearCache(); - } catch (ClientManagerException | TException e) { + } catch (final ClientManagerException | TException e) { future.setException(e); } } else { try { StorageEngine.getInstance().clearCache(); tsStatus = RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); - } catch (Exception e) { + } catch (final Exception e) { tsStatus = RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index 16e32688c35d9..d4ac4b5c59717 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.commons.cluster.NodeStatus; +import org.apache.iotdb.commons.schema.cache.CacheClearOptions; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; @@ -133,7 +134,7 @@ public interface IConfigTaskExecutor { SettableFuture flush(TFlushReq tFlushReq, boolean onCluster); - SettableFuture clearCache(boolean onCluster); + SettableFuture clearCache(boolean onCluster, Set options); SettableFuture setConfiguration(TSetConfigurationReq tSetConfigurationReq); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ClearCacheTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ClearCacheTask.java new file mode 100644 index 0000000000000..20b2878831dc9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ClearCacheTask.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; + +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ClearCache; + +import com.google.common.util.concurrent.ListenableFuture; + +public class ClearCacheTask implements IConfigTask { + + private final ClearCache clearCache; + + public ClearCacheTask(final ClearCache clearCache) { + this.clearCache = clearCache; + } + + @Override + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.clearCache(clearCache.isOnCluster()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ClearCacheTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ClearCacheTask.java index 55e6bcd8dc32a..217a5603b34e9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ClearCacheTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ClearCacheTask.java @@ -30,12 +30,12 @@ public class ClearCacheTask implements IConfigTask { private final ClearCacheStatement clearCacheStatement; - public ClearCacheTask(ClearCacheStatement clearCacheStatement) { + public ClearCacheTask(final ClearCacheStatement clearCacheStatement) { this.clearCacheStatement = clearCacheStatement; } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { // If the action is executed successfully, return the Future. // If your operation is async, you can return the corresponding future directly. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 103ffd9080924..932a7b4e7f4d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -355,6 +355,10 @@ protected R visitShowAINodes(ShowAINodes node, C context) { return visitStatement(node, context); } + protected R visitClearCache(ClearCache node, C context) { + return visitStatement(node, context); + } + protected R visitRenameTable(RenameTable node, C context) { return visitStatement(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ClearCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ClearCache.java index ce07528f129fb..6f9183da1fec4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ClearCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ClearCache.java @@ -48,6 +48,11 @@ public Set getOptions() { return options; } + @Override + public R accept(final AstVisitor visitor, final C context) { + return visitor.visitClearCache(this, context); + } + @Override public List getChildren() { return ImmutableList.of(); diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 5c1ad65b4fbdc..a9658b8c03ca5 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1484,7 +1484,7 @@ service IConfigNodeRPCService { common.TSStatus flush(common.TFlushReq req) /** Clear the cache of chunk, chunk metadata and timeseries metadata to release the memory footprint on all DataNodes */ - common.TSStatus clearCache() + common.TSStatus clearCache(set cacheClearOptions) /** Set configuration on specified node */ common.TSStatus setConfiguration(common.TSetConfigurationReq req) diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index bcccb14f8ab67..9a5f4fbb019d7 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -901,7 +901,7 @@ service IDataNodeRPCService { common.TSStatus stopRepairData() - common.TSStatus clearCache() + common.TSStatus clearCache(set cacheClearOptions) common.TShowConfigurationResp showConfiguration() From 57848dac2c1fa12d622469df67dac2393341b7ac Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 21:06:10 +0800 Subject: [PATCH 206/213] May complete --- ...oDnInternalServiceAsyncRequestManager.java | 5 ++- .../confignode/manager/ConfigManager.java | 10 +++--- .../iotdb/confignode/manager/IManager.java | 2 +- .../confignode/manager/node/NodeManager.java | 9 ++--- .../thrift/ConfigNodeRPCServiceProcessor.java | 6 ++-- .../db/protocol/client/ConfigNodeClient.java | 4 +-- .../impl/DataNodeInternalRPCServiceImpl.java | 36 +++++++++++++++++-- .../executor/ClusterConfigTaskExecutor.java | 12 +++---- .../fetcher/cache/TableDeviceCacheEntry.java | 23 +++++++++--- .../fetcher/cache/TableDeviceSchemaCache.java | 28 +++++++++++++++ .../service/DataNodeInternalRPCService.java | 4 +++ .../src/main/thrift/confignode.thrift | 2 +- 12 files changed, 110 insertions(+), 31 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java index 3fe2536a1d974..05b24ea11dbf8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java @@ -95,6 +95,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; /** Asynchronously send RPC requests to DataNodes. See queryengine.thrift for more details. */ @@ -104,6 +105,7 @@ public class CnToDnInternalServiceAsyncRequestManager private static final Logger LOGGER = LoggerFactory.getLogger(CnToDnInternalServiceAsyncRequestManager.class); + @SuppressWarnings("unchecked") @Override protected void initActionMapBuilder() { actionMapBuilder.put( @@ -223,7 +225,8 @@ protected void initActionMapBuilder() { client.flush((TFlushReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( CnToDnAsyncRequestType.CLEAR_CACHE, - (req, client, handler) -> client.clearCache((DataNodeTSStatusRPCHandler) handler)); + (req, client, handler) -> + client.clearCache((Set) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( CnToDnAsyncRequestType.START_REPAIR_DATA, (req, client, handler) -> client.startRepairData((DataNodeTSStatusRPCHandler) handler)); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 9d3f7fff9b8a9..0719cfe9f75d1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -1614,18 +1614,18 @@ public TSStatus merge() { } @Override - public TSStatus flush(TFlushReq req) { - TSStatus status = confirmLeader(); + public TSStatus flush(final TFlushReq req) { + final TSStatus status = confirmLeader(); return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() ? RpcUtils.squashResponseStatusList(nodeManager.flush(req)) : status; } @Override - public TSStatus clearCache() { - TSStatus status = confirmLeader(); + public TSStatus clearCache(final Set clearCacheOptions) { + final TSStatus status = confirmLeader(); return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() - ? RpcUtils.squashResponseStatusList(nodeManager.clearCache()) + ? RpcUtils.squashResponseStatusList(nodeManager.clearCache(clearCacheOptions)) : status; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 640c82d6653f4..97fc2e0ced30d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -557,7 +557,7 @@ TDataPartitionTableResp getOrCreateDataPartition( TSStatus flush(TFlushReq req); /** Clear cache on all DataNodes. */ - TSStatus clearCache(); + TSStatus clearCache(final Set clearCacheOptions); /** Set Configuration. */ TSStatus setConfiguration(TSetConfigurationReq req); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 49385131992a4..433a0748b8a7f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -881,11 +881,12 @@ public List flush(TFlushReq req) { return clientHandler.getResponseList(); } - public List clearCache() { - Map dataNodeLocationMap = + public List clearCache(final Set clearCacheOptions) { + final Map dataNodeLocationMap = configManager.getNodeManager().getRegisteredDataNodeLocations(); - DataNodeAsyncRequestContext clientHandler = - new DataNodeAsyncRequestContext<>(CnToDnAsyncRequestType.CLEAR_CACHE, dataNodeLocationMap); + final DataNodeAsyncRequestContext, TSStatus> clientHandler = + new DataNodeAsyncRequestContext<>( + CnToDnAsyncRequestType.CLEAR_CACHE, clearCacheOptions, dataNodeLocationMap); CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler); return clientHandler.getResponseList(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index d64e82026a40b..c5635843a828c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -925,12 +925,12 @@ public TSStatus flush(TFlushReq req) throws TException { } @Override - public TSStatus clearCache() { - return configManager.clearCache(); + public TSStatus clearCache(final Set clearCacheOptions) { + return configManager.clearCache(clearCacheOptions); } @Override - public TSStatus setConfiguration(TSetConfigurationReq req) throws TException { + public TSStatus setConfiguration(TSetConfigurationReq req) { return configManager.setConfiguration(req); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index f3c6a07307d1a..d92cb97e5c95e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -742,9 +742,9 @@ public TSStatus flush(TFlushReq req) throws TException { } @Override - public TSStatus clearCache() throws TException { + public TSStatus clearCache(final Set clearCacheOptions) throws TException { return executeRemoteCallWithRetry( - () -> client.clearCache(), status -> !updateConfigNodeLeader(status)); + () -> client.clearCache(clearCacheOptions), status -> !updateConfigNodeLeader(status)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 9a5e77989fa7c..673a97a797af1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -57,6 +57,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.cache.CacheClearOptions; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCType; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; @@ -283,6 +284,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -314,6 +316,9 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface private final SchemaEngine schemaEngine = SchemaEngine.getInstance(); private final StorageEngine storageEngine = StorageEngine.getInstance(); + private final TableDeviceSchemaCache tableDeviceSchemaCache = + TableDeviceSchemaCache.getInstance(); + private final DataNodeRegionManager regionManager = DataNodeRegionManager.getInstance(); private final DataNodeSpaceQuotaManager spaceQuotaManager = @@ -546,7 +551,7 @@ public TSStatus invalidateSchemaCache(final TInvalidateCacheReq req) { // clear table related cache final String database = req.getFullPath().substring(5); DataNodeTableCache.getInstance().invalid(database); - TableDeviceSchemaCache.getInstance().invalidate(database); + tableDeviceSchemaCache.invalidate(database); LOGGER.info("Schema cache of {} has been invalidated", req.getFullPath()); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } finally { @@ -1911,9 +1916,34 @@ public TSStatus flush(TFlushReq req) throws TException { } @Override - public TSStatus clearCache() throws TException { + public TSStatus clearCache(final Set clearCacheOptions) throws TException { + return clearCacheImpl( + clearCacheOptions.stream() + .map(i -> CacheClearOptions.values()[i]) + .collect(Collectors.toSet())); + } + + public TSStatus clearCacheImpl(final Set options) { try { - storageEngine.clearCache(); + if (options.contains(CacheClearOptions.DEFAULT) + || options.contains(CacheClearOptions.QUERY)) { + storageEngine.clearCache(); + } + if (options.contains(CacheClearOptions.QUERY) + && options.contains(CacheClearOptions.TABLE_ATTRIBUTE) + && options.contains(CacheClearOptions.TREE_SCHEMA)) { + tableDeviceSchemaCache.invalidateAll(); + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } + if (options.contains(CacheClearOptions.QUERY)) { + tableDeviceSchemaCache.invalidateLastCache(); + } + if (options.contains(CacheClearOptions.TABLE_ATTRIBUTE)) { + tableDeviceSchemaCache.invalidateAttributeCache(); + } + if (options.contains(CacheClearOptions.TREE_SCHEMA)) { + tableDeviceSchemaCache.invalidateTreeSchema(); + } } catch (final Exception e) { return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 3fed682eb0de4..08c50a7045d84 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -243,6 +243,7 @@ import org.apache.iotdb.db.schemaengine.template.TemplateAlterOperationType; import org.apache.iotdb.db.schemaengine.template.alter.TemplateAlterOperationUtil; import org.apache.iotdb.db.schemaengine.template.alter.TemplateExtendInfo; +import org.apache.iotdb.db.service.DataNodeInternalRPCService; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.compaction.repair.RepairTaskStatus; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleTaskManager; @@ -1070,17 +1071,14 @@ public SettableFuture clearCache( try (final ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { // Send request to some API server - tsStatus = client.clearCache(); + tsStatus = + client.clearCache( + options.stream().map(CacheClearOptions::ordinal).collect(Collectors.toSet())); } catch (final ClientManagerException | TException e) { future.setException(e); } } else { - try { - StorageEngine.getInstance().clearCache(); - tsStatus = RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); - } catch (final Exception e) { - tsStatus = RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); - } + tsStatus = DataNodeInternalRPCService.getInstance().getImpl().clearCacheImpl(options); } if (tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java index a15df1455ef8c..20cfff5e73bc5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java @@ -81,11 +81,12 @@ int updateAttribute( int invalidateAttribute() { final AtomicInteger size = new AtomicInteger(0); deviceSchema.updateAndGet( - map -> { - if (Objects.nonNull(map)) { - size.set(map.estimateSize()); + schema -> { + if (schema instanceof TableAttributeSchema) { + size.set(schema.estimateSize()); + return null; } - return null; + return schema; }); return size.get(); } @@ -143,6 +144,20 @@ IDeviceSchema getDeviceSchema() { return deviceSchema.get(); } + int invalidateTreeSchema() { + final AtomicInteger size = new AtomicInteger(0); + deviceSchema.updateAndGet( + schema -> { + if (schema instanceof TreeDeviceNormalSchema + || schema instanceof TreeDeviceTemplateSchema) { + size.set(schema.estimateSize()); + return null; + } + return schema; + }); + return size.get(); + } + /////////////////////////////// Last Cache /////////////////////////////// int updateLastCache( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java index a176eba869bbc..7acd9dcb3b370 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java @@ -520,6 +520,34 @@ public void invalidate(final String database, final String tableName) { } } + public void invalidateLastCache() { + readWriteLock.writeLock().lock(); + try { + dualKeyCache.update(tableId -> true, deviceID -> true, entry -> -entry.invalidateLastCache()); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + public void invalidateAttributeCache() { + readWriteLock.writeLock().lock(); + try { + dualKeyCache.update(tableId -> true, deviceID -> true, entry -> -entry.invalidateAttribute()); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + public void invalidateTreeSchema() { + readWriteLock.writeLock().lock(); + try { + dualKeyCache.update( + tableId -> true, deviceID -> true, entry -> -entry.invalidateTreeSchema()); + } finally { + readWriteLock.writeLock().unlock(); + } + } + public void invalidateAll() { dualKeyCache.invalidateAll(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeInternalRPCService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeInternalRPCService.java index 83e4869531f11..9c8be161d8547 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeInternalRPCService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeInternalRPCService.java @@ -88,6 +88,10 @@ public int getBindPort() { return IoTDBDescriptor.getInstance().getConfig().getInternalPort(); } + public DataNodeInternalRPCServiceImpl getImpl() { + return impl; + } + private static class DataNodeInternalRPCServiceHolder { private static final DataNodeInternalRPCService INSTANCE = new DataNodeInternalRPCService(); diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index a9658b8c03ca5..c444a35718210 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1483,7 +1483,7 @@ service IConfigNodeRPCService { /** Persist all the data points in the memory table of the database to the disk, and seal the data file on all DataNodes */ common.TSStatus flush(common.TFlushReq req) - /** Clear the cache of chunk, chunk metadata and timeseries metadata to release the memory footprint on all DataNodes */ + /** Clear the specific caches of all DataNodes */ common.TSStatus clearCache(set cacheClearOptions) /** Set configuration on specified node */ From 5d4468cff449b8550314ce6a9cba456f2f0f98e9 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 21:13:02 +0800 Subject: [PATCH 207/213] Update IoTDBSqlParser.g4 --- .../main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index 4bbe56b06d5c4..57d686de88e65 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -1035,7 +1035,7 @@ flush // Clear Cache clearCache - : CLEAR CACHE (ON (LOCAL | CLUSTER))? + : CLEAR (SCHEMA | QUERY | ALL)? CACHE (ON (LOCAL | CLUSTER))? ; // Set Configuration From deb798528b746c5ea5fec9f695faa45a6f7a6589 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 21:28:47 +0800 Subject: [PATCH 208/213] maycomplete --- .../execution/config/sys/ClearCacheTask.java | 3 ++- .../queryengine/plan/parser/ASTVisitor.java | 21 ++++++++++++++++++- .../statement/sys/ClearCacheStatement.java | 17 ++++++++++++--- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ClearCacheTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ClearCacheTask.java index 217a5603b34e9..bf5e332a29c2d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ClearCacheTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/ClearCacheTask.java @@ -39,6 +39,7 @@ public ListenableFuture execute(final IConfigTaskExecutor conf throws InterruptedException { // If the action is executed successfully, return the Future. // If your operation is async, you can return the corresponding future directly. - return configTaskExecutor.clearCache(clearCacheStatement.isOnCluster()); + return configTaskExecutor.clearCache( + clearCacheStatement.isOnCluster(), clearCacheStatement.getOptions()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 2139fe40d74de..6c0ee3db25421 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.cq.TimeoutPolicy; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.cache.CacheClearOptions; import org.apache.iotdb.commons.schema.filter.SchemaFilter; import org.apache.iotdb.commons.schema.filter.SchemaFilterFactory; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; @@ -242,6 +243,7 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -3349,8 +3351,25 @@ public Statement visitFlush(IoTDBSqlParser.FlushContext ctx) { @Override public Statement visitClearCache(IoTDBSqlParser.ClearCacheContext ctx) { - ClearCacheStatement clearCacheStatement = new ClearCacheStatement(StatementType.CLEAR_CACHE); + final ClearCacheStatement clearCacheStatement = + new ClearCacheStatement(StatementType.CLEAR_CACHE); clearCacheStatement.setOnCluster(ctx.LOCAL() == null); + + if (ctx.SCHEMA() != null) { + clearCacheStatement.setOptions(Collections.singleton(CacheClearOptions.TREE_SCHEMA)); + } else if (ctx.QUERY() != null) { + clearCacheStatement.setOptions(Collections.singleton(CacheClearOptions.QUERY)); + } else if (ctx.ALL() != null) { + clearCacheStatement.setOptions( + new HashSet<>( + Arrays.asList( + CacheClearOptions.TABLE_ATTRIBUTE, + CacheClearOptions.TREE_SCHEMA, + CacheClearOptions.QUERY))); + } else { + clearCacheStatement.setOptions(Collections.singleton(CacheClearOptions.DEFAULT)); + } + return clearCacheStatement; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ClearCacheStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ClearCacheStatement.java index 29f31d0204088..3b31739636617 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ClearCacheStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ClearCacheStatement.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.plan.statement.sys; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.cache.CacheClearOptions; import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; import org.apache.iotdb.db.queryengine.plan.statement.Statement; @@ -28,12 +29,14 @@ import java.util.Collections; import java.util.List; +import java.util.Set; public class ClearCacheStatement extends Statement implements IConfigStatement { private boolean onCluster; + private Set options; - public ClearCacheStatement(StatementType clearCacheType) { + public ClearCacheStatement(final StatementType clearCacheType) { this.statementType = clearCacheType; } @@ -41,10 +44,18 @@ public boolean isOnCluster() { return onCluster; } - public void setOnCluster(boolean onCluster) { + public void setOnCluster(final boolean onCluster) { this.onCluster = onCluster; } + public Set getOptions() { + return options; + } + + public void setOptions(final Set options) { + this.options = options; + } + @Override public QueryType getQueryType() { return QueryType.WRITE; @@ -56,7 +67,7 @@ public List getPaths() { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitClearCache(this, context); } } From 77c7693173ef235d0492ba51c6b295c17adc1266 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 21:40:53 +0800 Subject: [PATCH 209/213] Update ClientManagerTest.java --- .../org/apache/iotdb/commons/client/ClientManagerTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java index 81a439dd5f3bf..48bf893979e7c 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.exception.StartupException; +import org.apache.iotdb.commons.schema.cache.CacheClearOptions; import org.apache.iotdb.mpp.rpc.thrift.IDataNodeRPCService; import org.apache.commons.pool2.KeyedObjectPool; @@ -43,6 +44,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.NoSuchElementException; import java.util.concurrent.CountDownLatch; @@ -66,7 +68,7 @@ public void setUp() throws StartupException, TException { service = new MockInternalRPCService(endPoint); IDataNodeRPCService.Iface processor = mock(IDataNodeRPCService.Iface.class); // timeout method - when(processor.clearCache()) + when(processor.clearCache(Collections.singleton(CacheClearOptions.DEFAULT.ordinal()))) .thenAnswer( invocation -> { Thread.sleep(CONNECTION_TIMEOUT + 1000); @@ -530,7 +532,7 @@ public void syncClientTimeoutTest() throws Exception { // timeout RPC try (SyncDataNodeInternalServiceClient syncClient = syncClusterManager.borrowClient(endPoint)) { - syncClient.clearCache(); + syncClient.clearCache(Collections.singleton(CacheClearOptions.DEFAULT.ordinal())); Assert.fail("A timeout exception should occur here"); } catch (Exception ignored) { // no handling @@ -581,6 +583,7 @@ public void onError(Exception exception) { AtomicBoolean finalFailed1 = failed; CountDownLatch finalLatch1 = latch; asyncClient.clearCache( + Collections.singleton(CacheClearOptions.DEFAULT.ordinal()), new AsyncMethodCallback() { @Override public void onComplete(TSStatus response) { From 6e62401c8204b51495d7193453c1fb1408660db2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 21:45:06 +0800 Subject: [PATCH 210/213] Update ClearCacheTask.java --- .../execution/config/metadata/relational/ClearCacheTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ClearCacheTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ClearCacheTask.java index 20b2878831dc9..2c0a98059059b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ClearCacheTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ClearCacheTask.java @@ -37,6 +37,6 @@ public ClearCacheTask(final ClearCache clearCache) { @Override public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return configTaskExecutor.clearCache(clearCache.isOnCluster()); + return configTaskExecutor.clearCache(clearCache.isOnCluster(), clearCache.getOptions()); } } From b18f0f848443267f21709e406517db41d2c19a50 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 21:50:24 +0800 Subject: [PATCH 211/213] Update Coordinator.java --- .../java/org/apache/iotdb/db/queryengine/plan/Coordinator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java index 3ac30f633a7dd..37f72bd5aba60 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java @@ -48,6 +48,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.planner.TableModelPlanner; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AddColumn; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ClearCache; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DescribeTable; @@ -333,6 +334,7 @@ private IQueryExecution createQueryExecutionForTableModel( || statement instanceof ShowConfigNodes || statement instanceof ShowAINodes || statement instanceof Flush + || statement instanceof ClearCache || statement instanceof SetConfiguration || statement instanceof PipeStatement) { return new ConfigExecution( From 26f648fe9e138a8ee03877a75620fd3bf27d6d68 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 09:57:33 +0800 Subject: [PATCH 212/213] Update SchemaRegionMemoryImpl.java --- .../schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 89685a6e442e2..7361ff66d6a3a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -33,6 +33,7 @@ import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -249,7 +250,7 @@ public synchronized void init() throws MetadataException { deviceAttributeStore = new DeviceAttributeStore(regionStatistics); deviceAttributeCacheUpdater = - new DeviceAttributeCacheUpdater(regionStatistics, storageGroupFullPath.substring(5)); + new DeviceAttributeCacheUpdater(regionStatistics, PathUtils.unQualifyDatabaseName(storageGroupFullPath)); tagManager = new TagManager(schemaRegionDirPath, regionStatistics); mtree = new MTreeBelowSGMemoryImpl( From abaa5a828445555d391eeb70d00e15871c2c2725 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 09:59:31 +0800 Subject: [PATCH 213/213] Update SchemaRegionMemoryImpl.java --- .../schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 7361ff66d6a3a..28bc2101edb75 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -250,7 +250,8 @@ public synchronized void init() throws MetadataException { deviceAttributeStore = new DeviceAttributeStore(regionStatistics); deviceAttributeCacheUpdater = - new DeviceAttributeCacheUpdater(regionStatistics, PathUtils.unQualifyDatabaseName(storageGroupFullPath)); + new DeviceAttributeCacheUpdater( + regionStatistics, PathUtils.unQualifyDatabaseName(storageGroupFullPath)); tagManager = new TagManager(schemaRegionDirPath, regionStatistics); mtree = new MTreeBelowSGMemoryImpl(