Skip to content

Reduce string IDeviceID convert #12392

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
HTHou merged 2 commits into from
Apr 23, 2024
Merged

Reduce string IDeviceID convert #12392

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fd40e34
reduce to string
shuwenwei Apr 23, 2024
1eeccb7
modify parameter name
shuwenwei Apr 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions .../main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileInsertionDataContainer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,24 +144,24 @@ private Map<IDeviceID, List<String>> filterDeviceMeasurementsMapByPattern(
Map<IDeviceID, List<String>> originalDeviceMeasurementsMap) {
final Map<IDeviceID, List<String>> filteredDeviceMeasurementsMap = new HashMap<>();
for (Map.Entry<IDeviceID, List<String>> entry : originalDeviceMeasurementsMap.entrySet()) {
final String deviceId = entry.getKey().toString();
final IDeviceID deviceId = entry.getKey();
String deviceStr = deviceId.toString();

// case 1: for example, pattern is root.a.b or pattern is null and device is root.a.b.c
// in this case, all data can be matched without checking the measurements
if (Objects.isNull(pattern) || pattern.isRoot() || pattern.coversDevice(deviceId)) {
if (Objects.isNull(pattern) || pattern.isRoot() || pattern.coversDevice(deviceStr)) {
if (!entry.getValue().isEmpty()) {
filteredDeviceMeasurementsMap.put(
IDeviceID.Factory.DEFAULT_FACTORY.create(deviceId), entry.getValue());
filteredDeviceMeasurementsMap.put(deviceId, entry.getValue());
}
}

// case 2: for example, pattern is root.a.b.c and device is root.a.b
// in this case, we need to check the full path
else if (pattern.mayOverlapWithDevice(deviceId)) {
else if (pattern.mayOverlapWithDevice(deviceStr)) {
final List<String> filteredMeasurements = new ArrayList<>();

for (final String measurement : entry.getValue()) {
if (pattern.matchesMeasurement(deviceId, measurement)) {
if (pattern.matchesMeasurement(deviceStr, measurement)) {
filteredMeasurements.add(measurement);
} else {
// Parse pattern iff there are measurements filtered out
Expand All @@ -170,8 +170,7 @@ else if (pattern.mayOverlapWithDevice(deviceId)) {
}

if (!filteredMeasurements.isEmpty()) {
filteredDeviceMeasurementsMap.put(
IDeviceID.Factory.DEFAULT_FACTORY.create(deviceId), filteredMeasurements);
filteredDeviceMeasurementsMap.put(deviceId, filteredMeasurements);
}
}

Expand Down
13 changes: 8 additions & 5 deletions ...rg/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;

import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.compress.IUnCompressor;
import org.apache.tsfile.encoding.decoder.Decoder;
import org.apache.tsfile.enums.TSDataType;
Expand Down Expand Up @@ -109,6 +110,7 @@ public void scanTsFile() {

private void checkAlignedDeviceSeries(TsFileSequenceReader reader, IDeviceID device)
throws IOException {
String deviceStr = device.toString();
List<AlignedChunkMetadata> chunkMetadataList = reader.getAlignedChunkMetadata(device);
for (AlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) {
IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata();
Expand All @@ -133,7 +135,7 @@ private void checkAlignedDeviceSeries(TsFileSequenceReader reader, IDeviceID dev
Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType());
while (decoder.hasNext(uncompressedPageData)) {
long currentTime = decoder.readLong(uncompressedPageData);
checkPreviousTimeAndUpdate(device.toString(), currentTime);
checkPreviousTimeAndUpdate(deviceStr, currentTime);
}
}
}
Expand All @@ -149,16 +151,17 @@ private void checkNonAlignedDeviceSeries(TsFileSequenceReader reader, IDeviceID
measurementChunkMetadataListMapIterator.next();
for (Map.Entry<String, List<ChunkMetadata>> measurementChunkMetadataListEntry :
measurementChunkMetadataListMap.entrySet()) {
String measurement = measurementChunkMetadataListEntry.getKey();
String path =
device + TsFileConstant.PATH_SEPARATOR + measurementChunkMetadataListEntry.getKey();
List<ChunkMetadata> chunkMetadataList = measurementChunkMetadataListEntry.getValue();
checkSingleNonAlignedSeries(reader, measurement, chunkMetadataList);
checkSingleNonAlignedSeries(reader, path, chunkMetadataList);
previousTime = Long.MIN_VALUE;
}
}
}

private void checkSingleNonAlignedSeries(
TsFileSequenceReader reader, String measurement, List<ChunkMetadata> chunkMetadataList)
TsFileSequenceReader reader, String path, List<ChunkMetadata> chunkMetadataList)
throws IOException {
for (ChunkMetadata chunkMetadata : chunkMetadataList) {
if (chunkMetadata == null || chunkMetadata.getStatistics().getCount() == 0) {
Expand Down Expand Up @@ -187,7 +190,7 @@ private void checkSingleNonAlignedSeries(
TSDataType.INT64);
while (timeDecoder.hasNext(timeBuffer)) {
long currentTime = timeDecoder.readLong(timeBuffer);
checkPreviousTimeAndUpdate(measurement, currentTime);
checkPreviousTimeAndUpdate(path, currentTime);
}
}
}
Expand Down