Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[IOTDB-2930]Fix concurrent UnPin bug & Improve template implementation #5647

Merged
merged 5 commits into from
Apr 23, 2022
Merged

[IOTDB-2930]Fix concurrent UnPin bug & Improve template implementation #5647

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
013c9cd
rewrite mounted node check in memory mode
MarcosZyk Apr 23, 2022
c032ee8
rewrite mounted node check in schemaFile mode
MarcosZyk Apr 23, 2022
7270e9d
rewrite find measurement in template
MarcosZyk Apr 23, 2022
51503e3
fix concurrent unPin bug
MarcosZyk Apr 23, 2022
cb17ced
remove unnecessary code
MarcosZyk Apr 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion server/src/main/java/org/apache/iotdb/db/metadata/mtree/IMTreeBelowSG.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,8 @@ boolean isTemplateAppendable(Template tarTemplate, List<String> appendMeasuremen
* @return index on full path of the node which matches all measurements path with its
* upperTemplate.
*/
int getMountedNodeIndexOnMeasurementPath(PartialPath measurementPath) throws MetadataException;
int getMountedNodeIndexOnMeasurementPath(PartialPath devicePath, String[] measurements)
throws MetadataException;

List<String> getPathsSetOnTemplate(String templateName) throws MetadataException;

Expand Down
90 changes: 59 additions & 31 deletions server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTreeBelowSGCachedImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@

import static java.util.stream.Collectors.toList;
import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR;
import static org.apache.iotdb.db.metadata.MetadataConstant.ALL_RESULT_NODES;
import static org.apache.iotdb.db.metadata.lastCache.LastCacheManager.getLastTimeStamp;

Expand Down Expand Up @@ -111,7 +112,7 @@
public class MTreeBelowSGCachedImpl implements IMTreeBelowSG {

private CachedMTreeStore store;
private IStorageGroupMNode storageGroupMNode;
private volatile IStorageGroupMNode storageGroupMNode;
private int levelOfSG;

// region MTree initialization, clear and serialization
Expand Down Expand Up @@ -1289,53 +1290,80 @@ public boolean isPathExistsWithinTemplate(PartialPath path) throws MetadataExcep
* @return index on full path of the node which matches all measurements path with its
* upperTemplate.
*/
@Override
public int getMountedNodeIndexOnMeasurementPath(PartialPath measurementPath)
public int getMountedNodeIndexOnMeasurementPath(PartialPath devicePath, String[] measurements)
throws MetadataException {
String[] fullPathNodes = measurementPath.getNodes();
String[] nodes = devicePath.getNodes();
IMNode cur = storageGroupMNode;
IMNode child;
Template upperTemplate = cur.getSchemaTemplate();
int index = levelOfSG + 1;
boolean attemptToUseTemplate = false;

try {
for (int index = levelOfSG + 1; index < fullPathNodes.length; index++) {
// If there are nodes of target path on MTree, use it as possible.
for (; index < nodes.length; index++) {
upperTemplate = cur.getSchemaTemplate() != null ? cur.getSchemaTemplate() : upperTemplate;
child = store.getChild(cur, fullPathNodes[index]);
child = store.getChild(cur, nodes[index]);
if (child == null) {
if (upperTemplate != null) {
// for this fullPath, cur is the last node on MTree
// since upperTemplate exists, need to find the matched suffix path of fullPath and
// template
String suffixPath =
new PartialPath(Arrays.copyOfRange(fullPathNodes, index, fullPathNodes.length))
.toString();

// if suffix matches template, then fullPathNodes[index-1] should be the node to use
// template on MTree
if (upperTemplate.hasSchema(suffixPath)) {
return index - 1;
}

// if suffix doesn't match, but first node name matched, it's an overlap with template
// cast exception for now
if (upperTemplate.getDirectNode(fullPathNodes[index]) != null) {
throw new TemplateImcompatibeException(
measurementPath.getFullPath(), upperTemplate.getName(), fullPathNodes[index]);
}
} else {
if (upperTemplate == null) {
// no matched child, no template, need to create device node as logical device path
return fullPathNodes.length - 1;
return nodes.length;
} else {
attemptToUseTemplate = true;
break;
}
} else {
// has child on MTree
cur = child;
}
}
} finally {
if (index > levelOfSG + 1) {
unPinPath(cur);
}
}

if (!attemptToUseTemplate) {
// all nodes on path exist in MTree, device node should be the penultimate one
return fullPathNodes.length - 1;
} finally {
unPinPath(cur);
return nodes.length;
}

// The resting part of target path not exists on MTree, thus try to use template.
for (; index < nodes.length; index++) {
int fullPathLength = nodes.length - index + 1;
String[] suffixNodes = new String[fullPathLength];
System.arraycopy(nodes, index, suffixNodes, 0, nodes.length - index);
boolean hasAllMeasurements = true;

for (String measurement : measurements) {
// for this fullPath, cur is the last node on MTree
// since upperTemplate exists, need to find the matched suffix path of fullPath and
// template
suffixNodes[fullPathLength - 1] = measurement;
String suffixPath = String.join(String.valueOf(PATH_SEPARATOR), suffixNodes);

if (upperTemplate.hasSchema(suffixPath)) {
continue;
}

// if suffix doesn't match, but first node name matched, it's an overlap with template
// cast exception for now
if (upperTemplate.getDirectNode(nodes[index]) != null) {
throw new TemplateImcompatibeException(
devicePath.concatNode(measurement).getFullPath(),
upperTemplate.getName(),
nodes[index]);
}

hasAllMeasurements = false;
}

if (hasAllMeasurements) {
return index - 1;
}
}

return nodes.length;
}

@Override
Expand Down
87 changes: 56 additions & 31 deletions server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTreeBelowSGMemoryImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@

import static java.util.stream.Collectors.toList;
import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR;
import static org.apache.iotdb.db.metadata.MetadataConstant.ALL_RESULT_NODES;
import static org.apache.iotdb.db.metadata.lastCache.LastCacheManager.getLastTimeStamp;

Expand Down Expand Up @@ -111,7 +112,7 @@ public class MTreeBelowSGMemoryImpl implements IMTreeBelowSG {

// this implementation is based on memory, thus only MTree write operation must invoke MTreeStore
private MemMTreeStore store;
private IStorageGroupMNode storageGroupMNode;
private volatile IStorageGroupMNode storageGroupMNode;
private int levelOfSG;

// region MTree initialization, clear and serialization
Expand Down Expand Up @@ -1159,53 +1160,77 @@ public boolean isPathExistsWithinTemplate(PartialPath path) throws MetadataExcep
* created yet. The result is used for getDeviceNodeWithAutoCreate, which return corresponding
* IMNode on MTree.
*
* @return index on full path of the node which matches all measurements path with its
* @return index on full path of the node which matches all measurements' path with its
* upperTemplate.
*/
@Override
public int getMountedNodeIndexOnMeasurementPath(PartialPath measurementPath)
public int getMountedNodeIndexOnMeasurementPath(PartialPath devicePath, String[] measurements)
throws MetadataException {
String[] fullPathNodes = measurementPath.getNodes();
String[] nodes = devicePath.getNodes();
IMNode cur = storageGroupMNode;
IMNode child;
Template upperTemplate = cur.getSchemaTemplate();
int index;
boolean attemptToUseTemplate = false;

for (int index = levelOfSG + 1; index < fullPathNodes.length; index++) {
// If there are nodes of target path on MTree, use it as possible.
for (index = levelOfSG + 1; index < nodes.length; index++) {
upperTemplate = cur.getSchemaTemplate() != null ? cur.getSchemaTemplate() : upperTemplate;
child = cur.getChild(fullPathNodes[index]);
child = cur.getChild(nodes[index]);
if (child == null) {
if (upperTemplate != null) {
// for this fullPath, cur is the last node on MTree
// since upperTemplate exists, need to find the matched suffix path of fullPath and
// template
String suffixPath =
new PartialPath(Arrays.copyOfRange(fullPathNodes, index, fullPathNodes.length))
.toString();

// if suffix matches template, then fullPathNodes[index-1] should be the node to use
// template on MTree
if (upperTemplate.hasSchema(suffixPath)) {
return index - 1;
}

// if suffix doesn't match, but first node name matched, it's an overlap with template
// cast exception for now
if (upperTemplate.getDirectNode(fullPathNodes[index]) != null) {
throw new TemplateImcompatibeException(
measurementPath.getFullPath(), upperTemplate.getName(), fullPathNodes[index]);
}
} else {
if (upperTemplate == null) {
// no matched child, no template, need to create device node as logical device path
return fullPathNodes.length - 1;
return nodes.length;
} else {
attemptToUseTemplate = true;
break;
}
} else {
// has child on MTree
cur = child;
}
}

// all nodes on path exist in MTree, device node should be the penultimate one
return fullPathNodes.length - 1;
if (!attemptToUseTemplate) {
// all nodes on path exist in MTree, device node should be the penultimate one
return nodes.length;
}

// The resting part of target path not exists on MTree, thus try to use template.
for (; index < nodes.length; index++) {
int fullPathLength = nodes.length - index + 1;
String[] suffixNodes = new String[fullPathLength];
System.arraycopy(nodes, index, suffixNodes, 0, nodes.length - index);
boolean hasAllMeasurements = true;

for (String measurement : measurements) {
// for this fullPath, cur is the last node on MTree
// since upperTemplate exists, need to find the matched suffix path of fullPath and
// template
suffixNodes[fullPathLength - 1] = measurement;
String suffixPath = String.join(String.valueOf(PATH_SEPARATOR), suffixNodes);

if (upperTemplate.hasSchema(suffixPath)) {
continue;
}

// if suffix doesn't match, but first node name matched, it's an overlap with template
// cast exception for now
if (upperTemplate.getDirectNode(nodes[index]) != null) {
throw new TemplateImcompatibeException(
devicePath.concatNode(measurement).getFullPath(),
upperTemplate.getName(),
nodes[index]);
}

hasAllMeasurements = false;
}

if (hasAllMeasurements) {
return index - 1;
}
}

return nodes.length;
}

@Override
Expand Down
35 changes: 20 additions & 15 deletions server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/cache/CacheManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -427,24 +427,29 @@ public boolean unPinMNode(IMNode node) {
if (cacheEntry == null) {
return false;
}
cacheEntry.unPin();
if (!cacheEntry.isPinned()) {
memManager.releasePinnedMemResource(node);
IMNode parent = node.getParent();
while (!node.isStorageGroup()) {
node = parent;
parent = node.getParent();
cacheEntry = getCacheEntry(node);
cacheEntry.unPin();
if (cacheEntry.isPinned()) {
break;
}

return doUnPin(node);
}

private boolean doUnPin(IMNode node) {
CacheEntry cacheEntry = getCacheEntry(node);

boolean isPinStatusChanged = false;
synchronized (cacheEntry) {
cacheEntry.unPin();
if (!cacheEntry.isPinned()) {
isPinStatusChanged = true;
memManager.releasePinnedMemResource(node);
}
return true;
} else {
return false;
}

if (isPinStatusChanged) {
if (!node.isStorageGroup()) {
doUnPin(node.getParent());
}
}

return isPinStatusChanged;
}

@Override
Expand Down