Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
public class PartOfSpeechTrie {

static final String EMPTY_SYMBOL = "*";
static final String LEAF = "";

Map<String, Object> root = new HashMap<>();

Expand All @@ -37,6 +38,7 @@ public void add(String... items) {
(Map<String, Object>)node.computeIfAbsent(item, k -> new HashMap<>());
node = newNode;
}
node.put(LEAF, LEAF);
}

public boolean isPrefixOf(List<String> items, int begin, int end) {
Expand All @@ -47,17 +49,17 @@ public boolean isPrefixOf(List<String> items, int begin, int end) {
for (int i = begin; i < end; i++) {
String item = items.get(i);
if (EMPTY_SYMBOL.equals(item)) {
return node.isEmpty();
return node.containsKey(LEAF);
}
@SuppressWarnings("unchecked")
Map<String, Object> newNode = (Map<String, Object>)node.get(item);
node = newNode;
if (node == null) {
return false;
} else if (node.isEmpty()) {
} else if (node.containsKey(LEAF)) {
return true;
}
}
return true;
return node.containsKey(LEAF);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -91,4 +91,21 @@ public void testConjugationForm() throws IOException {
assertTokenStreamContents(tokenStream,
new String[] {"東京都", "東京", "都", "に", "行っ"});
}

public void testPrefixWithUnmatchedSubcategory() throws IOException {
String tags = "助詞,格助詞\n助詞,格助詞,引用\n";
factory.inform(new StringResourceLoader(tags));
tokenStream = factory.create(tokenStream);
assertTokenStreamContents(tokenStream,
new String[] {"東京都", "東京", "都", "行っ", "た"});
}

public void testTooLongCategory() throws IOException {
String tags = "名詞,固有名詞,地名,一般,一般\n";
factory.inform(new StringResourceLoader(tags));
tokenStream = factory.create(tokenStream);
assertTokenStreamContents(tokenStream,
new String[] {"東京都", "東京", "都", "に", "行っ", "た"});
}

}