diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/CategoryTable.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/CategoryTable.java index 7b9cc78129..48229674d5 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/CategoryTable.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/CategoryTable.java @@ -20,14 +20,17 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.IdentityHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import opennlp.tools.tokenize.lattice.MecabDictionary.Category; /** - * The {@code char.def} code point to category name mapping over the Unicode - * code point range. + * The {@code char.def} code point to category mappings over the Unicode code point + * range. The first category on a mapping supplies the unknown-word settings. Each + * listed category can keep a group running while following characters also list it. * *

The Basic Multilingual Plane is stored in a directly indexed array. The * supplementary planes are stored as a sorted, non-overlapping range table searched by @@ -35,13 +38,21 @@ */ final class CategoryTable { - private final Category[] bmp; + private final CategoryAssignment[] bmp; private final int[] rangeStart; private final int[] rangeEnd; - private final Category[] rangeCategory; + private final CategoryAssignment[] rangeCategory; - private CategoryTable(Category[] bmp, int[] rangeStart, int[] rangeEnd, - Category[] rangeCategory) { + /** + * Creates a table from resolved BMP entries and supplementary ranges. + * + * @param bmp The directly indexed BMP assignments. + * @param rangeStart The inclusive starts of the supplementary ranges. + * @param rangeEnd The inclusive upper bounds of the supplementary ranges. + * @param rangeCategory The assignment for each supplementary range. + */ + private CategoryTable(CategoryAssignment[] bmp, int[] rangeStart, int[] rangeEnd, + CategoryAssignment[] rangeCategory) { this.bmp = bmp; this.rangeStart = rangeStart; this.rangeEnd = rangeEnd; @@ -49,14 +60,12 @@ private CategoryTable(Category[] bmp, int[] rangeStart, int[] rangeEnd, } /** - * Looks up the category a {@code char.def} mapping gives a code point. The table - * contains the {@link Category} instances themselves, and two code points of one - * category share one instance, so categories may be compared by identity. + * Looks up the categories a {@code char.def} mapping gives a code point. * * @param codePoint The code point to classify. - * @return The category, or {@code null} when no mapping covers the code point. + * @return The assignment, or {@code null} when no mapping covers the code point. */ - Category categoryOf(int codePoint) { + CategoryAssignment categoriesOf(int codePoint) { if (codePoint <= Character.MAX_VALUE) { return bmp[codePoint]; } @@ -75,7 +84,62 @@ Category categoryOf(int codePoint) { return null; } - private static final String CHARACTER_DEFINITION_FILE = "char.def"; + /** + * The categories assigned to one code point, stored both in mapping order and as a + * mask over the dictionary's dense category ids. + */ + static final class CategoryAssignment { + + private final Category[] categories; + private final int categoryMask; + + /** + * Creates an assignment with the first entry as the primary category. + * + * @param categories The categories in mapping order. Must not be empty. + * @throws IllegalArgumentException Thrown if {@code categories} is {@code null} or + * empty. + */ + CategoryAssignment(Category[] categories) { + if (categories == null || categories.length == 0) { + throw new IllegalArgumentException("categories must not be null or empty"); + } + this.categories = categories.clone(); + int mask = 0; + for (final Category category : categories) { + mask |= 1 << category.id(); + } + this.categoryMask = mask; + } + + /** + * Returns the first category on the mapping. + * + * @return The category that supplies unknown-word settings. + */ + Category primary() { + return categories[0]; + } + + /** + * Computes the run end after comparing this assignment with the next character. + * MeCab's + * + * {@code seekToOtherType} replaces the current mask after each accepted + * character, so successive assignments must overlap. + * + * @param next The next character's assignment, or {@code null} at the end of text. + * @param nextRunEnd The run end calculated at the next character. + * @param characterEnd The exclusive end of the current character. + * @return {@code nextRunEnd} when the assignments intersect; + * {@code characterEnd} otherwise. + */ + int continuedRunEnd(CategoryAssignment next, int nextRunEnd, + int characterEnd) { + return next != null && (categoryMask & next.categoryMask) != 0 + ? nextRunEnd : characterEnd; + } + } /** * Collects {@code char.def} mappings in file order and builds a @@ -84,24 +148,39 @@ Category categoryOf(int codePoint) { */ static final class Builder { - private final String[] bmp = new String[Character.MAX_VALUE + 1]; + /** + * One mapping line retained for validation after all categories have been read. + * + * @param sourceStart The first code point on the mapping line, used in error + * messages even if a later mapping replaces it. + * @param categories The category names from the mapping, primary first. + */ + private record Mapping(int sourceStart, String[] categories) { + } + + private final String[][] bmp = new String[Character.MAX_VALUE + 1][]; private final List bounds = new ArrayList<>(); - private final List names = new ArrayList<>(); + private final List names = new ArrayList<>(); + private final List mappings = new ArrayList<>(); /** - * Records one inclusive code point range's category. + * Records one inclusive code point range's categories. * * @param from The first code point of the range. * @param to The last code point of the range, inclusive. - * @param category The category name to give the range. Must not be {@code null}. + * @param categories The category names to give the range, primary first. Must not + * be {@code null} or empty. */ - void map(int from, int to, String category) { + void map(int from, int to, String[] categories) { + // All positions written by this mapping store this array reference. build() + // relies on array identity to resolve one CategoryAssignment per mapping line. + mappings.add(new Mapping(from, categories)); for (int c = from; c <= Math.min(to, Character.MAX_VALUE); c++) { - bmp[c] = category; + bmp[c] = categories; } if (to > Character.MAX_VALUE) { bounds.add(new int[] {Math.max(from, Character.MAX_VALUE + 1), to}); - names.add(category); + names.add(categories); } } @@ -124,18 +203,18 @@ CategoryTable build(Map categories) throws IOException { } Arrays.sort(edges); final List intervals = new ArrayList<>(); - final List winners = new ArrayList<>(); + final List winners = new ArrayList<>(); for (int i = 0; i < edges.length - 1; i++) { if (edges[i] == edges[i + 1]) { continue; } - final String winner = lastCovering(edges[i]); + final String[] winner = lastCovering(edges[i]); if (winner == null) { continue; } final int previous = intervals.size() - 1; if (previous >= 0 && intervals.get(previous)[1] == edges[i] - 1 - && winners.get(previous).equals(winner)) { + && Arrays.equals(winners.get(previous), winner)) { intervals.get(previous)[1] = edges[i + 1] - 1; } else { intervals.add(new int[] {edges[i], edges[i + 1] - 1}); @@ -148,47 +227,68 @@ CategoryTable build(Map categories) throws IOException { starts[i] = intervals.get(i)[0]; ends[i] = intervals.get(i)[1]; } - final Category[] resolvedBmp = new Category[bmp.length]; + final Map resolvedAssignments = + new IdentityHashMap<>(); + // Validate mappings in file order, including mappings replaced across their + // ranges. This makes typo detection independent of range precedence. + for (final Mapping mapping : mappings) { + resolve(mapping.categories(), categories, resolvedAssignments, + mapping.sourceStart()); + } + final CategoryAssignment[] resolvedBmp = new CategoryAssignment[bmp.length]; for (int c = 0; c < bmp.length; c++) { if (bmp[c] != null) { - resolvedBmp[c] = resolve(bmp[c], categories, c); + resolvedBmp[c] = resolve(bmp[c], categories, resolvedAssignments, c); } } - final Category[] resolvedRanges = new Category[winners.size()]; + final CategoryAssignment[] resolvedRanges = new CategoryAssignment[winners.size()]; for (int i = 0; i < winners.size(); i++) { - resolvedRanges[i] = resolve(winners.get(i), categories, starts[i]); + resolvedRanges[i] = resolve(winners.get(i), categories, resolvedAssignments, + starts[i]); } return new CategoryTable(resolvedBmp, starts, ends, resolvedRanges); } /** - * Resolves a mapped category name against the defined categories. A mapping to an + * Resolves mapped category names using the defined categories. A mapping to an * undefined category fails at load and names the offending code point. * - * @param name The category name a mapping line gave. + * @param names The category names on a mapping line, primary first. * @param categories The defined categories, keyed by name. + * @param resolvedAssignments Previously resolved mapping lines, indexed by their + * shared category-name arrays. * @param codePoint A code point the mapping covers, for the error message. - * @return The resolved category. Not {@code null}. - * @throws IOException Thrown if no category of that name was defined. + * @return The resolved assignment. Not {@code null}. + * @throws IOException Thrown if any named category was not defined. */ - private Category resolve(String name, Map categories, - int codePoint) throws IOException { - final Category category = categories.get(name); - if (category == null) { - throw new IOException(String.format( - CHARACTER_DEFINITION_FILE + " maps U+%04X to the undefined category %s", - codePoint, name)); + private CategoryAssignment resolve(String[] names, Map categories, + Map resolvedAssignments, int codePoint) + throws IOException { + final CategoryAssignment cached = resolvedAssignments.get(names); + if (cached != null) { + return cached; + } + final Category[] resolved = new Category[names.length]; + for (int i = 0; i < names.length; i++) { + resolved[i] = categories.get(names[i]); + if (resolved[i] == null) { + throw new IOException(String.format(Locale.ROOT, + MecabDictionary.CHAR_DEF + " declaration at U+%04X names the" + + " undefined category %s", codePoint, names[i])); + } } - return category; + final CategoryAssignment assignment = new CategoryAssignment(resolved); + resolvedAssignments.put(names, assignment); + return assignment; } /** * Finds the category of the last recorded range covering a code point. * * @param codePoint The code point to look up. - * @return The category name, or {@code null} when no recorded range covers it. + * @return The category names, or {@code null} when no stored range covers it. */ - private String lastCovering(int codePoint) { + private String[] lastCovering(int codePoint) { for (int i = bounds.size() - 1; i >= 0; i--) { final int[] range = bounds.get(i); if (codePoint >= range[0] && codePoint <= range[1]) { diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/DoubleArrayLexicon.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/DoubleArrayLexicon.java index 51382e0721..fb7b805c94 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/DoubleArrayLexicon.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/DoubleArrayLexicon.java @@ -17,6 +17,7 @@ package opennlp.tools.tokenize.lattice; +import java.util.ArrayDeque; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -42,6 +43,14 @@ final class DoubleArrayLexicon { private final int[] codeOf; private final List[] values; + /** + * Creates a lexicon from completed double-array tables and entry lists. + * + * @param base The transition offsets. + * @param check The parent state for each occupied index. + * @param codeOf The dense label for each UTF-16 character. + * @param values The entries for each surface. + */ private DoubleArrayLexicon(int[] base, int[] check, int[] codeOf, List[] values) { this.base = base; @@ -124,10 +133,10 @@ void prefixMatches(String text, int from, int to, } /** - * The recursive sorted-range builder: each call places one node's children by - * finding a base at which every child label uses a free slot, then recurses - * per child range. A moving watermark keeps the free-slot search near-linear over - * real lexicons. + * Builds the trie from sorted surface ranges. Each node places children at a common + * free base, and a moving watermark makes that search near-linear over real + * lexicons. The traversal uses an explicit stack so a long surface cannot exhaust + * the thread stack. */ private static final class Builder { @@ -142,6 +151,12 @@ private static final class Builder { private int watermark = ROOT + 1; private int valueIndex; + /** + * Initializes storage for the sorted surfaces and their dense character codes. + * + * @param surfaces The sorted surface forms. + * @param codeOf The dense label for each UTF-16 character. + */ private Builder(String[] surfaces, int[] codeOf) { this.surfaces = surfaces; this.codeOf = codeOf; @@ -151,7 +166,18 @@ private Builder(String[] surfaces, int[] codeOf) { } /** - * Places the children of one trie node. + * A surface range with children waiting to be placed. + * + * @param left The first surface in the range. + * @param right The exclusive end of the range. + * @param depth The character depth of the node. + * @param state The node's array index. + */ + private record PendingNode(int left, int right, int depth, int state) { + } + + /** + * Places one trie and the descendants without consuming the thread stack. * * @param left The first surface of the node's range. * @param right The exclusive last surface of the node's range. @@ -159,50 +185,55 @@ private Builder(String[] surfaces, int[] codeOf) { * @param state The node's own slot. */ private void insert(int left, int right, int depth, int state) { - // gather the distinct child labels of this range, terminator first - final int[] labels = new int[right - left]; - int labelCount = 0; - int previous = -2; - for (int k = left; k < right; k++) { - final int label = surfaces[k].length() == depth - ? 0 : codeOf[surfaces[k].charAt(depth)]; - if (label != previous) { - labels[labelCount++] = label; - previous = label; - } - } - final int found = findBase(labels, labelCount); - base[state] = found; - for (int k = 0; k < labelCount; k++) { - final int child = found + labels[k]; - check[child] = state; - if (child > high) { - high = child; + final ArrayDeque pending = new ArrayDeque<>(); + pending.push(new PendingNode(left, right, depth, state)); + while (!pending.isEmpty()) { + final PendingNode node = pending.pop(); + final int[] labels = new int[node.right() - node.left()]; + int labelCount = 0; + int previous = -2; + for (int k = node.left(); k < node.right(); k++) { + final int label = surfaces[k].length() == node.depth() + ? 0 : codeOf[surfaces[k].charAt(node.depth())]; + if (label != previous) { + labels[labelCount++] = label; + previous = label; + } } - } - // recurse over each child's sub-range - int start = left; - for (int k = 0; k < labelCount; k++) { - final int label = labels[k]; - int end = start; - while (end < right && (surfaces[end].length() == depth - ? 0 : codeOf[surfaces[end].charAt(depth)]) == label) { - end++; + final int found = findBase(labels, labelCount); + base[node.state()] = found; + for (int k = 0; k < labelCount; k++) { + final int child = found + labels[k]; + check[child] = node.state(); + if (child > high) { + high = child; + } } - final int child = found + label; - if (label == 0) { - base[child] = -(++valueIndex); - } else { - insert(start, end, depth + 1, child); + + int childEnd = node.right(); + for (int k = labelCount - 1; k >= 0; k--) { + final int label = labels[k]; + int childStart = childEnd - 1; + while (childStart > node.left() + && (surfaces[childStart - 1].length() == node.depth() + ? 0 : codeOf[surfaces[childStart - 1].charAt(node.depth())]) == label) { + childStart--; + } + final int child = found + label; + if (label == 0) { + base[child] = -(++valueIndex); + } else { + pending.push(new PendingNode( + childStart, childEnd, node.depth() + 1, child)); + } + childEnd = childStart; } - start = end; } } /** - * Finds the lowest base at which every label uses a free slot. Labels - * arrive in surface-character order, not numeric order, so the smallest and - * largest label are computed rather than assumed positional. + * Finds the lowest base at which all labels use free indices. Labels are in + * surface-character order, so this method computes the numeric bounds. * * @param labels The child labels to place. * @param labelCount How many leading elements of {@code labels} are in use. diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index c88b243c74..d8ddcdcdb4 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -21,6 +21,7 @@ import java.util.List; import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.tokenize.lattice.CategoryTable.CategoryAssignment; import opennlp.tools.tokenize.lattice.MecabDictionary.Category; import opennlp.tools.tokenize.lattice.MecabDictionary.WordEntry; import opennlp.tools.util.Span; @@ -30,13 +31,14 @@ * Dictionary-driven segmentation for languages written without spaces: a Viterbi * search over the word lattice of a {@link MecabDictionary}, minimizing the sum of * word costs and connection costs. This is the segmentation approach behind Japanese - * and Korean morphological analysis; the same decoder serves both, since the language - * lives entirely in the user-supplied dictionary. + * and Korean morphological analysis. A single decoder serves both because the supplied + * dictionary provides the language-specific data. * *

Unknown text is handled through the dictionary's character categories: where the * lexicon has no entry, or a category always invokes them, unknown-word candidates are * generated per category template, grouping runs of same-category characters when the - * category says so. Whitespace never joins a morpheme and is never reported as one. + * category requests it. A multi-category run continues while successive assignments + * overlap. Whitespace cannot join or appear as a morpheme. * Every reported span is in original text coordinates.

* *

{@link #analyze(String)} returns full morphemes with their dictionary features; @@ -80,6 +82,14 @@ private static final class Node { private Node previous; private Node nextEndingHere; + /** + * Creates one lattice candidate. + * + * @param start The candidate start in original text coordinates. + * @param end The exclusive candidate end. + * @param entry The dictionary entry. + * @param unknown Whether unknown-word handling generated the candidate. + */ private Node(int start, int end, WordEntry entry, boolean unknown) { this.start = start; this.end = end; @@ -171,7 +181,7 @@ private void decode(String text, int from, int to, List morphemes) { // Each element heads the chain of nodes ending at that position. final Node[] endingAt = new Node[length + 1]; - final Category[] categoryAt = new Category[length]; + final CategoryAssignment[] categoryAt = new CategoryAssignment[length]; final int[] runEndAt = new int[length]; computeCategoryRuns(text, from, to, categoryAt, runEndAt); @@ -243,32 +253,31 @@ private void relax(Node candidate, Node predecessors) { } /** - * Fills the per-position category and same-category run end for one stretch, in one - * right-to-left pass over its code points. Positions inside a surrogate pair keep a - * {@code null} category; no candidate ever starts there. + * Computes the per-position categories and connected-category run end for one + * stretch, in one right-to-left pass over the code points. Positions inside a + * surrogate sequence have a {@code null} assignment; candidates do not start there. * * @param text The text being segmented. * @param from The stretch start. * @param to The exclusive stretch end. - * @param categoryAt Receives each position's category, indexed by {@code + * @param categoryAt Receives each position's categories, indexed by {@code * position - from}. - * @param runEndAt Receives each position's exclusive same-category run end, indexed - * the same way. + * @param runEndAt Receives each position's exclusive connected-category run end, + * indexed the same way. */ private void computeCategoryRuns(String text, int from, int to, - Category[] categoryAt, int[] runEndAt) { - int next = -1; + CategoryAssignment[] categoryAt, int[] runEndAt) { + CategoryAssignment next = null; + int nextRunEnd = to; for (int position = to; position > from; ) { final int codePoint = text.codePointBefore(position); position -= Character.charCount(codePoint); final int index = position - from; - categoryAt[index] = dictionary.categoryOf(codePoint); - if (next >= 0 && categoryAt[next] == categoryAt[index]) { - runEndAt[index] = runEndAt[next]; - } else { - runEndAt[index] = next >= 0 ? next + from : to; - } - next = index; + categoryAt[index] = dictionary.categoriesOf(codePoint); + runEndAt[index] = categoryAt[index].continuedRunEnd(next, nextRunEnd, + position + Character.charCount(codePoint)); + next = categoryAt[index]; + nextRunEnd = runEndAt[index]; } } @@ -279,17 +288,17 @@ private void computeCategoryRuns(String text, int from, int to, * @param from The stretch start. * @param to The exclusive stretch end, which no candidate may reach past. * @param offset The candidate start, relative to {@code from}. - * @param positionCategory The category of that position, or {@code null} for a - * position inside a surrogate pair. - * @param positionRunEnd The exclusive end of the same-category run starting there, - * meaningful only when {@code positionCategory} is not + * @param positionCategories The categories of that position, or {@code null} for a + * position inside a surrogate sequence. + * @param positionRunEnd The exclusive end of the overlapping-category run starting + * there. Used only when {@code positionCategories} is not * {@code null}. * @param candidates Receives the candidates. Must be empty on entry. * @throws IllegalStateException Thrown if neither the lexicon, the position's * category, nor the {@code DEFAULT} template offers a candidate. */ private void candidates(String text, int from, int to, int offset, - Category positionCategory, int positionRunEnd, List candidates) { + CategoryAssignment positionCategories, int positionRunEnd, List candidates) { final int position = from + offset; dictionary.prefixMatches(text, position, to, (length, entries) -> { for (final WordEntry entry : entries) { @@ -301,14 +310,14 @@ private void candidates(String text, int from, int to, int offset, final int codePoint = text.codePointAt(position); final Category category; final int runEnd; - if (positionCategory == null) { + if (positionCategories == null) { // Only a lexicon surface ending inside a surrogate pair can make such a // position reachable; classify the stray code unit on the spot so the lattice // stays connected. category = dictionary.categoryOf(codePoint); runEnd = position + Character.charCount(codePoint); } else { - category = positionCategory; + category = positionCategories.primary(); runEnd = positionRunEnd; } if (!lexiconMatch || category.invoke()) { @@ -318,10 +327,12 @@ private void candidates(String text, int from, int to, int offset, } } if (candidates.isEmpty()) { - // Neither the lexicon nor the character's category produced a candidate here, so a - // single-character entry from the DEFAULT template keeps the lattice connected. - final List fallback = - dictionary.unknownEntries(MecabDictionary.DEFAULT_CATEGORY); + // A category with no grouped or fixed-length candidate still provides one + // character. Incomplete dictionaries fall back to the DEFAULT template. + List fallback = dictionary.unknownEntries(category.name()); + if (fallback == null) { + fallback = dictionary.unknownEntries(MecabDictionary.DEFAULT_CATEGORY); + } if (fallback != null) { for (final WordEntry entry : fallback) { candidates.add( @@ -338,14 +349,13 @@ private void candidates(String text, int from, int to, int offset, /** * Emits unknown-word candidates per the category's grouping and length settings. * - *

Every candidate stays inside the same-category run, so an unknown word never - * glues characters of different categories together, and every length counts whole - * characters rather than code units.

+ *

Candidates remain inside a run connected by overlapping category assignments. + * Lengths count code points, not UTF-16 code units.

* * @param candidates Receives the candidates. * @param text The text being segmented. * @param position The position the candidates start at. - * @param runEnd The exclusive end of the same-category run starting at + * @param runEnd The exclusive end of the connected-category run starting at * {@code position}. * @param category The category of that run. * @param templates The category's unknown-word templates. diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index 1e0127b1fd..ebd7c9f34b 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -25,12 +25,15 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.BitSet; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import opennlp.tools.tokenize.lattice.CategoryTable.CategoryAssignment; import opennlp.tools.util.ResourceLimits; import opennlp.tools.util.StringUtil; @@ -49,15 +52,16 @@ * through this one reader, with the feature columns passed through untouched because * their schemas differ.

* - *

Each instance keeps about 0.75 MB of category tables keyed by the 16-bit code-unit - * space, so load once and share. Lexicon CSV files under the dictionary directory are + *

Each instance uses about 0.75 MB for category tables indexed by the 16-bit + * code-unit space, so load once and share. Lexicon CSV files under the directory are * read in sorted path order so tie-breaking is stable across file systems. Connection - * costs must cover every declared matrix cell; missing pairs are rejected rather than - * treated as cost zero. Matrix dimensions and the lexicon entry count are bounded by - * {@link ResourceLimits#MAX_ENTRIES}, and the matrix cell count by - * {@link ResourceLimits#MAX_MATRIX_CELLS}. Lexicon CSV fields may be - * MeCab-quoted with {@code ""} escapes. An {@code unk.def} template must name a - * category {@code char.def} defined.

+ * costs must cover all matrix cells; missing and duplicate entries are + * rejected instead of being treated as cost zero. Matrix dimensions and the lexicon + * entry count are bounded by {@link ResourceLimits#MAX_ENTRIES}, and the matrix cell + * count by {@link ResourceLimits#MAX_MATRIX_CELLS}. Lexicon CSV fields may be + * MeCab-quoted with {@code ""} escapes. Word and connection costs must fit in a signed + * 16-bit integer. An {@code unk.def} template must name a category defined by + * {@code char.def}.

* *

Instances are immutable and safe to share between threads.

* @@ -73,16 +77,30 @@ public final class MecabDictionary { static final String DEFAULT_CATEGORY = "DEFAULT"; private static final String MATRIX_DEF = "matrix.def"; - private static final String CHAR_DEF = "char.def"; + static final String CHAR_DEF = "char.def"; private static final String UNK_DEF = "unk.def"; + /** + * Maximum category count accepted by MeCab's + * + * character-property compiler. + */ + private static final int MAX_CATEGORY_COUNT = 17; + + /** + * Maximum value of MeCab's + * + * 4-bit category length field. + */ + private static final int MAX_CATEGORY_LENGTH = 15; + static final String LEXICON_EXTENSION = ".csv"; static final String DEFINITION_EXTENSION = ".def"; static final String CONFIGURATION_FILE = "dicrc"; private static final String LEXICON_GLOB = "*" + LEXICON_EXTENSION; private static final char COMMENT_MARKER = '#'; - /** The prefix a {@code char.def} code point field carries, in either letter case. */ + /** The code point prefix used by {@code char.def}, in either letter case. */ private static final String HEX_PREFIX = "0x"; /** The separator between the two ends of a {@code char.def} code point range. */ @@ -108,14 +126,15 @@ record WordEntry(int leftId, int rightId, int cost, List features) { /** * One character category's unknown-word behavior from {@code char.def}. * + * @param id The dense zero-based category id. * @param name The category name. * @param invoke Whether unknown-word candidates are generated even where the lexicon * matched. - * @param group Whether a whole run of same-category characters is offered as one - * candidate. + * @param group Whether a full run connected by overlapping category assignments is + * emitted as one candidate. * @param length How many leading characters of the run are offered as candidates. */ - record Category(String name, boolean invoke, boolean group, int length) { + record Category(int id, String name, boolean invoke, boolean group, int length) { } /** Receives one common-prefix match during {@link #prefixMatches}. */ @@ -134,9 +153,19 @@ interface PrefixMatchConsumer { private final short[] connectionCosts; private final int rightSize; private final CategoryTable categoryTable; - private final Category defaultCategory; + private final CategoryAssignment defaultCategories; private final Map> unknownEntries; + /** + * Creates an immutable dictionary from parsed lexicon and definition data. + * + * @param lexicon The surface lexicon. + * @param connectionCosts The flattened connection matrix. + * @param rightSize The number of left-context columns in each matrix record. + * @param categories The character categories by name. + * @param categoryTable The code point category assignments. + * @param unknownEntries The unknown-word templates by category. + */ private MecabDictionary(DoubleArrayLexicon lexicon, short[] connectionCosts, int rightSize, Map categories, CategoryTable categoryTable, Map> unknownEntries) { @@ -144,7 +173,10 @@ private MecabDictionary(DoubleArrayLexicon lexicon, this.connectionCosts = connectionCosts; this.rightSize = rightSize; this.categoryTable = categoryTable; - this.defaultCategory = categories.get(DEFAULT_CATEGORY); + final Category defaultCategory = Objects.requireNonNull( + categories.get(DEFAULT_CATEGORY), "DEFAULT category"); + this.defaultCategories = new CategoryAssignment( + new Category[] {defaultCategory}); final Map> copy = new HashMap<>(unknownEntries.size()); for (final Map.Entry> entry : unknownEntries.entrySet()) { copy.put(entry.getKey(), List.copyOf(entry.getValue())); @@ -259,6 +291,10 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc + " format defines"); } final int index = right * rightSize + left; + if (filled.get(index)) { + throw new IOException("duplicate " + MATRIX_DEF + " entry " + right + " " + + left + " at line " + lineNumber); + } costs[index] = (short) cost; filled.set(index); } @@ -304,7 +340,7 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } /** - * Reads one lexicon-format CSV file, rejecting any entry whose context ids the + * Parses one lexicon-format CSV file, rejecting entries with context ids the * connection matrix cannot be indexed with. * * @param file The file to read. @@ -316,9 +352,9 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc * ids. * @param entryCount A one-element running total of entries read so far, shared across * the lexicon files of one load. - * @throws IOException Thrown if the file is missing, an entry is malformed, an - * entry's context id is outside the matrix dimensions, or the running entry - * count exceeds {@link ResourceLimits#MAX_ENTRIES}. + * @throws IOException Thrown if the file is missing, an entry is malformed or has + * an empty surface, an entry's context id is outside the matrix dimensions, + * or the running entry count exceeds {@link ResourceLimits#MAX_ENTRIES}. */ private static void readLexicon(Path file, Charset charset, Map> target, int leftSize, int rightSize, int[] entryCount) @@ -340,7 +376,8 @@ private static void readLexicon(Path file, Charset charset, } final String surface = fields.get(0); if (surface.isEmpty()) { - continue; + throw new IOException("malformed entry at " + file + " line " + lineNumber + + ": surface must not be empty"); } final int leftId = parseInt(fields.get(1), file.toString(), lineNumber); final int rightId = parseInt(fields.get(2), file.toString(), lineNumber); @@ -359,8 +396,13 @@ private static void readLexicon(Path file, Charset charset, + ResourceLimits.MAX_ENTRIES); } entryCount[0]++; - final WordEntry entry = new WordEntry(leftId, rightId, - parseInt(fields.get(3), file.toString(), lineNumber), + final int cost = parseInt(fields.get(3), file.toString(), lineNumber); + if (cost < Short.MIN_VALUE || cost > Short.MAX_VALUE) { + throw new IOException("malformed entry at " + file + " line " + lineNumber + + ": word cost " + cost + + " is outside the 16-bit range the format defines"); + } + final WordEntry entry = new WordEntry(leftId, rightId, cost, List.copyOf(fields.subList(4, fields.size()))); target.computeIfAbsent(surface, key -> new ArrayList<>(1)).add(entry); } @@ -376,8 +418,9 @@ private static void readLexicon(Path file, Charset charset, * @param categories Receives the defined categories, keyed by name. * @param categoryTable Receives the code point to category name mappings. * @throws IOException Thrown if the file is missing, a line is malformed, a code - * point is outside the Unicode range, a range descends, or the file defines - * no {@code DEFAULT} category. + * point is outside the Unicode range, a range descends, a category is + * duplicated, category count or length exceeds the MeCab format, or the file + * defines no {@code DEFAULT} category. */ private static void readCharacterDefinition(Path file, Charset charset, Map categories, CategoryTable.Builder categoryTable) @@ -417,7 +460,7 @@ private static void readCharacterDefinition(Path file, Charset charset, throw new IOException("code point range descends at " + file + " line " + lineNumber); } - categoryTable.map(from, to, fields[1]); + categoryTable.map(from, to, Arrays.copyOfRange(fields, 1, fields.length)); } else { if (fields.length < 4) { throw new IOException( @@ -428,13 +471,22 @@ private static void readCharacterDefinition(Path file, Charset charset, "malformed category flag at " + file + " line " + lineNumber); } final int length = parseInt(fields[3], file.toString(), lineNumber); - if (length < 0) { + if (length < 0 || length > MAX_CATEGORY_LENGTH) { throw new IOException( - "category LENGTH must not be negative at " + file + " line " - + lineNumber); + "category LENGTH must be between 0 and " + MAX_CATEGORY_LENGTH + " at " + + file + " line " + lineNumber); } - categories.put(fields[0], new Category(fields[0], - FLAG_ON.equals(fields[1]), FLAG_ON.equals(fields[2]), length)); + if (categories.containsKey(fields[0])) { + throw new IOException("duplicate " + CHAR_DEF + " category " + + fields[0] + " at line " + lineNumber); + } + if (categories.size() >= MAX_CATEGORY_COUNT) { + throw new IOException(CHAR_DEF + " defines " + (categories.size() + 1) + + " categories; MeCab supports at most " + MAX_CATEGORY_COUNT); + } + final Category category = new Category(categories.size(), fields[0], + FLAG_ON.equals(fields[1]), FLAG_ON.equals(fields[2]), length); + categories.put(fields[0], category); } } } @@ -469,17 +521,29 @@ int connectionCost(int rightId, int leftId) { } /** - * Classifies a character by code point, so that a character outside the Basic - * Multilingual Plane is classified as the one character it is rather than as its two - * surrogates. + * Classifies a character by code point, so a character outside the Basic Multilingual + * Plane is classified once, not as separate surrogate code units. * * @param codePoint The code point to classify. * @return Its category, falling back to {@code DEFAULT} when no {@code char.def} * mapping covers the code point. Never {@code null}. */ Category categoryOf(int codePoint) { - final Category category = categoryTable.categoryOf(codePoint); - return category != null ? category : defaultCategory; + return categoriesOf(codePoint).primary(); + } + + /** + * Classifies a character into all categories assigned by {@code char.def}. The first + * category supplies unknown-word behavior and all categories participate in run + * grouping. + * + * @param codePoint The code point to classify. + * @return The category assignment, falling back to {@code DEFAULT} when no mapping + * covers the code point. + */ + CategoryAssignment categoriesOf(int codePoint) { + final CategoryAssignment categories = categoryTable.categoriesOf(codePoint); + return categories != null ? categories : defaultCategories; } /** @@ -496,7 +560,7 @@ List unknownEntries(String category) { * Removes a trailing {@code #} comment from a {@code char.def} line. * * @param line The raw line. - * @return The line up to but excluding the first {@code #}, or the whole line when + * @return The line up to but excluding the first {@code #}, or the complete line when * there is none. */ private static String stripComment(String line) { diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index 2d97010724..91f65bcafc 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -22,9 +22,11 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.Charset; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -40,9 +42,9 @@ * Frequency-driven segmentation for Chinese and similar scripts: a Viterbi search that * maximizes the summed log-probability of the words in a user-supplied frequency * lexicon, with unlisted characters falling back to single-character words. This is the - * unigram model behind common Chinese segmenters; it uses no connection costs, so it - * is lighter than the {@link LatticeTokenizer} and fits lexicons that list only words - * and counts. + * unigram model behind common Chinese segmenters. It omits the connection costs used + * by {@link LatticeTokenizer}, making it suitable for lexicons containing words and + * counts. * *

The lexicon format is one entry per line: the word, its count, and optionally a * tag, separated by whitespace. The lexicon file is user-supplied; no lexicon data is @@ -69,6 +71,13 @@ private static final class WordTrie { private final WordTrie[] nodes; private final double logProbability; + /** + * Creates an immutable trie node. + * + * @param keys The sorted child labels. + * @param nodes The child nodes, parallel to {@code keys}. + * @param logProbability The word score, or {@link Double#NaN} for a nonterminal node. + */ private WordTrie(char[] keys, WordTrie[] nodes, double logProbability) { this.keys = keys; this.nodes = nodes; @@ -92,22 +101,61 @@ private static final class WordTrieBuilder { private final Map children = new HashMap<>(); private double logProbability = Double.NaN; + private WordTrie built; - private WordTrie freeze() { - final char[] keys = new char[children.size()]; - int i = 0; - for (final Character key : children.keySet()) { - keys[i++] = key; - } - Arrays.sort(keys); - final WordTrie[] nodes = new WordTrie[keys.length]; - for (int k = 0; k < keys.length; k++) { - nodes[k] = children.get(keys[k]).freeze(); + /** + * One pending post-order traversal step. + * + * @param node The mutable node. + * @param childrenBuilt Whether the node's children have been copied. + */ + private record BuildStep(WordTrieBuilder node, boolean childrenBuilt) { + } + + /** + * Copies the mutable tree without consuming the thread stack. Each mutable node + * uses a step before child visits and a step afterward, preserving post-order + * construction for surfaces of any supported length. + */ + private WordTrie build() { + final ArrayDeque pending = new ArrayDeque<>(); + pending.push(new BuildStep(this, false)); + while (!pending.isEmpty()) { + final BuildStep step = pending.pop(); + final WordTrieBuilder node = step.node(); + if (!step.childrenBuilt()) { + pending.push(new BuildStep(node, true)); + for (final WordTrieBuilder child : node.children.values()) { + pending.push(new BuildStep(child, false)); + } + continue; + } + final char[] keys = new char[node.children.size()]; + int i = 0; + for (final Character key : node.children.keySet()) { + keys[i++] = key; + } + Arrays.sort(keys); + final WordTrie[] nodes = new WordTrie[keys.length]; + for (int k = 0; k < keys.length; k++) { + final WordTrieBuilder child = node.children.get(keys[k]); + nodes[k] = child.built; + child.built = null; + } + node.built = new WordTrie(keys, nodes, node.logProbability); } - return new WordTrie(keys, nodes, logProbability); + final WordTrie result = built; + built = null; + return result; } } + /** + * Creates a segmenter from a word trie and the unknown-character score. + * + * @param trie The word trie. + * @param unknownLogProbability The score for an unlisted character. + */ private UnigramSegmenter(WordTrie trie, double unknownLogProbability) { this.trie = trie; this.unknownLogProbability = unknownLogProbability; @@ -184,8 +232,10 @@ private static UnigramSegmenter loadInternal(InputStream lexiconStream, Charset } final Map counts = new HashMap<>(); long total = 0; - final BufferedReader reader = - new BufferedReader(new InputStreamReader(lexiconStream, charset)); + final BufferedReader reader = new BufferedReader(new InputStreamReader( + lexiconStream, charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT))); int lineNumber = 0; String raw; while ((raw = reader.readLine()) != null) { @@ -244,7 +294,7 @@ private static UnigramSegmenter loadInternal(InputStream lexiconStream, Charset // Charge an unlisted character half of one count out of the total, which makes it // rarer than any listed word: every listed count is at least one. final double unknown = Math.log(0.5) - logTotal; - return new UnigramSegmenter(root.freeze(), unknown); + return new UnigramSegmenter(root.build(), unknown); } /** diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index 21525c6073..c0cf0a6445 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -18,6 +18,7 @@ package opennlp.tools.tokenize.lattice; import java.io.IOException; +import java.nio.charset.MalformedInputException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -49,6 +50,9 @@ public class LatticeTokenizerTest { private static final String CHAR_DEF = "char.def"; private static final String UNK_DEF = "unk.def"; + /** UTF-8 lead byte with the required continuation byte omitted. */ + private static final byte TRUNCATED_UTF8_LEAD_BYTE = (byte) 0xC3; + /** A one by one connection matrix charging cost zero, for single-context fixtures. */ private static final String UNIT_MATRIX = "1 1\n0 0 0\n"; @@ -709,8 +713,8 @@ void testMappingToUndefinedCategoryFailsLoud(@TempDir Path ghost) throws IOExcep final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(ghost)); - Assertions.assertEquals("char.def maps U+0100 to the undefined category GHOST", - e.getMessage()); + Assertions.assertEquals("char.def declaration at U+0100 names the undefined" + + " category GHOST", e.getMessage()); } /** @@ -823,4 +827,275 @@ void testMatrixRowContextIdsOutsideDimensionsFailLoud(@TempDir Path broken) Assertions.assertEquals("malformed matrix.def line 2: context ids 2 0 are outside" + " the declared dimensions 1 1", e.getMessage()); } + + @Test + void testZeroLengthCategoryUsesItsUnknownTemplate(@TempDir Path dictionary) + throws IOException { + write(dictionary, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(dictionary, MATRIX_DEF, UNIT_MATRIX); + write(dictionary, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, + "SINGLE 1 0 0", + "", + "0x2460 SINGLE", + "")); + write(dictionary, UNK_DEF, String.join("\n", + DEFAULT_UNKNOWN_TEMPLATE, + "SINGLE,0,0,1000,symbol,single", + "")); + + final Morpheme morpheme = new LatticeTokenizer(MecabDictionary.load(dictionary)) + .analyze("\u2460").get(0); + + Assertions.assertEquals(List.of("symbol", "single"), morpheme.features()); + } + + @Test + void testRejectsEmptyLexiconSurface(@TempDir Path dictionary) throws IOException { + writeUnitMatrixDictionary(dictionary); + write(dictionary, LEXICON_CSV, String.join("\n", + "\u6771,0,0,3000,noun", + ",0,0,3000,noun", + "")); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(dictionary)); + + Assertions.assertEquals("malformed entry at " + dictionary.resolve(LEXICON_CSV) + + " line 2: surface must not be empty", e.getMessage()); + } + + @ParameterizedTest(name = "word cost {0}") + @ValueSource(ints = {-32769, 32768}) + void testRejectsLexiconCostOutsideShortRange(int cost, @TempDir Path dictionary) + throws IOException { + writeUnitMatrixDictionary(dictionary); + write(dictionary, LEXICON_CSV, "\u6771,0,0," + cost + ",noun\n"); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(dictionary)); + + Assertions.assertEquals("malformed entry at " + dictionary.resolve(LEXICON_CSV) + + " line 1: word cost " + cost + + " is outside the 16-bit range the format defines", e.getMessage()); + } + + @Test + void testRejectsDuplicateCharacterCategory(@TempDir Path dictionary) + throws IOException { + write(dictionary, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(dictionary, MATRIX_DEF, UNIT_MATRIX); + write(dictionary, CHAR_DEF, DEFAULT_CATEGORY_LINE + "\nDEFAULT 1 0 2\n"); + write(dictionary, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(dictionary)); + + Assertions.assertEquals("duplicate char.def category DEFAULT at line 2", + e.getMessage()); + } + + @Test + void testRejectsDuplicateMatrixEntry(@TempDir Path dictionary) throws IOException { + writeUnitMatrixDictionary(dictionary); + write(dictionary, MATRIX_DEF, "1 1\n0 0 1\n0 0 2\n"); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(dictionary)); + + Assertions.assertEquals("duplicate matrix.def entry 0 0 at line 3", e.getMessage()); + } + + @Test + void testRejectsMalformedDictionaryEncoding(@TempDir Path dictionary) + throws IOException { + writeUnitMatrixDictionary(dictionary); + Files.write(dictionary.resolve(LEXICON_CSV), + new byte[] {TRUNCATED_UTF8_LEAD_BYTE, ',', '0', ',', '0', ',', '1', '\n'}); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(dictionary)); + + Assertions.assertInstanceOf(MalformedInputException.class, e); + Assertions.assertEquals("Input length = 1", e.getMessage()); + } + + @Test + void testLongLexiconSurfaceLoads(@TempDir Path dictionary) throws IOException { + writeUnitMatrixDictionary(dictionary); + final String surface = "a".repeat(20_000); + write(dictionary, LEXICON_CSV, surface + ",0,0,3000,fixture\n"); + + final LatticeTokenizer longSurfaceTokenizer = + new LatticeTokenizer(MecabDictionary.load(dictionary)); + + Assertions.assertArrayEquals(new String[] {surface}, + longSurfaceTokenizer.tokenize(surface)); + } + + @Test + void testSecondaryCharacterCategoryExtendsUnknownRun(@TempDir Path dictionary) + throws IOException { + write(dictionary, LEXICON_CSV, "\u6771,0,0,6000,noun\n"); + write(dictionary, MATRIX_DEF, UNIT_MATRIX); + write(dictionary, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, + "KANJI 1 0 1", + "KANJINUMERIC 1 1 0", + "", + "0x4E00 KANJINUMERIC KANJI", + "0x5C71 KANJI", + "")); + write(dictionary, UNK_DEF, String.join("\n", + DEFAULT_UNKNOWN_TEMPLATE, + "KANJI,0,0,5000,noun,unknown", + "KANJINUMERIC,0,0,1000,number,unknown", + "")); + + final List morphemes = new LatticeTokenizer( + MecabDictionary.load(dictionary)).analyze("\u4E00\u5C71"); + + Assertions.assertEquals(1, morphemes.size()); + Assertions.assertEquals("\u4E00\u5C71", morphemes.get(0).surface()); + Assertions.assertEquals(List.of("number", "unknown"), morphemes.get(0).features()); + } + + /** + * Verifies MeCab's category-chain grouping. Character A has X and Y, B has X, and C + * has Y. A and B intersect, while B and C do not, so the initial unknown word covers + * {@code ab}. + */ + @Test + void testMultipleCategoriesUsePairwiseGrouping(@TempDir Path dictionary) + throws IOException { + write(dictionary, LEXICON_CSV, "z,0,0,6000,fixture\n"); + write(dictionary, MATRIX_DEF, UNIT_MATRIX); + write(dictionary, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, + "X 1 1 0", + "Y 1 1 0", + "", + "0x0061 X Y", + "0x0062 X", + "0x0063 Y", + "")); + write(dictionary, UNK_DEF, String.join("\n", + DEFAULT_UNKNOWN_TEMPLATE, + "X,0,0,1000,x,unknown", + "Y,0,0,1000,y,unknown", + "")); + + final LatticeTokenizer groupingTokenizer = + new LatticeTokenizer(MecabDictionary.load(dictionary)); + + Assertions.assertArrayEquals(new String[] {"ab", "c"}, + groupingTokenizer.tokenize("abc")); + } + + /** + * Verifies that an intermediate multi-category character can connect a run. + * Character A has X, B has X and Y, and C has Y. MeCab advances the active + * assignment at each position, allowing B to connect both portions. + */ + @Test + void testCategoryOverlapCanConnectRun(@TempDir Path dictionary) + throws IOException { + write(dictionary, LEXICON_CSV, "z,0,0,6000,fixture\n"); + write(dictionary, MATRIX_DEF, UNIT_MATRIX); + write(dictionary, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, + "X 1 1 0", + "Y 1 1 0", + "", + "0x0061 X", + "0x0062 X Y", + "0x0063 Y", + "")); + write(dictionary, UNK_DEF, String.join("\n", + DEFAULT_UNKNOWN_TEMPLATE, + "X,0,0,1000,x,unknown", + "Y,0,0,1000,y,unknown", + "")); + + final LatticeTokenizer groupingTokenizer = + new LatticeTokenizer(MecabDictionary.load(dictionary)); + + Assertions.assertArrayEquals(new String[] {"abc"}, + groupingTokenizer.tokenize("abc")); + } + + @Test + void testRejectsTooManyCharacterCategories(@TempDir Path dictionary) + throws IOException { + write(dictionary, LEXICON_CSV, "z,0,0,6000,fixture\n"); + write(dictionary, MATRIX_DEF, UNIT_MATRIX); + final StringBuilder charDef = new StringBuilder(DEFAULT_CATEGORY_LINE).append('\n'); + for (int i = 1; i < 18; i++) { + charDef.append('C').append(i).append(" 0 0 1\n"); + } + write(dictionary, CHAR_DEF, charDef.toString()); + write(dictionary, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(dictionary)); + + Assertions.assertEquals("char.def defines 18 categories; MeCab supports at most 17", + e.getMessage()); + } + + @Test + void testRejectsCharacterCategoryLengthAboveMecabLimit(@TempDir Path dictionary) + throws IOException { + write(dictionary, LEXICON_CSV, "z,0,0,6000,fixture\n"); + write(dictionary, MATRIX_DEF, UNIT_MATRIX); + write(dictionary, CHAR_DEF, "DEFAULT 0 1 16\n"); + write(dictionary, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(dictionary)); + + Assertions.assertEquals("category LENGTH must be between 0 and 15 at " + + dictionary.resolve(CHAR_DEF) + " line 1", e.getMessage()); + } + + @Test + void testRejectsUndefinedSecondaryCharacterCategory(@TempDir Path dictionary) + throws IOException { + write(dictionary, LEXICON_CSV, "\u6771,0,0,6000,noun\n"); + write(dictionary, MATRIX_DEF, UNIT_MATRIX); + write(dictionary, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, + "KANJINUMERIC 1 1 0", + "", + "0x4E00 KANJINUMERIC GHOST", + "")); + write(dictionary, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(dictionary)); + + Assertions.assertEquals("char.def declaration at U+4E00 names the undefined" + + " category GHOST", e.getMessage()); + } + + @Test + void testRejectsUndefinedCategoryOnShadowedMapping(@TempDir Path dictionary) + throws IOException { + write(dictionary, LEXICON_CSV, "z,0,0,6000,fixture\n"); + write(dictionary, MATRIX_DEF, UNIT_MATRIX); + write(dictionary, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, + "LATIN 1 1 0", + "", + "0x0061 GHOST", + "0x0061 LATIN", + "")); + write(dictionary, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(dictionary)); + + Assertions.assertEquals("char.def declaration at U+0061 names the undefined" + + " category GHOST", e.getMessage()); + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index bfcd2dc8ab..c6e00cf1a0 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -20,6 +20,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.MalformedInputException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; @@ -41,6 +42,9 @@ */ public class UnigramSegmenterTest { + /** UTF-8 lead byte with the required continuation byte omitted. */ + private static final byte TRUNCATED_UTF8_LEAD_BYTE = (byte) 0xC3; + private static final String LEXICON = String.join("\n", "\u6211 5000 r", "\u6765\u5230 2000 v", @@ -240,4 +244,27 @@ void testLeadingIdeographicSpaceIsTrimmed() throws IOException { StandardCharsets.UTF_8); Assertions.assertArrayEquals(new String[] {"\u6211"}, loaded.tokenize("\u6211")); } + + @Test + void testRejectsMalformedLexiconEncoding() { + final byte[] malformed = {'w', TRUNCATED_UTF8_LEAD_BYTE, ' ', '1', '\n'}; + + final IOException e = Assertions.assertThrows(IOException.class, + () -> UnigramSegmenter.load( + new ByteArrayInputStream(malformed), StandardCharsets.UTF_8)); + + Assertions.assertInstanceOf(MalformedInputException.class, e); + Assertions.assertEquals("Input length = 1", e.getMessage()); + } + + @Test + void testLongLexiconWordLoads() throws IOException { + final String word = "a".repeat(20_000); + + final UnigramSegmenter loaded = UnigramSegmenter.load( + new ByteArrayInputStream((word + " 1\n").getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + + Assertions.assertArrayEquals(new String[] {word}, loaded.tokenize(word)); + } } diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index 9ab156f520..32ee38e1f4 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -558,22 +558,35 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { opennlp.install.max.entry.bytes and opennlp.install.max.total.bytes for larger dictionaries such as UniDic. - Load also rejects an unk.def template for a category - char.def did not define, and accepts MeCab-quoted CSV fields. + Dictionary loading rejects empty lexicon surfaces, duplicate + matrix.def entries, duplicate char.def + categories, costs outside the signed 16-bit range, malformed text in the + selected encoding, and unk.def templates for undefined + categories. MeCab-quoted CSV fields are accepted. + A char.def may define up to 17 categories, and category + lengths range from 0 through 15, matching the MeCab character-property + compiler. + If a char.def mapping lists multiple categories, the first + provides unknown-word settings. Grouping continues while successive + assignments overlap. Unknown words use their category's unk.def + template first, with DEFAULT used when no category template exists. Tar headers are checksum-validated before extraction. Files are staged on the target filesystem and published after the archive passes validation. The installer does not replace files already present in the target directory. - - + morphemes = tokenizer.analyze(text); -UnigramSegmenter segmenter = UnigramSegmenter.load(Path.of("words.txt")); -String[] words = segmenter.tokenize(text);]]> +UnigramSegmenter segmenter = UnigramSegmenter.load(Path.of("chinese-words.txt")); +String[] words = segmenter.tokenize("我来到北京清华大学");]]>