Permalink
Find file Copy path
300 lines (259 sloc) 13.8 KB
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.standard;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/**
* This class implements Word Break rules from the Unicode Text Segmentation
* algorithm, as specified in
* <a href="http://unicode.org/reports/tr29/">Unicode Standard Annex #29</a>.
* <p>
* Tokens produced are of the following types:
* <ul>
* <li>&lt;ALPHANUM&gt;: A sequence of alphabetic and numeric characters</li>
* <li>&lt;NUM&gt;: A number</li>
* <li>&lt;SOUTHEAST_ASIAN&gt;: A sequence of characters from South and Southeast
* Asian languages, including Thai, Lao, Myanmar, and Khmer</li>
* <li>&lt;IDEOGRAPHIC&gt;: A single CJKV ideographic character</li>
* <li>&lt;HIRAGANA&gt;: A single hiragana character</li>
* <li>&lt;KATAKANA&gt;: A sequence of katakana characters</li>
* <li>&lt;HANGUL&gt;: A sequence of Hangul characters</li>
* <li>&lt;EMOJI&gt;: A sequence of Emoji characters</li>
* </ul>
*/
@SuppressWarnings("fallthrough")
%%
%unicode 9.0
%integer
%final
%public
%class StandardTokenizerImpl
%function getNextToken
%char
%buffer 255
//////////////////////////////////////////////////////////////////////////
// Begin Emoji Macros - see documentation below, near the EMOJI_TYPE rule
// TODO: Remove this include file when JFlex supports these properties directly (in Unicode 11.0+)
%include ../../../../../../data/jflex/UnicodeEmojiProperties.jflex
// UAX#29 WB4. X (Extend | Format | ZWJ)* --> X
//
// \uFE0E (Text Presentation Selector) and \uFE0F (Emoji Presentation Selector) - included in \p{WB:Extend}
// - are explicitly excluded here so that we can properly handle Emoji sequences.
//
ExtFmtZwjSansPresSel = [[\p{WB:Format}\p{WB:Extend}\p{WB:ZWJ}]--[\uFE0E\uFE0F]]*
KeyCapBaseChar = [0-9#*]
KeyCapBaseCharEx = {KeyCapBaseChar} {ExtFmtZwjSansPresSel}
KeyCap = \u20E3
KeyCapEx = {KeyCap} {ExtFmtZwjSansPresSel}
// # \u3030 = WAVY DASH; \u303D = PART ALTERNATION MARK
AccidentalEmoji = [©®™\u3030\u303D]
EmojiRKAM = ( \p{WB:Regional_Indicator} | {KeyCapBaseChar} | {AccidentalEmoji} | {Emoji_Modifier} )
// Unlike Unicode properties, macros are not allowed in character classes, so we achieve set difference
// by applying DeMorgan: the expression that matches everything of 'a' not matched by 'b' is: !(!a|b)
// TODO: Convert this expression to character class difference when JFlex supports the properties directly (in Unicode 11.0+)
EmojiSansRKAM = !( ! {Emoji} | {EmojiRKAM} )
EmojiChar = ( {Extended_Pictographic} | {EmojiSansRKAM} )
EmojiCharEx = {EmojiChar} {ExtFmtZwjSansPresSel}
EmojiModifierBaseEx = {Emoji_Modifier_Base} {ExtFmtZwjSansPresSel}
EmojiModifierEx = {Emoji_Modifier} {ExtFmtZwjSansPresSel}
EmojiPresentationSelector = \uFE0F
EmojiCharOrPresSeqOrModSeq = ( \p{WB:ZWJ}* {EmojiCharEx} {EmojiPresentationSelector}? ) | ( ( \p{WB:ZWJ}* {EmojiModifierBaseEx} )? {EmojiModifierEx} )
TagSpec = [\u{E0020}-\u{E007E}]
TagTerm = \u{E007F}
// End Emoji Macros
//////////////////////////////////////////////////////////////////////////
// UAX#29 WB4. X (Extend | Format | ZWJ)* --> X
//
ExtFmtZwj = [\p{WB:Format}\p{WB:Extend}\p{WB:ZWJ}]*
HangulEx = [\p{Script:Hangul}&&[\p{WB:ALetter}\p{WB:Hebrew_Letter}]] {ExtFmtZwj}
AHLetterEx = [\p{WB:ALetter}\p{WB:Hebrew_Letter}] {ExtFmtZwj}
NumericEx = [\p{WB:Numeric}[\p{Blk:HalfAndFullForms}&&\p{Nd}]] {ExtFmtZwj}
KatakanaEx = \p{WB:Katakana} {ExtFmtZwj}
MidLetterEx = [\p{WB:MidLetter}\p{WB:MidNumLet}\p{WB:SingleQuote}] {ExtFmtZwj}
MidNumericEx = [\p{WB:MidNum}\p{WB:MidNumLet}\p{WB:SingleQuote}] {ExtFmtZwj}
ExtendNumLetEx = \p{WB:ExtendNumLet} {ExtFmtZwj}
HanEx = \p{Script:Han} {ExtFmtZwj}
HiraganaEx = \p{Script:Hiragana} {ExtFmtZwj}
SingleQuoteEx = \p{WB:Single_Quote} {ExtFmtZwj}
DoubleQuoteEx = \p{WB:Double_Quote} {ExtFmtZwj}
HebrewLetterEx = \p{WB:Hebrew_Letter} {ExtFmtZwj}
RegionalIndicatorEx = \p{WB:Regional_Indicator} {ExtFmtZwj}
ComplexContextEx = \p{LB:Complex_Context} {ExtFmtZwj}
%{
/** Alphanumeric sequences */
public static final int WORD_TYPE = StandardTokenizer.ALPHANUM;
/** Numbers */
public static final int NUMERIC_TYPE = StandardTokenizer.NUM;
/**
* Chars in class \p{Line_Break = Complex_Context} are from South East Asian
* scripts (Thai, Lao, Myanmar, Khmer, etc.). Sequences of these are kept
* together as as a single token rather than broken up, because the logic
* required to break them at word boundaries is too complex for UAX#29.
* <p>
* See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA
*/
public static final int SOUTH_EAST_ASIAN_TYPE = StandardTokenizer.SOUTHEAST_ASIAN;
/** Ideographic token type */
public static final int IDEOGRAPHIC_TYPE = StandardTokenizer.IDEOGRAPHIC;
/** Hiragana token type */
public static final int HIRAGANA_TYPE = StandardTokenizer.HIRAGANA;
/** Katakana token type */
public static final int KATAKANA_TYPE = StandardTokenizer.KATAKANA;
/** Hangul token type */
public static final int HANGUL_TYPE = StandardTokenizer.HANGUL;
/** Emoji token type */
public static final int EMOJI_TYPE = StandardTokenizer.EMOJI;
/** Character count processed so far */
public final int yychar()
{
return yychar;
}
/**
* Fills CharTermAttribute with the current token text.
*/
public final void getText(CharTermAttribute t) {
t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
}
/**
* Sets the scanner buffer size in chars
*/
public final void setBufferSize(int numChars) {
ZZ_BUFFERSIZE = numChars;
char[] newZzBuffer = new char[ZZ_BUFFERSIZE];
System.arraycopy(zzBuffer, 0, newZzBuffer, 0, Math.min(zzBuffer.length, ZZ_BUFFERSIZE));
zzBuffer = newZzBuffer;
}
%}
%%
// UAX#29 WB1. sot ÷ Any
// WB2. Any ÷ eot
//
<<EOF>> { return YYEOF; }
// Instead of these: UAX#29 WB3c. ZWJ × (Glue_After_Zwj | EBG)
// WB14. (E_Base | EBG) × E_Modifier
// WB15. ^ (RI RI)* RI × RI
// WB16. [^RI] (RI RI)* RI × RI
//
// We use the "emoji_sequence" rule from http://www.unicode.org/reports/tr51/tr51-14.html (Unicode 11.0)
// and the Emoji data from http://unicode.org/Public/emoji/11.0/emoji-data.txt (in included file UnicodeEmojiProperties.jflex)
//
// emoji_sequence :=
// Top-level EBNF Expanded #1 Expanded #2 Expanded #3
// --------------------- ---------------------------- ----------------------------- ----------------------------------------------
// emoji_core_sequence emoji_combining_sequence emoji_character ( \p{Emoji}
// | emoji_presentation_sequence | \p{Emoji} \uFE0F
// | emoji_keycap_sequence | [0-9#*] \u{FE0F 20E3} [1]
// | emoji_modifier_sequence | \p{Emoji_Modifier_Base} \p{Emoji_Modifier}
// | emoji_flag_sequence | \p{WB:Regional_Indicator}{2} )
//
// | emoji_zwj_sequence emoji_zwj_element emoji_character ( \p{Emoji}
// | emoji_presentation_sequence | \p{Emoji} \uFE0F
// | emoji_modifier_sequence | \p{Emoji_Modifier_Base} \p{Emoji_Modifier} )
// ( ZWJ emoji_zwj_element )+ ( \p{WB:ZWJ} ^^ )+
//
// | emoji_tag_sequence tag_base emoji_character ( \p{Emoji}
// | emoji_presentation_sequence | \p{Emoji} \uFE0F
// | emoji_modifier_sequence | \p{Emoji_Modifier_Base} \p{Emoji_Modifier} )
// tag_spec [\u{E0020}-\u{E007E}]+
// tag_term \u{E007F}
//
// [1] https://unicode.org/Public/emoji/11.0/emoji-test.txt includes key cap sequences
// WITHOUT \uFE0F (emoji presentation indicator), annotating them as "non-fully-qualified";
// TR#51 says about non-fully-qualified *ZWJ sequences* that implementations may
// choose whether to support them for segmentation. This implementation will
// recognize /[0-9#*]\u20E3/ - i.e. without \uFE0F - as Emoji.
//
// See also: http://www.unicode.org/L2/L2016/16315-handling-seg-emoji.pdf
// https://docs.google.com/document/d/1yDZ5TUZNVVKaM9zYCCLbRIAKGNZANsAGl0bcNzGGvn8
//
// In particular, the above docs recommend a modified UAX#29 WB3c rule (covered by TR#51's "emoji_zwj_sequence"):
//
// WB3c′ ZWJ × ​(Extended_Pictographic | EmojiNRK)
//
{EmojiCharOrPresSeqOrModSeq} ( ( \p{WB:ZWJ} {EmojiCharOrPresSeqOrModSeq} )* | {TagSpec}+ {TagTerm} )
| {KeyCapBaseCharEx} {EmojiPresentationSelector}? {KeyCapEx}
| {RegionalIndicatorEx}{2}
{ return EMOJI_TYPE; }
// UAX#29 WB8. Numeric × Numeric
// WB11. Numeric (MidNum | MidNumLetQ) × Numeric
// WB12. Numeric × (MidNum | MidNumLetQ) Numeric
// WB13a. (AHLetter | Numeric | Katakana | ExtendNumLet) × ExtendNumLet
// WB13b. ExtendNumLet × (AHLetter | Numeric | Katakana)
//
{ExtendNumLetEx}* {NumericEx} ( ( {ExtendNumLetEx}* | {MidNumericEx} ) {NumericEx} )* {ExtendNumLetEx}*
{ return NUMERIC_TYPE; }
// subset of the below for typing purposes only!
{HangulEx}+
{ return HANGUL_TYPE; }
{KatakanaEx}+
{ return KATAKANA_TYPE; }
// UAX#29 WB5. AHLetter × AHLetter
// WB6. AHLetter × (MidLetter | MidNumLetQ) AHLetter
// WB7. AHLetter (MidLetter | MidNumLetQ) × AHLetter
// WB7a. Hebrew_Letter × Single_Quote
// WB7b. Hebrew_Letter × Double_Quote Hebrew_Letter
// WB7c. Hebrew_Letter Double_Quote × Hebrew_Letter
// WB9. AHLetter × Numeric
// WB10. Numeric × AHLetter
// WB13. Katakana × Katakana
// WB13a. (ALetter | Hebrew_Letter | Numeric | Katakana | ExtendNumLet) × ExtendNumLet
// WB13b. ExtendNumLet × (ALetter | Hebrew_Letter | Numeric | Katakana)
//
{ExtendNumLetEx}* ( {KatakanaEx} ( {ExtendNumLetEx}* {KatakanaEx} )*
| ( {HebrewLetterEx} ( {SingleQuoteEx} | {DoubleQuoteEx} {HebrewLetterEx} )
| {NumericEx} ( ( {ExtendNumLetEx}* | {MidNumericEx} ) {NumericEx} )*
| {AHLetterEx} ( ( {ExtendNumLetEx}* | {MidLetterEx} ) {AHLetterEx} )*
)+
)
({ExtendNumLetEx}+ ( {KatakanaEx} ( {ExtendNumLetEx}* {KatakanaEx} )*
| ( {HebrewLetterEx} ( {SingleQuoteEx} | {DoubleQuoteEx} {HebrewLetterEx} )
| {NumericEx} ( ( {ExtendNumLetEx}* | {MidNumericEx} ) {NumericEx} )*
| {AHLetterEx} ( ( {ExtendNumLetEx}* | {MidLetterEx} ) {AHLetterEx} )*
)+
)
)*
{ExtendNumLetEx}*
{ return WORD_TYPE; }
// From UAX #29:
//
// [C]haracters with the Line_Break property values of Contingent_Break (CB),
// Complex_Context (SA/South East Asian), and XX (Unknown) are assigned word
// boundary property values based on criteria outside of the scope of this
// annex. That means that satisfactory treatment of languages like Chinese
// or Thai requires special handling.
//
// In Unicode 9.0, only one character has the \p{Line_Break = Contingent_Break}
// property: U+FFFC (  ) OBJECT REPLACEMENT CHARACTER.
//
// In the ICU implementation of UAX#29, \p{Line_Break = Complex_Context}
// character sequences (from South East Asian scripts like Thai, Myanmar, Khmer,
// Lao, etc.) are kept together. This grammar does the same below.
//
// See also the Unicode Line Breaking Algorithm:
//
// http://www.unicode.org/reports/tr14/#SA
//
{ComplexContextEx}+ { return SOUTH_EAST_ASIAN_TYPE; }
// UAX#29 WB999. Any ÷ Any
//
{HanEx} { return IDEOGRAPHIC_TYPE; }
{HiraganaEx} { return HIRAGANA_TYPE; }
// UAX#29 WB3. CR × LF
// WB3a. (Newline | CR | LF) ÷
// WB3b. ÷ (Newline | CR | LF)
// WB999. Any ÷ Any
//
[^] { /* Break so we don't hit fall-through warning: */ break; /* Not numeric, word, ideographic, hiragana, emoji or SE Asian -- ignore it. */ }