ICUFoldingFilter.cs

// Lucene version compatibility level 7.1.0
using ICU4N.Text;
using J2N;
using Lucene.Net.Support;
using System.Reflection;

namespace Lucene.Net.Analysis.Icu
{
    /*
     * Licensed to the Apache Software Foundation (ASF) under one or more
     * contributor license agreements.  See the NOTICE file distributed with
     * this work for additional information regarding copyright ownership.
     * The ASF licenses this file to You under the Apache License, Version 2.0
     * (the "License"); you may not use this file except in compliance with
     * the License.  You may obtain a copy of the License at
     *
     *     http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

    /// <summary>
    /// A <see cref="TokenFilter"/> that applies search term folding to Unicode text,
    /// applying foldings from UTR#30 Character Foldings.
    /// </summary>
    /// <remarks>
    /// This filter applies the following foldings from the report to unicode text:
    /// <list type="bullet">
    ///     <item><description>Accent removal</description></item>
    ///     <item><description>Case folding</description></item>
    ///     <item><description>Canonical duplicates folding</description></item>
    ///     <item><description>Dashes folding</description></item>
    ///     <item><description>Diacritic removal (including stroke, hook, descender)</description></item>
    ///     <item><description>Greek letterforms folding</description></item>
    ///     <item><description>Han Radical folding</description></item>
    ///     <item><description>Hebrew Alternates folding</description></item>
    ///     <item><description>Jamo folding</description></item>
    ///     <item><description>Letterforms folding</description></item>
    ///     <item><description>Math symbol folding</description></item>
    ///     <item><description>Multigraph Expansions: All</description></item>
    ///     <item><description>Native digit folding</description></item>
    ///     <item><description>No-break folding</description></item>
    ///     <item><description>Overline folding</description></item>
    ///     <item><description>Positional forms folding</description></item>
    ///     <item><description>Small forms folding</description></item>
    ///     <item><description>Space folding</description></item>
    ///     <item><description>Spacing Accents folding</description></item>
    ///     <item><description>Subscript folding</description></item>
    ///     <item><description>Superscript folding</description></item>
    ///     <item><description>Suzhou Numeral folding</description></item>
    ///     <item><description>Symbol folding</description></item>
    ///     <item><description>Underline folding</description></item>
    ///     <item><description>Vertical forms folding</description></item>
    ///     <item><description>Width folding</description></item>
    /// </list>
    /// <para/>
    /// Additionally, Default Ignorables are removed, and text is normalized to NFKC.
    /// All foldings, case folding, and normalization mappings are applied recursively
    /// to ensure a fully folded and normalized result.
    /// </remarks>
    [ExceptionToClassNameConvention]
    public sealed class ICUFoldingFilter : ICUNormalizer2Filter
    {
        // TODO: if the wrong version of the ICU jar is used, loading these data files may give a strange error.
        // maybe add an explicit check? http://icu-project.org/apiref/icu4j/com/ibm/icu/util/VersionInfo.html
        private static readonly Normalizer2 normalizer = Normalizer2.GetInstance(
            typeof(ICUFoldingFilter).FindAndGetManifestResourceStream("utr30.nrm"),
            "utr30", Normalizer2Mode.Compose);

        /// <summary>
        /// Create a new <see cref="ICUFoldingFilter"/> on the specified input
        /// </summary>
        public ICUFoldingFilter(TokenStream input)
            : base(input, normalizer)
        {
        }
    }
}