索引模式可选分词结果最小颗粒度：#670

hankcs · Nov 22, 2017 · 706240f · 706240f · mamba8 · Nov 27, 2017
1 parent 520884b
commit 706240f
Show file tree

Hide file tree

Showing 7 changed files with 35 additions and 13 deletions.
diff --git a/src/main/java/com/hankcs/hanlp/seg/Config.java b/src/main/java/com/hankcs/hanlp/seg/Config.java
@@ -19,7 +19,7 @@ public class Config
     /**
      * 是否是索引分词（合理地最小分割）
      */
-    public boolean indexMode = false;
+    public int indexMode = 0;
     /**
      * 是否识别中国人名
      */

diff --git a/src/main/java/com/hankcs/hanlp/seg/Dijkstra/DijkstraSegment.java b/src/main/java/com/hankcs/hanlp/seg/Dijkstra/DijkstraSegment.java
@@ -47,7 +47,7 @@ public List<Term> segSentence(char[] sentence)
 
         if (config.useCustomDictionary)
         {
-            if (config.indexMode)
+            if (config.indexMode > 0)
                 combineByCustomDictionary(vertexList, wordNetAll);
             else combineByCustomDictionary(vertexList);
         }
@@ -107,7 +107,7 @@ public List<Term> segSentence(char[] sentence)
         }
 
         // 如果是索引模式则全切分
-        if (config.indexMode)
+        if (config.indexMode > 0)
         {
             return decorateResultForIndexMode(vertexList, wordNetAll);
         }

diff --git a/src/main/java/com/hankcs/hanlp/seg/NShort/NShortSegment.java b/src/main/java/com/hankcs/hanlp/seg/NShort/NShortSegment.java
@@ -117,7 +117,7 @@ public List<Term> segSentence(char[] sentence)
         }
 
         // 如果是索引模式则全切分
-        if (config.indexMode)
+        if (config.indexMode > 0)
         {
             return decorateResultForIndexMode(vertexList, wordNetAll);
         }
@@ -130,7 +130,7 @@ public List<Term> segSentence(char[] sentence)
 
         if (config.useCustomDictionary)
         {
-            if (config.indexMode)
+            if (config.indexMode > 0)
                 combineByCustomDictionary(vertexList, wordNetAll);
             else combineByCustomDictionary(vertexList);
         }

diff --git a/src/main/java/com/hankcs/hanlp/seg/Segment.java b/src/main/java/com/hankcs/hanlp/seg/Segment.java
@@ -172,7 +172,7 @@ protected static List<AtomNode> quickAtomSegment(char[] charArray, int start, in
                     if (offsetAtom+1 < end)
                     {
                         int nextType = CharType.get(charArray[offsetAtom+1]);
-                        if (nextType == CharType.CT_NUM) 
+                        if (nextType == CharType.CT_NUM)
                         {
                             continue;
                         }
@@ -354,7 +354,7 @@ protected void mergeNumberQuantifier(List<Vertex> termList, WordNet wordNetAll,
                 {
                     if ((cur.hasNature(Nature.q) || cur.hasNature(Nature.qv) || cur.hasNature(Nature.qt)))
                     {
-                        if (config.indexMode)
+                        if (config.indexMode > 0)
                         {
                             wordNetAll.add(line, new Vertex(sbQuantifier.toString(), new CoreDictionary.Attribute(Nature.m)));
                         }
@@ -454,7 +454,7 @@ public List<Term> seg(String text)
                 return Collections.emptyList();
             }
             List<Term> termList = new LinkedList<Term>();
-            if (config.offset || config.indexMode)  // 由于分割了句子，所以需要重新校正offset
+            if (config.offset || config.indexMode > 0)  // 由于分割了句子，所以需要重新校正offset
             {
                 int sentenceOffset = 0;
                 for (int i = 0; i < sentenceArray.length; ++i)
@@ -557,7 +557,21 @@ public List<List<Term>> seg2sentence(String text)
      */
     public Segment enableIndexMode(boolean enable)
     {
-        config.indexMode = enable;
+        config.indexMode = enable ? 2 : 0;
+        return this;
+    }
+
+    /**
+     * 索引模式下的最小切分颗粒度（设为1可以最小切分为单字）
+     *
+     * @param minimalLength 三字词及以上的词语将会被切分为大于等于此长度的子词语。默认取2。
+     * @return
+     */
+    public Segment enableIndexMode(int minimalLength)
+    {
+        if (minimalLength < 1) throw new IllegalArgumentException("最小长度应当大于等于1");
+        config.indexMode = minimalLength;
+
         return this;
     }
 

diff --git a/src/main/java/com/hankcs/hanlp/seg/Viterbi/ViterbiSegment.java b/src/main/java/com/hankcs/hanlp/seg/Viterbi/ViterbiSegment.java
@@ -52,7 +52,7 @@ protected List<Term> segSentence(char[] sentence)
 
         if (config.useCustomDictionary)
         {
-            if (config.indexMode)
+            if (config.indexMode > 0)
                 combineByCustomDictionary(vertexList, wordNetAll);
             else combineByCustomDictionary(vertexList);
         }
@@ -109,7 +109,7 @@ protected List<Term> segSentence(char[] sentence)
         }
 
         // 如果是索引模式则全切分
-        if (config.indexMode)
+        if (config.indexMode > 0)
         {
             return decorateResultForIndexMode(vertexList, wordNetAll);
         }

diff --git a/src/main/java/com/hankcs/hanlp/seg/WordBasedGenerativeModelSegment.java b/src/main/java/com/hankcs/hanlp/seg/WordBasedGenerativeModelSegment.java
@@ -471,7 +471,7 @@ public void hit(int begin, int end, CoreDictionary.Attribute value)
      * @param vertexList
      * @param wordNetAll
      */
-    protected static List<Term> decorateResultForIndexMode(List<Vertex> vertexList, WordNet wordNetAll)
+    protected List<Term> decorateResultForIndexMode(List<Vertex> vertexList, WordNet wordNetAll)
     {
         List<Term> termList = new LinkedList<Term>();
         int line = 1;
@@ -496,7 +496,7 @@ protected static List<Term> decorateResultForIndexMode(List<Vertex> vertexList,
                         Vertex smallVertex = iterator.next();
                         if (
                                 ((termMain.nature == Nature.mq && smallVertex.hasNature(Nature.q)) ||
-                                        smallVertex.realWord.length() > 1)
+                                        smallVertex.realWord.length() >= config.indexMode)
                                         && smallVertex != vertex // 防止重复添加
                                         && currentLine + smallVertex.realWord.length() <= line + vertex.realWord.length() // 防止超出边界
                             )

diff --git a/src/test/java/com/hankcs/demo/DemoIndexSegment.java b/src/test/java/com/hankcs/demo/DemoIndexSegment.java
@@ -29,5 +29,13 @@ public static void main(String[] args)
         {
             System.out.println(term + " [" + term.offset + ":" + (term.offset + term.word.length()) + "]");
         }
+
+        System.out.println("\n最细颗粒度切分：");
+        IndexTokenizer.SEGMENT.enableIndexMode(1);
+        termList = IndexTokenizer.segment("主副食品");
+        for (Term term : termList)
+        {
+            System.out.println(term + " [" + term.offset + ":" + (term.offset + term.word.length()) + "]");
+        }
     }
 }