add new api: Tokenize for location information of words

ChangeLog.md

@@ -3,6 +3,7 @@
 ## next version
 
 + upgrade cppjieba -> v4.7.0
++ add new api: Tokenize for location information of words
 
 ## v0.13.0
 

jieba.cpp

@@ -14,6 +14,17 @@ static char** ConvertWords(const std::vector<std::string>& words) {
   return res;
 }
 
+static Word* ConvertWords(const std::vector<cppjieba::Word>& words) {
+  Word* res = (Word*)malloc(sizeof(Word) * (words.size() + 1));
+  for (size_t i = 0; i < words.size(); i++) {
+    res[i].offset = words[i].offset;
+    res[i].len = words[i].word.size();
+  }
+  res[words.size()].offset = 0;
+  res[words.size()].len = 0;
+  return res;
+}
+
 Jieba NewJieba(const char* dict_path, const char* hmm_path, const char* user_dict) {
   return (Jieba)(new cppjieba::Jieba(dict_path, hmm_path, user_dict));
 }
@@ -53,3 +64,15 @@ char** Tag(Jieba x, const char* sentence) {
   }
   return ConvertWords(words);
 }
+
+Word* Tokenize(Jieba x, const char* sentence, TokenizeMode mode, int is_hmm_used) {
+  std::vector<cppjieba::Word> words;
+  switch (mode) {
+    case SearchMode:
+      ((cppjieba::Jieba*)x)->CutForSearch(sentence, words, is_hmm_used);
+      return ConvertWords(words);
+    default:
+      ((cppjieba::Jieba*)x)->Cut(sentence, words, is_hmm_used);
+      return ConvertWords(words);
+  }
+}

jieba.go

@@ -8,6 +8,19 @@ package gojieba
 import "C"
 import "unsafe"
 
+type TokenizeMode int
+
+const (
+	DefaultMode TokenizeMode = iota
+	SearchMode
+)
+
+type Word struct {
+	Str   string
+	Start int
+	End   int
+}
+
 type Jieba struct {
 	jieba C.Jieba
 }
@@ -72,3 +85,15 @@ func (x *Jieba) Tag(s string) []string {
 	res := cstrings(words)
 	return res
 }
+
+func (x *Jieba) Tokenize(s string, mode TokenizeMode, hmm bool) []Word {
+	c_int_hmm := 0
+	if hmm {
+		c_int_hmm = 1
+	}
+	cstr := C.CString(s)
+	defer C.free(unsafe.Pointer(cstr))
+	var words *C.Word = C.Tokenize(x.jieba, cstr, C.TokenizeMode(mode), C.int(c_int_hmm))
+	defer C.free(unsafe.Pointer(words))
+	return convertWords(s, words)
+}

jieba.h

@@ -1,10 +1,21 @@
 #ifndef CJIEBA_JIEBA_H
 #define CJIEBA_JIEBA_H
 
+#include <stdlib.h>
 #include "util.h"
 
 typedef void* Jieba;
 
+typedef struct {
+  size_t offset;
+  size_t len;
+} Word;
+
+typedef enum {
+  DefaultMode = 0,
+  SearchMode = 1,
+} TokenizeMode;
+
 Jieba NewJieba(const char* dict_path, const char* hmm_path, const char* user_dict);
 void FreeJieba(Jieba);
 
@@ -13,4 +24,6 @@ char** CutAll(Jieba handle, const char* sentence);
 char** CutForSearch(Jieba handle, const char* sentence, int is_hmm_used);
 char** Tag(Jieba handle, const char* sentence);
 
+Word* Tokenize(Jieba x, const char* sentence, TokenizeMode mode, int is_hmm_used);
+
 #endif // CJIEBA_JIEBA_H

jieba_test.go

@@ -2,6 +2,7 @@ package gojieba
 
 import (
 	"fmt"
+	"reflect"
 	"strings"
 	"testing"
 )
@@ -38,6 +39,16 @@ func ExampleJieba() {
 	fmt.Println(s)
 	fmt.Println("词性标注:", strings.Join(words, ","))
 
+	s = "长春市长春药店"
+	wordinfos := x.Tokenize(s, SearchMode, false)
+	fmt.Println(s)
+	fmt.Println("Tokenize:", wordinfos)
+
+	//s = "长春市长春药店"
+	//wordinfos := x.Tokenize(s, SearchMode, !use_hmm)
+	//fmt.Println(s)
+	//fmt.Println(wordinfos)
+
 	// Output:
 	// 我来到北京清华大学
 	// 全模式: 我/来到/北京/清华/清华大学/华大/大学
@@ -49,6 +60,8 @@ func ExampleJieba() {
 	// 搜索引擎模式: 小明/硕士/毕业/于/中国/中国科学院/科学/科学院/学院/计算所/，/后/在/日本/日本京都大学/京都/京都大学/大学/深造
 	// 长春市长春药店
 	// 词性标注: 长春市/ns,长春/ns,药店/n
+	// 长春市长春药店
+	// Tokenize: [{长春市 0 9} {长春 9 15} {药店 15 21}]
 }
 
 func TestJieba(t *testing.T) {
@@ -101,6 +114,17 @@ func TestJieba(t *testing.T) {
 	if expected != actual {
 		t.Error(actual)
 	}
+
+	s = "长春市长春药店"
+	wordinfos := x.Tokenize(s, SearchMode, false)
+	expectedwords := []Word{
+		Word{Str: "长春市", Start: 0, End: 9},
+		Word{Str: "长春", Start: 9, End: 15},
+		Word{Str: "药店", Start: 15, End: 21},
+	}
+	if !reflect.DeepEqual(wordinfos, expectedwords) {
+		t.Error()
+	}
 }
 
 func BenchmarkJieba(b *testing.B) {

util.go

@@ -1,5 +1,8 @@
 package gojieba
 
+/*
+#include "jieba.h"
+*/
 import "C"
 import "unsafe"
 
@@ -12,3 +15,18 @@ func cstrings(x **C.char) []string {
 	}
 	return s
 }
+
+func convertWords(s string, words *C.Word) []Word {
+	result := make([]Word, 0)
+	x := words
+	eltSize := unsafe.Sizeof(*x)
+	start := 0
+	end := 0
+	for (*x).len != 0 {
+		start = int((*x).offset)
+		end = start + int((*x).len)
+		result = append(result, Word{s[start:end], start, end})
+		x = (*C.Word)(unsafe.Pointer(uintptr(unsafe.Pointer(x)) + eltSize))
+	}
+	return result
+}