The search result lost fragments

[wolf1860]

I test the new version,it's great,but I met a little bit trouble when I tried to import gse-bleve to my project,the search result lost fragments,I can not figure it out.Here is my code:
step 1: I make the Index

const INDEX_DIR      = "gse.bleve2"

var (
	MyIndex bleve.Index
	opt = gse.Option{
		Index: INDEX_DIR,
		Dicts: "embed, zh", Stop: "",  //
		Opt: "search-hmm",Trim:"trim"}
)

func InitIndex() bool {
	index, indexErr := bleve.Open(INDEX_DIR)
	if indexErr != nil {
		if indexErr != bleve.ErrorIndexPathDoesNotExist {
			return false
		}
		newMapping := GetIndexMapping()
		if newMapping == nil {
			return false
		}
		newIndex, newIndexErr := bleve.New(INDEX_DIR, newMapping)
		if newIndexErr != nil {
			return false
		}
		index = newIndex
	}
	MyIndex = index
	return true
}

func GetIndexMapping() *mapping.IndexMappingImpl {
	//struct各个字段
	//自定义TokerFilter
	sfileMapping := mapping.NewDocumentMapping()
	//不需要分词的字段
	// a generic reusable mapping for keyword text
	keywordFieldMapping := bleve.NewTextFieldMapping()
	keywordFieldMapping.Analyzer = keyword.Name
	keywordFieldMapping.Name = keyword.Name //好像必须使用Name?英文检索是这样的
 
	textFieldMapping := gse.NewTextMap()  //mapping.NewTextFieldMapping()

	// a generic reusable mapping for numeric
	numericFieldMapping := bleve.NewNumericFieldMapping()

	// a generic reusable mapping for datetime
	//datetimeFieldMapping := bleve.NewDateTimeFieldMapping()
	/*
		目前没有索引的字段position_id，日期字段暂时没有用到（日期转成in64）
	*/
	//分词的字段
	sfileMapping.AddFieldMappingsAt("name", textFieldMapping)
	sfileMapping.AddFieldMappingsAt("description", textFieldMapping)
	sfileMapping.AddFieldMappingsAt("summary", textFieldMapping)
	sfileMapping.AddFieldMappingsAt("content", textFieldMapping)
	//关键字字段，日期的字段转成字符串，带到查询结果中
	sfileMapping.AddFieldMappingsAt("author", textFieldMapping)
	sfileMapping.AddFieldMappingsAt("specialty", textFieldMapping)
	sfileMapping.AddFieldMappingsAt("classify", textFieldMapping)

	sfileMapping.AddFieldMappingsAt("id", textFieldMapping)
	sfileMapping.AddFieldMappingsAt("ext", textFieldMapping)
	sfileMapping.AddFieldMappingsAt("input_user_id", textFieldMapping)

	sfileMapping.AddFieldMappingsAt("date_send_str", textFieldMapping)
	sfileMapping.AddFieldMappingsAt("date_effect_str", textFieldMapping)
	sfileMapping.AddFieldMappingsAt("date_cancel_str", textFieldMapping)
	sfileMapping.AddFieldMappingsAt("input_time_str", textFieldMapping)
	//数字字段
	sfileMapping.AddFieldMappingsAt("state", numericFieldMapping)
	sfileMapping.AddFieldMappingsAt("date_send", numericFieldMapping)
	sfileMapping.AddFieldMappingsAt("date_effect", numericFieldMapping)
	sfileMapping.AddFieldMappingsAt("date_cancel", numericFieldMapping)
	sfileMapping.AddFieldMappingsAt("input_time", numericFieldMapping)

	result,err := gse.NewMapping(opt)
	if err!=nil{
		fmt.Println("初始化出错：",err)
		return nil
	}
	result.AddDocumentMapping("myfile", sfileMapping)
	return result
}

step 2: Index some text

step3 :do search

type SearchContent struct {
	Content string `json:"content"`
	Page    int    `json:"page"`
}

func DoSearch(q query.Query, pageNo int) *bleve.SearchResult {
	req := bleve.NewSearchRequest(q)

	req.Highlight = bleve.NewHighlight()
	//[]string{"*"}返回全部字段
	req.Fields = []string{"id", "name", "summary", "description", "ext", "author",
		"specialty", "classify", "date_send_str", "date_effect_str",
		"date_cancel_str", "input_time_str", "input_user_id", "state"}
	req.Size = MyConfig.PageSize                //一次请求返回结果数量
	req.From = MyConfig.PageSize * (pageNo - 1) //返回结果起始序号
	res, err := MyIndex.Search(req)

	//fmt.Println(res.Total)
	if err != nil {
		panic(err)
	}

	return res
}

..............................
swords := &SearchContent{}
q := fmt.Sprintf("content:%s name:%s", swords.Content,swords.Content)
myquery := bleve.NewQueryStringQuery(q)
res := DoSearch(myquery, swords.Page)

..............................
Did anything I have missed?

[vcaesar]

You should give a detailed description what fragments lost? Some field?

[wolf1860]

Finally, I found the point. the key is text field mapping . code here:
textFieldMapping := gse.NewTextMap()
//textFieldMapping.IncludeTermVectors = true

if I set IncludeTermVectors = true , the mess code appeared ,or the search result's hits element will lost the fragments all. Did I make it clear?

By the way ,my code with "github.com/leopku/bleve-gse-tokenizer/v2" has been tested,it worked ok except that I can not get right result with "审计" searching from "审计工作"

[vcaesar]

I write a simple test, and it can't reproduce.
Some word can't be search, maybe the package uses API with mistake or not have that word.

[wolf1860]

It's time to close this issue.I upgraded gse-bleve and It worked fine. Thank u again!