Skip to content

Commit

Permalink
🎨 搜索支持排序 #6766
Browse files Browse the repository at this point in the history
  • Loading branch information
@88250
88250 committed Dec 2, 2022
1 parent dc5eaa7 commit fa5e607
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 107 deletions.
7 changes: 6 additions & 1 deletion kernel/api/search.go
View file
Expand Up @@ -220,12 +220,17 @@ func fullTextSearchBlock(c *gin.Context) {
if nil != methodArg {
method = int(methodArg.(float64))
}
orderByArg := arg["orderBy"]
var orderBy int // 0：按块类型（默认），1：按创建时间升序，2：按创建时间降序，3：按更新时间升序，4：按更新时间降序，5：按内容顺序（仅在按文档分组时）
if nil != orderByArg {
orderBy = int(orderByArg.(float64))
}
groupByArg := arg["groupBy"]
var groupBy int // 0：不分组，1：按文档分组
if nil != groupByArg {
groupBy = int(groupByArg.(float64))
}
blocks, matchedBlockCount, matchedRootCount := model.FullTextSearchBlock(query, boxes, paths, types, method, groupBy)
blocks, matchedBlockCount, matchedRootCount := model.FullTextSearchBlock(query, boxes, paths, types, method, orderBy, groupBy)
ret.Data = map[string]interface{}{
"blocks": blocks,
"matchedBlockCount": matchedBlockCount,
Expand Down
64 changes: 64 additions & 0 deletions kernel/model/graph.go
View file
Expand Up @@ -18,6 +18,8 @@ package model

import (
"bytes"
"github.com/88250/lute/parse"
"github.com/siyuan-note/siyuan/kernel/util"
"math"
"strings"
"unicode/utf8"
Expand Down Expand Up @@ -685,3 +687,65 @@ func nodeTitleLabel(node *GraphNode, blockContent string) {
node.Label = blockContent
}
}

func query2Stmt(queryStr string) (ret string) {
buf := bytes.Buffer{}
if util.IsIDPattern(queryStr) {
buf.WriteString("id = '" + queryStr + "'")
} else {
var tags []string
luteEngine := NewLute()
t := parse.Inline("", []byte(queryStr), luteEngine.ParseOptions)
ast.Walk(t.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
if ast.NodeTag == n.Type || (n.IsTextMarkType("tag")) {
tags = append(tags, n.Text())
}
return ast.WalkContinue
})

for _, tag := range tags {
queryStr = strings.ReplaceAll(queryStr, "#"+tag+"#", "")
}
parts := strings.Split(queryStr, " ")

for i, part := range parts {
if "" == part {
continue
}
part = strings.ReplaceAll(part, "'", "''")
buf.WriteString("(content LIKE '%" + part + "%'")
buf.WriteString(Conf.Search.NAMFilter(part))
buf.WriteString(")")
if i < len(parts)-1 {
buf.WriteString(" AND ")
}
}

if 0 < len(tags) {
if 0 < buf.Len() {
buf.WriteString(" OR ")
}
for i, tag := range tags {
buf.WriteString("(content LIKE '%#" + tag + "#%')")
if i < len(tags)-1 {
buf.WriteString(" AND ")
}
}
buf.WriteString(" OR ")
for i, tag := range tags {
buf.WriteString("ial LIKE '%tags=\"%" + tag + "%\"%'")
if i < len(tags)-1 {
buf.WriteString(" AND ")
}
}
}
}
if 1 > buf.Len() {
buf.WriteString("1=1")
}
ret = buf.String()
return
}
160 changes: 54 additions & 106 deletions kernel/model/search.go
View file
Expand Up @@ -312,8 +312,9 @@ func FindReplace(keyword, replacement string, ids []string, method int) (err err
return
}

func FullTextSearchBlock(query string, boxes, paths []string, types map[string]bool, method int, groupBy int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
func FullTextSearchBlock(query string, boxes, paths []string, types map[string]bool, method, orderBy, groupBy int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
// method：0：文本，1：查询语法，2：SQL，3：正则表达式
// orderBy: 0：按块类型（默认），1：按创建时间升序，2：按创建时间降序，3：按更新时间升序，4：按更新时间降序，5：按内容顺序（仅在按文档分组时）
// groupBy：0：不分组，1：按文档分组
query = strings.TrimSpace(query)
beforeLen := 36
Expand All @@ -324,7 +325,7 @@ func FullTextSearchBlock(query string, boxes, paths []string, types map[string]b
filter := buildTypeFilter(types)
boxFilter := buildBoxesFilter(boxes)
pathFilter := buildPathsFilter(paths)
blocks, matchedBlockCount, matchedRootCount = fullTextSearch(query, boxFilter, pathFilter, filter, beforeLen, true)
blocks, matchedBlockCount, matchedRootCount = fullTextSearchByQuerySyntax(query, boxFilter, pathFilter, filter, beforeLen)
case 2: // SQL
blocks, matchedBlockCount, matchedRootCount = searchBySQL(query, beforeLen)
case 3: // 正则表达式
Expand All @@ -336,7 +337,7 @@ func FullTextSearchBlock(query string, boxes, paths []string, types map[string]b
filter := buildTypeFilter(types)
boxFilter := buildBoxesFilter(boxes)
pathFilter := buildPathsFilter(paths)
blocks, matchedBlockCount, matchedRootCount = fullTextSearch(query, boxFilter, pathFilter, filter, beforeLen, false)
blocks, matchedBlockCount, matchedRootCount = fullTextSearchByKeyword(query, boxFilter, pathFilter, filter, beforeLen)
}

switch groupBy {
Expand Down Expand Up @@ -523,25 +524,48 @@ func fullTextSearchRefBlock(keyword string, beforeLen int) (ret []*Block) {
return
}

func fullTextSearchCount(query, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) {
func fullTextSearchByQuerySyntax(query, boxFilter, pathFilter, typeFilter string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
query = gulu.Str.RemoveInvisible(query)
if util.IsIDPattern(query) {
ret, _ := sql.Query("SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `blocks` WHERE `id` = '" + query + "'")
if 1 > len(ret) {
return
}
matchedBlockCount = int(ret[0]["matches"].(int64))
matchedRootCount = int(ret[0]["docs"].(int64))
ret, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen)
return
}
return fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter, beforeLen)
}

table := "blocks_fts" // 大小写敏感
if !Conf.Search.CaseSensitive {
table = "blocks_fts_case_insensitive"
func fullTextSearchByKeyword(query, boxFilter, pathFilter, typeFilter string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
query = gulu.Str.RemoveInvisible(query)
if util.IsIDPattern(query) {
ret, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen)
return
}
query = stringQuery(query)
return fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter, beforeLen)
}

stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `" + table + "` WHERE `" + table + "` MATCH '" + columnFilter() + ":(" + query + ")' AND type IN " + typeFilter
func fullTextSearchByRegexp(exp, boxFilter, pathFilter, typeFilter string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
exp = gulu.Str.RemoveInvisible(exp)
exp = regexp.QuoteMeta(exp)

fieldFilter := fieldRegexp(exp)
stmt := "SELECT * FROM `blocks` WHERE (" + fieldFilter + ") AND type IN " + typeFilter
stmt += boxFilter + pathFilter
stmt += " ORDER BY sort ASC LIMIT " + strconv.Itoa(Conf.Search.Limit)
blocks := sql.SelectBlocksRawStmt(stmt, Conf.Search.Limit)
ret = fromSQLBlocks(&blocks, "", beforeLen)
if 1 > len(ret) {
ret = []*Block{}
}

matchedBlockCount, matchedRootCount = fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter)
return
}

func fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) {
fieldFilter := fieldRegexp(exp)
stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `blocks` WHERE " + fieldFilter + " AND type IN " + typeFilter
stmt += boxFilter + pathFilter
stmt += " LIMIT " + strconv.Itoa(Conf.Search.Limit)
result, _ := sql.Query(stmt)
if 1 > len(result) {
return
Expand All @@ -551,17 +575,7 @@ func fullTextSearchCount(query, boxFilter, pathFilter, typeFilter string) (match
return
}

func fullTextSearch(query, boxFilter, pathFilter, typeFilter string, beforeLen int, querySyntax bool) (ret []*Block, matchedBlockCount, matchedRootCount int) {
query = gulu.Str.RemoveInvisible(query)
if util.IsIDPattern(query) {
ret, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen)
return
}

if !querySyntax {
query = stringQuery(query)
}

func fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
table := "blocks_fts" // 大小写敏感
if !Conf.Search.CaseSensitive {
table = "blocks_fts_case_insensitive"
Expand All @@ -587,29 +601,25 @@ func fullTextSearch(query, boxFilter, pathFilter, typeFilter string, beforeLen i
return
}

func fullTextSearchByRegexp(exp, boxFilter, pathFilter, typeFilter string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
exp = gulu.Str.RemoveInvisible(exp)
exp = regexp.QuoteMeta(exp)

fieldFilter := fieldRegexp(exp)
stmt := "SELECT * FROM `blocks` WHERE (" + fieldFilter + ") AND type IN " + typeFilter
stmt += boxFilter + pathFilter
stmt += " ORDER BY sort ASC LIMIT " + strconv.Itoa(Conf.Search.Limit)
blocks := sql.SelectBlocksRawStmt(stmt, Conf.Search.Limit)
ret = fromSQLBlocks(&blocks, "", beforeLen)
if 1 > len(ret) {
ret = []*Block{}
func fullTextSearchCount(query, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) {
query = gulu.Str.RemoveInvisible(query)
if util.IsIDPattern(query) {
ret, _ := sql.Query("SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `blocks` WHERE `id` = '" + query + "'")
if 1 > len(ret) {
return
}
matchedBlockCount = int(ret[0]["matches"].(int64))
matchedRootCount = int(ret[0]["docs"].(int64))
return
}

matchedBlockCount, matchedRootCount = fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter)
return
}
table := "blocks_fts" // 大小写敏感
if !Conf.Search.CaseSensitive {
table = "blocks_fts_case_insensitive"
}

func fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) {
fieldFilter := fieldRegexp(exp)
stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `blocks` WHERE " + fieldFilter + " AND type IN " + typeFilter
stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `" + table + "` WHERE `" + table + "` MATCH '" + columnFilter() + ":(" + query + ")' AND type IN " + typeFilter
stmt += boxFilter + pathFilter
stmt += " LIMIT " + strconv.Itoa(Conf.Search.Limit)
result, _ := sql.Query(stmt)
if 1 > len(result) {
return
Expand All @@ -619,68 +629,6 @@ func fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter string)
return
}

func query2Stmt(queryStr string) (ret string) {
buf := bytes.Buffer{}
if util.IsIDPattern(queryStr) {
buf.WriteString("id = '" + queryStr + "'")
} else {
var tags []string
luteEngine := NewLute()
t := parse.Inline("", []byte(queryStr), luteEngine.ParseOptions)
ast.Walk(t.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
if ast.NodeTag == n.Type || (n.IsTextMarkType("tag")) {
tags = append(tags, n.Text())
}
return ast.WalkContinue
})

for _, tag := range tags {
queryStr = strings.ReplaceAll(queryStr, "#"+tag+"#", "")
}
parts := strings.Split(queryStr, " ")

for i, part := range parts {
if "" == part {
continue
}
part = strings.ReplaceAll(part, "'", "''")
buf.WriteString("(content LIKE '%" + part + "%'")
buf.WriteString(Conf.Search.NAMFilter(part))
buf.WriteString(")")
if i < len(parts)-1 {
buf.WriteString(" AND ")
}
}

if 0 < len(tags) {
if 0 < buf.Len() {
buf.WriteString(" OR ")
}
for i, tag := range tags {
buf.WriteString("(content LIKE '%#" + tag + "#%')")
if i < len(tags)-1 {
buf.WriteString(" AND ")
}
}
buf.WriteString(" OR ")
for i, tag := range tags {
buf.WriteString("ial LIKE '%tags=\"%" + tag + "%\"%'")
if i < len(tags)-1 {
buf.WriteString(" AND ")
}
}
}
}
if 1 > buf.Len() {
buf.WriteString("1=1")
}
ret = buf.String()
return
}

func markSearch(text string, keyword string, beforeLen int) (marked string, score float64) {
if 0 == len(keyword) {
marked = text
Expand Down

0 comments on commit fa5e607

Please sign in to comment.