From fa5e60750add635c740cfa96592c1ab49fcbbad2 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 2 Dec 2022 17:30:16 +0800 Subject: [PATCH] =?UTF-8?q?:art:=20=E6=90=9C=E7=B4=A2=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E6=8E=92=E5=BA=8F=20https://github.com/siyuan-note/siyuan/issu?= =?UTF-8?q?es/6766?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/api/search.go | 7 +- kernel/model/graph.go | 64 +++++++++++++++++ kernel/model/search.go | 160 ++++++++++++++--------------------------- 3 files changed, 124 insertions(+), 107 deletions(-) diff --git a/kernel/api/search.go b/kernel/api/search.go index c5b342ac7d8..c71298fafeb 100644 --- a/kernel/api/search.go +++ b/kernel/api/search.go @@ -220,12 +220,17 @@ func fullTextSearchBlock(c *gin.Context) { if nil != methodArg { method = int(methodArg.(float64)) } + orderByArg := arg["orderBy"] + var orderBy int // 0：按块类型（默认），1：按创建时间升序，2：按创建时间降序，3：按更新时间升序，4：按更新时间降序，5：按内容顺序（仅在按文档分组时） + if nil != orderByArg { + orderBy = int(orderByArg.(float64)) + } groupByArg := arg["groupBy"] var groupBy int // 0：不分组，1：按文档分组 if nil != groupByArg { groupBy = int(groupByArg.(float64)) } - blocks, matchedBlockCount, matchedRootCount := model.FullTextSearchBlock(query, boxes, paths, types, method, groupBy) + blocks, matchedBlockCount, matchedRootCount := model.FullTextSearchBlock(query, boxes, paths, types, method, orderBy, groupBy) ret.Data = map[string]interface{}{ "blocks": blocks, "matchedBlockCount": matchedBlockCount, diff --git a/kernel/model/graph.go b/kernel/model/graph.go index 82ff0b45a9b..34465259bbb 100644 --- a/kernel/model/graph.go +++ b/kernel/model/graph.go @@ -18,6 +18,8 @@ package model import ( "bytes" + "github.com/88250/lute/parse" + "github.com/siyuan-note/siyuan/kernel/util" "math" "strings" "unicode/utf8" @@ -685,3 +687,65 @@ func nodeTitleLabel(node *GraphNode, blockContent string) { node.Label = blockContent } } + +func query2Stmt(queryStr string) (ret string) { + buf := bytes.Buffer{} + if util.IsIDPattern(queryStr) { + buf.WriteString("id = '" + queryStr + "'") + } else { + var tags []string + luteEngine := NewLute() + t := parse.Inline("", []byte(queryStr), luteEngine.ParseOptions) + ast.Walk(t.Root, func(n *ast.Node, entering bool) ast.WalkStatus { + if !entering { + return ast.WalkContinue + } + if ast.NodeTag == n.Type || (n.IsTextMarkType("tag")) { + tags = append(tags, n.Text()) + } + return ast.WalkContinue + }) + + for _, tag := range tags { + queryStr = strings.ReplaceAll(queryStr, "#"+tag+"#", "") + } + parts := strings.Split(queryStr, " ") + + for i, part := range parts { + if "" == part { + continue + } + part = strings.ReplaceAll(part, "'", "''") + buf.WriteString("(content LIKE '%" + part + "%'") + buf.WriteString(Conf.Search.NAMFilter(part)) + buf.WriteString(")") + if i < len(parts)-1 { + buf.WriteString(" AND ") + } + } + + if 0 < len(tags) { + if 0 < buf.Len() { + buf.WriteString(" OR ") + } + for i, tag := range tags { + buf.WriteString("(content LIKE '%#" + tag + "#%')") + if i < len(tags)-1 { + buf.WriteString(" AND ") + } + } + buf.WriteString(" OR ") + for i, tag := range tags { + buf.WriteString("ial LIKE '%tags=\"%" + tag + "%\"%'") + if i < len(tags)-1 { + buf.WriteString(" AND ") + } + } + } + } + if 1 > buf.Len() { + buf.WriteString("1=1") + } + ret = buf.String() + return +} diff --git a/kernel/model/search.go b/kernel/model/search.go index f60132beb05..535ed0b34ea 100644 --- a/kernel/model/search.go +++ b/kernel/model/search.go @@ -312,8 +312,9 @@ func FindReplace(keyword, replacement string, ids []string, method int) (err err return } -func FullTextSearchBlock(query string, boxes, paths []string, types map[string]bool, method int, groupBy int) (ret []*Block, matchedBlockCount, matchedRootCount int) { +func FullTextSearchBlock(query string, boxes, paths []string, types map[string]bool, method, orderBy, groupBy int) (ret []*Block, matchedBlockCount, matchedRootCount int) { // method：0：文本，1：查询语法，2：SQL，3：正则表达式 + // orderBy: 0：按块类型（默认），1：按创建时间升序，2：按创建时间降序，3：按更新时间升序，4：按更新时间降序，5：按内容顺序（仅在按文档分组时） // groupBy：0：不分组，1：按文档分组 query = strings.TrimSpace(query) beforeLen := 36 @@ -324,7 +325,7 @@ func FullTextSearchBlock(query string, boxes, paths []string, types map[string]b filter := buildTypeFilter(types) boxFilter := buildBoxesFilter(boxes) pathFilter := buildPathsFilter(paths) - blocks, matchedBlockCount, matchedRootCount = fullTextSearch(query, boxFilter, pathFilter, filter, beforeLen, true) + blocks, matchedBlockCount, matchedRootCount = fullTextSearchByQuerySyntax(query, boxFilter, pathFilter, filter, beforeLen) case 2: // SQL blocks, matchedBlockCount, matchedRootCount = searchBySQL(query, beforeLen) case 3: // 正则表达式 @@ -336,7 +337,7 @@ func FullTextSearchBlock(query string, boxes, paths []string, types map[string]b filter := buildTypeFilter(types) boxFilter := buildBoxesFilter(boxes) pathFilter := buildPathsFilter(paths) - blocks, matchedBlockCount, matchedRootCount = fullTextSearch(query, boxFilter, pathFilter, filter, beforeLen, false) + blocks, matchedBlockCount, matchedRootCount = fullTextSearchByKeyword(query, boxFilter, pathFilter, filter, beforeLen) } switch groupBy { @@ -523,25 +524,48 @@ func fullTextSearchRefBlock(keyword string, beforeLen int) (ret []*Block) { return } -func fullTextSearchCount(query, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) { +func fullTextSearchByQuerySyntax(query, boxFilter, pathFilter, typeFilter string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) { query = gulu.Str.RemoveInvisible(query) if util.IsIDPattern(query) { - ret, _ := sql.Query("SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `blocks` WHERE `id` = '" + query + "'") - if 1 > len(ret) { - return - } - matchedBlockCount = int(ret[0]["matches"].(int64)) - matchedRootCount = int(ret[0]["docs"].(int64)) + ret, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen) return } + return fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter, beforeLen) +} - table := "blocks_fts" // 大小写敏感 - if !Conf.Search.CaseSensitive { - table = "blocks_fts_case_insensitive" +func fullTextSearchByKeyword(query, boxFilter, pathFilter, typeFilter string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) { + query = gulu.Str.RemoveInvisible(query) + if util.IsIDPattern(query) { + ret, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen) + return } + query = stringQuery(query) + return fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter, beforeLen) +} - stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `" + table + "` WHERE `" + table + "` MATCH '" + columnFilter() + ":(" + query + ")' AND type IN " + typeFilter +func fullTextSearchByRegexp(exp, boxFilter, pathFilter, typeFilter string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) { + exp = gulu.Str.RemoveInvisible(exp) + exp = regexp.QuoteMeta(exp) + + fieldFilter := fieldRegexp(exp) + stmt := "SELECT * FROM `blocks` WHERE (" + fieldFilter + ") AND type IN " + typeFilter stmt += boxFilter + pathFilter + stmt += " ORDER BY sort ASC LIMIT " + strconv.Itoa(Conf.Search.Limit) + blocks := sql.SelectBlocksRawStmt(stmt, Conf.Search.Limit) + ret = fromSQLBlocks(&blocks, "", beforeLen) + if 1 > len(ret) { + ret = []*Block{} + } + + matchedBlockCount, matchedRootCount = fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter) + return +} + +func fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) { + fieldFilter := fieldRegexp(exp) + stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `blocks` WHERE " + fieldFilter + " AND type IN " + typeFilter + stmt += boxFilter + pathFilter + stmt += " LIMIT " + strconv.Itoa(Conf.Search.Limit) result, _ := sql.Query(stmt) if 1 > len(result) { return @@ -551,17 +575,7 @@ func fullTextSearchCount(query, boxFilter, pathFilter, typeFilter string) (match return } -func fullTextSearch(query, boxFilter, pathFilter, typeFilter string, beforeLen int, querySyntax bool) (ret []*Block, matchedBlockCount, matchedRootCount int) { - query = gulu.Str.RemoveInvisible(query) - if util.IsIDPattern(query) { - ret, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen) - return - } - - if !querySyntax { - query = stringQuery(query) - } - +func fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) { table := "blocks_fts" // 大小写敏感 if !Conf.Search.CaseSensitive { table = "blocks_fts_case_insensitive" @@ -587,29 +601,25 @@ func fullTextSearch(query, boxFilter, pathFilter, typeFilter string, beforeLen i return } -func fullTextSearchByRegexp(exp, boxFilter, pathFilter, typeFilter string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) { - exp = gulu.Str.RemoveInvisible(exp) - exp = regexp.QuoteMeta(exp) - - fieldFilter := fieldRegexp(exp) - stmt := "SELECT * FROM `blocks` WHERE (" + fieldFilter + ") AND type IN " + typeFilter - stmt += boxFilter + pathFilter - stmt += " ORDER BY sort ASC LIMIT " + strconv.Itoa(Conf.Search.Limit) - blocks := sql.SelectBlocksRawStmt(stmt, Conf.Search.Limit) - ret = fromSQLBlocks(&blocks, "", beforeLen) - if 1 > len(ret) { - ret = []*Block{} +func fullTextSearchCount(query, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) { + query = gulu.Str.RemoveInvisible(query) + if util.IsIDPattern(query) { + ret, _ := sql.Query("SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `blocks` WHERE `id` = '" + query + "'") + if 1 > len(ret) { + return + } + matchedBlockCount = int(ret[0]["matches"].(int64)) + matchedRootCount = int(ret[0]["docs"].(int64)) + return } - matchedBlockCount, matchedRootCount = fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter) - return -} + table := "blocks_fts" // 大小写敏感 + if !Conf.Search.CaseSensitive { + table = "blocks_fts_case_insensitive" + } -func fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) { - fieldFilter := fieldRegexp(exp) - stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `blocks` WHERE " + fieldFilter + " AND type IN " + typeFilter + stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `" + table + "` WHERE `" + table + "` MATCH '" + columnFilter() + ":(" + query + ")' AND type IN " + typeFilter stmt += boxFilter + pathFilter - stmt += " LIMIT " + strconv.Itoa(Conf.Search.Limit) result, _ := sql.Query(stmt) if 1 > len(result) { return @@ -619,68 +629,6 @@ func fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter string) return } -func query2Stmt(queryStr string) (ret string) { - buf := bytes.Buffer{} - if util.IsIDPattern(queryStr) { - buf.WriteString("id = '" + queryStr + "'") - } else { - var tags []string - luteEngine := NewLute() - t := parse.Inline("", []byte(queryStr), luteEngine.ParseOptions) - ast.Walk(t.Root, func(n *ast.Node, entering bool) ast.WalkStatus { - if !entering { - return ast.WalkContinue - } - if ast.NodeTag == n.Type || (n.IsTextMarkType("tag")) { - tags = append(tags, n.Text()) - } - return ast.WalkContinue - }) - - for _, tag := range tags { - queryStr = strings.ReplaceAll(queryStr, "#"+tag+"#", "") - } - parts := strings.Split(queryStr, " ") - - for i, part := range parts { - if "" == part { - continue - } - part = strings.ReplaceAll(part, "'", "''") - buf.WriteString("(content LIKE '%" + part + "%'") - buf.WriteString(Conf.Search.NAMFilter(part)) - buf.WriteString(")") - if i < len(parts)-1 { - buf.WriteString(" AND ") - } - } - - if 0 < len(tags) { - if 0 < buf.Len() { - buf.WriteString(" OR ") - } - for i, tag := range tags { - buf.WriteString("(content LIKE '%#" + tag + "#%')") - if i < len(tags)-1 { - buf.WriteString(" AND ") - } - } - buf.WriteString(" OR ") - for i, tag := range tags { - buf.WriteString("ial LIKE '%tags=\"%" + tag + "%\"%'") - if i < len(tags)-1 { - buf.WriteString(" AND ") - } - } - } - } - if 1 > buf.Len() { - buf.WriteString("1=1") - } - ret = buf.String() - return -} - func markSearch(text string, keyword string, beforeLen int) (marked string, score float64) { if 0 == len(keyword) { marked = text