Improve issue search (#2387)

* Improve issue indexer

* Fix new issue sqlite bug

* Different test indexer paths for each db

* Add integration indexer paths to make clean

.gitignore

@@ -53,5 +53,8 @@ coverage.all
 /integrations/gitea-integration-mysql
 /integrations/gitea-integration-pgsql
 /integrations/gitea-integration-sqlite
+/integrations/indexers-mysql
+/integrations/indexers-pgsql
+/integrations/indexers-sqlite
 /integrations/mysql.ini
 /integrations/pgsql.ini

Makefile

@@ -63,7 +63,11 @@ all: build
 .PHONY: clean
 clean:
 	$(GO) clean -i ./...
-	rm -rf $(EXECUTABLE) $(DIST) $(BINDATA) integrations*.test integrations/gitea-integration-pgsql/ integrations/gitea-integration-mysql/ integrations/gitea-integration-sqlite/ integrations/mysql.ini integrations/pgsql.ini
+	rm -rf $(EXECUTABLE) $(DIST) $(BINDATA) \
+		integrations*.test \
+		integrations/gitea-integration-pgsql/ integrations/gitea-integration-mysql/ integrations/gitea-integration-sqlite/ \
+		integrations/indexers-mysql/ integrations/indexers-pgsql integrations/indexers-sqlite \
+		integrations/mysql.ini integrations/pgsql.ini
 
 required-gofmt-version:
 	@$(GO) version  | grep -q '\(1.7\|1.8\)' || { echo "We require go version 1.7 or 1.8 to format code" >&2 && exit 1; }

integrations/integration_test.go

@@ -57,7 +57,14 @@ func TestMain(m *testing.M) {
 		fmt.Printf("Error initializing test database: %v\n", err)
 		os.Exit(1)
 	}
-	os.Exit(m.Run())
+	exitCode := m.Run()
+
+	if err = os.RemoveAll(setting.Indexer.IssuePath); err != nil {
+		fmt.Printf("os.RemoveAll: %v\n", err)
+		os.Exit(1)
+	}
+
+	os.Exit(exitCode)
 }
 
 func initIntegrationTest() {

integrations/issue_test.go

@@ -18,8 +18,10 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func getIssuesSelection(htmlDoc *HTMLDoc) *goquery.Selection {
-	return htmlDoc.doc.Find(".issue.list").Find("li").Find(".title")
+func getIssuesSelection(t testing.TB, htmlDoc *HTMLDoc) *goquery.Selection {
+	issueList := htmlDoc.doc.Find(".issue.list")
+	assert.EqualValues(t, 1, issueList.Length())
+	return issueList.Find("li").Find(".title")
 }
 
 func getIssue(t *testing.T, repoID int64, issueSelection *goquery.Selection) *models.Issue {
@@ -31,26 +33,37 @@ func getIssue(t *testing.T, repoID int64, issueSelection *goquery.Selection) *mo
 	return models.AssertExistsAndLoadBean(t, &models.Issue{RepoID: repoID, Index: int64(index)}).(*models.Issue)
 }
 
+func assertMatch(t testing.TB, issue *models.Issue, keyword string) {
+	matches := strings.Contains(strings.ToLower(issue.Title), keyword) ||
+		strings.Contains(strings.ToLower(issue.Content), keyword)
+	for _, comment := range issue.Comments {
+		matches = matches || strings.Contains(
+			strings.ToLower(comment.Content),
+			keyword,
+		)
+	}
+	assert.True(t, matches)
+}
+
 func TestNoLoginViewIssues(t *testing.T) {
 	prepareTestEnv(t)
 
 	req := NewRequest(t, "GET", "/user2/repo1/issues")
 	MakeRequest(t, req, http.StatusOK)
 }
 
-func TestNoLoginViewIssuesSortByType(t *testing.T) {
+func TestViewIssuesSortByType(t *testing.T) {
 	prepareTestEnv(t)
 
 	user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
 	repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
-	repo.Owner = models.AssertExistsAndLoadBean(t, &models.User{ID: repo.OwnerID}).(*models.User)
 
 	session := loginUser(t, user.Name)
 	req := NewRequest(t, "GET", repo.RelLink()+"/issues?type=created_by")
 	resp := session.MakeRequest(t, req, http.StatusOK)
 
 	htmlDoc := NewHTMLParser(t, resp.Body)
-	issuesSelection := getIssuesSelection(htmlDoc)
+	issuesSelection := getIssuesSelection(t, htmlDoc)
 	expectedNumIssues := models.GetCount(t,
 		&models.Issue{RepoID: repo.ID, PosterID: user.ID},
 		models.Cond("is_closed=?", false),
@@ -67,6 +80,26 @@ func TestNoLoginViewIssuesSortByType(t *testing.T) {
 	})
 }
 
+func TestViewIssuesKeyword(t *testing.T) {
+	prepareTestEnv(t)
+
+	repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
+
+	const keyword = "first"
+	req := NewRequestf(t, "GET", "%s/issues?q=%s", repo.RelLink(), keyword)
+	resp := MakeRequest(t, req, http.StatusOK)
+
+	htmlDoc := NewHTMLParser(t, resp.Body)
+	issuesSelection := getIssuesSelection(t, htmlDoc)
+	assert.EqualValues(t, 1, issuesSelection.Length())
+	issuesSelection.Each(func(_ int, selection *goquery.Selection) {
+		issue := getIssue(t, repo.ID, selection)
+		assert.False(t, issue.IsClosed)
+		assert.False(t, issue.IsPull)
+		assertMatch(t, issue, keyword)
+	})
+}
+
 func TestNoLoginViewIssue(t *testing.T) {
 	prepareTestEnv(t)
 

integrations/mysql.ini.tmpl

@@ -10,6 +10,9 @@ PASSWD   = {{TEST_MYSQL_PASSWORD}}
 SSL_MODE = disable
 PATH     = data/gitea.db
 
+[indexer]
+ISSUE_INDEXER_PATH = integrations/indexers-mysql/issues.bleve
+
 [repository]
 ROOT = integrations/gitea-integration-mysql/gitea-repositories
 

integrations/pgsql.ini.tmpl

@@ -10,6 +10,9 @@ PASSWD   = {{TEST_PGSQL_PASSWORD}}
 SSL_MODE = disable
 PATH     = data/gitea.db
 
+[indexer]
+ISSUE_INDEXER_PATH = integrations/indexers-pgsql/issues.bleve
+
 [repository]
 ROOT = integrations/gitea-integration-pgsql/gitea-repositories
 

integrations/sqlite.ini

@@ -5,6 +5,9 @@ RUN_MODE = prod
 DB_TYPE  = sqlite3
 PATH     = :memory:
 
+[indexer]
+ISSUE_INDEXER_PATH = integrations/indexers-sqlite/issues.bleve
+
 [repository]
 ROOT = integrations/gitea-integration-sqlite/gitea-repositories
 

models/fixtures/issue.yml

@@ -5,7 +5,7 @@
   poster_id: 1
   assignee_id: 1
   name: issue1
-  content: content1
+  content: content for the first issue
   is_closed: false
   is_pull: false
   num_comments: 2
@@ -18,7 +18,7 @@
   index: 2
   poster_id: 1
   name: issue2
-  content: content2
+  content: content for the second issue
   milestone_id: 1
   is_closed: false
   is_pull: true
@@ -32,7 +32,7 @@
   index: 3
   poster_id: 1
   name: issue3
-  content: content4
+  content: content for the third issue
   is_closed: false
   is_pull: true
   created_unix: 946684820
@@ -44,7 +44,7 @@
   index: 1
   poster_id: 2
   name: issue4
-  content: content4
+  content: content for the fourth issue
   is_closed: true
   is_pull: false
 
@@ -54,7 +54,7 @@
   index: 4
   poster_id: 2
   name: issue5
-  content: content5
+  content: content for the fifth issue
   is_closed: true
   is_pull: false
 -

models/issue.go

@@ -155,6 +155,17 @@ func (issue *Issue) loadPullRequest(e Engine) (err error) {
 	return nil
 }
 
+func (issue *Issue) loadComments(e Engine) (err error) {
+	if issue.Comments != nil {
+		return nil
+	}
+	issue.Comments, err = findComments(e, FindCommentsOptions{
+		IssueID: issue.ID,
+		Type:    CommentTypeUnknown,
+	})
+	return err
+}
+
 func (issue *Issue) loadAttributes(e Engine) (err error) {
 	if err = issue.loadRepo(e); err != nil {
 		return
@@ -191,14 +202,8 @@ func (issue *Issue) loadAttributes(e Engine) (err error) {
 		}
 	}
 
-	if issue.Comments == nil {
-		issue.Comments, err = findComments(e, FindCommentsOptions{
-			IssueID: issue.ID,
-			Type:    CommentTypeUnknown,
-		})
-		if err != nil {
-			return fmt.Errorf("getCommentsByIssueID [%d]: %v", issue.ID, err)
-		}
+	if err = issue.loadComments(e); err != nil {
+		return
 	}
 
 	return nil
@@ -577,7 +582,7 @@ func updateIssueCols(e Engine, issue *Issue, cols ...string) error {
 	if _, err := e.Id(issue.ID).Cols(cols...).Update(issue); err != nil {
 		return err
 	}
-	UpdateIssueIndexer(issue)
+	UpdateIssueIndexer(issue.ID)
 	return nil
 }
 
@@ -907,8 +912,6 @@ func newIssue(e *xorm.Session, doer *User, opts NewIssueOptions) (err error) {
 		return err
 	}
 
-	UpdateIssueIndexer(opts.Issue)
-
 	if len(opts.Attachments) > 0 {
 		attachments, err := getAttachmentsByUUIDs(e, opts.Attachments)
 		if err != nil {
@@ -947,6 +950,8 @@ func NewIssue(repo *Repository, issue *Issue, labelIDs []int64, uuids []string)
 		return fmt.Errorf("Commit: %v", err)
 	}
 
+	UpdateIssueIndexer(issue.ID)
+
 	if err = NotifyWatchers(&Action{
 		ActUserID: issue.Poster.ID,
 		ActUser:   issue.Poster,
@@ -1448,7 +1453,7 @@ func updateIssue(e Engine, issue *Issue) error {
 	if err != nil {
 		return err
 	}
-	UpdateIssueIndexer(issue)
+	UpdateIssueIndexer(issue.ID)
 	return nil
 }
 

models/issue_comment.go

@@ -520,7 +520,14 @@ func CreateComment(opts *CreateCommentOptions) (comment *Comment, err error) {
 		return nil, err
 	}
 
-	return comment, sess.Commit()
+	if err = sess.Commit(); err != nil {
+		return nil, err
+	}
+
+	if opts.Type == CommentTypeComment {
+		UpdateIssueIndexer(opts.Issue.ID)
+	}
+	return comment, nil
 }
 
 // CreateIssueComment creates a plain issue comment.
@@ -645,8 +652,12 @@ func GetCommentsByRepoIDSince(repoID, since int64) ([]*Comment, error) {
 
 // UpdateComment updates information of comment.
 func UpdateComment(c *Comment) error {
-	_, err := x.Id(c.ID).AllCols().Update(c)
-	return err
+	if _, err := x.Id(c.ID).AllCols().Update(c); err != nil {
+		return err
+	} else if c.Type == CommentTypeComment {
+		UpdateIssueIndexer(c.IssueID)
+	}
+	return nil
 }
 
 // DeleteComment deletes the comment
@@ -672,5 +683,10 @@ func DeleteComment(comment *Comment) error {
 		return err
 	}
 
-	return sess.Commit()
+	if err := sess.Commit(); err != nil {
+		return err
+	} else if comment.Type == CommentTypeComment {
+		UpdateIssueIndexer(comment.IssueID)
+	}
+	return nil
 }