Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor issue indexer, add some testing and fix a bug #6131

Merged
merged 4 commits into from
Feb 21, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ coverage.all
/integrations/pgsql.ini
/integrations/mssql.ini
/node_modules
/modules/indexer/issues/indexers


# Snapcraft
Expand Down
5 changes: 5 additions & 0 deletions models/issue.go
Original file line number Diff line number Diff line change
Expand Up @@ -1231,6 +1231,11 @@ func getIssueIDsByRepoID(e Engine, repoID int64) ([]int64, error) {
return ids, err
}

// GetIssueIDsByRepoID returns all issue ids by repo id
func GetIssueIDsByRepoID(repoID int64) ([]int64, error) {
return getIssueIDsByRepoID(x, repoID)
}

// GetIssuesByIDs return issues with the given IDs.
func GetIssuesByIDs(issueIDs []int64) ([]*Issue, error) {
return getIssuesByIDs(x, issueIDs)
Expand Down
148 changes: 0 additions & 148 deletions models/issue_indexer.go

This file was deleted.

4 changes: 0 additions & 4 deletions models/unit_tests.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,6 @@ func MainTest(m *testing.M, pathToGiteaRoot string) {
fatalTestError("Error creating test engine: %v\n", err)
}

if err = InitIssueIndexer(); err != nil {
fatalTestError("Error InitIssueIndexer: %v\n", err)
}

setting.AppURL = "https://try.gitea.io/"
setting.RunUser = "runuser"
setting.SSH.Port = 3000
Expand Down
148 changes: 148 additions & 0 deletions modules/indexer/issues/indexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,15 @@

package issues

import (
"fmt"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
)

// IndexerData data stored in the issue indexer
type IndexerData struct {
ID int64
Expand Down Expand Up @@ -34,3 +43,142 @@ type Indexer interface {
Delete(ids ...int64) error
Search(kw string, repoID int64, limit, start int) (*SearchResult, error)
}

var (
// issueIndexerUpdateQueue queue of issue ids to be updated
issueIndexerUpdateQueue Queue
issueIndexer Indexer
)

// InitIssueIndexer initialize issue indexer, syncReindex is true then reindex until
// all issue index done.
func InitIssueIndexer(syncReindex bool) error {
var populate bool
switch setting.Indexer.IssueType {
case "bleve":
issueIndexer = NewBleveIndexer(setting.Indexer.IssuePath)
exist, err := issueIndexer.Init()
if err != nil {
return err
}
populate = !exist
default:
return fmt.Errorf("unknow issue indexer type: %s", setting.Indexer.IssueType)
}

var err error
switch setting.Indexer.IssueIndexerQueueType {
case setting.LevelQueueType:
issueIndexerUpdateQueue, err = NewLevelQueue(
issueIndexer,
setting.Indexer.IssueIndexerQueueDir,
setting.Indexer.IssueIndexerQueueBatchNumber)
if err != nil {
return err
}
case setting.ChannelQueueType:
issueIndexerUpdateQueue = NewChannelQueue(issueIndexer, setting.Indexer.IssueIndexerQueueBatchNumber)
default:
return fmt.Errorf("Unsupported indexer queue type: %v", setting.Indexer.IssueIndexerQueueType)
}

go issueIndexerUpdateQueue.Run()

if populate {
if syncReindex {
populateIssueIndexer()
} else {
go populateIssueIndexer()
}
}

return nil
}

// populateIssueIndexer populate the issue indexer with issue data
func populateIssueIndexer() {
for page := 1; ; page++ {
repos, _, err := models.SearchRepositoryByName(&models.SearchRepoOptions{
Page: page,
PageSize: models.RepositoryListDefaultPageSize,
OrderBy: models.SearchOrderByID,
Private: true,
Collaborate: util.OptionalBoolFalse,
})
if err != nil {
log.Error(4, "SearchRepositoryByName: %v", err)
continue
}
if len(repos) == 0 {
return
}

for _, repo := range repos {
is, err := models.Issues(&models.IssuesOptions{
RepoIDs: []int64{repo.ID},
IsClosed: util.OptionalBoolNone,
IsPull: util.OptionalBoolNone,
})
if err != nil {
log.Error(4, "Issues: %v", err)
continue
}
if err = models.IssueList(is).LoadDiscussComments(); err != nil {
log.Error(4, "LoadComments: %v", err)
continue
}
for _, issue := range is {
UpdateIssueIndexer(issue)
}
}
}
}

// UpdateIssueIndexer add/update an issue to the issue indexer
func UpdateIssueIndexer(issue *models.Issue) {
var comments []string
for _, comment := range issue.Comments {
if comment.Type == models.CommentTypeComment {
comments = append(comments, comment.Content)
}
}
issueIndexerUpdateQueue.Push(&IndexerData{
ID: issue.ID,
RepoID: issue.RepoID,
Title: issue.Title,
Content: issue.Content,
Comments: comments,
})
}

// DeleteRepoIssueIndexer deletes repo's all issues indexes
func DeleteRepoIssueIndexer(repo *models.Repository) {
var ids []int64
ids, err := models.GetIssueIDsByRepoID(repo.ID)
if err != nil {
log.Error(4, "getIssueIDsByRepoID failed: %v", err)
return
}

if len(ids) <= 0 {
return
}

issueIndexerUpdateQueue.Push(&IndexerData{
IDs: ids,
IsDelete: true,
})
}

// SearchIssuesByKeyword search issue ids by keywords and repo id
func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
var issueIDs []int64
res, err := issueIndexer.Search(keyword, repoID, 1000, 0)
if err != nil {
return nil, err
}
for _, r := range res.Hits {
issueIDs = append(issueIDs, r.ID)
}
return issueIDs, nil
}
51 changes: 51 additions & 0 deletions modules/indexer/issues/indexer_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package issues

import (
"fmt"
"os"
"path/filepath"
"testing"
"time"

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/setting"

"github.com/stretchr/testify/assert"
)

func fatalTestError(fmtStr string, args ...interface{}) {
fmt.Fprintf(os.Stderr, fmtStr, args...)
os.Exit(1)
}

func TestMain(m *testing.M) {
models.MainTest(m, filepath.Join("..", "..", ".."))
}

func TestSearchIssues(t *testing.T) {
assert.NoError(t, models.PrepareTestDatabase())

os.RemoveAll(setting.Indexer.IssueIndexerQueueDir)
os.RemoveAll(setting.Indexer.IssuePath)
if err := InitIssueIndexer(true); err != nil {
fatalTestError("Error InitIssueIndexer: %v\n", err)
}

time.Sleep(10 * time.Second)

ids, err := SearchIssuesByKeyword(1, "issue2")
assert.NoError(t, err)
assert.EqualValues(t, []int64{2}, ids)

ids, err = SearchIssuesByKeyword(1, "first")
assert.NoError(t, err)
assert.EqualValues(t, []int64{1}, ids)

ids, err = SearchIssuesByKeyword(1, "for")
assert.NoError(t, err)
assert.EqualValues(t, []int64{1, 2, 3, 5}, ids)
}
Loading