From 44da40c5018c5d627261259a4c66efd74bbbade5 Mon Sep 17 00:00:00 2001 From: vcaesar Date: Sat, 10 Feb 2018 21:31:25 +0800 Subject: [PATCH 1/4] update storage func code --- engine.go | 110 +++++++++++++++++++++++---------------------- indexer_worker.go | 1 + net/grpc/search.go | 6 ++- 3 files changed, 61 insertions(+), 56 deletions(-) diff --git a/engine.go b/engine.go index 4328802..541180e 100755 --- a/engine.go +++ b/engine.go @@ -137,17 +137,15 @@ func (engine *Engine) Ranker(options types.EngineOpts) { // InitStorage initialize the persistent storage channel func (engine *Engine) InitStorage() { - if engine.initOptions.UseStorage { - engine.storageIndexDocChans = - make([]chan storageIndexDocReq, - engine.initOptions.StorageShards) - for shard := 0; shard < engine.initOptions.StorageShards; shard++ { - engine.storageIndexDocChans[shard] = make( - chan storageIndexDocReq) - } - engine.storageInitChan = make( - chan bool, engine.initOptions.StorageShards) - } + engine.storageIndexDocChans = + make([]chan storageIndexDocReq, + engine.initOptions.StorageShards) + for shard := 0; shard < engine.initOptions.StorageShards; shard++ { + engine.storageIndexDocChans[shard] = make( + chan storageIndexDocReq) + } + engine.storageInitChan = make( + chan bool, engine.initOptions.StorageShards) } // CheckMem check the memory when the memory is larger @@ -171,58 +169,58 @@ func (engine *Engine) CheckMem() { // Storage start the persistent storage work connection func (engine *Engine) Storage() { - if engine.initOptions.UseStorage { - err := os.MkdirAll(engine.initOptions.StorageFolder, 0700) - if err != nil { - log.Fatal("Can not create directory", engine.initOptions.StorageFolder) - } + // if engine.initOptions.UseStorage { + err := os.MkdirAll(engine.initOptions.StorageFolder, 0700) + if err != nil { + log.Fatal("Can not create directory", engine.initOptions.StorageFolder) + } - // 打开或者创建数据库 - engine.dbs = make([]storage.Storage, engine.initOptions.StorageShards) - for shard := 0; shard < engine.initOptions.StorageShards; shard++ { - dbPath := engine.initOptions.StorageFolder + "/" + - StorageFilePrefix + "." + strconv.Itoa(shard) + // 打开或者创建数据库 + engine.dbs = make([]storage.Storage, engine.initOptions.StorageShards) + for shard := 0; shard < engine.initOptions.StorageShards; shard++ { + dbPath := engine.initOptions.StorageFolder + "/" + + StorageFilePrefix + "." + strconv.Itoa(shard) - db, err := storage.OpenStorage(dbPath, engine.initOptions.StorageEngine) - if db == nil || err != nil { - log.Fatal("Unable to open database", dbPath, ": ", err) - } - engine.dbs[shard] = db + db, err := storage.OpenStorage(dbPath, engine.initOptions.StorageEngine) + if db == nil || err != nil { + log.Fatal("Unable to open database", dbPath, ": ", err) } + engine.dbs[shard] = db + } - // 从数据库中恢复 - for shard := 0; shard < engine.initOptions.StorageShards; shard++ { - go engine.storageInitWorker(shard) - } + // 从数据库中恢复 + for shard := 0; shard < engine.initOptions.StorageShards; shard++ { + go engine.storageInitWorker(shard) + } - // 等待恢复完成 - for shard := 0; shard < engine.initOptions.StorageShards; shard++ { - <-engine.storageInitChan - } - for { - runtime.Gosched() - if engine.numIndexingReqs == engine.numDocsIndexed { - break - } + // 等待恢复完成 + for shard := 0; shard < engine.initOptions.StorageShards; shard++ { + <-engine.storageInitChan + } + for { + runtime.Gosched() + if engine.numIndexingReqs == engine.numDocsIndexed { + break } + } - // 关闭并重新打开数据库 - for shard := 0; shard < engine.initOptions.StorageShards; shard++ { - engine.dbs[shard].Close() - dbPath := engine.initOptions.StorageFolder + "/" + - StorageFilePrefix + "." + strconv.Itoa(shard) + // 关闭并重新打开数据库 + for shard := 0; shard < engine.initOptions.StorageShards; shard++ { + engine.dbs[shard].Close() + dbPath := engine.initOptions.StorageFolder + "/" + + StorageFilePrefix + "." + strconv.Itoa(shard) - db, err := storage.OpenStorage(dbPath, engine.initOptions.StorageEngine) - if db == nil || err != nil { - log.Fatal("Unable to open database", dbPath, ": ", err) - } - engine.dbs[shard] = db + db, err := storage.OpenStorage(dbPath, engine.initOptions.StorageEngine) + if db == nil || err != nil { + log.Fatal("Unable to open database", dbPath, ": ", err) } + engine.dbs[shard] = db + } - for shard := 0; shard < engine.initOptions.StorageShards; shard++ { - go engine.storageIndexDocWorker(shard) - } + for shard := 0; shard < engine.initOptions.StorageShards; shard++ { + go engine.storageIndexDocWorker(shard) } + // } } // Init initialize the engine @@ -270,7 +268,9 @@ func (engine *Engine) Init(options types.EngineOpts) { engine.CheckMem() // 初始化持久化存储通道 - engine.InitStorage() + if engine.initOptions.UseStorage { + engine.InitStorage() + } // 启动分词器 for iThread := 0; iThread < options.NumSegmenterThreads; iThread++ { @@ -293,7 +293,9 @@ func (engine *Engine) Init(options types.EngineOpts) { } // 启动持久化存储工作协程 - engine.Storage() + if engine.initOptions.UseStorage { + engine.Storage() + } atomic.AddUint64(&engine.numDocsStored, engine.numIndexingReqs) } diff --git a/indexer_worker.go b/indexer_worker.go index 76b6cb1..198491d 100755 --- a/indexer_worker.go +++ b/indexer_worker.go @@ -49,6 +49,7 @@ func (engine *Engine) indexerAddDocWorker(shard int) { if request.doc != nil { atomic.AddUint64(&engine.numTokenIndexAdded, uint64(len(request.doc.Keywords))) + atomic.AddUint64(&engine.numDocsIndexed, 1) } if request.forceUpdate { diff --git a/net/grpc/search.go b/net/grpc/search.go index e0dc958..db5f8e5 100644 --- a/net/grpc/search.go +++ b/net/grpc/search.go @@ -108,8 +108,10 @@ func rpcSearch(sea com.SearchArgs) *pb.SearchReply { return rep } -var rpcdata []*pb.SearchReply -var rpcwg sync.WaitGroup +var ( + rpcdata []*pb.SearchReply + rpcwg sync.WaitGroup +) // WgRpc rpc func WgRpc(address string, sea com.SearchArgs) { From 5431969f2ac15a53ef3a18932bb87720568bf4f0 Mon Sep 17 00:00:00 2001 From: vcaesar Date: Tue, 13 Feb 2018 22:31:27 +0800 Subject: [PATCH 2/4] update circle to 1.9.4 --- circle.yml | 2 +- examples/store/main.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index ca28f24..f7fb55e 100644 --- a/circle.yml +++ b/circle.yml @@ -8,7 +8,7 @@ version: 2 jobs: build: docker: - - image: golang:1.9.3 + - image: golang:1.9.4 working_directory: /go/src/github.com/go-ego/riot steps: - checkout diff --git a/examples/store/main.go b/examples/store/main.go index 672ef7e..87999a4 100644 --- a/examples/store/main.go +++ b/examples/store/main.go @@ -96,4 +96,6 @@ func main() { }}) fmt.Println("search---------", sea, "; docs=", sea.Docs) + + // os.RemoveAll("riot-index") } From e19d2610c271a66556d30eb7a1ebe013025f2f04 Mon Sep 17 00:00:00 2001 From: vcaesar Date: Sat, 17 Feb 2018 21:55:26 +0800 Subject: [PATCH 3/4] update travis support go 1.10 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4d9348a..451ec8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ go: # - 1.7.x # - 1.8.x - 1.9.x + - 1.10.x # - tip install: From 8bef667917a3a42a4d1fbb70e83c08cf7562cf70 Mon Sep 17 00:00:00 2001 From: vcaesar Date: Sun, 18 Feb 2018 19:51:04 +0800 Subject: [PATCH 4/4] fix travis ci --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 451ec8a..9860b7c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ go: # - 1.7.x # - 1.8.x - 1.9.x - - 1.10.x + - "1.10" # - tip install: