fix: change default params

.env.sample

@@ -1,12 +1,15 @@
 ZINC_FIRST_ADMIN_USER=admin
 ZINC_FIRST_ADMIN_PASSWORD=admin
 ZINC_SERVER_MODE=cluster
-ZINC_ICE_COMPRESSOR=s2
+ZINC_ICE_COMPRESSOR=zstd
 ZINC_S3_BUCKET=zinc-dev-misc1
 ZINC_ETCD_USERNAME=root
 ZINC_ETCD_ENDPOINTS=localhost:2379
 ZINC_ETCD_PASSWORD=iwapfOK1wv
+ZINC_BATCH_SIZE=1024
 ZINC_WAL_SYNC_INTERVAL=1s
+ZINC_READ_GORUTINE_NUM=10
+ZINC_SHARD_NUM=3
 ZINC_PROFILER=false
 ZINC_PROFILER_SERVER="https://pyroscope.dev.zincsearch.com/"
-ZINC_PROFILER_FRIENDLY_PROFILE_ID="parallel"
+ZINC_PROFILER_FRIENDLY_PROFILE_ID="parallel"

go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe
 	github.com/swaggo/gin-swagger v1.5.1
 	github.com/swaggo/swag v1.8.4
-	github.com/zinclabs/wal v1.2.2
+	github.com/zinclabs/wal v1.2.4
 	github.com/zsais/go-gin-prometheus v0.1.0
 	go.etcd.io/bbolt v1.3.6
 	go.etcd.io/etcd/client/v3 v3.5.4

go.sum

@@ -457,12 +457,6 @@ github.com/swaggo/gin-swagger v1.5.1/go.mod h1:Cbj/MlHApPOjZdf4joWFXLLgmZVPyh54G
 github.com/swaggo/swag v1.8.1/go.mod h1:ugemnJsPZm/kRwFUnzBlbHRd0JY9zE1M4F+uy2pAaPQ=
 github.com/swaggo/swag v1.8.4 h1:oGB351qH1JqUqK1tsMYEE5qTBbPk394BhsZxmUfebcI=
 github.com/swaggo/swag v1.8.4/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg=
-github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo=
-github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
-github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
-github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
-github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/tinylru v1.1.0 h1:XY6IUfzVTU9rpwdhKUF6nQdChgCdGjkMfLzbWyiau6I=
 github.com/tidwall/tinylru v1.1.0/go.mod h1:3+bX+TJ2baOLMWTnlyNWHh4QMnFyARg2TLTQ6OFbzw8=
 github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
@@ -507,8 +501,8 @@ github.com/zinclabs/bluge_segment_api v1.0.0 h1:GJvPxdzR7KjwdxmcKleQLvtIYi/J7Q7e
 github.com/zinclabs/bluge_segment_api v1.0.0/go.mod h1:mYfPVUdXLZ4iXsicXMER+RcI/avwphjMOi8nhN9HDLA=
 github.com/zinclabs/ice v1.1.3 h1:LNfncdxQw2ix6P1T2ISmhO+6BFRa27qyTTfK0PitF2c=
 github.com/zinclabs/ice v1.1.3/go.mod h1:wTwGEe30mQnSLaR1ezxu4E80GcwO6EyOww67KpJtIiw=
-github.com/zinclabs/wal v1.2.2 h1:YsAIZlMclgImKDWON4+bWEYIMnwnhOfjqebY0e0S0xA=
-github.com/zinclabs/wal v1.2.2/go.mod h1:07xTJV3UHj8Y9LWPsZXzietInH8xUDTb5h7ROHCO+vw=
+github.com/zinclabs/wal v1.2.4 h1:9i8lV0loNp/cbfvp6/fpvdoY7jlHSytNYbLeCVOzpZM=
+github.com/zinclabs/wal v1.2.4/go.mod h1:pZe8LFjElrAxuIqLgz0f7e9mEy1j0D6x+DDkNG9TPcM=
 github.com/zsais/go-gin-prometheus v0.1.0 h1:bkLv1XCdzqVgQ36ScgRi09MA2UC1t3tAB6nsfErsGO4=
 github.com/zsais/go-gin-prometheus v0.1.0/go.mod h1:Slirjzuz8uM8Cw0jmPNqbneoqcUtY2GGjn2bEd4NRLY=
 go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=

pkg/bluge/search/search.go

@@ -26,6 +26,7 @@ import (
 	"github.com/blugelabs/bluge/search/aggregations"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/zinclabs/zinc/pkg/config"
 	"github.com/zinclabs/zinc/pkg/meta"
 	"github.com/zinclabs/zinc/pkg/uquery"
 )
@@ -57,8 +58,8 @@ func MultiSearch(
 	bucketAggs["duration"] = aggregations.Duration()
 
 	eg := &errgroup.Group{}
-	eg.SetLimit(10)
-	docs := make(chan *search.DocumentMatch, len(readers)*2)
+	eg.SetLimit(config.Global.ReadGorutineNum)
+	docs := make(chan *search.DocumentMatch, len(readers)*10)
 	aggs := make(chan *search.Bucket, len(readers))
 
 	docList := &DocumentList{

pkg/core/index_document.go

@@ -34,9 +34,9 @@ func (index *Index) CreateDocument(docID string, doc map[string]interface{}, upd
 		return err
 	}
 
-	secondShardID := ShardNoNeedLatest
+	secondShardID := ShardIDNeedLatest
 	if update {
-		secondShardID = ShardNoNeedUpdate
+		secondShardID = ShardIDNeedUpdate
 	}
 	data, err := shard.CheckDocument(docID, doc, update, secondShardID)
 	if err != nil {

pkg/core/index_shards.go

@@ -34,8 +34,8 @@ import (
 )
 
 const (
-	ShardNoNeedLatest int64 = -1 // get lastest shardID
-	ShardNoNeedUpdate int64 = -2 // get all shardIDs
+	ShardIDNeedLatest int64 = -1 // get lastest shardID
+	ShardIDNeedUpdate int64 = -2 // get all shardIDs
 )
 
 // IndexShard first layer shard by fixed number shards for index.

pkg/core/index_shards_wal.go

@@ -347,7 +347,7 @@ func (w *walMergeDocs) WriteToShard(shard *IndexShard, shardID int64, batch *blu
 	var writer *bluge.Writer
 	otherWriters := make([]*bluge.Writer, 0)
 	otherBatch := blugeindex.NewBatch()
-	if shardID == ShardNoNeedLatest {
+	if shardID == ShardIDNeedLatest {
 		shardID = shard.GetLatestShardID()
 	}
 	if shardID >= 0 {

pkg/core/index_shards_wallist.go

@@ -16,6 +16,7 @@
 package core
 
 import (
+	"runtime"
 	"sync"
 	"time"
 
@@ -98,5 +99,8 @@ func (t *IndexShardWALList) ConsumeWAL() {
 		for name := range indexClosed {
 			t.Remove(name)
 		}
+
+		// force gc
+		runtime.GC()
 	}
 }

pkg/wal/wal.go

@@ -35,14 +35,13 @@ func Open(indexName string) (*Log, error) {
 	l := new(Log)
 	l.name = indexName
 	opt := &wal.Options{
-		NoSync:           true,       // Fsync after every write
-		SegmentSize:      16777216,   // 16 MB log segment files.
-		LogFormat:        wal.Binary, // Binary format is small and fast.
-		SegmentCacheSize: 2,          // Number of cached in-memory segments
-		NoCopy:           true,       // Make a new copy of data for every Read call.
-		DirPerms:         0750,       // Permissions for the created directories
-		FilePerms:        0640,       // Permissions for the created data files
-		FillID:           true,       // Allow writes with a zero ID
+		NoSync:           true,     // Fsync after every write
+		SegmentSize:      16777216, // 16 MB log segment files.
+		SegmentCacheSize: 2,        // Number of cached in-memory segments
+		NoCopy:           true,     // Make a new copy of data for every Read call.
+		DirPerms:         0750,     // Permissions for the created directories
+		FilePerms:        0640,     // Permissions for the created data files
+		FillID:           true,     // Allow writes with a zero ID
 	}
 	l.log, err = wal.Open(path.Join(config.Global.DataPath, indexName, "wal"), opt)
 	if err != nil {