Merge branch 'main' into dependabot/go_modules/github.com/minio/minio…

…-go/v7-7.0.34
zincsearch · Aug 3, 2022 · a1f751f · a1f751f
2 parents c5acac5 + cac58ce
commit a1f751f
Show file tree

Hide file tree

Showing 16 changed files with 406 additions and 26 deletions.
diff --git a/.env.sample b/.env.sample
@@ -1,12 +1,15 @@
 ZINC_FIRST_ADMIN_USER=admin
 ZINC_FIRST_ADMIN_PASSWORD=admin
 ZINC_SERVER_MODE=cluster
-ZINC_ICE_COMPRESSOR=s2
+ZINC_ICE_COMPRESSOR=zstd
 ZINC_S3_BUCKET=zinc-dev-misc1
 ZINC_ETCD_USERNAME=root
 ZINC_ETCD_ENDPOINTS=localhost:2379
 ZINC_ETCD_PASSWORD=iwapfOK1wv
+ZINC_BATCH_SIZE=1024
 ZINC_WAL_SYNC_INTERVAL=1s
+ZINC_READ_GORUTINE_NUM=10
+ZINC_SHARD_NUM=3
 ZINC_PROFILER=false
 ZINC_PROFILER_SERVER="https://pyroscope.dev.zincsearch.com/"
-ZINC_PROFILER_FRIENDLY_PROFILE_ID="parallel"
+ZINC_PROFILER_FRIENDLY_PROFILE_ID="parallel"
diff --git a/docs/docs.go b/docs/docs.go
@@ -540,6 +540,59 @@ const docTemplate = `{
                 }
             }
         },
+        "/api/{index}/_bulkv2": {
+            "post": {
+                "consumes": [
+                    "application/json"
+                ],
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "Document"
+                ],
+                "summary": "Bulkv2 documents",
+                "operationId": "Bulkv2",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "description": "Index",
+                        "name": "index",
+                        "in": "path",
+                        "required": true
+                    },
+                    {
+                        "description": "Query",
+                        "name": "query",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/meta.JSONIngest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/meta.HTTPResponseRecordCount"
+                        }
+                    },
+                    "400": {
+                        "description": "Bad Request",
+                        "schema": {
+                            "$ref": "#/definitions/meta.HTTPResponseError"
+                        }
+                    },
+                    "500": {
+                        "description": "Internal Server Error",
+                        "schema": {
+                            "$ref": "#/definitions/meta.HTTPResponseError"
+                        }
+                    }
+                }
+            }
+        },
         "/api/{index}/_doc": {
             "post": {
                 "consumes": [
@@ -2231,6 +2284,21 @@ const docTemplate = `{
                 }
             }
         },
+        "meta.JSONIngest": {
+            "type": "object",
+            "properties": {
+                "index": {
+                    "type": "string"
+                },
+                "records": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "additionalProperties": true
+                    }
+                }
+            }
+        },
         "meta.Mappings": {
             "type": "object",
             "properties": {

diff --git a/docs/swagger.json b/docs/swagger.json
@@ -537,6 +537,59 @@
                 }
             }
         },
+        "/api/{index}/_bulkv2": {
+            "post": {
+                "consumes": [
+                    "application/json"
+                ],
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "Document"
+                ],
+                "summary": "Bulkv2 documents",
+                "operationId": "Bulkv2",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "description": "Index",
+                        "name": "index",
+                        "in": "path",
+                        "required": true
+                    },
+                    {
+                        "description": "Query",
+                        "name": "query",
+                        "in": "body",
+                        "required": true,
+                        "schema": {
+                            "$ref": "#/definitions/meta.JSONIngest"
+                        }
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/meta.HTTPResponseRecordCount"
+                        }
+                    },
+                    "400": {
+                        "description": "Bad Request",
+                        "schema": {
+                            "$ref": "#/definitions/meta.HTTPResponseError"
+                        }
+                    },
+                    "500": {
+                        "description": "Internal Server Error",
+                        "schema": {
+                            "$ref": "#/definitions/meta.HTTPResponseError"
+                        }
+                    }
+                }
+            }
+        },
         "/api/{index}/_doc": {
             "post": {
                 "consumes": [
@@ -2228,6 +2281,21 @@
                 }
             }
         },
+        "meta.JSONIngest": {
+            "type": "object",
+            "properties": {
+                "index": {
+                    "type": "string"
+                },
+                "records": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "additionalProperties": true
+                    }
+                }
+            }
+        },
         "meta.Mappings": {
             "type": "object",
             "properties": {

diff --git a/docs/swagger.yaml b/docs/swagger.yaml
@@ -498,6 +498,16 @@ definitions:
       updated_at:
         type: string
     type: object
+  meta.JSONIngest:
+    properties:
+      index:
+        type: string
+      records:
+        items:
+          additionalProperties: true
+          type: object
+        type: array
+    type: object
   meta.Mappings:
     properties:
       properties:
@@ -1174,6 +1184,41 @@ paths:
       summary: Analyze
       tags:
       - Index
+  /api/{index}/_bulkv2:
+    post:
+      consumes:
+      - application/json
+      operationId: Bulkv2
+      parameters:
+      - description: Index
+        in: path
+        name: index
+        required: true
+        type: string
+      - description: Query
+        in: body
+        name: query
+        required: true
+        schema:
+          $ref: '#/definitions/meta.JSONIngest'
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/meta.HTTPResponseRecordCount'
+        "400":
+          description: Bad Request
+          schema:
+            $ref: '#/definitions/meta.HTTPResponseError'
+        "500":
+          description: Internal Server Error
+          schema:
+            $ref: '#/definitions/meta.HTTPResponseError'
+      summary: Bulkv2 documents
+      tags:
+      - Document
   /api/{index}/_doc:
     post:
       consumes:

diff --git a/go.mod b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe
 	github.com/swaggo/gin-swagger v1.5.1
 	github.com/swaggo/swag v1.8.4
-	github.com/zinclabs/wal v1.2.2
+	github.com/zinclabs/wal v1.2.4
 	github.com/zsais/go-gin-prometheus v0.1.0
 	go.etcd.io/bbolt v1.3.6
 	go.etcd.io/etcd/client/v3 v3.5.4

diff --git a/go.sum b/go.sum
@@ -454,12 +454,6 @@ github.com/swaggo/gin-swagger v1.5.1/go.mod h1:Cbj/MlHApPOjZdf4joWFXLLgmZVPyh54G
 github.com/swaggo/swag v1.8.1/go.mod h1:ugemnJsPZm/kRwFUnzBlbHRd0JY9zE1M4F+uy2pAaPQ=
 github.com/swaggo/swag v1.8.4 h1:oGB351qH1JqUqK1tsMYEE5qTBbPk394BhsZxmUfebcI=
 github.com/swaggo/swag v1.8.4/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg=
-github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo=
-github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
-github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
-github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
-github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/tinylru v1.1.0 h1:XY6IUfzVTU9rpwdhKUF6nQdChgCdGjkMfLzbWyiau6I=
 github.com/tidwall/tinylru v1.1.0/go.mod h1:3+bX+TJ2baOLMWTnlyNWHh4QMnFyARg2TLTQ6OFbzw8=
 github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
@@ -504,8 +498,8 @@ github.com/zinclabs/bluge_segment_api v1.0.0 h1:GJvPxdzR7KjwdxmcKleQLvtIYi/J7Q7e
 github.com/zinclabs/bluge_segment_api v1.0.0/go.mod h1:mYfPVUdXLZ4iXsicXMER+RcI/avwphjMOi8nhN9HDLA=
 github.com/zinclabs/ice v1.1.3 h1:LNfncdxQw2ix6P1T2ISmhO+6BFRa27qyTTfK0PitF2c=
 github.com/zinclabs/ice v1.1.3/go.mod h1:wTwGEe30mQnSLaR1ezxu4E80GcwO6EyOww67KpJtIiw=
-github.com/zinclabs/wal v1.2.2 h1:YsAIZlMclgImKDWON4+bWEYIMnwnhOfjqebY0e0S0xA=
-github.com/zinclabs/wal v1.2.2/go.mod h1:07xTJV3UHj8Y9LWPsZXzietInH8xUDTb5h7ROHCO+vw=
+github.com/zinclabs/wal v1.2.4 h1:9i8lV0loNp/cbfvp6/fpvdoY7jlHSytNYbLeCVOzpZM=
+github.com/zinclabs/wal v1.2.4/go.mod h1:pZe8LFjElrAxuIqLgz0f7e9mEy1j0D6x+DDkNG9TPcM=
 github.com/zsais/go-gin-prometheus v0.1.0 h1:bkLv1XCdzqVgQ36ScgRi09MA2UC1t3tAB6nsfErsGO4=
 github.com/zsais/go-gin-prometheus v0.1.0/go.mod h1:Slirjzuz8uM8Cw0jmPNqbneoqcUtY2GGjn2bEd4NRLY=
 go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=

diff --git a/pkg/bluge/search/search.go b/pkg/bluge/search/search.go
@@ -26,6 +26,7 @@ import (
 	"github.com/blugelabs/bluge/search/aggregations"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/zinclabs/zinc/pkg/config"
 	"github.com/zinclabs/zinc/pkg/meta"
 	"github.com/zinclabs/zinc/pkg/uquery"
 )
@@ -57,8 +58,8 @@ func MultiSearch(
 	bucketAggs["duration"] = aggregations.Duration()
 
 	eg := &errgroup.Group{}
-	eg.SetLimit(10)
-	docs := make(chan *search.DocumentMatch, len(readers)*2)
+	eg.SetLimit(config.Global.ReadGorutineNum)
+	docs := make(chan *search.DocumentMatch, len(readers)*10)
 	aggs := make(chan *search.Bucket, len(readers))
 
 	docList := &DocumentList{

diff --git a/pkg/core/index_document.go b/pkg/core/index_document.go
@@ -34,9 +34,9 @@ func (index *Index) CreateDocument(docID string, doc map[string]interface{}, upd
 		return err
 	}
 
-	secondShardID := ShardNoNeedLatest
+	secondShardID := ShardIDNeedLatest
 	if update {
-		secondShardID = ShardNoNeedUpdate
+		secondShardID = ShardIDNeedUpdate
 	}
 	data, err := shard.CheckDocument(docID, doc, update, secondShardID)
 	if err != nil {

diff --git a/pkg/core/index_shards.go b/pkg/core/index_shards.go
@@ -34,8 +34,8 @@ import (
 )
 
 const (
-	ShardNoNeedLatest int64 = -1 // get lastest shardID
-	ShardNoNeedUpdate int64 = -2 // get all shardIDs
+	ShardIDNeedLatest int64 = -1 // get lastest shardID
+	ShardIDNeedUpdate int64 = -2 // get all shardIDs
 )
 
 // IndexShard first layer shard by fixed number shards for index.

diff --git a/pkg/core/index_shards_wal.go b/pkg/core/index_shards_wal.go
@@ -347,7 +347,7 @@ func (w *walMergeDocs) WriteToShard(shard *IndexShard, shardID int64, batch *blu
 	var writer *bluge.Writer
 	otherWriters := make([]*bluge.Writer, 0)
 	otherBatch := blugeindex.NewBatch()
-	if shardID == ShardNoNeedLatest {
+	if shardID == ShardIDNeedLatest {
 		shardID = shard.GetLatestShardID()
 	}
 	if shardID >= 0 {

diff --git a/pkg/core/index_shards_wallist.go b/pkg/core/index_shards_wallist.go
@@ -16,6 +16,7 @@
 package core
 
 import (
+	"runtime"
 	"sync"
 	"time"
 
@@ -98,5 +99,8 @@ func (t *IndexShardWALList) ConsumeWAL() {
 		for name := range indexClosed {
 			t.Remove(name)
 		}
+
+		// force gc
+		runtime.GC()
 	}
 }