Use int encoder for string lengths.

xichen2020 · Jan 25, 2019 · 5823b26 · 5823b26
1 parent 5f99eb3
commit 5823b26
Show file tree

Hide file tree

Showing 20 changed files with 488 additions and 115 deletions.
diff --git a/generated/generics/generate.go b/generated/generics/generate.go
@@ -75,6 +75,9 @@
 //go:generate sh -c "cat $GOPATH/src/$PACKAGE/x/proto/template/encode.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/x/proto/encode_string_array.gen.go -pkg=proto gen \"GenericEncodeProtoMessage=*encodingpb.StringArray EncodeValue=EncodeStringArray\""
 //go:generate sh -c "cat $GOPATH/src/$PACKAGE/x/proto/template/decode.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/x/proto/decode_string_array.gen.go -pkg=proto gen \"GenericDecodeProtoMessage=*encodingpb.StringArray DecodeValue=DecodeStringArray DecodeValueRaw=DecodeStringArrayRaw\""
 
+//go:generate sh -c "cat $GOPATH/src/$PACKAGE/x/proto/template/encode.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/x/proto/encode_string_lengths.gen.go -pkg=proto gen \"GenericEncodeProtoMessage=*encodingpb.StringLengths EncodeValue=EncodeStringLengths\""
+//go:generate sh -c "cat $GOPATH/src/$PACKAGE/x/proto/template/decode.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/x/proto/decode_string_lengths.gen.go -pkg=proto gen \"GenericDecodeProtoMessage=*encodingpb.StringLengths DecodeValue=DecodeStringLengths DecodeValueRaw=DecodeStringLengthsRaw\""
+
 //go:generate sh -c "cat $GOPATH/src/$PACKAGE/x/proto/template/encode.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/x/proto/encode_int_meta.gen.go -pkg=proto gen \"GenericEncodeProtoMessage=*encodingpb.IntMeta EncodeValue=EncodeIntMeta\""
 //go:generate sh -c "cat $GOPATH/src/$PACKAGE/x/proto/template/decode.go | awk '/^package/{i++}i' | genny -out=$GOPATH/src/$PACKAGE/x/proto/decode_int_meta.gen.go -pkg=proto gen \"GenericDecodeProtoMessage=*encodingpb.IntMeta DecodeValue=DecodeIntMeta DecodeValueRaw=DecodeIntMetaRaw\""
 

diff --git a/generated/proto/encodingpb/string.pb.go b/generated/proto/encodingpb/string.pb.go
diff --git a/generated/proto/encodingpb/string.proto b/generated/proto/encodingpb/string.proto
@@ -17,3 +17,7 @@ message StringMeta {
 message StringArray {
     repeated string data = 1;
 }
+
+message StringLengths {
+    bytes data = 1;
+}
diff --git a/parser/json/value/bucketized_kv_array_pool.gen.go b/parser/json/value/bucketized_kv_array_pool.gen.go
@@ -97,7 +97,7 @@ func (p *BucketizedKVArrayPool) Init(alloc func(capacity int) KVArray) {
 			SetMetricsScope(scope.Tagged(map[string]string{
 				"bucket-capacity": fmt.Sprintf("%d", capacity),
 			}))
-		opts.SetInstrumentOptions(iOpts)
+		opts = opts.SetInstrumentOptions(iOpts)
 
 		buckets[i].capacity = capacity
 		buckets[i].pool = NewKVArrayPool(opts)

diff --git a/parser/json/value/bucketized_value_array_pool.gen.go b/parser/json/value/bucketized_value_array_pool.gen.go
@@ -97,7 +97,7 @@ func (p *BucketizedArrayPool) Init(alloc func(capacity int) Array) {
 			SetMetricsScope(scope.Tagged(map[string]string{
 				"bucket-capacity": fmt.Sprintf("%d", capacity),
 			}))
-		opts.SetInstrumentOptions(iOpts)
+		opts = opts.SetInstrumentOptions(iOpts)
 
 		buckets[i].capacity = capacity
 		buckets[i].pool = NewArrayPool(opts)

diff --git a/persist/fs/options.go b/persist/fs/options.go
@@ -6,6 +6,7 @@ import (
 
 	"github.com/m3db/m3x/clock"
 	"github.com/m3db/m3x/instrument"
+	"github.com/xichen2020/eventdb/x/pool"
 )
 
 const (
@@ -52,6 +53,7 @@ type Options struct {
 	writeBufferSize        int
 	mmapEnableHugePages    bool
 	mmapHugePagesThreshold int64
+	intArrayPool           *pool.BucketizedIntArrayPool
 }
 
 // NewOptions provide a new set of options.
@@ -202,3 +204,15 @@ func (o *Options) SetMmapHugePagesThreshold(v int64) *Options {
 func (o *Options) MmapHugePagesThreshold() int64 {
 	return o.mmapHugePagesThreshold
 }
+
+// SetIntArrayPool sets the int array pool.
+func (o *Options) SetIntArrayPool(v *pool.BucketizedIntArrayPool) *Options {
+	opts := *o
+	opts.intArrayPool = v
+	return &opts
+}
+
+// IntArrayPool returns the int array pool.
+func (o *Options) IntArrayPool() *pool.BucketizedIntArrayPool {
+	return o.intArrayPool
+}
diff --git a/services/eventdb/config/database.go b/services/eventdb/config/database.go
@@ -104,13 +104,6 @@ func (c *DatabaseConfiguration) NewOptions(instrumentOpts instrument.Options) (*
 	if c.SegmentUnloadAfterUnreadFor != nil {
 		opts = opts.SetSegmentUnloadAfterUnreadFor(*c.SegmentUnloadAfterUnreadFor)
 	}
-	fsOpts := c.PersistManager.NewFileSystemOptions(
-		opts.FilePathPrefix(),
-		opts.FieldPathSeparator(),
-	)
-	persistManager := fs.NewPersistManager(fsOpts)
-	fieldRetriever := fs.NewFieldRetriever(fsOpts)
-	opts = opts.SetPersistManager(persistManager).SetFieldRetriever(fieldRetriever)
 
 	// Initialize various pools.
 	if c.ContextPool != nil {
@@ -158,6 +151,14 @@ func (c *DatabaseConfiguration) NewOptions(instrumentOpts instrument.Options) (*
 		stringArrayPool.Init(func(capacity int) []string { return make([]string, 0, capacity) })
 		opts = opts.SetStringArrayPool(stringArrayPool)
 	}
+
+	fsOpts := c.PersistManager.NewFileSystemOptions(
+		opts.FilePathPrefix(),
+		opts.FieldPathSeparator(),
+	).SetIntArrayPool(opts.IntArrayPool())
+	persistManager := fs.NewPersistManager(fsOpts)
+	fieldRetriever := fs.NewFieldRetriever(fsOpts)
+	opts = opts.SetPersistManager(persistManager).SetFieldRetriever(fieldRetriever)
 	return opts, nil
 }