feat(ragdoll-data): 完成定长记录和有元素大小限制内存池的初步开发和测试

cli/main.go → client/cli/main.go

@@ -6,7 +6,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/Trinoooo/eggie_kv/cli/handle"
+	"github.com/Trinoooo/eggie_kv/client/cli/handle"
 	"io"
 	"log"
 	"net/http"

errs/error.go

@@ -43,45 +43,45 @@ func GetCode(err error) int64 {
 }
 
 const (
-	UnknownErrCode                 = 0
-	InvalidParamErrCode            = 100001
-	JsonMarshalErrCode             = 100002
-	JsonUnmarshalErrCode           = 100003
-	UnsupportedOperatorTypeErrCode = 100004
-	OpenFileErrCode                = 100005
-	DirNotExistErrCode             = 100006
-	FileNoPermissionErrCode        = 100007
-	FileStatErrCode                = 100008
-	MkdirErrCode                   = 100009
-	ReadFileErrCode                = 100010
-	WriteFileErrCode               = 100011
-	SyncFileErrCode                = 100012
-	CloseFileErrCode               = 100013
-	FileIntegrityErrCode           = 100014
-	ExecCmdErrCode                 = 100015
-	GetWdErrCode                   = 100016
-	WalkDirErrCode                 = 100017
-	FileClosedErrCode              = 100018
-	SeekFileErrCode                = 100019
-	SegmentFullErrCode             = 100020
-	NotFoundErrCode                = 100021
-	CorruptErrCode                 = 100022
-	WalFullErrCode                 = 100023
-	BackgroundErrCode              = 100024
-	ReachBlockIdxLimitErrCode      = 100025
-	GetErrCode                     = 100026
-	SetErrCode                     = 100027
-	BuildCoreErrCode               = 200001
-	ParseIntErrCode                = 100028
-	RenameFileErrCode              = 100029
-	TruncateFileErrCode            = 100030
-	RemoveFileErrCode              = 100031
-	FlockFileErrCode               = 100032
-	CreateTempFileErrCode          = 100033
-	CopyFileErrCode                = 100034
-	CoreNotFoundErrCode            = 100035
-	ReadSocketErrCode              = 100036
-	WriteSocketErrCode             = 100037
+	UnknownErrCode                 int64 = 0
+	InvalidParamErrCode            int64 = 100001
+	JsonMarshalErrCode             int64 = 100002
+	JsonUnmarshalErrCode           int64 = 100003
+	UnsupportedOperatorTypeErrCode int64 = 100004
+	OpenFileErrCode                int64 = 100005
+	DirNotExistErrCode             int64 = 100006
+	FileNoPermissionErrCode        int64 = 100007
+	FileStatErrCode                int64 = 100008
+	MkdirErrCode                   int64 = 100009
+	ReadFileErrCode                int64 = 100010
+	WriteFileErrCode               int64 = 100011
+	SyncFileErrCode                int64 = 100012
+	CloseFileErrCode               int64 = 100013
+	FileIntegrityErrCode           int64 = 100014
+	ExecCmdErrCode                 int64 = 100015
+	GetWdErrCode                   int64 = 100016
+	WalkDirErrCode                 int64 = 100017
+	FileClosedErrCode              int64 = 100018
+	SeekFileErrCode                int64 = 100019
+	SegmentFullErrCode             int64 = 100020
+	NotFoundErrCode                int64 = 100021
+	CorruptErrCode                 int64 = 100022
+	WalFullErrCode                 int64 = 100023
+	BackgroundErrCode              int64 = 100024
+	ReachBlockIdxLimitErrCode      int64 = 100025
+	GetErrCode                     int64 = 100026
+	SetErrCode                     int64 = 100027
+	BuildCoreErrCode               int64 = 200001
+	ParseIntErrCode                int64 = 100028
+	RenameFileErrCode              int64 = 100029
+	TruncateFileErrCode            int64 = 100030
+	RemoveFileErrCode              int64 = 100031
+	FlockFileErrCode               int64 = 100032
+	CreateTempFileErrCode          int64 = 100033
+	CopyFileErrCode                int64 = 100034
+	CoreNotFoundErrCode            int64 = 100035
+	ReadSocketErrCode              int64 = 100036
+	WriteSocketErrCode             int64 = 100037
 )
 
 func NewUnknownErr() *KvErr {

go.mod

@@ -1,24 +1,27 @@
 module github.com/Trinoooo/eggie_kv
 
-go 1.19
+go 1.21
 
 require (
 	github.com/bytedance/gopkg v0.0.0-20231219111115-a5eedbe96960
 	github.com/chzyer/readline v1.5.1
 	github.com/luci/go-render v0.0.0-20160219211803-9a04cc21af0f
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/spf13/viper v1.18.2
+	github.com/stretchr/testify v1.8.4
 	github.com/urfave/cli/v2 v2.26.0
 	go.uber.org/zap v1.26.0
 )
 
 require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect

go.sum

@@ -11,14 +11,19 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/luci/go-render v0.0.0-20160219211803-9a04cc21af0f h1:WVPqVsbUsrzAebTEgWRAZMdDOfkFx06iyhbIoyMgtkE=
 github.com/luci/go-render v0.0.0-20160219211803-9a04cc21af0f/go.mod h1:aS446i8akEg0DAtNKTVYpNpLPMc0SzsZ0RtGhjl0uFM=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
@@ -31,7 +36,9 @@ github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6
 github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
@@ -63,6 +70,7 @@ github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6S
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
+go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
@@ -83,6 +91,7 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

storage/core/ragdoll/data/consts.go

@@ -0,0 +1,13 @@
+package data
+
+const (
+	offsetToIdx       = 0
+	offsetToNext      = 8
+	offsetToLength    = 16
+	offsetToReserved  = 17
+	offsetToSignature = 32
+	offsetToData      = 48
+
+	lengthOfSignature = 16
+	lengthOfRecord    = 256
+)

storage/core/ragdoll/data/fixed_length_record.go

@@ -0,0 +1,87 @@
+package data
+
+import (
+	"crypto/md5"
+	"encoding/binary"
+	"github.com/Trinoooo/eggie_kv/consts"
+	"github.com/Trinoooo/eggie_kv/errs"
+	"github.com/Trinoooo/eggie_kv/storage/core/ragdoll/logs"
+	"go.uber.org/zap"
+)
+
+// fixedLengthRecord 定长记录
+// 内存结构 & 文件内存储结构：
+// ---------------------------------------------
+// | 	idx - 8字节		| 	next - 8字节			|
+// ---------------------------------------------
+// | 	length - 1字节	| 	reserved - 15字节	|
+// ---------------------------------------------
+// | 	signature - 16字节						|
+// ---------------------------------------------
+// | 	data - 224字节							|
+// ---------------------------------------------
+type fixedLengthRecord struct {
+	idx    int64 // idx 全局索引
+	next   int64 // next 片内下一条记录索引
+	length int8  // length 记录有效长度
+	// reserved  [15]byte  // reserved 保留字段，向后扩展
+	signature [16]byte // signature 记录签名
+	data      []byte   // data 记录数据内容
+}
+
+// newFixedLengthRecord 初始化空定长记录
+func newFixedLengthRecord() *fixedLengthRecord {
+	return &fixedLengthRecord{}
+}
+
+func (flr *fixedLengthRecord) fill(idx, next int64, length int8, data []byte) {
+	flr.idx = idx
+	flr.next = next
+	flr.length = length
+	flr.data = data
+	flr.sign()
+}
+
+func (flr *fixedLengthRecord) marshal() []byte {
+	buf := make([]byte, lengthOfRecord)
+	binary.PutVarint(buf, flr.idx)
+	binary.PutVarint(buf[offsetToNext:], flr.next)
+	buf[offsetToLength] = byte(flr.length)
+	for i := 0; i < lengthOfSignature; i++ {
+		buf[offsetToSignature+i] = flr.signature[i]
+	}
+	copy(buf[offsetToData:], flr.data)
+	return buf
+}
+
+func (flr *fixedLengthRecord) unmarshal(data []byte) error {
+	if lodata := len(data); lodata != lengthOfRecord {
+		e := errs.NewInvalidParamErr()
+		logs.Error(
+			e.Error(),
+			zap.String(consts.LogFieldParams, "len(data)"),
+			zap.Int(consts.LogFieldValue, lodata),
+		)
+		return e
+	}
+
+	flr.idx, _ = binary.Varint(data)
+	flr.next, _ = binary.Varint(data[offsetToNext:])
+	flr.length = int8(data[offsetToLength])
+	arr := [16]byte{}
+	for i := 0; i < lengthOfSignature; i++ {
+		arr[i] = data[offsetToSignature+i]
+	}
+	flr.signature = arr
+	flr.data = data[offsetToData : flr.length+offsetToData]
+	return nil
+}
+
+func (flr *fixedLengthRecord) sign() {
+	dataToSign := make([]byte, 240)
+	binary.PutVarint(dataToSign, flr.idx)
+	binary.PutVarint(dataToSign[offsetToNext:], flr.next)
+	dataToSign[offsetToLength] = byte(flr.length)
+	copy(dataToSign[offsetToSignature:], flr.data)
+	flr.signature = md5.Sum(dataToSign)
+}

storage/core/ragdoll/data/manager.go

@@ -0,0 +1,40 @@
+package data
+
+type Manager struct {
+}
+
+func NewManager() *Manager {
+	return nil
+}
+
+func (m *Manager) Write(data []byte) (int64, error) {
+	return 0, nil
+}
+
+func (m *Manager) MultiWrite(dataList [][]byte) ([]int64, error) {
+	return nil, nil
+}
+
+func (m *Manager) Remove(idx int64) error {
+	return nil
+}
+
+func (m *Manager) MultiRemove(idxList []int64) error {
+	return nil
+}
+
+func (m *Manager) Modify(idx int64, data []byte) error {
+	return nil
+}
+
+func (m *Manager) MultiModify(map[int64][]byte) error {
+	return nil
+}
+
+func (m *Manager) Read(idx int64) ([]byte, error) {
+	return nil, nil
+}
+
+func (m *Manager) MultiRead(idxList []int64) (map[int64][]byte, error) {
+	return nil, nil
+}