实现了持久化的GET方法

consts.go

@@ -0,0 +1,15 @@
+/**
+ * @Author: JameyWoo
+ * @Email: 2622075127wjh@gmail.com
+ * @Date: 2021/1/21
+ * @Desc: 保存一些全局的常量, 后续改为配置
+ * @Copyright (c) 2020, JameyWoo. All rights reserved.
+ */
+
+package TinyBase
+
+const (
+	// memStore所占内存的阈值, 如果到达了该阈值则将其持久化. 暂定 1024B = 1KB
+	maxMemSize int = 1 << 15
+	deleted string = "__deleted__"
+)

db.go

@@ -10,11 +10,12 @@ package TinyBase
 
 import (
 	"bufio"
+	"encoding/binary"
 	"errors"
+	"fmt"
 	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"os"
-	"strconv"
 	"time"
 )
 
@@ -56,7 +57,13 @@ func (db *DB) Close() {
 }
 
 func (db *DB) Get(key string) (string, error) {
-	return db.eng.Get(key)
+	value, err := db.eng.Get(key)
+	// 如果没有得到value
+	if err == nil {
+		return value, err
+	}
+	// 从磁盘上获取value
+	return db.diskGet(key)
 }
 
 func (db *DB) Put(kv KeyValue) error {
@@ -125,10 +132,13 @@ func (db *DB) flush() error {
 	fileStr := ""
 	for key, val := range db.eng.memStore {
 		// 编码, [keyLen, key, valueLen, value]
-		fileStr += strconv.Itoa(len(key)) + key + strconv.Itoa(len(val)) + val
+		keyLen, valueLen := make([]byte, 4), make([]byte, 4)
+		binary.LittleEndian.PutUint32(keyLen, uint32(len(key)))
+		binary.LittleEndian.PutUint32(valueLen, uint32(len(val)))
+		fileStr += string(keyLen) + key + string(valueLen) + val
 	}
-	// 创建一个新文件
-	newFile, err := os.Create(flushPath + strconv.Itoa(fileId) + ".flush")
+	// 创建一个新文件, 前面补零
+	newFile, err := os.Create(flushPath + fmt.Sprintf("%010d", fileId) + ".flush")
 	if err != nil {
 		logrus.Fatal(err)
 		return err
@@ -138,11 +148,45 @@ func (db *DB) flush() error {
 		logrus.Fatal(err)
 		return err
 	}
+	_ = newFile.Sync()
 
 	if byteLen != len(fileStr) {
 		err = errors.New("byteLen != len(fileStr)")
 		logrus.Fatal(err)
 		return err
 	}
 	return nil
+}
+
+func (db *DB) diskGet(key string) (string, error) {
+	// 从磁盘上获取目录及文件, 然后一个一个读取
+	flushPath := db.dir + "/flush_files/"
+	_, err := os.Stat(flushPath)
+	if os.IsNotExist(err) {
+		// create
+		os.Mkdir(flushPath, os.ModePerm)
+	}
+	files, _ := ioutil.ReadDir(flushPath)  // 编号从0开始
+	for ii := 0; ii < len(files); ii++ {
+		bytes, err := ioutil.ReadFile(flushPath + files[ii].Name())
+		//logrus.Info(flushPath + files[ii].Name())
+		if err != nil {
+			logrus.Error(err)
+		}
+		// 解码
+		i := 0
+		for i < len(bytes) {
+			keyLen := int(binary.LittleEndian.Uint32(bytes[i: i + 4]))
+			theKey := string(bytes[i + 4: i + 4 + keyLen])
+			//logrus.Info("theKey: ", theKey)
+			i = i + 4 + keyLen
+			valLen := int(binary.LittleEndian.Uint32(bytes[i: i + 4]))
+			if theKey == key {
+				return string(bytes[i + 4: i + 4 + valLen]), nil
+			} else {
+				i = i + 4 + valLen
+			}
+		}
+	}
+	return "", GetEmptyError
 }

engine.go

@@ -1,16 +1,9 @@
 package TinyBase
 
 import (
-	"errors"
 	"sort"
 )
 
-const (
-	// memStore所占内存的阈值, 如果到达了该阈值则将其持久化. 暂定 1024B = 1KB
-	maxMemSize int = 1 << 10
-	deleted string = "__deleted__"
-)
-
 // the memory Engine
 type Engine struct {
 	memStore map[string]string
@@ -29,7 +22,7 @@ func NewEngine() *Engine {
 func (e *Engine) Get(key string) (string, error) {
 	m, ok := e.memStore[key]
 	if !ok {
-		return "", errors.New("no such element")
+		return "", GetEmptyError
 	}
 	return m, nil
 }

myerrors.go

@@ -0,0 +1,17 @@
+/**
+ * @Author: JameyWoo
+ * @Email: 2622075127wjh@gmail.com
+ * @Date: 2021/1/21
+ * @Desc: 定义一些错误
+ * @Copyright (c) 2020, JameyWoo. All rights reserved.
+ */
+
+package TinyBase
+
+import "errors"
+
+var GetEmptyError error
+
+func init() {
+	GetEmptyError = errors.New("GetEmptyError: no such element")
+}

readme.md

@@ -39,5 +39,5 @@ A persistent, LSM tree structured key value database engine implemented by go la
 - [ ] 考虑增量设计(偏移增量等)
 - [ ] 考虑键值分离的结构, 类似boltDB
 - [ ] 使用标准使用接口改造项目调用方式
-- [ ] 实现可变长度整形 varint 从而更好地压缩
+- [ ] 实现可变长度整形 varint 从而更好地压缩 (binary中有实现)
 - [ ] 实现字符串压缩算法

test/db_test.go

@@ -24,21 +24,30 @@ func TestDbOpen(t *testing.T) {
 	}
 	db.Put(TinyBase.KeyValue{Key: "hello", Value: "world"})
 	val, _ := db.Get("hello")
-	logrus.Info(val)
+	logrus.Info("hello: ", val)
 	db.Close()
 }
 
-// 测试flush的写入
+// 测试flush的写入以及读取
 func TestFlush(t *testing.T) {
 	db, err := TinyBase.Open("db1")
 	if err != nil {
 		logrus.Error(err)
 	}
-	for i := 0; i < 1000; i++ {
+	for i := 0; i < 11000; i++ {
 		db.Put(TinyBase.KeyValue{Key: strconv.Itoa(i) + "_key", Value: strconv.Itoa(i) + "_value"})
 	}
-	val, _ := db.Get("100_key")
-	logrus.Info(val)
+	val, err := db.Get("100_key")
+	if err != nil {
+		logrus.Error(err)
+	}
+	logrus.Info("val:", val)
+
+	val, err = db.Get("10000_key")
+	if err != nil {
+		logrus.Error(err)
+	}
+	logrus.Info("val:", val)
 	db.Close()
 }