Merge branch 'release/0.1.0'

.travis.yml

@@ -1,9 +1,8 @@
 language: go
 sudo: false
 go:
-- 1.7
-- 1.8
-- tip
+- 1.7.x
+- 1.8.x
 env:
   global:
   - GO15VENDOREXPERIMENT=1
@@ -22,4 +21,4 @@ after_success:
 - goveralls -service=travis-ci
 notifications:
   slack:
-    secure: NLDyic6tUdurGVb6/wEdBQZLgoMpS9OVgVNomYH/TrcVZKrD+J2tkjJtgGTqw4+rkQhO7Ip9KyuSSYjRd8l6yGBpOlXykEUcunlILo72td+2ZGCojdsbDJhSdtgkpKozMbRniAzhp4aal2NzDPMypNvJqeyr6/F2zksYUSL4Jchx0juJ2gCUiTxrHuCHtq18Tg2l1mzcEAfAlpj9vMive6bM03wAalA/NcGl8hYe3f05yEWcGMDwaO2I84oG1fKGy0IlS3oneODfzT/JZzFkz08btcZeWCtKNro0g9XG2i5IQb5250K9DwbknxggKgZR72r1vnv6fwfT8iE4xMd+Vv3eYGNHsdrHsDWHlC8mjURWzRMGnxA9v12H1JobfffyuZBH4ZowfCgbsrcxlt/9hopTZEQ6gGYRPl/ZNVMPH1zbaveTi0oNbkSgA4waQ3efIIEbubcra+VAV401QmyNL2js2x2ph20rUquP3xZpKMbAV6ouduhNzHuvm1XSNQD2FbOI3tM9+Cb/Wm0AFZrM007mEx673SA6wKqm3G1oTUnUo7JMv/9nP4VQ5mQlR3VUt1UiZ+Xo/ZvLAACIrhhdHQKU1kCSq9E6MCHR+ETWk55iRraRyCdAK1mbt+6/l2qRHhgnyDWrIufHuhPVcFY01okc969bKNCTk3nqaQzEQt4=
+    secure: Chj8qKwfCJ5n0yqE7rIN80QL3W+s1PzCtUHuK7BGQtE4ZkUa8eQ0ElEtERIsdeUjNXZUkfxZWZyilVXOXl2KvXDD2f6FSZJ2r5z1ZGgFvQwpul/p5h6fCw8z/iapS8w8/3dkY0tV66/IzktEM6Vca2yv20s4b5KuNYT1GKRH/aBU6Fvpj63BwmIK4YwmJlm8KkEg5OlDNtu6KMpS4MVPlJG8cNll1DmU1EwFg4w+JAdRo2ENZLCa3GqiDh3phLNVkFTjDn773FxMQhlo8jAz7HjUAukat5OvZRuGt0aHf4ZoWzxBRlYpATwgGhGzdvt6y6m2Xmxkhu6VAprJf+m7fd+T49CGWVkL5+BDKQdpVEaZUkiPqpojRjFrUe79zrzwSsohHLqHjqlBHE8Egc2HRBi632FKT4TMU4g32VJiWGRM2YQBtsXwGS01hfU0f2YZ0I9jKwUNPvYcmotVmHKS69JLGOcH3NPaSkCVrpMzXsyLvYHjMdlucPV9TrS9s7Sktor4jDOIThBQ8J8OwYDQ6RtnPhRrjGOpKR8CvglhtCk3ADdtf5IZTEqbZvvvAqklz5/l6qbGQiM+EoYf34H5aDCCqWss7s88BolEax1t7pE8Ba35vzOaL7FIOCDBWzOggRsp8rjipicuYn3rgSek76NW7gMBre0eYS5wmWgLk90=

.vscode/tasks.json

@@ -14,7 +14,7 @@
         },
         {
             "taskName": "test",
-            "args": ["-v", "-cover", "-race"],
+            "args": ["-v", "-cover", "-race", "-timeout", "10s"],
             "isTestCommand": true
         }
     ]

encode.go

@@ -0,0 +1,153 @@
+package tdb
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type fileEncode uint32
+
+func encodeFileFromUnix(unixtime uint32) fileEncode {
+	t := time.Unix(int64(unixtime), 0)
+	year, month, day := t.Date()
+	hour := t.Hour()
+	return encodeFile(year, int(month), day, hour)
+}
+
+func encodeFile(year, month, day, hour int) fileEncode {
+	var encoded uint32
+	encoded += uint32(year) * 1e5
+	encoded += uint32(month) * 1e3
+	encoded += uint32(day) * 10
+	if hour >= 12 {
+		encoded += 5
+	}
+	return fileEncode(encoded)
+}
+
+func encodeFromPath(folder, filename string) (fileEncode, error) {
+	var encoded fileEncode
+
+	dir := filepath.Base(folder)
+	if len(dir) != 6 {
+		return encoded, errors.New("folder is wrong to encode")
+	}
+
+	fName := getFileName(filename)
+	if len(fName) != 3 {
+		return encoded, errors.New("filename is wrong to encode")
+	}
+
+	dirV, err := strconv.Atoi(dir)
+	if err != nil {
+		return encoded, err
+	}
+
+	fNameV, err := strconv.Atoi(fName)
+	if err != nil {
+		return encoded, err
+	}
+	return fileEncode(dirV*1e3 + fNameV), nil
+}
+
+func (f fileEncode) year() int {
+	return int(uint32(f) / 1e5)
+}
+
+func (f fileEncode) month() int {
+	return int(uint32(f)/1e3) % 100
+}
+
+func (f fileEncode) day() int {
+	return int(uint32(f)/10) % 100
+}
+
+func (f fileEncode) isAM() bool {
+	return f%10 == 0
+}
+
+// get subFolder folder and file name
+func (f fileEncode) path() (string, string) {
+	folder := strconv.Itoa(int(uint32(f) / 1e3))
+	fileName := strconv.Itoa(int(uint32(f) % 1e3))
+	if len(fileName) == 2 {
+		fileName = strings.Join([]string{"0", fileName}, "")
+	}
+	return folder, fileName
+}
+
+// get original unix time
+func (f fileEncode) origin() uint32 {
+	fileOrigin := time.Date(f.year(), time.Month(f.month()), f.day(), 0, 0, 0, 0, time.Local).Unix()
+	if !f.isAM() {
+		fileOrigin = fileOrigin + 43200
+	}
+	return uint32(fileOrigin)
+}
+
+// implement the sort interface
+type fileEncodeSlice []fileEncode
+
+func (s fileEncodeSlice) Len() int           { return len(s) }
+func (s fileEncodeSlice) Less(i, j int) bool { return s[i] < s[j] }
+func (s fileEncodeSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
+// fileRange used for query
+type fileRange struct {
+	start fileEncode
+	end   fileEncode
+}
+
+// create a new fileRange instance
+// startUnix == 0 means from very beginning; endUnix == 0 means to very end.
+func newFileRange(startUnix uint32, endUnix uint32) (*fileRange, error) {
+	if startUnix != 0 && endUnix != 0 && startUnix > endUnix {
+		return nil, ErrRange
+	}
+
+	one := &fileRange{
+		start: encodeFileFromUnix(startUnix),
+		end:   encodeFileFromUnix(endUnix),
+	}
+	// set to 0 if startUnix == 0
+	if startUnix == 0 {
+		one.start = fileEncode(0)
+	}
+	// set to max uint32 if endUnix == 0
+	if endUnix == 0 {
+		one.end = fileEncode(math.MaxUint32)
+	}
+	return one, nil
+}
+
+// folder should be the year-month folder like "201703"
+func (f *fileRange) folderInRange(folder string) (bool, error) {
+	v, err := strconv.Atoi(folder)
+	if err != nil {
+		return false, err
+	}
+
+	t := uint32(v)
+	if t >= uint32(f.start/1000) && t <= uint32(f.end/1000) {
+		return true, nil
+	}
+	return false, nil
+}
+
+// file should be the file encoded format like "201702135"
+func (f *fileRange) fileInRange(file fileEncode) (bool, error) {
+	t := uint32(file)
+	if t >= uint32(f.start) && t <= uint32(f.end) {
+		return true, nil
+	}
+	return false, nil
+}
+
+func encodeAliasAndFile(folder string, file fileEncode) string {
+	return fmt.Sprintf("%s%d", filepath.Base(folder), file)
+}

encode_test.go

@@ -0,0 +1,106 @@
+package tdb
+
+import (
+	"math"
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFileEncode(t *testing.T) {
+	encoded := encodeFile(2017, 4, 23, 5)
+	assert.Equal(t, fileEncode(201704230), encoded, "file encode wrong")
+	assert.True(t, encoded.isAM(), "should be in am")
+
+	encoded = encodeFile(2017, 12, 1, 23)
+	assert.Equal(t, fileEncode(201712015), encoded, "file encode wrong")
+
+	assert.Equal(t, 2017, encoded.year(), "encode year wrong")
+	assert.Equal(t, 12, encoded.month(), "encode month wrong")
+	assert.Equal(t, 1, encoded.day(), "encode month wrong")
+	assert.False(t, encoded.isAM(), "should be in pm")
+
+	folder, filename := encoded.path()
+	assert.Equal(t, "201712", folder, "folder is wrong")
+	assert.Equal(t, "015", filename, "folder is wrong")
+}
+
+func TestFileEncodeFromPath(t *testing.T) {
+	encoded, err := encodeFromPath("201705", "025.idx")
+	assert.NoError(t, err, "should have no error when encoding from path")
+	assert.Equal(t, fileEncode(201705025), encoded, "file encode wrong")
+
+	_, err = encodeFromPath("201705a", "025.idx")
+	assert.NotNil(t, err, "should have error when folder name length is not 6")
+
+	_, err = encodeFromPath("201705", "0025.idx")
+	assert.NotNil(t, err, "should have error when file name length is not 3")
+
+	_, err = encodeFromPath("20170a", "025.idx")
+	assert.NotNil(t, err, "bad folder name when encoding")
+
+	_, err = encodeFromPath("201705", "a25.idx")
+	assert.NotNil(t, err, "bad file name when encoding")
+}
+
+func TestSortingFileEncode(t *testing.T) {
+	var fileEncodes []fileEncode
+	fileEncodes = append(fileEncodes, fileEncode(3))
+	fileEncodes = append(fileEncodes, fileEncode(2))
+	fileEncodes = append(fileEncodes, fileEncode(1))
+
+	s := fileEncodeSlice(fileEncodes)
+	sort.Sort(s)
+
+	assert.Equal(t, fileEncode(1), fileEncodes[0], "first value wrong")
+	assert.Equal(t, fileEncode(2), fileEncodes[1], "second value wrong")
+	assert.Equal(t, fileEncode(3), fileEncodes[2], "third value wrong")
+}
+
+func TestFileRange(t *testing.T) {
+	fRange, err := newFileRange(0, 0)
+	assert.NoError(t, err, "error creating file range")
+	assert.Equal(t, fileEncode(0), fRange.start, "start should be 0")
+	assert.Equal(t, fileEncode(math.MaxUint32), fRange.end, "end should be the max uint32")
+
+	fRange, err = newFileRange(2, 1)
+	assert.Equal(t, ErrRange, err, "should have error when start > end")
+	assert.Nil(t, fRange, "filerange should be nil")
+
+	fRange, err = newFileRange(0, 0)
+	inRange, err := fRange.folderInRange("badfolder")
+	assert.False(t, inRange, "should not in range.")
+	assert.NotNil(t, err, "can't convert to int")
+
+	// from 201704020 to 201902255 (UTC)
+	fRange, err = newFileRange(1491134201, 1551134201)
+
+	// test folder range
+	inRange, err = fRange.folderInRange("201712")
+	assert.True(t, inRange, "should in range")
+	assert.NoError(t, err, "should have no error")
+
+	inRange, err = fRange.folderInRange("201903")
+	assert.False(t, inRange, "should not in range")
+	assert.NoError(t, err, "should have no error")
+
+	inRange, err = fRange.folderInRange("201705120")
+	assert.False(t, inRange, "should not in range")
+	assert.NoError(t, err, "should have no error")
+
+	// test file range
+	inRange, err = fRange.fileInRange(fileEncode(201801250))
+	assert.NoError(t, err, "should have no error")
+	assert.True(t, inRange, "should in range")
+
+	inRange, err = fRange.fileInRange(fileEncode(202001250))
+	assert.NoError(t, err, "should have no error")
+	assert.False(t, inRange, "should in range")
+}
+
+func TestEncodeAliasAndFile(t *testing.T) {
+	alias := "lkfj/abcdef"
+	file := fileEncode(201501205)
+	assert.Equal(t, "abcdef201501205", encodeAliasAndFile(alias, file), "encode wrong")
+}

errors.go

@@ -5,8 +5,12 @@ import (
 )
 
 var (
-	// ErrDBPathNotFolder path for TDB is not a folder
-	ErrDBPathNotFolder = errors.New("path for TDB is not a folder")
-	// ErrInfoFilePath path for info file is not a file
-	ErrInfoFilePath = errors.New("path for info file is not a file")
+	// ErrPathNotFolder path is not a folder
+	ErrPathNotFolder = errors.New("path for is not a folder")
+	// ErrPathNotFile path is not a file
+	ErrPathNotFile = errors.New("path is not a file")
+	// ErrNotExist something not exist
+	ErrNotExist = errors.New("not exist")
+	// ErrRange the range is wrong
+	ErrRange = errors.New("range is wrong")
 )

glide.yaml

@@ -3,6 +3,11 @@ import:
 - package: github.com/golang/protobuf
   subpackages:
   - proto
+- package: golang.org/x/sync
+  subpackages:
+  - errgroup
+- package: github.com/drkaka/ulid
+- package: github.com/tracerun/locker
 testImport:
 - package: github.com/stretchr/testify
   subpackages: