diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index ff21613586b5..0124e104d881 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1471,16 +1471,19 @@ PATH = ;; if the cache enabled ;ENABLED = true ;; -;; Either "memory", "redis", or "memcache", default is "memory" +;; Either "memory", "ledis", "redis", or "memcache", default is "memory" ;ADAPTER = memory ;; ;; For "memory" only, GC interval in seconds, default is 60 ;INTERVAL = 60 ;; -;; For "redis" and "memcache", connection host address +;; The same as ADAPTER_CONFIG but Deprecated +;HOST = +;; For "ledis", "redis" and "memcache", connection host address +;; ledis: data_dir=./data/caches,db=0 ;; redis: network=tcp,addr=:6379,password=macaron,db=0,pool_size=100,idle_timeout=180 ;; memcache: `127.0.0.1:11211` -;HOST = +;ADAPTER_CONFIG = ;; ;; Time to keep items in cache if not used, default is 16 hours. ;; Setting it to 0 disables caching diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 67b375481608..d64297b01fb0 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -590,9 +590,11 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type ## Cache (`cache`) - `ENABLED`: **true**: Enable the cache. -- `ADAPTER`: **memory**: Cache engine adapter, either `memory`, `redis`, or `memcache`. +- `ADAPTER`: **memory**: Cache engine adapter, either `memory`, `ledis`, `redis`, or `memcache`. - `INTERVAL`: **60**: Garbage Collection interval (sec), for memory cache only. -- `HOST`: **\**: Connection string for `redis` and `memcache`. +- `HOST`: DEPRECATED. The same as `ADAPTER_CONFIG`. +- `ADAPTER_CONFIG`: **\**: Connection string for `ledis`, `redis` and `memcache`. + - ledis: `data_dir=./data/caches,db=0`. It's a local disk cache. - Redis: `redis://:macaron@127.0.0.1:6379/0?pool_size=100&idle_timeout=180s` - Memcache: `127.0.0.1:9090;127.0.0.1:9091` - `ITEM_TTL`: **16h**: Time to keep items in cache if not used, Setting it to 0 disables caching. diff --git a/go.sum b/go.sum index 110e2e6834ef..fa42a9c6ffba 100644 --- a/go.sum +++ b/go.sum @@ -229,6 +229,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76 h1:Lgdd/Qp96Qj8jqLpq2cI1I1X7BJnu06efS+XkhRoLUQ= github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= @@ -263,6 +264,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/editorconfig/editorconfig-core-go/v2 v2.4.2 h1:1lkDpSoAaFLrgYTVJ/eNCV+lkDSv/j9Wm0jcvDfVVEo= github.com/editorconfig/editorconfig-core-go/v2 v2.4.2/go.mod h1:IXeWRVO4LZRoNunhHh/oP6BQvTs94nB2pNvbw32l8tQ= +github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= @@ -954,9 +956,12 @@ github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 h1:pXY9qYc/MP5zdvqWEUH6SjNiu7VhSjuVFTFiTcphaLU= github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= github.com/siddontang/go-snappy v0.0.0-20140704025258-d8f7bb82a96d/go.mod h1:vq0tzqLRu6TS7Id0wMo2N5QzJoKedVeovOpHjnykSzY= +github.com/siddontang/ledisdb v0.0.0-20190202134119-8ceb77e66a92 h1:qvsJwGToa8rxb42cDRhkbKeX2H5N8BH+s2aUikGt8mI= github.com/siddontang/ledisdb v0.0.0-20190202134119-8ceb77e66a92/go.mod h1:mF1DpOSOUiJRMR+FDqaqu3EBqrybQtrDDszLUZ6oxPg= +github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d h1:NVwnfyR3rENtlz62bcrkXME3INVUa4lcdGt+opvxExs= github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z92TR1JKMkLLoaOQk++LVnOKL3ScbJ8GNGA= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= diff --git a/modules/cache/cache.go b/modules/cache/cache.go index 3a2732c34366..4364f555c997 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -10,19 +10,20 @@ import ( "code.gitea.io/gitea/modules/setting" - mc "gitea.com/go-chi/cache" + "gitea.com/go-chi/cache" + _ "gitea.com/go-chi/cache/ledis" // ledis plugin for cache _ "gitea.com/go-chi/cache/memcache" // memcache plugin for cache ) var ( - conn mc.Cache + conn cache.Cache ) -func newCache(cacheConfig setting.Cache) (mc.Cache, error) { - return mc.NewCacher(mc.Options{ +func newCache(cacheConfig setting.Cache) (cache.Cache, error) { + return cache.NewCacher(cache.Options{ Adapter: cacheConfig.Adapter, - AdapterConfig: cacheConfig.Conn, + AdapterConfig: cacheConfig.AdapterConfig, Interval: cacheConfig.Interval, }) } @@ -41,7 +42,7 @@ func NewContext() error { } // GetCache returns the currently configured cache -func GetCache() mc.Cache { +func GetCache() cache.Cache { return conn } diff --git a/modules/setting/cache.go b/modules/setting/cache.go index 7bfea919618e..2208a5ec5f2d 100644 --- a/modules/setting/cache.go +++ b/modules/setting/cache.go @@ -13,11 +13,12 @@ import ( // Cache represents cache settings type Cache struct { - Enabled bool - Adapter string - Interval int - Conn string - TTL time.Duration `ini:"ITEM_TTL"` + Enabled bool + Adapter string + AdapterConfig string + Interval int + Conn string + TTL time.Duration `ini:"ITEM_TTL"` } var ( @@ -58,11 +59,16 @@ func newCacheService() { log.Fatal("Failed to map Cache settings: %v", err) } - CacheService.Adapter = sec.Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"}) + CacheService.Adapter = sec.Key("ADAPTER").In("memory", []string{"memory", "ledis", "redis", "memcache"}) + CacheService.AdapterConfig = sec.Key("ADAPTER_CONFIG").String() switch CacheService.Adapter { case "memory": + case "ledis": case "redis", "memcache": - CacheService.Conn = strings.Trim(sec.Key("HOST").String(), "\" ") + if CacheService.AdapterConfig == "" { + CacheService.AdapterConfig = sec.Key("HOST").String() + } + CacheService.AdapterConfig = strings.Trim(CacheService.AdapterConfig, "\" ") case "": // disable cache CacheService.Enabled = false default: diff --git a/vendor/gitea.com/go-chi/cache/ledis/ledis.go b/vendor/gitea.com/go-chi/cache/ledis/ledis.go new file mode 100644 index 000000000000..8111d2fcb17c --- /dev/null +++ b/vendor/gitea.com/go-chi/cache/ledis/ledis.go @@ -0,0 +1,185 @@ +// Copyright 2014 The Macaron Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package cache + +import ( + "fmt" + "log" + "strings" + "time" + + "github.com/siddontang/ledisdb/config" + "github.com/siddontang/ledisdb/ledis" + "github.com/unknwon/com" + "gopkg.in/ini.v1" + + "gitea.com/go-chi/cache" +) + +var defaultHSetName = []byte("MacaronCache") + +// LedisCacher represents a ledis cache adapter implementation. +type LedisCacher struct { + c *ledis.DB + interval int +} + +// Put puts value into cache with key and expire time. +// If expired is 0, it lives forever. +func (c *LedisCacher) Put(key string, val interface{}, expire int64) (err error) { + if expire == 0 { + if err = c.c.Set([]byte(key), []byte(com.ToStr(val))); err != nil { + return err + } + _, err = c.c.HSet([]byte(key), defaultHSetName, []byte("0")) + return err + } + + if err = c.c.SetEX([]byte(key), expire, []byte(com.ToStr(val))); err != nil { + return err + } + _, err = c.c.HSet([]byte(key), defaultHSetName, []byte(com.ToStr(time.Now().Add(time.Duration(expire)*time.Second).Unix()))) + return err +} + +// Get gets cached value by given key. +func (c *LedisCacher) Get(key string) interface{} { + val, err := c.c.Get([]byte(key)) + if err != nil || len(val) == 0 { + return nil + } + return string(val) +} + +// Delete deletes cached value by given key. +func (c *LedisCacher) Delete(key string) (err error) { + if _, err = c.c.Del([]byte(key)); err != nil { + return err + } + _, err = c.c.HDel(defaultHSetName, []byte(key)) + return err +} + +// Incr increases cached int-type value by given key as a counter. +func (c *LedisCacher) Incr(key string) error { + if !c.IsExist(key) { + return fmt.Errorf("key '%s' not exist", key) + } + _, err := c.c.Incr([]byte(key)) + return err +} + +// Decr decreases cached int-type value by given key as a counter. +func (c *LedisCacher) Decr(key string) error { + if !c.IsExist(key) { + return fmt.Errorf("key '%s' not exist", key) + } + _, err := c.c.Decr([]byte(key)) + return err +} + +// IsExist returns true if cached value exists. +func (c *LedisCacher) IsExist(key string) bool { + count, err := c.c.Exists([]byte(key)) + if err == nil && count > 0 { + return true + } + c.c.HDel(defaultHSetName, []byte(key)) + return false +} + +// Flush deletes all cached data. +func (c *LedisCacher) Flush() error { + // FIXME: there must be something wrong, shouldn't use this one. + _, err := c.c.FlushAll() + return err + + // keys, err := c.c.HKeys(defaultHSetName) + // if err != nil { + // return err + // } + // if _, err = c.c.Del(keys...); err != nil { + // return err + // } + // _, err = c.c.Del(defaultHSetName) + // return err +} + +func (c *LedisCacher) startGC() { + if c.interval < 1 { + return + } + + kvs, err := c.c.HGetAll(defaultHSetName) + if err != nil { + log.Printf("cache/redis: error garbage collecting(get): %v", err) + return + } + + now := time.Now().Unix() + for _, v := range kvs { + expire := com.StrTo(v.Value).MustInt64() + if expire == 0 || now < expire { + continue + } + + if err = c.Delete(string(v.Field)); err != nil { + log.Printf("cache/redis: error garbage collecting(delete): %v", err) + continue + } + } + + time.AfterFunc(time.Duration(c.interval)*time.Second, func() { c.startGC() }) +} + +// StartAndGC starts GC routine based on config string settings. +// AdapterConfig: data_dir=./app.db,db=0 +func (c *LedisCacher) StartAndGC(opts cache.Options) error { + c.interval = opts.Interval + + cfg, err := ini.Load([]byte(strings.Replace(opts.AdapterConfig, ",", "\n", -1))) + if err != nil { + return err + } + + db := 0 + opt := new(config.Config) + for k, v := range cfg.Section("").KeysHash() { + switch k { + case "data_dir": + opt.DataDir = v + case "db": + db = com.StrTo(v).MustInt() + default: + return fmt.Errorf("session/ledis: unsupported option '%s'", k) + } + } + + l, err := ledis.Open(opt) + if err != nil { + return fmt.Errorf("session/ledis: error opening db: %v", err) + } + c.c, err = l.Select(db) + if err != nil { + return err + } + + go c.startGC() + return nil +} + +func init() { + cache.Register("ledis", &LedisCacher{}) +} diff --git a/vendor/gitea.com/go-chi/cache/ledis/ledis.goconvey b/vendor/gitea.com/go-chi/cache/ledis/ledis.goconvey new file mode 100644 index 000000000000..8485e986e458 --- /dev/null +++ b/vendor/gitea.com/go-chi/cache/ledis/ledis.goconvey @@ -0,0 +1 @@ +ignore \ No newline at end of file diff --git a/vendor/github.com/cupcake/rdb/.gitignore b/vendor/github.com/cupcake/rdb/.gitignore new file mode 100644 index 000000000000..fcc1e666714f --- /dev/null +++ b/vendor/github.com/cupcake/rdb/.gitignore @@ -0,0 +1,25 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe + +# Project-specific files +diff diff --git a/vendor/github.com/cupcake/rdb/.travis.yml b/vendor/github.com/cupcake/rdb/.travis.yml new file mode 100644 index 000000000000..49c6fb8f79cd --- /dev/null +++ b/vendor/github.com/cupcake/rdb/.travis.yml @@ -0,0 +1,6 @@ +language: go +go: + - 1.1 + - tip +before_install: + - go get gopkg.in/check.v1 diff --git a/vendor/github.com/cupcake/rdb/LICENCE b/vendor/github.com/cupcake/rdb/LICENCE new file mode 100644 index 000000000000..50257901b124 --- /dev/null +++ b/vendor/github.com/cupcake/rdb/LICENCE @@ -0,0 +1,21 @@ +Copyright (c) 2012 Jonathan Rudenberg +Copyright (c) 2012 Sripathi Krishnan + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/cupcake/rdb/README.md b/vendor/github.com/cupcake/rdb/README.md new file mode 100644 index 000000000000..5c19212cd767 --- /dev/null +++ b/vendor/github.com/cupcake/rdb/README.md @@ -0,0 +1,17 @@ +# rdb [![Build Status](https://travis-ci.org/cupcake/rdb.png?branch=master)](https://travis-ci.org/cupcake/rdb) + +rdb is a Go package that implements parsing and encoding of the +[Redis](http://redis.io) [RDB file +format](https://github.com/sripathikrishnan/redis-rdb-tools/blob/master/docs/RDB_File_Format.textile). + +This package was heavily inspired by +[redis-rdb-tools](https://github.com/sripathikrishnan/redis-rdb-tools) by +[Sripathi Krishnan](https://github.com/sripathikrishnan). + +[**Documentation**](http://godoc.org/github.com/cupcake/rdb) + +## Installation + +``` +go get github.com/cupcake/rdb +``` diff --git a/vendor/github.com/cupcake/rdb/crc64/crc64.go b/vendor/github.com/cupcake/rdb/crc64/crc64.go new file mode 100644 index 000000000000..54fed9c5a291 --- /dev/null +++ b/vendor/github.com/cupcake/rdb/crc64/crc64.go @@ -0,0 +1,64 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package crc64 implements the Jones coefficients with an init value of 0. +package crc64 + +import "hash" + +// Redis uses the CRC64 variant with "Jones" coefficients and init value of 0. +// +// Specification of this CRC64 variant follows: +// Name: crc-64-jones +// Width: 64 bits +// Poly: 0xad93d23594c935a9 +// Reflected In: True +// Xor_In: 0xffffffffffffffff +// Reflected_Out: True +// Xor_Out: 0x0 + +var table = [256]uint64{0x0000000000000000, 0x7ad870c830358979, 0xf5b0e190606b12f2, 0x8f689158505e9b8b, 0xc038e5739841b68f, 0xbae095bba8743ff6, 0x358804e3f82aa47d, 0x4f50742bc81f2d04, 0xab28ecb46814fe75, 0xd1f09c7c5821770c, 0x5e980d24087fec87, 0x24407dec384a65fe, 0x6b1009c7f05548fa, 0x11c8790fc060c183, 0x9ea0e857903e5a08, 0xe478989fa00bd371, 0x7d08ff3b88be6f81, 0x07d08ff3b88be6f8, 0x88b81eabe8d57d73, 0xf2606e63d8e0f40a, 0xbd301a4810ffd90e, 0xc7e86a8020ca5077, 0x4880fbd87094cbfc, 0x32588b1040a14285, 0xd620138fe0aa91f4, 0xacf86347d09f188d, 0x2390f21f80c18306, 0x594882d7b0f40a7f, 0x1618f6fc78eb277b, 0x6cc0863448deae02, 0xe3a8176c18803589, 0x997067a428b5bcf0, 0xfa11fe77117cdf02, 0x80c98ebf2149567b, 0x0fa11fe77117cdf0, 0x75796f2f41224489, 0x3a291b04893d698d, 0x40f16bccb908e0f4, 0xcf99fa94e9567b7f, 0xb5418a5cd963f206, 0x513912c379682177, 0x2be1620b495da80e, 0xa489f35319033385, 0xde51839b2936bafc, 0x9101f7b0e12997f8, 0xebd98778d11c1e81, 0x64b116208142850a, 0x1e6966e8b1770c73, 0x8719014c99c2b083, 0xfdc17184a9f739fa, 0x72a9e0dcf9a9a271, 0x08719014c99c2b08, 0x4721e43f0183060c, 0x3df994f731b68f75, 0xb29105af61e814fe, 0xc849756751dd9d87, 0x2c31edf8f1d64ef6, 0x56e99d30c1e3c78f, 0xd9810c6891bd5c04, 0xa3597ca0a188d57d, 0xec09088b6997f879, 0x96d1784359a27100, 0x19b9e91b09fcea8b, 0x636199d339c963f2, 0xdf7adabd7a6e2d6f, 0xa5a2aa754a5ba416, 0x2aca3b2d1a053f9d, 0x50124be52a30b6e4, 0x1f423fcee22f9be0, 0x659a4f06d21a1299, 0xeaf2de5e82448912, 0x902aae96b271006b, 0x74523609127ad31a, 0x0e8a46c1224f5a63, 0x81e2d7997211c1e8, 0xfb3aa75142244891, 0xb46ad37a8a3b6595, 0xceb2a3b2ba0eecec, 0x41da32eaea507767, 0x3b024222da65fe1e, 0xa2722586f2d042ee, 0xd8aa554ec2e5cb97, 0x57c2c41692bb501c, 0x2d1ab4dea28ed965, 0x624ac0f56a91f461, 0x1892b03d5aa47d18, 0x97fa21650afae693, 0xed2251ad3acf6fea, 0x095ac9329ac4bc9b, 0x7382b9faaaf135e2, 0xfcea28a2faafae69, 0x8632586aca9a2710, 0xc9622c4102850a14, 0xb3ba5c8932b0836d, 0x3cd2cdd162ee18e6, 0x460abd1952db919f, 0x256b24ca6b12f26d, 0x5fb354025b277b14, 0xd0dbc55a0b79e09f, 0xaa03b5923b4c69e6, 0xe553c1b9f35344e2, 0x9f8bb171c366cd9b, 0x10e3202993385610, 0x6a3b50e1a30ddf69, 0x8e43c87e03060c18, 0xf49bb8b633338561, 0x7bf329ee636d1eea, 0x012b592653589793, 0x4e7b2d0d9b47ba97, 0x34a35dc5ab7233ee, 0xbbcbcc9dfb2ca865, 0xc113bc55cb19211c, 0x5863dbf1e3ac9dec, 0x22bbab39d3991495, 0xadd33a6183c78f1e, 0xd70b4aa9b3f20667, 0x985b3e827bed2b63, 0xe2834e4a4bd8a21a, 0x6debdf121b863991, 0x1733afda2bb3b0e8, 0xf34b37458bb86399, 0x8993478dbb8deae0, 0x06fbd6d5ebd3716b, 0x7c23a61ddbe6f812, 0x3373d23613f9d516, 0x49aba2fe23cc5c6f, 0xc6c333a67392c7e4, 0xbc1b436e43a74e9d, 0x95ac9329ac4bc9b5, 0xef74e3e19c7e40cc, 0x601c72b9cc20db47, 0x1ac40271fc15523e, 0x5594765a340a7f3a, 0x2f4c0692043ff643, 0xa02497ca54616dc8, 0xdafce7026454e4b1, 0x3e847f9dc45f37c0, 0x445c0f55f46abeb9, 0xcb349e0da4342532, 0xb1eceec59401ac4b, 0xfebc9aee5c1e814f, 0x8464ea266c2b0836, 0x0b0c7b7e3c7593bd, 0x71d40bb60c401ac4, 0xe8a46c1224f5a634, 0x927c1cda14c02f4d, 0x1d148d82449eb4c6, 0x67ccfd4a74ab3dbf, 0x289c8961bcb410bb, 0x5244f9a98c8199c2, 0xdd2c68f1dcdf0249, 0xa7f41839ecea8b30, 0x438c80a64ce15841, 0x3954f06e7cd4d138, 0xb63c61362c8a4ab3, 0xcce411fe1cbfc3ca, 0x83b465d5d4a0eece, 0xf96c151de49567b7, 0x76048445b4cbfc3c, 0x0cdcf48d84fe7545, 0x6fbd6d5ebd3716b7, 0x15651d968d029fce, 0x9a0d8ccedd5c0445, 0xe0d5fc06ed698d3c, 0xaf85882d2576a038, 0xd55df8e515432941, 0x5a3569bd451db2ca, 0x20ed197575283bb3, 0xc49581ead523e8c2, 0xbe4df122e51661bb, 0x3125607ab548fa30, 0x4bfd10b2857d7349, 0x04ad64994d625e4d, 0x7e7514517d57d734, 0xf11d85092d094cbf, 0x8bc5f5c11d3cc5c6, 0x12b5926535897936, 0x686de2ad05bcf04f, 0xe70573f555e26bc4, 0x9ddd033d65d7e2bd, 0xd28d7716adc8cfb9, 0xa85507de9dfd46c0, 0x273d9686cda3dd4b, 0x5de5e64efd965432, 0xb99d7ed15d9d8743, 0xc3450e196da80e3a, 0x4c2d9f413df695b1, 0x36f5ef890dc31cc8, 0x79a59ba2c5dc31cc, 0x037deb6af5e9b8b5, 0x8c157a32a5b7233e, 0xf6cd0afa9582aa47, 0x4ad64994d625e4da, 0x300e395ce6106da3, 0xbf66a804b64ef628, 0xc5bed8cc867b7f51, 0x8aeeace74e645255, 0xf036dc2f7e51db2c, 0x7f5e4d772e0f40a7, 0x05863dbf1e3ac9de, 0xe1fea520be311aaf, 0x9b26d5e88e0493d6, 0x144e44b0de5a085d, 0x6e963478ee6f8124, 0x21c640532670ac20, 0x5b1e309b16452559, 0xd476a1c3461bbed2, 0xaeaed10b762e37ab, 0x37deb6af5e9b8b5b, 0x4d06c6676eae0222, 0xc26e573f3ef099a9, 0xb8b627f70ec510d0, 0xf7e653dcc6da3dd4, 0x8d3e2314f6efb4ad, 0x0256b24ca6b12f26, 0x788ec2849684a65f, 0x9cf65a1b368f752e, 0xe62e2ad306bafc57, 0x6946bb8b56e467dc, 0x139ecb4366d1eea5, 0x5ccebf68aecec3a1, 0x2616cfa09efb4ad8, 0xa97e5ef8cea5d153, 0xd3a62e30fe90582a, 0xb0c7b7e3c7593bd8, 0xca1fc72bf76cb2a1, 0x45775673a732292a, 0x3faf26bb9707a053, 0x70ff52905f188d57, 0x0a2722586f2d042e, 0x854fb3003f739fa5, 0xff97c3c80f4616dc, 0x1bef5b57af4dc5ad, 0x61372b9f9f784cd4, 0xee5fbac7cf26d75f, 0x9487ca0fff135e26, 0xdbd7be24370c7322, 0xa10fceec0739fa5b, 0x2e675fb4576761d0, 0x54bf2f7c6752e8a9, 0xcdcf48d84fe75459, 0xb71738107fd2dd20, 0x387fa9482f8c46ab, 0x42a7d9801fb9cfd2, 0x0df7adabd7a6e2d6, 0x772fdd63e7936baf, 0xf8474c3bb7cdf024, 0x829f3cf387f8795d, 0x66e7a46c27f3aa2c, 0x1c3fd4a417c62355, 0x935745fc4798b8de, 0xe98f353477ad31a7, 0xa6df411fbfb21ca3, 0xdc0731d78f8795da, 0x536fa08fdfd90e51, 0x29b7d047efec8728} + +func crc64(crc uint64, b []byte) uint64 { + for _, v := range b { + crc = table[byte(crc)^v] ^ (crc >> 8) + } + return crc +} + +func Digest(b []byte) uint64 { + return crc64(0, b) +} + +type digest struct { + crc uint64 +} + +func New() hash.Hash64 { + return &digest{} +} + +func (h *digest) Write(p []byte) (int, error) { + h.crc = crc64(h.crc, p) + return len(p), nil +} + +// Encode in little endian +func (d *digest) Sum(in []byte) []byte { + s := d.Sum64() + in = append(in, byte(s)) + in = append(in, byte(s>>8)) + in = append(in, byte(s>>16)) + in = append(in, byte(s>>24)) + in = append(in, byte(s>>32)) + in = append(in, byte(s>>40)) + in = append(in, byte(s>>48)) + in = append(in, byte(s>>56)) + return in +} + +func (d *digest) Sum64() uint64 { return d.crc } +func (d *digest) BlockSize() int { return 1 } +func (d *digest) Size() int { return 8 } +func (d *digest) Reset() { d.crc = 0 } diff --git a/vendor/github.com/cupcake/rdb/decoder.go b/vendor/github.com/cupcake/rdb/decoder.go new file mode 100644 index 000000000000..dd3993b5cebc --- /dev/null +++ b/vendor/github.com/cupcake/rdb/decoder.go @@ -0,0 +1,824 @@ +// Package rdb implements parsing and encoding of the Redis RDB file format. +package rdb + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "io" + "math" + "strconv" + + "github.com/cupcake/rdb/crc64" +) + +// A Decoder must be implemented to parse a RDB file. +type Decoder interface { + // StartRDB is called when parsing of a valid RDB file starts. + StartRDB() + // StartDatabase is called when database n starts. + // Once a database starts, another database will not start until EndDatabase is called. + StartDatabase(n int) + // AUX field + Aux(key, value []byte) + // ResizeDB hint + ResizeDatabase(dbSize, expiresSize uint32) + // Set is called once for each string key. + Set(key, value []byte, expiry int64) + // StartHash is called at the beginning of a hash. + // Hset will be called exactly length times before EndHash. + StartHash(key []byte, length, expiry int64) + // Hset is called once for each field=value pair in a hash. + Hset(key, field, value []byte) + // EndHash is called when there are no more fields in a hash. + EndHash(key []byte) + // StartSet is called at the beginning of a set. + // Sadd will be called exactly cardinality times before EndSet. + StartSet(key []byte, cardinality, expiry int64) + // Sadd is called once for each member of a set. + Sadd(key, member []byte) + // EndSet is called when there are no more fields in a set. + EndSet(key []byte) + // StartList is called at the beginning of a list. + // Rpush will be called exactly length times before EndList. + // If length of the list is not known, then length is -1 + StartList(key []byte, length, expiry int64) + // Rpush is called once for each value in a list. + Rpush(key, value []byte) + // EndList is called when there are no more values in a list. + EndList(key []byte) + // StartZSet is called at the beginning of a sorted set. + // Zadd will be called exactly cardinality times before EndZSet. + StartZSet(key []byte, cardinality, expiry int64) + // Zadd is called once for each member of a sorted set. + Zadd(key []byte, score float64, member []byte) + // EndZSet is called when there are no more members in a sorted set. + EndZSet(key []byte) + // EndDatabase is called at the end of a database. + EndDatabase(n int) + // EndRDB is called when parsing of the RDB file is complete. + EndRDB() +} + +// Decode parses a RDB file from r and calls the decode hooks on d. +func Decode(r io.Reader, d Decoder) error { + decoder := &decode{d, make([]byte, 8), bufio.NewReader(r)} + return decoder.decode() +} + +// Decode a byte slice from the Redis DUMP command. The dump does not contain the +// database, key or expiry, so they must be included in the function call (but +// can be zero values). +func DecodeDump(dump []byte, db int, key []byte, expiry int64, d Decoder) error { + err := verifyDump(dump) + if err != nil { + return err + } + + decoder := &decode{d, make([]byte, 8), bytes.NewReader(dump[1:])} + decoder.event.StartRDB() + decoder.event.StartDatabase(db) + + err = decoder.readObject(key, ValueType(dump[0]), expiry) + + decoder.event.EndDatabase(db) + decoder.event.EndRDB() + return err +} + +type byteReader interface { + io.Reader + io.ByteReader +} + +type decode struct { + event Decoder + intBuf []byte + r byteReader +} + +type ValueType byte + +const ( + TypeString ValueType = 0 + TypeList ValueType = 1 + TypeSet ValueType = 2 + TypeZSet ValueType = 3 + TypeHash ValueType = 4 + + TypeHashZipmap ValueType = 9 + TypeListZiplist ValueType = 10 + TypeSetIntset ValueType = 11 + TypeZSetZiplist ValueType = 12 + TypeHashZiplist ValueType = 13 + TypeListQuicklist ValueType = 14 +) + +const ( + rdb6bitLen = 0 + rdb14bitLen = 1 + rdb32bitLen = 2 + rdbEncVal = 3 + + rdbFlagAux = 0xfa + rdbFlagResizeDB = 0xfb + rdbFlagExpiryMS = 0xfc + rdbFlagExpiry = 0xfd + rdbFlagSelectDB = 0xfe + rdbFlagEOF = 0xff + + rdbEncInt8 = 0 + rdbEncInt16 = 1 + rdbEncInt32 = 2 + rdbEncLZF = 3 + + rdbZiplist6bitlenString = 0 + rdbZiplist14bitlenString = 1 + rdbZiplist32bitlenString = 2 + + rdbZiplistInt16 = 0xc0 + rdbZiplistInt32 = 0xd0 + rdbZiplistInt64 = 0xe0 + rdbZiplistInt24 = 0xf0 + rdbZiplistInt8 = 0xfe + rdbZiplistInt4 = 15 +) + +func (d *decode) decode() error { + err := d.checkHeader() + if err != nil { + return err + } + + d.event.StartRDB() + + var db uint32 + var expiry int64 + firstDB := true + for { + objType, err := d.r.ReadByte() + if err != nil { + return err + } + switch objType { + case rdbFlagAux: + auxKey, err := d.readString() + if err != nil { + return err + } + auxVal, err := d.readString() + if err != nil { + return err + } + d.event.Aux(auxKey, auxVal) + case rdbFlagResizeDB: + dbSize, _, err := d.readLength() + if err != nil { + return err + } + expiresSize, _, err := d.readLength() + if err != nil { + return err + } + d.event.ResizeDatabase(dbSize, expiresSize) + case rdbFlagExpiryMS: + _, err := io.ReadFull(d.r, d.intBuf) + if err != nil { + return err + } + expiry = int64(binary.LittleEndian.Uint64(d.intBuf)) + case rdbFlagExpiry: + _, err := io.ReadFull(d.r, d.intBuf[:4]) + if err != nil { + return err + } + expiry = int64(binary.LittleEndian.Uint32(d.intBuf)) * 1000 + case rdbFlagSelectDB: + if !firstDB { + d.event.EndDatabase(int(db)) + } + db, _, err = d.readLength() + if err != nil { + return err + } + d.event.StartDatabase(int(db)) + case rdbFlagEOF: + d.event.EndDatabase(int(db)) + d.event.EndRDB() + return nil + default: + key, err := d.readString() + if err != nil { + return err + } + err = d.readObject(key, ValueType(objType), expiry) + if err != nil { + return err + } + expiry = 0 + } + } + + panic("not reached") +} + +func (d *decode) readObject(key []byte, typ ValueType, expiry int64) error { + switch typ { + case TypeString: + value, err := d.readString() + if err != nil { + return err + } + d.event.Set(key, value, expiry) + case TypeList: + length, _, err := d.readLength() + if err != nil { + return err + } + d.event.StartList(key, int64(length), expiry) + for i := uint32(0); i < length; i++ { + value, err := d.readString() + if err != nil { + return err + } + d.event.Rpush(key, value) + } + d.event.EndList(key) + case TypeListQuicklist: + length, _, err := d.readLength() + if err != nil { + return err + } + d.event.StartList(key, int64(-1), expiry) + for i := uint32(0); i < length; i++ { + d.readZiplist(key, 0, false) + } + d.event.EndList(key) + case TypeSet: + cardinality, _, err := d.readLength() + if err != nil { + return err + } + d.event.StartSet(key, int64(cardinality), expiry) + for i := uint32(0); i < cardinality; i++ { + member, err := d.readString() + if err != nil { + return err + } + d.event.Sadd(key, member) + } + d.event.EndSet(key) + case TypeZSet: + cardinality, _, err := d.readLength() + if err != nil { + return err + } + d.event.StartZSet(key, int64(cardinality), expiry) + for i := uint32(0); i < cardinality; i++ { + member, err := d.readString() + if err != nil { + return err + } + score, err := d.readFloat64() + if err != nil { + return err + } + d.event.Zadd(key, score, member) + } + d.event.EndZSet(key) + case TypeHash: + length, _, err := d.readLength() + if err != nil { + return err + } + d.event.StartHash(key, int64(length), expiry) + for i := uint32(0); i < length; i++ { + field, err := d.readString() + if err != nil { + return err + } + value, err := d.readString() + if err != nil { + return err + } + d.event.Hset(key, field, value) + } + d.event.EndHash(key) + case TypeHashZipmap: + return d.readZipmap(key, expiry) + case TypeListZiplist: + return d.readZiplist(key, expiry, true) + case TypeSetIntset: + return d.readIntset(key, expiry) + case TypeZSetZiplist: + return d.readZiplistZset(key, expiry) + case TypeHashZiplist: + return d.readZiplistHash(key, expiry) + default: + return fmt.Errorf("rdb: unknown object type %d for key %s", typ, key) + } + return nil +} + +func (d *decode) readZipmap(key []byte, expiry int64) error { + var length int + zipmap, err := d.readString() + if err != nil { + return err + } + buf := newSliceBuffer(zipmap) + lenByte, err := buf.ReadByte() + if err != nil { + return err + } + if lenByte >= 254 { // we need to count the items manually + length, err = countZipmapItems(buf) + length /= 2 + if err != nil { + return err + } + } else { + length = int(lenByte) + } + d.event.StartHash(key, int64(length), expiry) + for i := 0; i < length; i++ { + field, err := readZipmapItem(buf, false) + if err != nil { + return err + } + value, err := readZipmapItem(buf, true) + if err != nil { + return err + } + d.event.Hset(key, field, value) + } + d.event.EndHash(key) + return nil +} + +func readZipmapItem(buf *sliceBuffer, readFree bool) ([]byte, error) { + length, free, err := readZipmapItemLength(buf, readFree) + if err != nil { + return nil, err + } + if length == -1 { + return nil, nil + } + value, err := buf.Slice(length) + if err != nil { + return nil, err + } + _, err = buf.Seek(int64(free), 1) + return value, err +} + +func countZipmapItems(buf *sliceBuffer) (int, error) { + n := 0 + for { + strLen, free, err := readZipmapItemLength(buf, n%2 != 0) + if err != nil { + return 0, err + } + if strLen == -1 { + break + } + _, err = buf.Seek(int64(strLen)+int64(free), 1) + if err != nil { + return 0, err + } + n++ + } + _, err := buf.Seek(0, 0) + return n, err +} + +func readZipmapItemLength(buf *sliceBuffer, readFree bool) (int, int, error) { + b, err := buf.ReadByte() + if err != nil { + return 0, 0, err + } + switch b { + case 253: + s, err := buf.Slice(5) + if err != nil { + return 0, 0, err + } + return int(binary.BigEndian.Uint32(s)), int(s[4]), nil + case 254: + return 0, 0, fmt.Errorf("rdb: invalid zipmap item length") + case 255: + return -1, 0, nil + } + var free byte + if readFree { + free, err = buf.ReadByte() + } + return int(b), int(free), err +} + +func (d *decode) readZiplist(key []byte, expiry int64, addListEvents bool) error { + ziplist, err := d.readString() + if err != nil { + return err + } + buf := newSliceBuffer(ziplist) + length, err := readZiplistLength(buf) + if err != nil { + return err + } + if addListEvents { + d.event.StartList(key, length, expiry) + } + for i := int64(0); i < length; i++ { + entry, err := readZiplistEntry(buf) + if err != nil { + return err + } + d.event.Rpush(key, entry) + } + if addListEvents { + d.event.EndList(key) + } + return nil +} + +func (d *decode) readZiplistZset(key []byte, expiry int64) error { + ziplist, err := d.readString() + if err != nil { + return err + } + buf := newSliceBuffer(ziplist) + cardinality, err := readZiplistLength(buf) + if err != nil { + return err + } + cardinality /= 2 + d.event.StartZSet(key, cardinality, expiry) + for i := int64(0); i < cardinality; i++ { + member, err := readZiplistEntry(buf) + if err != nil { + return err + } + scoreBytes, err := readZiplistEntry(buf) + if err != nil { + return err + } + score, err := strconv.ParseFloat(string(scoreBytes), 64) + if err != nil { + return err + } + d.event.Zadd(key, score, member) + } + d.event.EndZSet(key) + return nil +} + +func (d *decode) readZiplistHash(key []byte, expiry int64) error { + ziplist, err := d.readString() + if err != nil { + return err + } + buf := newSliceBuffer(ziplist) + length, err := readZiplistLength(buf) + if err != nil { + return err + } + length /= 2 + d.event.StartHash(key, length, expiry) + for i := int64(0); i < length; i++ { + field, err := readZiplistEntry(buf) + if err != nil { + return err + } + value, err := readZiplistEntry(buf) + if err != nil { + return err + } + d.event.Hset(key, field, value) + } + d.event.EndHash(key) + return nil +} + +func readZiplistLength(buf *sliceBuffer) (int64, error) { + buf.Seek(8, 0) // skip the zlbytes and zltail + lenBytes, err := buf.Slice(2) + if err != nil { + return 0, err + } + return int64(binary.LittleEndian.Uint16(lenBytes)), nil +} + +func readZiplistEntry(buf *sliceBuffer) ([]byte, error) { + prevLen, err := buf.ReadByte() + if err != nil { + return nil, err + } + if prevLen == 254 { + buf.Seek(4, 1) // skip the 4-byte prevlen + } + + header, err := buf.ReadByte() + if err != nil { + return nil, err + } + switch { + case header>>6 == rdbZiplist6bitlenString: + return buf.Slice(int(header & 0x3f)) + case header>>6 == rdbZiplist14bitlenString: + b, err := buf.ReadByte() + if err != nil { + return nil, err + } + return buf.Slice((int(header&0x3f) << 8) | int(b)) + case header>>6 == rdbZiplist32bitlenString: + lenBytes, err := buf.Slice(4) + if err != nil { + return nil, err + } + return buf.Slice(int(binary.BigEndian.Uint32(lenBytes))) + case header == rdbZiplistInt16: + intBytes, err := buf.Slice(2) + if err != nil { + return nil, err + } + return []byte(strconv.FormatInt(int64(int16(binary.LittleEndian.Uint16(intBytes))), 10)), nil + case header == rdbZiplistInt32: + intBytes, err := buf.Slice(4) + if err != nil { + return nil, err + } + return []byte(strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))), 10)), nil + case header == rdbZiplistInt64: + intBytes, err := buf.Slice(8) + if err != nil { + return nil, err + } + return []byte(strconv.FormatInt(int64(binary.LittleEndian.Uint64(intBytes)), 10)), nil + case header == rdbZiplistInt24: + intBytes := make([]byte, 4) + _, err := buf.Read(intBytes[1:]) + if err != nil { + return nil, err + } + return []byte(strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))>>8), 10)), nil + case header == rdbZiplistInt8: + b, err := buf.ReadByte() + return []byte(strconv.FormatInt(int64(int8(b)), 10)), err + case header>>4 == rdbZiplistInt4: + return []byte(strconv.FormatInt(int64(header&0x0f)-1, 10)), nil + } + + return nil, fmt.Errorf("rdb: unknown ziplist header byte: %d", header) +} + +func (d *decode) readIntset(key []byte, expiry int64) error { + intset, err := d.readString() + if err != nil { + return err + } + buf := newSliceBuffer(intset) + intSizeBytes, err := buf.Slice(4) + if err != nil { + return err + } + intSize := binary.LittleEndian.Uint32(intSizeBytes) + + if intSize != 2 && intSize != 4 && intSize != 8 { + return fmt.Errorf("rdb: unknown intset encoding: %d", intSize) + } + + lenBytes, err := buf.Slice(4) + if err != nil { + return err + } + cardinality := binary.LittleEndian.Uint32(lenBytes) + + d.event.StartSet(key, int64(cardinality), expiry) + for i := uint32(0); i < cardinality; i++ { + intBytes, err := buf.Slice(int(intSize)) + if err != nil { + return err + } + var intString string + switch intSize { + case 2: + intString = strconv.FormatInt(int64(int16(binary.LittleEndian.Uint16(intBytes))), 10) + case 4: + intString = strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))), 10) + case 8: + intString = strconv.FormatInt(int64(int64(binary.LittleEndian.Uint64(intBytes))), 10) + } + d.event.Sadd(key, []byte(intString)) + } + d.event.EndSet(key) + return nil +} + +func (d *decode) checkHeader() error { + header := make([]byte, 9) + _, err := io.ReadFull(d.r, header) + if err != nil { + return err + } + + if !bytes.Equal(header[:5], []byte("REDIS")) { + return fmt.Errorf("rdb: invalid file format") + } + + version, _ := strconv.ParseInt(string(header[5:]), 10, 64) + if version < 1 || version > 7 { + return fmt.Errorf("rdb: invalid RDB version number %d", version) + } + + return nil +} + +func (d *decode) readString() ([]byte, error) { + length, encoded, err := d.readLength() + if err != nil { + return nil, err + } + if encoded { + switch length { + case rdbEncInt8: + i, err := d.readUint8() + return []byte(strconv.FormatInt(int64(int8(i)), 10)), err + case rdbEncInt16: + i, err := d.readUint16() + return []byte(strconv.FormatInt(int64(int16(i)), 10)), err + case rdbEncInt32: + i, err := d.readUint32() + return []byte(strconv.FormatInt(int64(int32(i)), 10)), err + case rdbEncLZF: + clen, _, err := d.readLength() + if err != nil { + return nil, err + } + ulen, _, err := d.readLength() + if err != nil { + return nil, err + } + compressed := make([]byte, clen) + _, err = io.ReadFull(d.r, compressed) + if err != nil { + return nil, err + } + decompressed := lzfDecompress(compressed, int(ulen)) + if len(decompressed) != int(ulen) { + return nil, fmt.Errorf("decompressed string length %d didn't match expected length %d", len(decompressed), ulen) + } + return decompressed, nil + } + } + + str := make([]byte, length) + _, err = io.ReadFull(d.r, str) + return str, err +} + +func (d *decode) readUint8() (uint8, error) { + b, err := d.r.ReadByte() + return uint8(b), err +} + +func (d *decode) readUint16() (uint16, error) { + _, err := io.ReadFull(d.r, d.intBuf[:2]) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint16(d.intBuf), nil +} + +func (d *decode) readUint32() (uint32, error) { + _, err := io.ReadFull(d.r, d.intBuf[:4]) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint32(d.intBuf), nil +} + +func (d *decode) readUint64() (uint64, error) { + _, err := io.ReadFull(d.r, d.intBuf) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint64(d.intBuf), nil +} + +func (d *decode) readUint32Big() (uint32, error) { + _, err := io.ReadFull(d.r, d.intBuf[:4]) + if err != nil { + return 0, err + } + return binary.BigEndian.Uint32(d.intBuf), nil +} + +// Doubles are saved as strings prefixed by an unsigned +// 8 bit integer specifying the length of the representation. +// This 8 bit integer has special values in order to specify the following +// conditions: +// 253: not a number +// 254: + inf +// 255: - inf +func (d *decode) readFloat64() (float64, error) { + length, err := d.readUint8() + if err != nil { + return 0, err + } + switch length { + case 253: + return math.NaN(), nil + case 254: + return math.Inf(0), nil + case 255: + return math.Inf(-1), nil + default: + floatBytes := make([]byte, length) + _, err := io.ReadFull(d.r, floatBytes) + if err != nil { + return 0, err + } + f, err := strconv.ParseFloat(string(floatBytes), 64) + return f, err + } + + panic("not reached") +} + +func (d *decode) readLength() (uint32, bool, error) { + b, err := d.r.ReadByte() + if err != nil { + return 0, false, err + } + // The first two bits of the first byte are used to indicate the length encoding type + switch (b & 0xc0) >> 6 { + case rdb6bitLen: + // When the first two bits are 00, the next 6 bits are the length. + return uint32(b & 0x3f), false, nil + case rdb14bitLen: + // When the first two bits are 01, the next 14 bits are the length. + bb, err := d.r.ReadByte() + if err != nil { + return 0, false, err + } + return (uint32(b&0x3f) << 8) | uint32(bb), false, nil + case rdbEncVal: + // When the first two bits are 11, the next object is encoded. + // The next 6 bits indicate the encoding type. + return uint32(b & 0x3f), true, nil + default: + // When the first two bits are 10, the next 6 bits are discarded. + // The next 4 bytes are the length. + length, err := d.readUint32Big() + return length, false, err + } + + panic("not reached") +} + +func verifyDump(d []byte) error { + if len(d) < 10 { + return fmt.Errorf("rdb: invalid dump length") + } + version := binary.LittleEndian.Uint16(d[len(d)-10:]) + if version != uint16(Version) { + return fmt.Errorf("rdb: invalid version %d, expecting %d", version, Version) + } + + if binary.LittleEndian.Uint64(d[len(d)-8:]) != crc64.Digest(d[:len(d)-8]) { + return fmt.Errorf("rdb: invalid CRC checksum") + } + + return nil +} + +func lzfDecompress(in []byte, outlen int) []byte { + out := make([]byte, outlen) + for i, o := 0, 0; i < len(in); { + ctrl := int(in[i]) + i++ + if ctrl < 32 { + for x := 0; x <= ctrl; x++ { + out[o] = in[i] + i++ + o++ + } + } else { + length := ctrl >> 5 + if length == 7 { + length = length + int(in[i]) + i++ + } + ref := o - ((ctrl & 0x1f) << 8) - int(in[i]) - 1 + i++ + for x := 0; x <= length+1; x++ { + out[o] = out[ref] + ref++ + o++ + } + } + } + return out +} diff --git a/vendor/github.com/cupcake/rdb/encoder.go b/vendor/github.com/cupcake/rdb/encoder.go new file mode 100644 index 000000000000..7902a7d314d7 --- /dev/null +++ b/vendor/github.com/cupcake/rdb/encoder.go @@ -0,0 +1,130 @@ +package rdb + +import ( + "encoding/binary" + "fmt" + "hash" + "io" + "math" + "strconv" + + "github.com/cupcake/rdb/crc64" +) + +const Version = 6 + +type Encoder struct { + w io.Writer + crc hash.Hash +} + +func NewEncoder(w io.Writer) *Encoder { + e := &Encoder{crc: crc64.New()} + e.w = io.MultiWriter(w, e.crc) + return e +} + +func (e *Encoder) EncodeHeader() error { + _, err := fmt.Fprintf(e.w, "REDIS%04d", Version) + return err +} + +func (e *Encoder) EncodeFooter() error { + e.w.Write([]byte{rdbFlagEOF}) + _, err := e.w.Write(e.crc.Sum(nil)) + return err +} + +func (e *Encoder) EncodeDumpFooter() error { + binary.Write(e.w, binary.LittleEndian, uint16(Version)) + _, err := e.w.Write(e.crc.Sum(nil)) + return err +} + +func (e *Encoder) EncodeDatabase(n int) error { + e.w.Write([]byte{rdbFlagSelectDB}) + return e.EncodeLength(uint32(n)) +} + +func (e *Encoder) EncodeExpiry(expiry uint64) error { + b := make([]byte, 9) + b[0] = rdbFlagExpiryMS + binary.LittleEndian.PutUint64(b[1:], expiry) + _, err := e.w.Write(b) + return err +} + +func (e *Encoder) EncodeType(v ValueType) error { + _, err := e.w.Write([]byte{byte(v)}) + return err +} + +func (e *Encoder) EncodeString(s []byte) error { + written, err := e.encodeIntString(s) + if written { + return err + } + e.EncodeLength(uint32(len(s))) + _, err = e.w.Write(s) + return err +} + +func (e *Encoder) EncodeLength(l uint32) (err error) { + switch { + case l < 1<<6: + _, err = e.w.Write([]byte{byte(l)}) + case l < 1<<14: + _, err = e.w.Write([]byte{byte(l>>8) | rdb14bitLen<<6, byte(l)}) + default: + b := make([]byte, 5) + b[0] = rdb32bitLen << 6 + binary.BigEndian.PutUint32(b[1:], l) + _, err = e.w.Write(b) + } + return +} + +func (e *Encoder) EncodeFloat(f float64) (err error) { + switch { + case math.IsNaN(f): + _, err = e.w.Write([]byte{253}) + case math.IsInf(f, 1): + _, err = e.w.Write([]byte{254}) + case math.IsInf(f, -1): + _, err = e.w.Write([]byte{255}) + default: + b := []byte(strconv.FormatFloat(f, 'g', 17, 64)) + e.w.Write([]byte{byte(len(b))}) + _, err = e.w.Write(b) + } + return +} + +func (e *Encoder) encodeIntString(b []byte) (written bool, err error) { + s := string(b) + i, err := strconv.ParseInt(s, 10, 32) + if err != nil { + return + } + // if the stringified parsed int isn't exactly the same, we can't encode it as an int + if s != strconv.FormatInt(i, 10) { + return + } + switch { + case i >= math.MinInt8 && i <= math.MaxInt8: + _, err = e.w.Write([]byte{rdbEncVal << 6, byte(int8(i))}) + case i >= math.MinInt16 && i <= math.MaxInt16: + b := make([]byte, 3) + b[0] = rdbEncVal<<6 | rdbEncInt16 + binary.LittleEndian.PutUint16(b[1:], uint16(int16(i))) + _, err = e.w.Write(b) + case i >= math.MinInt32 && i <= math.MaxInt32: + b := make([]byte, 5) + b[0] = rdbEncVal<<6 | rdbEncInt32 + binary.LittleEndian.PutUint32(b[1:], uint32(int32(i))) + _, err = e.w.Write(b) + default: + return + } + return true, err +} diff --git a/vendor/github.com/cupcake/rdb/nopdecoder/nop_decoder.go b/vendor/github.com/cupcake/rdb/nopdecoder/nop_decoder.go new file mode 100644 index 000000000000..de93a6973fed --- /dev/null +++ b/vendor/github.com/cupcake/rdb/nopdecoder/nop_decoder.go @@ -0,0 +1,24 @@ +package nopdecoder + +// NopDecoder may be embedded in a real Decoder to avoid implementing methods. +type NopDecoder struct{} + +func (d NopDecoder) StartRDB() {} +func (d NopDecoder) StartDatabase(n int) {} +func (d NopDecoder) Aux(key, value []byte) {} +func (d NopDecoder) ResizeDatabase(dbSize, expiresSize uint32) {} +func (d NopDecoder) EndDatabase(n int) {} +func (d NopDecoder) EndRDB() {} +func (d NopDecoder) Set(key, value []byte, expiry int64) {} +func (d NopDecoder) StartHash(key []byte, length, expiry int64) {} +func (d NopDecoder) Hset(key, field, value []byte) {} +func (d NopDecoder) EndHash(key []byte) {} +func (d NopDecoder) StartSet(key []byte, cardinality, expiry int64) {} +func (d NopDecoder) Sadd(key, member []byte) {} +func (d NopDecoder) EndSet(key []byte) {} +func (d NopDecoder) StartList(key []byte, length, expiry int64) {} +func (d NopDecoder) Rpush(key, value []byte) {} +func (d NopDecoder) EndList(key []byte) {} +func (d NopDecoder) StartZSet(key []byte, cardinality, expiry int64) {} +func (d NopDecoder) Zadd(key []byte, score float64, member []byte) {} +func (d NopDecoder) EndZSet(key []byte) {} diff --git a/vendor/github.com/cupcake/rdb/slice_buffer.go b/vendor/github.com/cupcake/rdb/slice_buffer.go new file mode 100644 index 000000000000..b3e12a02c6c0 --- /dev/null +++ b/vendor/github.com/cupcake/rdb/slice_buffer.go @@ -0,0 +1,67 @@ +package rdb + +import ( + "errors" + "io" +) + +type sliceBuffer struct { + s []byte + i int +} + +func newSliceBuffer(s []byte) *sliceBuffer { + return &sliceBuffer{s, 0} +} + +func (s *sliceBuffer) Slice(n int) ([]byte, error) { + if s.i+n > len(s.s) { + return nil, io.EOF + } + b := s.s[s.i : s.i+n] + s.i += n + return b, nil +} + +func (s *sliceBuffer) ReadByte() (byte, error) { + if s.i >= len(s.s) { + return 0, io.EOF + } + b := s.s[s.i] + s.i++ + return b, nil +} + +func (s *sliceBuffer) Read(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + if s.i >= len(s.s) { + return 0, io.EOF + } + n := copy(b, s.s[s.i:]) + s.i += n + return n, nil +} + +func (s *sliceBuffer) Seek(offset int64, whence int) (int64, error) { + var abs int64 + switch whence { + case 0: + abs = offset + case 1: + abs = int64(s.i) + offset + case 2: + abs = int64(len(s.s)) + offset + default: + return 0, errors.New("invalid whence") + } + if abs < 0 { + return 0, errors.New("negative position") + } + if abs >= 1<<31 { + return 0, errors.New("position out of range") + } + s.i = int(abs) + return abs, nil +} diff --git a/vendor/github.com/edsrzf/mmap-go/.gitignore b/vendor/github.com/edsrzf/mmap-go/.gitignore new file mode 100644 index 000000000000..9aa02c1ed3d0 --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/.gitignore @@ -0,0 +1,8 @@ +*.out +*.5 +*.6 +*.8 +*.swp +_obj +_test +testdata diff --git a/vendor/github.com/edsrzf/mmap-go/LICENSE b/vendor/github.com/edsrzf/mmap-go/LICENSE new file mode 100644 index 000000000000..8f05f338ac5a --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2011, Evan Shaw +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/vendor/github.com/edsrzf/mmap-go/README.md b/vendor/github.com/edsrzf/mmap-go/README.md new file mode 100644 index 000000000000..4cc2bfe1c8ce --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/README.md @@ -0,0 +1,12 @@ +mmap-go +======= + +mmap-go is a portable mmap package for the [Go programming language](http://golang.org). +It has been tested on Linux (386, amd64), OS X, and Windows (386). It should also +work on other Unix-like platforms, but hasn't been tested with them. I'm interested +to hear about the results. + +I haven't been able to add more features without adding significant complexity, +so mmap-go doesn't support mprotect, mincore, and maybe a few other things. +If you're running on a Unix-like platform and need some of these features, +I suggest Gustavo Niemeyer's [gommap](http://labix.org/gommap). diff --git a/vendor/github.com/edsrzf/mmap-go/mmap.go b/vendor/github.com/edsrzf/mmap-go/mmap.go new file mode 100644 index 000000000000..29655bd222ca --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/mmap.go @@ -0,0 +1,117 @@ +// Copyright 2011 Evan Shaw. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file defines the common package interface and contains a little bit of +// factored out logic. + +// Package mmap allows mapping files into memory. It tries to provide a simple, reasonably portable interface, +// but doesn't go out of its way to abstract away every little platform detail. +// This specifically means: +// * forked processes may or may not inherit mappings +// * a file's timestamp may or may not be updated by writes through mappings +// * specifying a size larger than the file's actual size can increase the file's size +// * If the mapped file is being modified by another process while your program's running, don't expect consistent results between platforms +package mmap + +import ( + "errors" + "os" + "reflect" + "unsafe" +) + +const ( + // RDONLY maps the memory read-only. + // Attempts to write to the MMap object will result in undefined behavior. + RDONLY = 0 + // RDWR maps the memory as read-write. Writes to the MMap object will update the + // underlying file. + RDWR = 1 << iota + // COPY maps the memory as copy-on-write. Writes to the MMap object will affect + // memory, but the underlying file will remain unchanged. + COPY + // If EXEC is set, the mapped memory is marked as executable. + EXEC +) + +const ( + // If the ANON flag is set, the mapped memory will not be backed by a file. + ANON = 1 << iota +) + +// MMap represents a file mapped into memory. +type MMap []byte + +// Map maps an entire file into memory. +// If ANON is set in flags, f is ignored. +func Map(f *os.File, prot, flags int) (MMap, error) { + return MapRegion(f, -1, prot, flags, 0) +} + +// MapRegion maps part of a file into memory. +// The offset parameter must be a multiple of the system's page size. +// If length < 0, the entire file will be mapped. +// If ANON is set in flags, f is ignored. +func MapRegion(f *os.File, length int, prot, flags int, offset int64) (MMap, error) { + if offset%int64(os.Getpagesize()) != 0 { + return nil, errors.New("offset parameter must be a multiple of the system's page size") + } + + var fd uintptr + if flags&ANON == 0 { + fd = uintptr(f.Fd()) + if length < 0 { + fi, err := f.Stat() + if err != nil { + return nil, err + } + length = int(fi.Size()) + } + } else { + if length <= 0 { + return nil, errors.New("anonymous mapping requires non-zero length") + } + fd = ^uintptr(0) + } + return mmap(length, uintptr(prot), uintptr(flags), fd, offset) +} + +func (m *MMap) header() *reflect.SliceHeader { + return (*reflect.SliceHeader)(unsafe.Pointer(m)) +} + +func (m *MMap) addrLen() (uintptr, uintptr) { + header := m.header() + return header.Data, uintptr(header.Len) +} + +// Lock keeps the mapped region in physical memory, ensuring that it will not be +// swapped out. +func (m MMap) Lock() error { + return m.lock() +} + +// Unlock reverses the effect of Lock, allowing the mapped region to potentially +// be swapped out. +// If m is already unlocked, aan error will result. +func (m MMap) Unlock() error { + return m.unlock() +} + +// Flush synchronizes the mapping's contents to the file's contents on disk. +func (m MMap) Flush() error { + return m.flush() +} + +// Unmap deletes the memory mapped region, flushes any remaining changes, and sets +// m to nil. +// Trying to read or write any remaining references to m after Unmap is called will +// result in undefined behavior. +// Unmap should only be called on the slice value that was originally returned from +// a call to Map. Calling Unmap on a derived slice may cause errors. +func (m *MMap) Unmap() error { + err := m.unmap() + *m = nil + return err +} diff --git a/vendor/github.com/edsrzf/mmap-go/mmap_unix.go b/vendor/github.com/edsrzf/mmap-go/mmap_unix.go new file mode 100644 index 000000000000..25b13e51fdff --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/mmap_unix.go @@ -0,0 +1,51 @@ +// Copyright 2011 Evan Shaw. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux openbsd solaris netbsd + +package mmap + +import ( + "golang.org/x/sys/unix" +) + +func mmap(len int, inprot, inflags, fd uintptr, off int64) ([]byte, error) { + flags := unix.MAP_SHARED + prot := unix.PROT_READ + switch { + case inprot© != 0: + prot |= unix.PROT_WRITE + flags = unix.MAP_PRIVATE + case inprot&RDWR != 0: + prot |= unix.PROT_WRITE + } + if inprot&EXEC != 0 { + prot |= unix.PROT_EXEC + } + if inflags&ANON != 0 { + flags |= unix.MAP_ANON + } + + b, err := unix.Mmap(int(fd), off, len, prot, flags) + if err != nil { + return nil, err + } + return b, nil +} + +func (m MMap) flush() error { + return unix.Msync([]byte(m), unix.MS_SYNC) +} + +func (m MMap) lock() error { + return unix.Mlock([]byte(m)) +} + +func (m MMap) unlock() error { + return unix.Munlock([]byte(m)) +} + +func (m MMap) unmap() error { + return unix.Munmap([]byte(m)) +} diff --git a/vendor/github.com/edsrzf/mmap-go/mmap_windows.go b/vendor/github.com/edsrzf/mmap-go/mmap_windows.go new file mode 100644 index 000000000000..7910da257703 --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/mmap_windows.go @@ -0,0 +1,143 @@ +// Copyright 2011 Evan Shaw. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mmap + +import ( + "errors" + "os" + "sync" + + "golang.org/x/sys/windows" +) + +// mmap on Windows is a two-step process. +// First, we call CreateFileMapping to get a handle. +// Then, we call MapviewToFile to get an actual pointer into memory. +// Because we want to emulate a POSIX-style mmap, we don't want to expose +// the handle -- only the pointer. We also want to return only a byte slice, +// not a struct, so it's convenient to manipulate. + +// We keep this map so that we can get back the original handle from the memory address. + +type addrinfo struct { + file windows.Handle + mapview windows.Handle +} + +var handleLock sync.Mutex +var handleMap = map[uintptr]*addrinfo{} + +func mmap(len int, prot, flags, hfile uintptr, off int64) ([]byte, error) { + flProtect := uint32(windows.PAGE_READONLY) + dwDesiredAccess := uint32(windows.FILE_MAP_READ) + switch { + case prot© != 0: + flProtect = windows.PAGE_WRITECOPY + dwDesiredAccess = windows.FILE_MAP_COPY + case prot&RDWR != 0: + flProtect = windows.PAGE_READWRITE + dwDesiredAccess = windows.FILE_MAP_WRITE + } + if prot&EXEC != 0 { + flProtect <<= 4 + dwDesiredAccess |= windows.FILE_MAP_EXECUTE + } + + // The maximum size is the area of the file, starting from 0, + // that we wish to allow to be mappable. It is the sum of + // the length the user requested, plus the offset where that length + // is starting from. This does not map the data into memory. + maxSizeHigh := uint32((off + int64(len)) >> 32) + maxSizeLow := uint32((off + int64(len)) & 0xFFFFFFFF) + // TODO: Do we need to set some security attributes? It might help portability. + h, errno := windows.CreateFileMapping(windows.Handle(hfile), nil, flProtect, maxSizeHigh, maxSizeLow, nil) + if h == 0 { + return nil, os.NewSyscallError("CreateFileMapping", errno) + } + + // Actually map a view of the data into memory. The view's size + // is the length the user requested. + fileOffsetHigh := uint32(off >> 32) + fileOffsetLow := uint32(off & 0xFFFFFFFF) + addr, errno := windows.MapViewOfFile(h, dwDesiredAccess, fileOffsetHigh, fileOffsetLow, uintptr(len)) + if addr == 0 { + return nil, os.NewSyscallError("MapViewOfFile", errno) + } + handleLock.Lock() + handleMap[addr] = &addrinfo{ + file: windows.Handle(hfile), + mapview: h, + } + handleLock.Unlock() + + m := MMap{} + dh := m.header() + dh.Data = addr + dh.Len = len + dh.Cap = dh.Len + + return m, nil +} + +func (m MMap) flush() error { + addr, len := m.addrLen() + errno := windows.FlushViewOfFile(addr, len) + if errno != nil { + return os.NewSyscallError("FlushViewOfFile", errno) + } + + handleLock.Lock() + defer handleLock.Unlock() + handle, ok := handleMap[addr] + if !ok { + // should be impossible; we would've errored above + return errors.New("unknown base address") + } + + errno = windows.FlushFileBuffers(handle.file) + return os.NewSyscallError("FlushFileBuffers", errno) +} + +func (m MMap) lock() error { + addr, len := m.addrLen() + errno := windows.VirtualLock(addr, len) + return os.NewSyscallError("VirtualLock", errno) +} + +func (m MMap) unlock() error { + addr, len := m.addrLen() + errno := windows.VirtualUnlock(addr, len) + return os.NewSyscallError("VirtualUnlock", errno) +} + +func (m MMap) unmap() error { + err := m.flush() + if err != nil { + return err + } + + addr := m.header().Data + // Lock the UnmapViewOfFile along with the handleMap deletion. + // As soon as we unmap the view, the OS is free to give the + // same addr to another new map. We don't want another goroutine + // to insert and remove the same addr into handleMap while + // we're trying to remove our old addr/handle pair. + handleLock.Lock() + defer handleLock.Unlock() + err = windows.UnmapViewOfFile(addr) + if err != nil { + return err + } + + handle, ok := handleMap[addr] + if !ok { + // should be impossible; we would've errored above + return errors.New("unknown base address") + } + delete(handleMap, addr) + + e := windows.CloseHandle(windows.Handle(handle.mapview)) + return os.NewSyscallError("CloseHandle", e) +} diff --git a/vendor/github.com/siddontang/go/LICENSE b/vendor/github.com/siddontang/go/LICENSE new file mode 100644 index 000000000000..80511a0a784d --- /dev/null +++ b/vendor/github.com/siddontang/go/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2014 siddontang + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/siddontang/go/filelock/LICENSE b/vendor/github.com/siddontang/go/filelock/LICENSE new file mode 100644 index 000000000000..fec05ce12959 --- /dev/null +++ b/vendor/github.com/siddontang/go/filelock/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 The LevelDB-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/siddontang/go/filelock/file_lock_generic.go b/vendor/github.com/siddontang/go/filelock/file_lock_generic.go new file mode 100644 index 000000000000..53c292acbdff --- /dev/null +++ b/vendor/github.com/siddontang/go/filelock/file_lock_generic.go @@ -0,0 +1,17 @@ +// Copyright 2012 The LevelDB-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows + +package filelock + +import ( + "fmt" + "io" + "runtime" +) + +func Lock(name string) (io.Closer, error) { + return nil, fmt.Errorf("leveldb/db: file locking is not implemented on %s/%s", runtime.GOOS, runtime.GOARCH) +} diff --git a/vendor/github.com/siddontang/go/filelock/file_lock_solaris.go b/vendor/github.com/siddontang/go/filelock/file_lock_solaris.go new file mode 100644 index 000000000000..56ff3e2ceef2 --- /dev/null +++ b/vendor/github.com/siddontang/go/filelock/file_lock_solaris.go @@ -0,0 +1,43 @@ +// Copyright 2014 The LevelDB-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build solaris + +package filelock + +import ( + "io" + "os" + "syscall" +) + +// lockCloser hides all of an os.File's methods, except for Close. +type lockCloser struct { + f *os.File +} + +func (l lockCloser) Close() error { + return l.f.Close() +} + +func Lock(name string) (io.Closer, error) { + f, err := os.Create(name) + if err != nil { + return nil, err + } + + spec := syscall.Flock_t{ + Type: syscall.F_WRLCK, + Whence: int16(os.SEEK_SET), + Start: 0, + Len: 0, // 0 means to lock the entire file. + Pid: int32(os.Getpid()), + } + if err := syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &spec); err != nil { + f.Close() + return nil, err + } + + return lockCloser{f}, nil +} diff --git a/vendor/github.com/siddontang/go/filelock/file_lock_unix.go b/vendor/github.com/siddontang/go/filelock/file_lock_unix.go new file mode 100644 index 000000000000..f70ae6192c59 --- /dev/null +++ b/vendor/github.com/siddontang/go/filelock/file_lock_unix.go @@ -0,0 +1,51 @@ +// Copyright 2014 The LevelDB-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd + +package filelock + +import ( + "io" + "os" + "syscall" +) + +// lockCloser hides all of an os.File's methods, except for Close. +type lockCloser struct { + f *os.File +} + +func (l lockCloser) Close() error { + return l.f.Close() +} + +func Lock(name string) (io.Closer, error) { + f, err := os.Create(name) + if err != nil { + return nil, err + } + + /* + Some people tell me FcntlFlock does not exist, so use flock here + */ + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + f.Close() + return nil, err + } + + // spec := syscall.Flock_t{ + // Type: syscall.F_WRLCK, + // Whence: int16(os.SEEK_SET), + // Start: 0, + // Len: 0, // 0 means to lock the entire file. + // Pid: int32(os.Getpid()), + // } + // if err := syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &spec); err != nil { + // f.Close() + // return nil, err + // } + + return lockCloser{f}, nil +} diff --git a/vendor/github.com/siddontang/go/filelock/file_lock_windows.go b/vendor/github.com/siddontang/go/filelock/file_lock_windows.go new file mode 100644 index 000000000000..5d3e4ba2029a --- /dev/null +++ b/vendor/github.com/siddontang/go/filelock/file_lock_windows.go @@ -0,0 +1,36 @@ +// Copyright 2013 The LevelDB-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package filelock + +import ( + "io" + "syscall" +) + +// lockCloser hides all of an syscall.Handle's methods, except for Close. +type lockCloser struct { + fd syscall.Handle +} + +func (l lockCloser) Close() error { + return syscall.Close(l.fd) +} + +func Lock(name string) (io.Closer, error) { + p, err := syscall.UTF16PtrFromString(name) + if err != nil { + return nil, err + } + fd, err := syscall.CreateFile(p, + syscall.GENERIC_READ|syscall.GENERIC_WRITE, + 0, nil, syscall.CREATE_ALWAYS, + syscall.FILE_ATTRIBUTE_NORMAL, + 0, + ) + if err != nil { + return nil, err + } + return lockCloser{fd: fd}, nil +} diff --git a/vendor/github.com/siddontang/go/hack/hack.go b/vendor/github.com/siddontang/go/hack/hack.go new file mode 100644 index 000000000000..74ee83cbf5d5 --- /dev/null +++ b/vendor/github.com/siddontang/go/hack/hack.go @@ -0,0 +1,27 @@ +package hack + +import ( + "reflect" + "unsafe" +) + +// no copy to change slice to string +// use your own risk +func String(b []byte) (s string) { + pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) + pstring.Data = pbytes.Data + pstring.Len = pbytes.Len + return +} + +// no copy to change string to slice +// use your own risk +func Slice(s string) (b []byte) { + pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) + pbytes.Data = pstring.Data + pbytes.Len = pstring.Len + pbytes.Cap = pstring.Len + return +} diff --git a/vendor/github.com/siddontang/go/ioutil2/ioutil.go b/vendor/github.com/siddontang/go/ioutil2/ioutil.go new file mode 100644 index 000000000000..35c0ad3cad51 --- /dev/null +++ b/vendor/github.com/siddontang/go/ioutil2/ioutil.go @@ -0,0 +1,39 @@ +// Copyright 2012, Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ioutil2 + +import ( + "io" + "io/ioutil" + "os" + "path" +) + +// Write file to temp and atomically move when everything else succeeds. +func WriteFileAtomic(filename string, data []byte, perm os.FileMode) error { + dir, name := path.Dir(filename), path.Base(filename) + f, err := ioutil.TempFile(dir, name) + if err != nil { + return err + } + n, err := f.Write(data) + f.Close() + if err == nil && n < len(data) { + err = io.ErrShortWrite + } else { + err = os.Chmod(f.Name(), perm) + } + if err != nil { + os.Remove(f.Name()) + return err + } + return os.Rename(f.Name(), filename) +} + +// Check file exists or not +func FileExists(name string) bool { + _, err := os.Stat(name) + return !os.IsNotExist(err) +} diff --git a/vendor/github.com/siddontang/go/ioutil2/sectionwriter.go b/vendor/github.com/siddontang/go/ioutil2/sectionwriter.go new file mode 100644 index 000000000000..c02ab0d5fd1f --- /dev/null +++ b/vendor/github.com/siddontang/go/ioutil2/sectionwriter.go @@ -0,0 +1,69 @@ +package ioutil2 + +import ( + "errors" + "io" +) + +var ErrExceedLimit = errors.New("write exceed limit") + +func NewSectionWriter(w io.WriterAt, off int64, n int64) *SectionWriter { + return &SectionWriter{w, off, off, off + n} +} + +type SectionWriter struct { + w io.WriterAt + base int64 + off int64 + limit int64 +} + +func (s *SectionWriter) Write(p []byte) (n int, err error) { + if s.off >= s.limit { + return 0, ErrExceedLimit + } + + if max := s.limit - s.off; int64(len(p)) > max { + return 0, ErrExceedLimit + } + + n, err = s.w.WriteAt(p, s.off) + s.off += int64(n) + return +} + +var errWhence = errors.New("Seek: invalid whence") +var errOffset = errors.New("Seek: invalid offset") + +func (s *SectionWriter) Seek(offset int64, whence int) (int64, error) { + switch whence { + default: + return 0, errWhence + case 0: + offset += s.base + case 1: + offset += s.off + case 2: + offset += s.limit + } + if offset < s.base { + return 0, errOffset + } + s.off = offset + return offset - s.base, nil +} + +func (s *SectionWriter) WriteAt(p []byte, off int64) (n int, err error) { + if off < 0 || off >= s.limit-s.base { + return 0, errOffset + } + off += s.base + if max := s.limit - off; int64(len(p)) > max { + return 0, ErrExceedLimit + } + + return s.w.WriteAt(p, off) +} + +// Size returns the size of the section in bytes. +func (s *SectionWriter) Size() int64 { return s.limit - s.base } diff --git a/vendor/github.com/siddontang/go/log/doc.go b/vendor/github.com/siddontang/go/log/doc.go new file mode 100644 index 000000000000..81a60ee853bc --- /dev/null +++ b/vendor/github.com/siddontang/go/log/doc.go @@ -0,0 +1,21 @@ +// log package supplies more advanced features than go orign log package. +// +// It supports log different level: trace, debug, info, warn, error, fatal. +// +// It also supports different log handlers which you can log to stdout, file, socket, etc... +// +// Use +// +// import "github.com/siddontang/go/log" +// +// //log with different level +// log.Info("hello world") +// log.Error("hello world") +// +// //create a logger with specified handler +// h := NewStreamHandler(os.Stdout) +// l := log.NewDefault(h) +// l.Info("hello world") +// l.Infof("%s %d", "hello", 123) +// +package log diff --git a/vendor/github.com/siddontang/go/log/filehandler.go b/vendor/github.com/siddontang/go/log/filehandler.go new file mode 100644 index 000000000000..2c158e2cf569 --- /dev/null +++ b/vendor/github.com/siddontang/go/log/filehandler.go @@ -0,0 +1,221 @@ +package log + +import ( + "fmt" + "os" + "path" + "time" +) + +//FileHandler writes log to a file. +type FileHandler struct { + fd *os.File +} + +func NewFileHandler(fileName string, flag int) (*FileHandler, error) { + dir := path.Dir(fileName) + os.Mkdir(dir, 0777) + + f, err := os.OpenFile(fileName, flag, 0) + if err != nil { + return nil, err + } + + h := new(FileHandler) + + h.fd = f + + return h, nil +} + +func (h *FileHandler) Write(b []byte) (n int, err error) { + return h.fd.Write(b) +} + +func (h *FileHandler) Close() error { + return h.fd.Close() +} + +//RotatingFileHandler writes log a file, if file size exceeds maxBytes, +//it will backup current file and open a new one. +// +//max backup file number is set by backupCount, it will delete oldest if backups too many. +type RotatingFileHandler struct { + fd *os.File + + fileName string + maxBytes int + curBytes int + backupCount int +} + +func NewRotatingFileHandler(fileName string, maxBytes int, backupCount int) (*RotatingFileHandler, error) { + dir := path.Dir(fileName) + os.MkdirAll(dir, 0777) + + h := new(RotatingFileHandler) + + if maxBytes <= 0 { + return nil, fmt.Errorf("invalid max bytes") + } + + h.fileName = fileName + h.maxBytes = maxBytes + h.backupCount = backupCount + + var err error + h.fd, err = os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + return nil, err + } + + f, err := h.fd.Stat() + if err != nil { + return nil, err + } + h.curBytes = int(f.Size()) + + return h, nil +} + +func (h *RotatingFileHandler) Write(p []byte) (n int, err error) { + h.doRollover() + n, err = h.fd.Write(p) + h.curBytes += n + return +} + +func (h *RotatingFileHandler) Close() error { + if h.fd != nil { + return h.fd.Close() + } + return nil +} + +func (h *RotatingFileHandler) doRollover() { + + if h.curBytes < h.maxBytes { + return + } + + f, err := h.fd.Stat() + if err != nil { + return + } + + if h.maxBytes <= 0 { + return + } else if f.Size() < int64(h.maxBytes) { + h.curBytes = int(f.Size()) + return + } + + if h.backupCount > 0 { + h.fd.Close() + + for i := h.backupCount - 1; i > 0; i-- { + sfn := fmt.Sprintf("%s.%d", h.fileName, i) + dfn := fmt.Sprintf("%s.%d", h.fileName, i+1) + + os.Rename(sfn, dfn) + } + + dfn := fmt.Sprintf("%s.1", h.fileName) + os.Rename(h.fileName, dfn) + + h.fd, _ = os.OpenFile(h.fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + h.curBytes = 0 + f, err := h.fd.Stat() + if err != nil { + return + } + h.curBytes = int(f.Size()) + } +} + +//TimeRotatingFileHandler writes log to a file, +//it will backup current and open a new one, with a period time you sepecified. +// +//refer: http://docs.python.org/2/library/logging.handlers.html. +//same like python TimedRotatingFileHandler. +type TimeRotatingFileHandler struct { + fd *os.File + + baseName string + interval int64 + suffix string + rolloverAt int64 +} + +const ( + WhenSecond = iota + WhenMinute + WhenHour + WhenDay +) + +func NewTimeRotatingFileHandler(baseName string, when int8, interval int) (*TimeRotatingFileHandler, error) { + dir := path.Dir(baseName) + os.Mkdir(dir, 0777) + + h := new(TimeRotatingFileHandler) + + h.baseName = baseName + + switch when { + case WhenSecond: + h.interval = 1 + h.suffix = "2006-01-02_15-04-05" + case WhenMinute: + h.interval = 60 + h.suffix = "2006-01-02_15-04" + case WhenHour: + h.interval = 3600 + h.suffix = "2006-01-02_15" + case WhenDay: + h.interval = 3600 * 24 + h.suffix = "2006-01-02" + default: + return nil, fmt.Errorf("invalid when_rotate: %d", when) + } + + h.interval = h.interval * int64(interval) + + var err error + h.fd, err = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + return nil, err + } + + fInfo, _ := h.fd.Stat() + h.rolloverAt = fInfo.ModTime().Unix() + h.interval + + return h, nil +} + +func (h *TimeRotatingFileHandler) doRollover() { + //refer http://hg.python.org/cpython/file/2.7/Lib/logging/handlers.py + now := time.Now() + + if h.rolloverAt <= now.Unix() { + fName := h.baseName + now.Format(h.suffix) + h.fd.Close() + e := os.Rename(h.baseName, fName) + if e != nil { + panic(e) + } + + h.fd, _ = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + + h.rolloverAt = time.Now().Unix() + h.interval + } +} + +func (h *TimeRotatingFileHandler) Write(b []byte) (n int, err error) { + h.doRollover() + return h.fd.Write(b) +} + +func (h *TimeRotatingFileHandler) Close() error { + return h.fd.Close() +} diff --git a/vendor/github.com/siddontang/go/log/handler.go b/vendor/github.com/siddontang/go/log/handler.go new file mode 100644 index 000000000000..4dc086f45c0a --- /dev/null +++ b/vendor/github.com/siddontang/go/log/handler.go @@ -0,0 +1,48 @@ +package log + +import ( + "io" +) + +//Handler writes logs to somewhere +type Handler interface { + Write(p []byte) (n int, err error) + Close() error +} + +//StreamHandler writes logs to a specified io Writer, maybe stdout, stderr, etc... +type StreamHandler struct { + w io.Writer +} + +func NewStreamHandler(w io.Writer) (*StreamHandler, error) { + h := new(StreamHandler) + + h.w = w + + return h, nil +} + +func (h *StreamHandler) Write(b []byte) (n int, err error) { + return h.w.Write(b) +} + +func (h *StreamHandler) Close() error { + return nil +} + +//NullHandler does nothing, it discards anything. +type NullHandler struct { +} + +func NewNullHandler() (*NullHandler, error) { + return new(NullHandler), nil +} + +func (h *NullHandler) Write(b []byte) (n int, err error) { + return len(b), nil +} + +func (h *NullHandler) Close() { + +} diff --git a/vendor/github.com/siddontang/go/log/log.go b/vendor/github.com/siddontang/go/log/log.go new file mode 100644 index 000000000000..371f6016871e --- /dev/null +++ b/vendor/github.com/siddontang/go/log/log.go @@ -0,0 +1,343 @@ +package log + +import ( + "fmt" + "os" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +//log level, from low to high, more high means more serious +const ( + LevelTrace = iota + LevelDebug + LevelInfo + LevelWarn + LevelError + LevelFatal +) + +const ( + Ltime = 1 << iota //time format "2006/01/02 15:04:05" + Lfile //file.go:123 + Llevel //[Trace|Debug|Info...] +) + +var LevelName [6]string = [6]string{"Trace", "Debug", "Info", "Warn", "Error", "Fatal"} + +const TimeFormat = "2006/01/02 15:04:05" + +const maxBufPoolSize = 16 + +type atomicInt32 int32 + +func (i *atomicInt32) Set(n int) { + atomic.StoreInt32((*int32)(i), int32(n)) +} + +func (i *atomicInt32) Get() int { + return int(atomic.LoadInt32((*int32)(i))) +} + +type Logger struct { + level atomicInt32 + flag int + + hMutex sync.Mutex + handler Handler + + bufMutex sync.Mutex + bufs [][]byte + + closed atomicInt32 +} + +//new a logger with specified handler and flag +func New(handler Handler, flag int) *Logger { + var l = new(Logger) + + l.level.Set(LevelInfo) + l.handler = handler + + l.flag = flag + + l.closed.Set(0) + + l.bufs = make([][]byte, 0, 16) + + return l +} + +//new a default logger with specified handler and flag: Ltime|Lfile|Llevel +func NewDefault(handler Handler) *Logger { + return New(handler, Ltime|Lfile|Llevel) +} + +func newStdHandler() *StreamHandler { + h, _ := NewStreamHandler(os.Stdout) + return h +} + +var std = NewDefault(newStdHandler()) + +func (l *Logger) popBuf() []byte { + l.bufMutex.Lock() + var buf []byte + if len(l.bufs) == 0 { + buf = make([]byte, 0, 1024) + } else { + buf = l.bufs[len(l.bufs)-1] + l.bufs = l.bufs[0 : len(l.bufs)-1] + } + l.bufMutex.Unlock() + + return buf +} + +func (l *Logger) putBuf(buf []byte) { + l.bufMutex.Lock() + if len(l.bufs) < maxBufPoolSize { + buf = buf[0:0] + l.bufs = append(l.bufs, buf) + } + l.bufMutex.Unlock() +} + +func (l *Logger) Close() { + if l.closed.Get() == 1 { + return + } + l.closed.Set(1) + + l.handler.Close() +} + +//set log level, any log level less than it will not log +func (l *Logger) SetLevel(level int) { + l.level.Set(level) +} + +// name can be in ["trace", "debug", "info", "warn", "error", "fatal"] +func (l *Logger) SetLevelByName(name string) { + name = strings.ToLower(name) + switch name { + case "trace": + l.SetLevel(LevelTrace) + case "debug": + l.SetLevel(LevelDebug) + case "info": + l.SetLevel(LevelInfo) + case "warn": + l.SetLevel(LevelWarn) + case "error": + l.SetLevel(LevelError) + case "fatal": + l.SetLevel(LevelFatal) + } +} + +func (l *Logger) SetHandler(h Handler) { + if l.closed.Get() == 1 { + return + } + + l.hMutex.Lock() + if l.handler != nil { + l.handler.Close() + } + l.handler = h + l.hMutex.Unlock() +} + +func (l *Logger) Output(callDepth int, level int, format string, v ...interface{}) { + if l.closed.Get() == 1 { + // closed + return + } + + if l.level.Get() > level { + // higher level can be logged + return + } + + var s string + if format == "" { + s = fmt.Sprint(v...) + } else { + s = fmt.Sprintf(format, v...) + } + + buf := l.popBuf() + + if l.flag&Ltime > 0 { + now := time.Now().Format(TimeFormat) + buf = append(buf, '[') + buf = append(buf, now...) + buf = append(buf, "] "...) + } + + if l.flag&Lfile > 0 { + _, file, line, ok := runtime.Caller(callDepth) + if !ok { + file = "???" + line = 0 + } else { + for i := len(file) - 1; i > 0; i-- { + if file[i] == '/' { + file = file[i+1:] + break + } + } + } + + buf = append(buf, file...) + buf = append(buf, ':') + + buf = strconv.AppendInt(buf, int64(line), 10) + buf = append(buf, ' ') + } + + if l.flag&Llevel > 0 { + buf = append(buf, '[') + buf = append(buf, LevelName[level]...) + buf = append(buf, "] "...) + } + + buf = append(buf, s...) + + if s[len(s)-1] != '\n' { + buf = append(buf, '\n') + } + + // l.msg <- buf + + l.hMutex.Lock() + l.handler.Write(buf) + l.hMutex.Unlock() + l.putBuf(buf) +} + +//log with Trace level +func (l *Logger) Trace(v ...interface{}) { + l.Output(2, LevelTrace, "", v...) +} + +//log with Debug level +func (l *Logger) Debug(v ...interface{}) { + l.Output(2, LevelDebug, "", v...) +} + +//log with info level +func (l *Logger) Info(v ...interface{}) { + l.Output(2, LevelInfo, "", v...) +} + +//log with warn level +func (l *Logger) Warn(v ...interface{}) { + l.Output(2, LevelWarn, "", v...) +} + +//log with error level +func (l *Logger) Error(v ...interface{}) { + l.Output(2, LevelError, "", v...) +} + +//log with fatal level +func (l *Logger) Fatal(v ...interface{}) { + l.Output(2, LevelFatal, "", v...) +} + +//log with Trace level +func (l *Logger) Tracef(format string, v ...interface{}) { + l.Output(2, LevelTrace, format, v...) +} + +//log with Debug level +func (l *Logger) Debugf(format string, v ...interface{}) { + l.Output(2, LevelDebug, format, v...) +} + +//log with info level +func (l *Logger) Infof(format string, v ...interface{}) { + l.Output(2, LevelInfo, format, v...) +} + +//log with warn level +func (l *Logger) Warnf(format string, v ...interface{}) { + l.Output(2, LevelWarn, format, v...) +} + +//log with error level +func (l *Logger) Errorf(format string, v ...interface{}) { + l.Output(2, LevelError, format, v...) +} + +//log with fatal level +func (l *Logger) Fatalf(format string, v ...interface{}) { + l.Output(2, LevelFatal, format, v...) +} + +func SetLevel(level int) { + std.SetLevel(level) +} + +// name can be in ["trace", "debug", "info", "warn", "error", "fatal"] +func SetLevelByName(name string) { + std.SetLevelByName(name) +} + +func SetHandler(h Handler) { + std.SetHandler(h) +} + +func Trace(v ...interface{}) { + std.Output(2, LevelTrace, "", v...) +} + +func Debug(v ...interface{}) { + std.Output(2, LevelDebug, "", v...) +} + +func Info(v ...interface{}) { + std.Output(2, LevelInfo, "", v...) +} + +func Warn(v ...interface{}) { + std.Output(2, LevelWarn, "", v...) +} + +func Error(v ...interface{}) { + std.Output(2, LevelError, "", v...) +} + +func Fatal(v ...interface{}) { + std.Output(2, LevelFatal, "", v...) +} + +func Tracef(format string, v ...interface{}) { + std.Output(2, LevelTrace, format, v...) +} + +func Debugf(format string, v ...interface{}) { + std.Output(2, LevelDebug, format, v...) +} + +func Infof(format string, v ...interface{}) { + std.Output(2, LevelInfo, format, v...) +} + +func Warnf(format string, v ...interface{}) { + std.Output(2, LevelWarn, format, v...) +} + +func Errorf(format string, v ...interface{}) { + std.Output(2, LevelError, format, v...) +} + +func Fatalf(format string, v ...interface{}) { + std.Output(2, LevelFatal, format, v...) +} diff --git a/vendor/github.com/siddontang/go/log/sockethandler.go b/vendor/github.com/siddontang/go/log/sockethandler.go new file mode 100644 index 000000000000..3e7494d9501b --- /dev/null +++ b/vendor/github.com/siddontang/go/log/sockethandler.go @@ -0,0 +1,65 @@ +package log + +import ( + "encoding/binary" + "net" + "time" +) + +//SocketHandler writes log to a connectionl. +//Network protocol is simple: log length + log | log length + log. log length is uint32, bigendian. +//you must implement your own log server, maybe you can use logd instead simply. +type SocketHandler struct { + c net.Conn + protocol string + addr string +} + +func NewSocketHandler(protocol string, addr string) (*SocketHandler, error) { + s := new(SocketHandler) + + s.protocol = protocol + s.addr = addr + + return s, nil +} + +func (h *SocketHandler) Write(p []byte) (n int, err error) { + if err = h.connect(); err != nil { + return + } + + buf := make([]byte, len(p)+4) + + binary.BigEndian.PutUint32(buf, uint32(len(p))) + + copy(buf[4:], p) + + n, err = h.c.Write(buf) + if err != nil { + h.c.Close() + h.c = nil + } + return +} + +func (h *SocketHandler) Close() error { + if h.c != nil { + h.c.Close() + } + return nil +} + +func (h *SocketHandler) connect() error { + if h.c != nil { + return nil + } + + var err error + h.c, err = net.DialTimeout(h.protocol, h.addr, 20*time.Second) + if err != nil { + return err + } + + return nil +} diff --git a/vendor/github.com/siddontang/go/num/bytes.go b/vendor/github.com/siddontang/go/num/bytes.go new file mode 100644 index 000000000000..1f3def74ac73 --- /dev/null +++ b/vendor/github.com/siddontang/go/num/bytes.go @@ -0,0 +1,67 @@ +package num + +import ( + "encoding/binary" +) + +//all are bigendian format + +func BytesToUint16(b []byte) uint16 { + return binary.BigEndian.Uint16(b) +} + +func Uint16ToBytes(u uint16) []byte { + buf := make([]byte, 2) + binary.BigEndian.PutUint16(buf, u) + return buf +} + +func BytesToUint32(b []byte) uint32 { + return binary.BigEndian.Uint32(b) +} + +func Uint32ToBytes(u uint32) []byte { + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf, u) + return buf +} + +func BytesToUint64(b []byte) uint64 { + return binary.BigEndian.Uint64(b) +} + +func Uint64ToBytes(u uint64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, u) + return buf +} + +func BytesToInt16(b []byte) int16 { + return int16(binary.BigEndian.Uint16(b)) +} + +func Int16ToBytes(u int16) []byte { + buf := make([]byte, 2) + binary.BigEndian.PutUint16(buf, uint16(u)) + return buf +} + +func BytesToInt32(b []byte) int32 { + return int32(binary.BigEndian.Uint32(b)) +} + +func Int32ToBytes(u int32) []byte { + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf, uint32(u)) + return buf +} + +func BytesToInt64(b []byte) int64 { + return int64(binary.BigEndian.Uint64(b)) +} + +func Int64ToBytes(u int64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(u)) + return buf +} diff --git a/vendor/github.com/siddontang/go/num/cmp.go b/vendor/github.com/siddontang/go/num/cmp.go new file mode 100644 index 000000000000..78f8d4f14af3 --- /dev/null +++ b/vendor/github.com/siddontang/go/num/cmp.go @@ -0,0 +1,161 @@ +package num + +func MinUint(a uint, b uint) uint { + if a > b { + return b + } else { + return a + } +} + +func MaxUint(a uint, b uint) uint { + if a > b { + return a + } else { + return b + } +} + +func MinInt(a int, b int) int { + if a > b { + return b + } else { + return a + } +} + +func MaxInt(a int, b int) int { + if a > b { + return a + } else { + return b + } +} + +func MinUint8(a uint8, b uint8) uint8 { + if a > b { + return b + } else { + return a + } +} + +func MaxUint8(a uint8, b uint8) uint8 { + if a > b { + return a + } else { + return b + } +} + +func MinInt8(a int8, b int8) int8 { + if a > b { + return b + } else { + return a + } +} + +func MaxInt8(a int8, b int8) int8 { + if a > b { + return a + } else { + return b + } +} + +func MinUint16(a uint16, b uint16) uint16 { + if a > b { + return b + } else { + return a + } +} + +func MaxUint16(a uint16, b uint16) uint16 { + if a > b { + return a + } else { + return b + } +} + +func MinInt16(a int16, b int16) int16 { + if a > b { + return b + } else { + return a + } +} + +func MaxInt16(a int16, b int16) int16 { + if a > b { + return a + } else { + return b + } +} + +func MinUint32(a uint32, b uint32) uint32 { + if a > b { + return b + } else { + return a + } +} + +func MaxUint32(a uint32, b uint32) uint32 { + if a > b { + return a + } else { + return b + } +} + +func MinInt32(a int32, b int32) int32 { + if a > b { + return b + } else { + return a + } +} + +func MaxInt32(a int32, b int32) int32 { + if a > b { + return a + } else { + return b + } +} + +func MinUint64(a uint64, b uint64) uint64 { + if a > b { + return b + } else { + return a + } +} + +func MaxUint64(a uint64, b uint64) uint64 { + if a > b { + return a + } else { + return b + } +} + +func MinInt64(a int64, b int64) int64 { + if a > b { + return b + } else { + return a + } +} + +func MaxInt64(a int64, b int64) int64 { + if a > b { + return a + } else { + return b + } +} diff --git a/vendor/github.com/siddontang/go/num/str.go b/vendor/github.com/siddontang/go/num/str.go new file mode 100644 index 000000000000..4b304817b86d --- /dev/null +++ b/vendor/github.com/siddontang/go/num/str.go @@ -0,0 +1,157 @@ +package num + +import ( + "strconv" +) + +func ParseUint(s string) (uint, error) { + if v, err := strconv.ParseUint(s, 10, 0); err != nil { + return 0, err + } else { + return uint(v), nil + } +} + +func ParseUint8(s string) (uint8, error) { + if v, err := strconv.ParseUint(s, 10, 8); err != nil { + return 0, err + } else { + return uint8(v), nil + } +} + +func ParseUint16(s string) (uint16, error) { + if v, err := strconv.ParseUint(s, 10, 16); err != nil { + return 0, err + } else { + return uint16(v), nil + } +} + +func ParseUint32(s string) (uint32, error) { + if v, err := strconv.ParseUint(s, 10, 32); err != nil { + return 0, err + } else { + return uint32(v), nil + } +} + +func ParseUint64(s string) (uint64, error) { + return strconv.ParseUint(s, 10, 64) +} + +func ParseInt(s string) (int, error) { + if v, err := strconv.ParseInt(s, 10, 0); err != nil { + return 0, err + } else { + return int(v), nil + } +} + +func ParseInt8(s string) (int8, error) { + if v, err := strconv.ParseInt(s, 10, 8); err != nil { + return 0, err + } else { + return int8(v), nil + } +} + +func ParseInt16(s string) (int16, error) { + if v, err := strconv.ParseInt(s, 10, 16); err != nil { + return 0, err + } else { + return int16(v), nil + } +} + +func ParseInt32(s string) (int32, error) { + if v, err := strconv.ParseInt(s, 10, 32); err != nil { + return 0, err + } else { + return int32(v), nil + } +} + +func ParseInt64(s string) (int64, error) { + return strconv.ParseInt(s, 10, 64) +} + +func FormatInt(v int) string { + return strconv.FormatInt(int64(v), 10) +} + +func FormatInt8(v int8) string { + return strconv.FormatInt(int64(v), 10) +} + +func FormatInt16(v int16) string { + return strconv.FormatInt(int64(v), 10) +} + +func FormatInt32(v int32) string { + return strconv.FormatInt(int64(v), 10) +} + +func FormatInt64(v int64) string { + return strconv.FormatInt(int64(v), 10) +} + +func FormatUint(v uint) string { + return strconv.FormatUint(uint64(v), 10) +} + +func FormatUint8(v uint8) string { + return strconv.FormatUint(uint64(v), 10) +} + +func FormatUint16(v uint16) string { + return strconv.FormatUint(uint64(v), 10) +} + +func FormatUint32(v uint32) string { + return strconv.FormatUint(uint64(v), 10) +} + +func FormatUint64(v uint64) string { + return strconv.FormatUint(uint64(v), 10) +} + +func FormatIntToSlice(v int) []byte { + return strconv.AppendInt(nil, int64(v), 10) +} + +func FormatInt8ToSlice(v int8) []byte { + return strconv.AppendInt(nil, int64(v), 10) +} + +func FormatInt16ToSlice(v int16) []byte { + return strconv.AppendInt(nil, int64(v), 10) +} + +func FormatInt32ToSlice(v int32) []byte { + return strconv.AppendInt(nil, int64(v), 10) +} + +func FormatInt64ToSlice(v int64) []byte { + return strconv.AppendInt(nil, int64(v), 10) +} + +func FormatUintToSlice(v uint) []byte { + return strconv.AppendUint(nil, uint64(v), 10) +} + +func FormatUint8ToSlice(v uint8) []byte { + return strconv.AppendUint(nil, uint64(v), 10) +} + +func FormatUint16ToSlice(v uint16) []byte { + return strconv.AppendUint(nil, uint64(v), 10) +} + +func FormatUint32ToSlice(v uint32) []byte { + return strconv.AppendUint(nil, uint64(v), 10) +} + +func FormatUint64ToSlice(v uint64) []byte { + return strconv.AppendUint(nil, uint64(v), 10) +} diff --git a/vendor/github.com/siddontang/go/snappy/LICENSE b/vendor/github.com/siddontang/go/snappy/LICENSE new file mode 100644 index 000000000000..6050c10f4c8b --- /dev/null +++ b/vendor/github.com/siddontang/go/snappy/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/siddontang/go/snappy/decode.go b/vendor/github.com/siddontang/go/snappy/decode.go new file mode 100644 index 000000000000..d93c1b9dbfd7 --- /dev/null +++ b/vendor/github.com/siddontang/go/snappy/decode.go @@ -0,0 +1,124 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package snappy + +import ( + "encoding/binary" + "errors" +) + +// ErrCorrupt reports that the input is invalid. +var ErrCorrupt = errors.New("snappy: corrupt input") + +// DecodedLen returns the length of the decoded block. +func DecodedLen(src []byte) (int, error) { + v, _, err := decodedLen(src) + return v, err +} + +// decodedLen returns the length of the decoded block and the number of bytes +// that the length header occupied. +func decodedLen(src []byte) (blockLen, headerLen int, err error) { + v, n := binary.Uvarint(src) + if n == 0 { + return 0, 0, ErrCorrupt + } + if uint64(int(v)) != v { + return 0, 0, errors.New("snappy: decoded block is too large") + } + return int(v), n, nil +} + +// Decode returns the decoded form of src. The returned slice may be a sub- +// slice of dst if dst was large enough to hold the entire decoded block. +// Otherwise, a newly allocated slice will be returned. +// It is valid to pass a nil dst. +func Decode(dst, src []byte) ([]byte, error) { + dLen, s, err := decodedLen(src) + if err != nil { + return nil, err + } + if len(dst) < dLen { + dst = make([]byte, dLen) + } + + var d, offset, length int + for s < len(src) { + switch src[s] & 0x03 { + case tagLiteral: + x := uint(src[s] >> 2) + switch { + case x < 60: + s += 1 + case x == 60: + s += 2 + if s > len(src) { + return nil, ErrCorrupt + } + x = uint(src[s-1]) + case x == 61: + s += 3 + if s > len(src) { + return nil, ErrCorrupt + } + x = uint(src[s-2]) | uint(src[s-1])<<8 + case x == 62: + s += 4 + if s > len(src) { + return nil, ErrCorrupt + } + x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16 + case x == 63: + s += 5 + if s > len(src) { + return nil, ErrCorrupt + } + x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24 + } + length = int(x + 1) + if length <= 0 { + return nil, errors.New("snappy: unsupported literal length") + } + if length > len(dst)-d || length > len(src)-s { + return nil, ErrCorrupt + } + copy(dst[d:], src[s:s+length]) + d += length + s += length + continue + + case tagCopy1: + s += 2 + if s > len(src) { + return nil, ErrCorrupt + } + length = 4 + int(src[s-2])>>2&0x7 + offset = int(src[s-2])&0xe0<<3 | int(src[s-1]) + + case tagCopy2: + s += 3 + if s > len(src) { + return nil, ErrCorrupt + } + length = 1 + int(src[s-3])>>2 + offset = int(src[s-2]) | int(src[s-1])<<8 + + case tagCopy4: + return nil, errors.New("snappy: unsupported COPY_4 tag") + } + + end := d + length + if offset > d || end > len(dst) { + return nil, ErrCorrupt + } + for ; d < end; d++ { + dst[d] = dst[d-offset] + } + } + if d != dLen { + return nil, ErrCorrupt + } + return dst[:d], nil +} diff --git a/vendor/github.com/siddontang/go/snappy/encode.go b/vendor/github.com/siddontang/go/snappy/encode.go new file mode 100644 index 000000000000..b2371db11c8f --- /dev/null +++ b/vendor/github.com/siddontang/go/snappy/encode.go @@ -0,0 +1,174 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package snappy + +import ( + "encoding/binary" +) + +// We limit how far copy back-references can go, the same as the C++ code. +const maxOffset = 1 << 15 + +// emitLiteral writes a literal chunk and returns the number of bytes written. +func emitLiteral(dst, lit []byte) int { + i, n := 0, uint(len(lit)-1) + switch { + case n < 60: + dst[0] = uint8(n)<<2 | tagLiteral + i = 1 + case n < 1<<8: + dst[0] = 60<<2 | tagLiteral + dst[1] = uint8(n) + i = 2 + case n < 1<<16: + dst[0] = 61<<2 | tagLiteral + dst[1] = uint8(n) + dst[2] = uint8(n >> 8) + i = 3 + case n < 1<<24: + dst[0] = 62<<2 | tagLiteral + dst[1] = uint8(n) + dst[2] = uint8(n >> 8) + dst[3] = uint8(n >> 16) + i = 4 + case int64(n) < 1<<32: + dst[0] = 63<<2 | tagLiteral + dst[1] = uint8(n) + dst[2] = uint8(n >> 8) + dst[3] = uint8(n >> 16) + dst[4] = uint8(n >> 24) + i = 5 + default: + panic("snappy: source buffer is too long") + } + if copy(dst[i:], lit) != len(lit) { + panic("snappy: destination buffer is too short") + } + return i + len(lit) +} + +// emitCopy writes a copy chunk and returns the number of bytes written. +func emitCopy(dst []byte, offset, length int) int { + i := 0 + for length > 0 { + x := length - 4 + if 0 <= x && x < 1<<3 && offset < 1<<11 { + dst[i+0] = uint8(offset>>8)&0x07<<5 | uint8(x)<<2 | tagCopy1 + dst[i+1] = uint8(offset) + i += 2 + break + } + + x = length + if x > 1<<6 { + x = 1 << 6 + } + dst[i+0] = uint8(x-1)<<2 | tagCopy2 + dst[i+1] = uint8(offset) + dst[i+2] = uint8(offset >> 8) + i += 3 + length -= x + } + return i +} + +// Encode returns the encoded form of src. The returned slice may be a sub- +// slice of dst if dst was large enough to hold the entire encoded block. +// Otherwise, a newly allocated slice will be returned. +// It is valid to pass a nil dst. +func Encode(dst, src []byte) ([]byte, error) { + if n := MaxEncodedLen(len(src)); len(dst) < n { + dst = make([]byte, n) + } + + // The block starts with the varint-encoded length of the decompressed bytes. + d := binary.PutUvarint(dst, uint64(len(src))) + + // Return early if src is short. + if len(src) <= 4 { + if len(src) != 0 { + d += emitLiteral(dst[d:], src) + } + return dst[:d], nil + } + + // Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive. + const maxTableSize = 1 << 14 + shift, tableSize := uint(32-8), 1<<8 + for tableSize < maxTableSize && tableSize < len(src) { + shift-- + tableSize *= 2 + } + var table [maxTableSize]int + + // Iterate over the source bytes. + var ( + s int // The iterator position. + t int // The last position with the same hash as s. + lit int // The start position of any pending literal bytes. + ) + for s+3 < len(src) { + // Update the hash table. + b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3] + h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24 + p := &table[(h*0x1e35a7bd)>>shift] + // We need to to store values in [-1, inf) in table. To save + // some initialization time, (re)use the table's zero value + // and shift the values against this zero: add 1 on writes, + // subtract 1 on reads. + t, *p = *p-1, s+1 + // If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte. + if t < 0 || s-t >= maxOffset || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] { + s++ + continue + } + // Otherwise, we have a match. First, emit any pending literal bytes. + if lit != s { + d += emitLiteral(dst[d:], src[lit:s]) + } + // Extend the match to be as long as possible. + s0 := s + s, t = s+4, t+4 + for s < len(src) && src[s] == src[t] { + s++ + t++ + } + // Emit the copied bytes. + d += emitCopy(dst[d:], s-t, s-s0) + lit = s + } + + // Emit any final pending literal bytes and return. + if lit != len(src) { + d += emitLiteral(dst[d:], src[lit:]) + } + return dst[:d], nil +} + +// MaxEncodedLen returns the maximum length of a snappy block, given its +// uncompressed length. +func MaxEncodedLen(srcLen int) int { + // Compressed data can be defined as: + // compressed := item* literal* + // item := literal* copy + // + // The trailing literal sequence has a space blowup of at most 62/60 + // since a literal of length 60 needs one tag byte + one extra byte + // for length information. + // + // Item blowup is trickier to measure. Suppose the "copy" op copies + // 4 bytes of data. Because of a special check in the encoding code, + // we produce a 4-byte copy only if the offset is < 65536. Therefore + // the copy op takes 3 bytes to encode, and this type of item leads + // to at most the 62/60 blowup for representing literals. + // + // Suppose the "copy" op copies 5 bytes of data. If the offset is big + // enough, it will take 5 bytes to encode the copy op. Therefore the + // worst case here is a one-byte literal followed by a five-byte copy. + // That is, 6 bytes of input turn into 7 bytes of "compressed" data. + // + // This last factor dominates the blowup, so the final estimate is: + return 32 + srcLen + srcLen/6 +} diff --git a/vendor/github.com/siddontang/go/snappy/snappy.go b/vendor/github.com/siddontang/go/snappy/snappy.go new file mode 100644 index 000000000000..2f1b790d0b71 --- /dev/null +++ b/vendor/github.com/siddontang/go/snappy/snappy.go @@ -0,0 +1,38 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package snappy implements the snappy block-based compression format. +// It aims for very high speeds and reasonable compression. +// +// The C++ snappy implementation is at http://code.google.com/p/snappy/ +package snappy + +/* +Each encoded block begins with the varint-encoded length of the decoded data, +followed by a sequence of chunks. Chunks begin and end on byte boundaries. The +first byte of each chunk is broken into its 2 least and 6 most significant bits +called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag. +Zero means a literal tag. All other values mean a copy tag. + +For literal tags: + - If m < 60, the next 1 + m bytes are literal bytes. + - Otherwise, let n be the little-endian unsigned integer denoted by the next + m - 59 bytes. The next 1 + n bytes after that are literal bytes. + +For copy tags, length bytes are copied from offset bytes ago, in the style of +Lempel-Ziv compression algorithms. In particular: + - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12). + The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10 + of the offset. The next byte is bits 0-7 of the offset. + - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65). + The length is 1 + m. The offset is the little-endian unsigned integer + denoted by the next 2 bytes. + - For l == 3, this tag is a legacy format that is no longer supported. +*/ +const ( + tagLiteral = 0x00 + tagCopy1 = 0x01 + tagCopy2 = 0x02 + tagCopy4 = 0x03 +) diff --git a/vendor/github.com/siddontang/go/sync2/atomic.go b/vendor/github.com/siddontang/go/sync2/atomic.go new file mode 100644 index 000000000000..382fc20dfec7 --- /dev/null +++ b/vendor/github.com/siddontang/go/sync2/atomic.go @@ -0,0 +1,146 @@ +// Copyright 2013, Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sync2 + +import ( + "sync" + "sync/atomic" + "time" +) + +type AtomicInt32 int32 + +func (i *AtomicInt32) Add(n int32) int32 { + return atomic.AddInt32((*int32)(i), n) +} + +func (i *AtomicInt32) Set(n int32) { + atomic.StoreInt32((*int32)(i), n) +} + +func (i *AtomicInt32) Get() int32 { + return atomic.LoadInt32((*int32)(i)) +} + +func (i *AtomicInt32) CompareAndSwap(oldval, newval int32) (swapped bool) { + return atomic.CompareAndSwapInt32((*int32)(i), oldval, newval) +} + +type AtomicUint32 uint32 + +func (i *AtomicUint32) Add(n uint32) uint32 { + return atomic.AddUint32((*uint32)(i), n) +} + +func (i *AtomicUint32) Set(n uint32) { + atomic.StoreUint32((*uint32)(i), n) +} + +func (i *AtomicUint32) Get() uint32 { + return atomic.LoadUint32((*uint32)(i)) +} + +func (i *AtomicUint32) CompareAndSwap(oldval, newval uint32) (swapped bool) { + return atomic.CompareAndSwapUint32((*uint32)(i), oldval, newval) +} + +type AtomicInt64 int64 + +func (i *AtomicInt64) Add(n int64) int64 { + return atomic.AddInt64((*int64)(i), n) +} + +func (i *AtomicInt64) Set(n int64) { + atomic.StoreInt64((*int64)(i), n) +} + +func (i *AtomicInt64) Get() int64 { + return atomic.LoadInt64((*int64)(i)) +} + +func (i *AtomicInt64) CompareAndSwap(oldval, newval int64) (swapped bool) { + return atomic.CompareAndSwapInt64((*int64)(i), oldval, newval) +} + +type AtomicUint64 uint64 + +func (i *AtomicUint64) Add(n uint64) uint64 { + return atomic.AddUint64((*uint64)(i), n) +} + +func (i *AtomicUint64) Set(n uint64) { + atomic.StoreUint64((*uint64)(i), n) +} + +func (i *AtomicUint64) Get() uint64 { + return atomic.LoadUint64((*uint64)(i)) +} + +func (i *AtomicUint64) CompareAndSwap(oldval, newval uint64) (swapped bool) { + return atomic.CompareAndSwapUint64((*uint64)(i), oldval, newval) +} + +type AtomicDuration int64 + +func (d *AtomicDuration) Add(duration time.Duration) time.Duration { + return time.Duration(atomic.AddInt64((*int64)(d), int64(duration))) +} + +func (d *AtomicDuration) Set(duration time.Duration) { + atomic.StoreInt64((*int64)(d), int64(duration)) +} + +func (d *AtomicDuration) Get() time.Duration { + return time.Duration(atomic.LoadInt64((*int64)(d))) +} + +func (d *AtomicDuration) CompareAndSwap(oldval, newval time.Duration) (swapped bool) { + return atomic.CompareAndSwapInt64((*int64)(d), int64(oldval), int64(newval)) +} + +// AtomicString gives you atomic-style APIs for string, but +// it's only a convenience wrapper that uses a mutex. So, it's +// not as efficient as the rest of the atomic types. +type AtomicString struct { + mu sync.Mutex + str string +} + +func (s *AtomicString) Set(str string) { + s.mu.Lock() + s.str = str + s.mu.Unlock() +} + +func (s *AtomicString) Get() string { + s.mu.Lock() + str := s.str + s.mu.Unlock() + return str +} + +func (s *AtomicString) CompareAndSwap(oldval, newval string) (swapped bool) { + s.mu.Lock() + defer s.mu.Unlock() + if s.str == oldval { + s.str = newval + return true + } + return false +} + +type AtomicBool int32 + +func (b *AtomicBool) Set(v bool) { + if v { + atomic.StoreInt32((*int32)(b), 1) + } else { + atomic.StoreInt32((*int32)(b), 0) + } +} + +func (b *AtomicBool) Get() bool { + return atomic.LoadInt32((*int32)(b)) == 1 +} diff --git a/vendor/github.com/siddontang/go/sync2/semaphore.go b/vendor/github.com/siddontang/go/sync2/semaphore.go new file mode 100644 index 000000000000..d310da7294c7 --- /dev/null +++ b/vendor/github.com/siddontang/go/sync2/semaphore.go @@ -0,0 +1,65 @@ +package sync2 + +import ( + "sync" + "sync/atomic" + "time" +) + +func NewSemaphore(initialCount int) *Semaphore { + res := &Semaphore{ + counter: int64(initialCount), + } + res.cond.L = &res.lock + return res +} + +type Semaphore struct { + lock sync.Mutex + cond sync.Cond + counter int64 +} + +func (s *Semaphore) Release() { + s.lock.Lock() + s.counter += 1 + if s.counter >= 0 { + s.cond.Signal() + } + s.lock.Unlock() +} + +func (s *Semaphore) Acquire() { + s.lock.Lock() + for s.counter < 1 { + s.cond.Wait() + } + s.counter -= 1 + s.lock.Unlock() +} + +func (s *Semaphore) AcquireTimeout(timeout time.Duration) bool { + done := make(chan bool, 1) + // Gate used to communicate between the threads and decide what the result + // is. If the main thread decides, we have timed out, otherwise we succeed. + decided := new(int32) + go func() { + s.Acquire() + if atomic.SwapInt32(decided, 1) == 0 { + done <- true + } else { + // If we already decided the result, and this thread did not win + s.Release() + } + }() + select { + case <-done: + return true + case <-time.NewTimer(timeout).C: + if atomic.SwapInt32(decided, 1) == 1 { + // The other thread already decided the result + return true + } + return false + } +} diff --git a/vendor/github.com/siddontang/ledisdb/LICENSE b/vendor/github.com/siddontang/ledisdb/LICENSE new file mode 100644 index 000000000000..7ece9fdf5a64 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 siddontang + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/siddontang/ledisdb/config/config-docker.toml b/vendor/github.com/siddontang/ledisdb/config/config-docker.toml new file mode 100644 index 000000000000..f37cf2897ad3 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/config/config-docker.toml @@ -0,0 +1,170 @@ +# LedisDB configuration + +# Server listen address +addr = "0.0.0.0:6380" + +# Unix socket permissions, 755 by default. +# Ignored for tcp socket. +addr_unixsocketperm = "0770" + +# Server http listen address, set empty to disable +http_addr = "0.0.0.0:11181" + +# Data store path, all ledisdb's data will be saved here +data_dir = "/datastore" + +# Set the number of databases. You can use `select dbindex` to choose a db. +# dbindex must be in [0, databases - 1]. +# Default databases is 16, maximum is 10240 now. +databases = 16 + +# Log server command, set empty to disable +access_log = "" + +# Set slaveof to enable replication from master, empty, no replication +# Any write operations except flushall and replication will be disabled in slave mode. +slaveof = "" + +# Readonly mode, slave server is always readonly even readonly = false +# for readonly mode, only replication and flushall can write +readonly = false + +# Choose which backend storage to use, now support: +# +# leveldb +# rocksdb +# goleveldb +# memory +# +db_name = "leveldb" + +# If not set, use data_dir/"db_name"_data +db_path = "" + +# Sync commit to disk if possible +# 0: no sync +# 1: sync every second +# 2: sync every commit +db_sync_commit = 0 + +# enable replication or not +use_replication = false + +# set connection buffer, you can increase them appropriately +# more size, more memory used +conn_read_buffer_size = 10240 +conn_write_buffer_size = 10240 + +# if connection receives no data after n seconds, it may be dead, close +# 0 to disable and not check +conn_keepalive_interval = 0 + +# checking TTL (time to live) data every n seconds +# if you set big, the expired data may not be deleted immediately +ttl_check_interval = 1 + +[leveldb] +# for leveldb and goleveldb +compression = false +block_size = 32768 +write_buffer_size = 67108864 +cache_size = 524288000 +max_open_files = 1024 +max_file_size = 33554432 + +[rocksdb] +# rocksdb has many many configurations, +# we only list little now, but may add more later. +# good luck! + +# 0:no, 1:snappy, 2:zlib, 3:bz2, 4:lz4, 5:lz4hc +compression = 0 +block_size = 65536 +write_buffer_size = 134217728 +cache_size = 1073741824 +max_open_files = 1024 +max_write_buffer_num = 6 +min_write_buffer_number_to_merge = 2 +num_levels = 7 +level0_file_num_compaction_trigger = 8 +level0_slowdown_writes_trigger = 16 +level0_stop_writes_trigger = 64 +target_file_size_base = 67108864 +target_file_size_multiplier = 1 +max_bytes_for_level_base = 536870912 +max_bytes_for_level_multiplier = 8 +disable_auto_compactions = false +disable_data_sync = false +use_fsync = false +background_theads = 16 +high_priority_background_threads = 1 +max_background_compactions = 15 +max_background_flushes = 1 +allow_os_buffer = true +enable_statistics = false +stats_dump_period_sec = 3600 +# dangerous to set true, write may got lost after a crash +# you can set true if replication opened, we may recover from replication log, +# but it is still not a easy work. +disable_wal = false +max_manifest_file_size = 20971520 + +[lmdb] +map_size = 524288000 +nosync = true + +[replication] +# Path to store replication information(write ahead log, commit log, etc.) +# if not set, use data_dir/rpl +path = "" + +# If sync is true, the new log must be sent to some slaves, and then commit. +# It will reduce performance but have better high availability. +sync = false + +# If sync is true, wait at last wait_sync_time milliseconds for slave syncing this log +wait_sync_time = 500 + +# If sync is true, wait at most min(wait_max_slave_acks, (n + 1) / 2) to promise syncing ok. +# n is slave number +# If 0, wait (n + 1) / 2 acks. +wait_max_slave_acks = 2 + +# store name: file, goleveldb +# change in runtime is very dangerous +store_name = "file" + +# Expire write ahead logs after the given days +expired_log_days = 7 + +# for file store, if 0, use default 256MB, max is 1G +max_log_file_size = 0 + +# for file store, if 0, use default 50 +max_log_file_num = 0 + +# for file store, use mmap for file read and write +use_mmap = true + +# Sync log to disk if possible +# 0: no sync +# 1: sync every second +# 2: sync every commit +sync_log = 0 + +# Compress the log or not +compression = false + +[snapshot] +# Path to store snapshot dump file +# if not set, use data_dir/snapshot +# snapshot file name format is dmp-2006-01-02T15:04:05.999999999 +path = "" + +# Reserve newest max_num snapshot dump files +max_num = 1 + +[tls] +enabled = false +certificate = "test.crt" +key = "test.key" \ No newline at end of file diff --git a/vendor/github.com/siddontang/ledisdb/config/config.go b/vendor/github.com/siddontang/ledisdb/config/config.go new file mode 100644 index 000000000000..31652be3266f --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/config/config.go @@ -0,0 +1,316 @@ +package config + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + "sync" + + "fmt" + + "github.com/pelletier/go-toml" + "github.com/siddontang/go/ioutil2" +) + +var ( + ErrNoConfigFile = errors.New("Running without a config file") +) + +const ( + DefaultAddr string = "127.0.0.1:6380" + + DefaultDBName string = "goleveldb" + + DefaultDataDir string = "./var" + + KB int = 1024 + MB int = KB * 1024 + GB int = MB * 1024 +) + +type LevelDBConfig struct { + Compression bool `toml:"compression"` + BlockSize int `toml:"block_size"` + WriteBufferSize int `toml:"write_buffer_size"` + CacheSize int `toml:"cache_size"` + MaxOpenFiles int `toml:"max_open_files"` + MaxFileSize int `toml:"max_file_size"` +} + +type RocksDBConfig struct { + Compression int `toml:"compression"` + BlockSize int `toml:"block_size"` + WriteBufferSize int `toml:"write_buffer_size"` + CacheSize int `toml:"cache_size"` + MaxOpenFiles int `toml:"max_open_files"` + MaxWriteBufferNum int `toml:"max_write_buffer_num"` + MinWriteBufferNumberToMerge int `toml:"min_write_buffer_number_to_merge"` + NumLevels int `toml:"num_levels"` + Level0FileNumCompactionTrigger int `toml:"level0_file_num_compaction_trigger"` + Level0SlowdownWritesTrigger int `toml:"level0_slowdown_writes_trigger"` + Level0StopWritesTrigger int `toml:"level0_stop_writes_trigger"` + TargetFileSizeBase int `toml:"target_file_size_base"` + TargetFileSizeMultiplier int `toml:"target_file_size_multiplier"` + MaxBytesForLevelBase int `toml:"max_bytes_for_level_base"` + MaxBytesForLevelMultiplier int `toml:"max_bytes_for_level_multiplier"` + DisableAutoCompactions bool `toml:"disable_auto_compactions"` + UseFsync bool `toml:"use_fsync"` + MaxBackgroundCompactions int `toml:"max_background_compactions"` + MaxBackgroundFlushes int `toml:"max_background_flushes"` + EnableStatistics bool `toml:"enable_statistics"` + StatsDumpPeriodSec int `toml:"stats_dump_period_sec"` + BackgroundThreads int `toml:"background_theads"` + HighPriorityBackgroundThreads int `toml:"high_priority_background_threads"` + DisableWAL bool `toml:"disable_wal"` + MaxManifestFileSize int `toml:"max_manifest_file_size"` +} + +type LMDBConfig struct { + MapSize int `toml:"map_size"` + NoSync bool `toml:"nosync"` +} + +type ReplicationConfig struct { + Path string `toml:"path"` + Sync bool `toml:"sync"` + WaitSyncTime int `toml:"wait_sync_time"` + WaitMaxSlaveAcks int `toml:"wait_max_slave_acks"` + ExpiredLogDays int `toml:"expired_log_days"` + StoreName string `toml:"store_name"` + MaxLogFileSize int64 `toml:"max_log_file_size"` + MaxLogFileNum int `toml:"max_log_file_num"` + SyncLog int `toml:"sync_log"` + Compression bool `toml:"compression"` + UseMmap bool `toml:"use_mmap"` + MasterPassword string `toml:"master_password"` +} + +type SnapshotConfig struct { + Path string `toml:"path"` + MaxNum int `toml:"max_num"` +} + +type TLS struct { + Enabled bool `toml:"enabled"` + Certificate string `toml:"certificate"` + Key string `toml:"key"` +} + +type AuthMethod func(c *Config, password string) bool + +type Config struct { + m sync.RWMutex `toml:"-"` + + AuthPassword string `toml:"auth_password"` + + //AuthMethod custom authentication method + AuthMethod AuthMethod `toml:"-"` + + FileName string `toml:"-"` + + // Addr can be empty to assign a local address dynamically + Addr string `toml:"addr"` + + AddrUnixSocketPerm string `toml:"addr_unixsocketperm"` + + HttpAddr string `toml:"http_addr"` + + SlaveOf string `toml:"slaveof"` + + Readonly bool `toml:"readonly"` + + DataDir string `toml:"data_dir"` + + Databases int `toml:"databases"` + + DBName string `toml:"db_name"` + DBPath string `toml:"db_path"` + DBSyncCommit int `toml:"db_sync_commit"` + + LevelDB LevelDBConfig `toml:"leveldb"` + RocksDB RocksDBConfig `toml:"rocksdb"` + + LMDB LMDBConfig `toml:"lmdb"` + + AccessLog string `toml:"access_log"` + + UseReplication bool `toml:"use_replication"` + Replication ReplicationConfig `toml:"replication"` + + Snapshot SnapshotConfig `toml:"snapshot"` + + ConnReadBufferSize int `toml:"conn_read_buffer_size"` + ConnWriteBufferSize int `toml:"conn_write_buffer_size"` + ConnKeepaliveInterval int `toml:"conn_keepalive_interval"` + + TTLCheckInterval int `toml:"ttl_check_interval"` + + //tls config + TLS TLS `toml:"tls"` +} + +func NewConfigWithFile(fileName string) (*Config, error) { + data, err := ioutil.ReadFile(fileName) + if err != nil { + return nil, err + } + + cfg, err := NewConfigWithData(data) + if err != nil { + return nil, err + } + + cfg.FileName = fileName + return cfg, nil +} + +func NewConfigWithData(data []byte) (*Config, error) { + cfg := NewConfigDefault() + + if err := toml.Unmarshal(data, cfg); err != nil { + return nil, fmt.Errorf("newConfigwithData: unmarashal: %s", err) + } + + cfg.adjust() + + return cfg, nil +} + +func NewConfigDefault() *Config { + cfg := new(Config) + + cfg.Addr = DefaultAddr + cfg.HttpAddr = "" + + cfg.DataDir = DefaultDataDir + + cfg.DBName = DefaultDBName + + cfg.SlaveOf = "" + cfg.Readonly = false + + // Disable Auth by default, by setting password to blank + cfg.AuthPassword = "" + + // default databases number + cfg.Databases = 16 + + // disable access log + cfg.AccessLog = "" + + cfg.LMDB.MapSize = 20 * MB + cfg.LMDB.NoSync = true + + cfg.UseReplication = false + cfg.Replication.WaitSyncTime = 500 + cfg.Replication.Compression = true + cfg.Replication.WaitMaxSlaveAcks = 2 + cfg.Replication.SyncLog = 0 + cfg.Replication.UseMmap = true + cfg.Snapshot.MaxNum = 1 + + cfg.RocksDB.EnableStatistics = false + cfg.RocksDB.UseFsync = false + cfg.RocksDB.DisableAutoCompactions = false + cfg.RocksDB.DisableWAL = false + + cfg.adjust() + + return cfg +} + +func getDefault(d int, s int) int { + if s <= 0 { + return d + } + + return s +} + +func (cfg *Config) adjust() { + cfg.LevelDB.adjust() + + cfg.RocksDB.adjust() + + cfg.Replication.ExpiredLogDays = getDefault(7, cfg.Replication.ExpiredLogDays) + cfg.Replication.MaxLogFileNum = getDefault(50, cfg.Replication.MaxLogFileNum) + cfg.ConnReadBufferSize = getDefault(4*KB, cfg.ConnReadBufferSize) + cfg.ConnWriteBufferSize = getDefault(4*KB, cfg.ConnWriteBufferSize) + cfg.TTLCheckInterval = getDefault(1, cfg.TTLCheckInterval) + cfg.Databases = getDefault(16, cfg.Databases) +} + +func (cfg *LevelDBConfig) adjust() { + cfg.CacheSize = getDefault(4*MB, cfg.CacheSize) + cfg.BlockSize = getDefault(4*KB, cfg.BlockSize) + cfg.WriteBufferSize = getDefault(4*MB, cfg.WriteBufferSize) + cfg.MaxOpenFiles = getDefault(1024, cfg.MaxOpenFiles) + cfg.MaxFileSize = getDefault(32*MB, cfg.MaxFileSize) +} + +func (cfg *RocksDBConfig) adjust() { + cfg.CacheSize = getDefault(4*MB, cfg.CacheSize) + cfg.BlockSize = getDefault(4*KB, cfg.BlockSize) + cfg.WriteBufferSize = getDefault(4*MB, cfg.WriteBufferSize) + cfg.MaxOpenFiles = getDefault(1024, cfg.MaxOpenFiles) + cfg.MaxWriteBufferNum = getDefault(2, cfg.MaxWriteBufferNum) + cfg.MinWriteBufferNumberToMerge = getDefault(1, cfg.MinWriteBufferNumberToMerge) + cfg.NumLevels = getDefault(7, cfg.NumLevels) + cfg.Level0FileNumCompactionTrigger = getDefault(4, cfg.Level0FileNumCompactionTrigger) + cfg.Level0SlowdownWritesTrigger = getDefault(16, cfg.Level0SlowdownWritesTrigger) + cfg.Level0StopWritesTrigger = getDefault(64, cfg.Level0StopWritesTrigger) + cfg.TargetFileSizeBase = getDefault(32*MB, cfg.TargetFileSizeBase) + cfg.TargetFileSizeMultiplier = getDefault(1, cfg.TargetFileSizeMultiplier) + cfg.MaxBytesForLevelBase = getDefault(32*MB, cfg.MaxBytesForLevelBase) + cfg.MaxBytesForLevelMultiplier = getDefault(1, cfg.MaxBytesForLevelMultiplier) + cfg.MaxBackgroundCompactions = getDefault(1, cfg.MaxBackgroundCompactions) + cfg.MaxBackgroundFlushes = getDefault(1, cfg.MaxBackgroundFlushes) + cfg.StatsDumpPeriodSec = getDefault(3600, cfg.StatsDumpPeriodSec) + cfg.BackgroundThreads = getDefault(2, cfg.BackgroundThreads) + cfg.HighPriorityBackgroundThreads = getDefault(1, cfg.HighPriorityBackgroundThreads) + cfg.MaxManifestFileSize = getDefault(20*MB, cfg.MaxManifestFileSize) +} + +func (cfg *Config) Dump(w io.Writer) error { + data, err := toml.Marshal(*cfg) + if err != nil { + return err + } + if _, err := w.Write(data); err != nil { + return err + } + + return nil +} + +func (cfg *Config) DumpFile(fileName string) error { + var b bytes.Buffer + + if err := cfg.Dump(&b); err != nil { + return err + } + + return ioutil2.WriteFileAtomic(fileName, b.Bytes(), 0644) +} + +func (cfg *Config) Rewrite() error { + if len(cfg.FileName) == 0 { + return ErrNoConfigFile + } + + return cfg.DumpFile(cfg.FileName) +} + +func (cfg *Config) GetReadonly() bool { + cfg.m.RLock() + b := cfg.Readonly + cfg.m.RUnlock() + return b +} + +func (cfg *Config) SetReadonly(b bool) { + cfg.m.Lock() + cfg.Readonly = b + cfg.m.Unlock() +} diff --git a/vendor/github.com/siddontang/ledisdb/config/config.toml b/vendor/github.com/siddontang/ledisdb/config/config.toml new file mode 100644 index 000000000000..a75409cb032c --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/config/config.toml @@ -0,0 +1,170 @@ +# LedisDB configuration + +# Server listen address +addr = "127.0.0.1:6380" + +# Unix socket permissions, 755 by default. +# Ignored for tcp socket. +addr_unixsocketperm = "0770" + +# Server http listen address, set empty to disable +http_addr = "127.0.0.1:11181" + +# Data store path, all ledisdb's data will be saved here +data_dir = "/tmp/ledis_server" + +# Set the number of databases. You can use `select dbindex` to choose a db. +# dbindex must be in [0, databases - 1]. +# Default databases is 16, maximum is 10240 now. +databases = 16 + +# Log server command, set empty to disable +access_log = "" + +# Set slaveof to enable replication from master, empty, no replication +# Any write operations except flushall and replication will be disabled in slave mode. +slaveof = "" + +# Readonly mode, slave server is always readonly even readonly = false +# for readonly mode, only replication and flushall can write +readonly = false + +# Choose which backend storage to use, now support: +# +# leveldb +# rocksdb +# goleveldb +# memory +# +db_name = "leveldb" + +# If not set, use data_dir/"db_name"_data +db_path = "" + +# Sync commit to disk if possible +# 0: no sync +# 1: sync every second +# 2: sync every commit +db_sync_commit = 0 + +# enable replication or not +use_replication = false + +# set connection buffer, you can increase them appropriately +# more size, more memory used +conn_read_buffer_size = 10240 +conn_write_buffer_size = 10240 + +# if connection receives no data after n seconds, it may be dead, close +# 0 to disable and not check +conn_keepalive_interval = 0 + +# checking TTL (time to live) data every n seconds +# if you set big, the expired data may not be deleted immediately +ttl_check_interval = 1 + +[leveldb] +# for leveldb and goleveldb +compression = false +block_size = 32768 +write_buffer_size = 67108864 +cache_size = 524288000 +max_open_files = 1024 +max_file_size = 33554432 + +[rocksdb] +# rocksdb has many many configurations, +# we only list little now, but may add more later. +# good luck! + +# 0:no, 1:snappy, 2:zlib, 3:bz2, 4:lz4, 5:lz4hc +compression = 0 +block_size = 65536 +write_buffer_size = 134217728 +cache_size = 1073741824 +max_open_files = 1024 +max_write_buffer_num = 6 +min_write_buffer_number_to_merge = 2 +num_levels = 7 +level0_file_num_compaction_trigger = 8 +level0_slowdown_writes_trigger = 16 +level0_stop_writes_trigger = 64 +target_file_size_base = 67108864 +target_file_size_multiplier = 1 +max_bytes_for_level_base = 536870912 +max_bytes_for_level_multiplier = 8 +disable_auto_compactions = false +disable_data_sync = false +use_fsync = false +background_theads = 16 +high_priority_background_threads = 1 +max_background_compactions = 15 +max_background_flushes = 1 +allow_os_buffer = true +enable_statistics = false +stats_dump_period_sec = 3600 +# dangerous to set true, write may got lost after a crash +# you can set true if replication opened, we may recover from replication log, +# but it is still not a easy work. +disable_wal = false +max_manifest_file_size = 20971520 + +[lmdb] +map_size = 524288000 +nosync = true + +[replication] +# Path to store replication information(write ahead log, commit log, etc.) +# if not set, use data_dir/rpl +path = "" + +# If sync is true, the new log must be sent to some slaves, and then commit. +# It will reduce performance but have better high availability. +sync = false + +# If sync is true, wait at last wait_sync_time milliseconds for slave syncing this log +wait_sync_time = 500 + +# If sync is true, wait at most min(wait_max_slave_acks, (n + 1) / 2) to promise syncing ok. +# n is slave number +# If 0, wait (n + 1) / 2 acks. +wait_max_slave_acks = 2 + +# store name: file, goleveldb +# change in runtime is very dangerous +store_name = "file" + +# Expire write ahead logs after the given days +expired_log_days = 7 + +# for file store, if 0, use default 256MB, max is 1G +max_log_file_size = 0 + +# for file store, if 0, use default 50 +max_log_file_num = 0 + +# for file store, use mmap for file read and write +use_mmap = true + +# Sync log to disk if possible +# 0: no sync +# 1: sync every second +# 2: sync every commit +sync_log = 0 + +# Compress the log or not +compression = false + +[snapshot] +# Path to store snapshot dump file +# if not set, use data_dir/snapshot +# snapshot file name format is dmp-2006-01-02T15:04:05.999999999 +path = "" + +# Reserve newest max_num snapshot dump files +max_num = 1 + +[tls] +enabled = true +certificate = "test.crt" +key = "test.key" \ No newline at end of file diff --git a/vendor/github.com/siddontang/ledisdb/ledis/batch.go b/vendor/github.com/siddontang/ledisdb/ledis/batch.go new file mode 100644 index 000000000000..6800dfe898f0 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/batch.go @@ -0,0 +1,139 @@ +package ledis + +import ( + "sync" + + "github.com/siddontang/go/log" + "github.com/siddontang/ledisdb/rpl" + "github.com/siddontang/ledisdb/store" +) + +type batch struct { + l *Ledis + + *store.WriteBatch + + sync.Locker + + // tx *Tx +} + +func (b *batch) Commit() error { + if b.l.cfg.GetReadonly() { + return ErrWriteInROnly + } + + return b.l.handleCommit(b.WriteBatch, b.WriteBatch) + + // if b.tx == nil { + // return b.l.handleCommit(b.WriteBatch, b.WriteBatch) + // } else { + // if b.l.r != nil { + // if err := b.tx.data.Append(b.WriteBatch.BatchData()); err != nil { + // return err + // } + // } + // return b.WriteBatch.Commit() + // } +} + +func (b *batch) Lock() { + b.Locker.Lock() +} + +func (b *batch) Unlock() { + b.WriteBatch.Rollback() + b.Locker.Unlock() +} + +func (b *batch) Put(key []byte, value []byte) { + b.WriteBatch.Put(key, value) +} + +func (b *batch) Delete(key []byte) { + b.WriteBatch.Delete(key) +} + +type dbBatchLocker struct { + l *sync.Mutex + wrLock *sync.RWMutex +} + +func (l *dbBatchLocker) Lock() { + l.wrLock.RLock() + l.l.Lock() +} + +func (l *dbBatchLocker) Unlock() { + l.l.Unlock() + l.wrLock.RUnlock() +} + +// type txBatchLocker struct { +// } + +// func (l *txBatchLocker) Lock() {} +// func (l *txBatchLocker) Unlock() {} + +// type multiBatchLocker struct { +// } + +// func (l *multiBatchLocker) Lock() {} +// func (l *multiBatchLocker) Unlock() {} + +func (l *Ledis) newBatch(wb *store.WriteBatch, locker sync.Locker) *batch { + b := new(batch) + b.l = l + b.WriteBatch = wb + + b.Locker = locker + + return b +} + +type commiter interface { + Commit() error +} + +type commitDataGetter interface { + Data() []byte +} + +func (l *Ledis) handleCommit(g commitDataGetter, c commiter) error { + l.commitLock.Lock() + + var err error + if l.r != nil { + var rl *rpl.Log + if rl, err = l.r.Log(g.Data()); err != nil { + l.commitLock.Unlock() + + log.Fatalf("write wal error %s", err.Error()) + return err + } + + l.propagate(rl) + + if err = c.Commit(); err != nil { + l.commitLock.Unlock() + + log.Fatalf("commit error %s", err.Error()) + l.noticeReplication() + return err + } + + if err = l.r.UpdateCommitID(rl.ID); err != nil { + l.commitLock.Unlock() + + log.Fatalf("update commit id error %s", err.Error()) + l.noticeReplication() + return err + } + } else { + err = c.Commit() + } + + l.commitLock.Unlock() + + return err +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/const.go b/vendor/github.com/siddontang/ledisdb/ledis/const.go new file mode 100644 index 000000000000..610dada9893b --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/const.go @@ -0,0 +1,150 @@ +package ledis + +import ( + "errors" +) + +// Version is for version +const Version = "0.5" + +// DataType is defined for the different types +type DataType byte + +// for out use +const ( + KV DataType = iota + LIST + HASH + SET + ZSET +) + +func (d DataType) String() string { + switch d { + case KV: + return KVName + case LIST: + return ListName + case HASH: + return HashName + case SET: + return SetName + case ZSET: + return ZSetName + default: + return "unknown" + } +} + +// For different type name +const ( + KVName = "KV" + ListName = "LIST" + HashName = "HASH" + SetName = "SET" + ZSetName = "ZSET" +) + +// for backend store +const ( + NoneType byte = 0 + KVType byte = 1 + HashType byte = 2 + HSizeType byte = 3 + ListType byte = 4 + LMetaType byte = 5 + ZSetType byte = 6 + ZSizeType byte = 7 + ZScoreType byte = 8 + // BitType byte = 9 + // BitMetaType byte = 10 + SetType byte = 11 + SSizeType byte = 12 + + maxDataType byte = 100 + + /* + I make a big mistake about TTL time key format and have to use a new one (change 101 to 103). + You must run the ledis-upgrade-ttl to upgrade db. + */ + ObsoleteExpTimeType byte = 101 + ExpMetaType byte = 102 + ExpTimeType byte = 103 + + MetaType byte = 201 +) + +// TypeName is the map of type -> name +var TypeName = map[byte]string{ + KVType: "kv", + HashType: "hash", + HSizeType: "hsize", + ListType: "list", + LMetaType: "lmeta", + ZSetType: "zset", + ZSizeType: "zsize", + ZScoreType: "zscore", + // BitType: "bit", + // BitMetaType: "bitmeta", + SetType: "set", + SSizeType: "ssize", + ExpTimeType: "exptime", + ExpMetaType: "expmeta", +} + +const ( + defaultScanCount int = 10 +) + +var ( + errKeySize = errors.New("invalid key size") + errValueSize = errors.New("invalid value size") + errHashFieldSize = errors.New("invalid hash field size") + errSetMemberSize = errors.New("invalid set member size") + errZSetMemberSize = errors.New("invalid zset member size") + errExpireValue = errors.New("invalid expire value") + errListIndex = errors.New("invalid list index") +) + +// For different const size configuration +const ( + // max allowed databases + MaxDatabases int = 10240 + + // max key size + MaxKeySize int = 1024 + + // max hash field size + MaxHashFieldSize int = 1024 + + // max zset member size + MaxZSetMemberSize int = 1024 + + // max set member size + MaxSetMemberSize int = 1024 + + // max value size + MaxValueSize int = 1024 * 1024 * 1024 +) + +// For different common errors +var ( + ErrScoreMiss = errors.New("zset score miss") + ErrWriteInROnly = errors.New("write not support in readonly mode") + ErrRplInRDWR = errors.New("replication not support in read write mode") + ErrRplNotSupport = errors.New("replication not support") +) + +// const ( +// DBAutoCommit uint8 = 0x0 +// DBInTransaction uint8 = 0x1 +// DBInMulti uint8 = 0x2 +// ) + +// For bit operation +const ( + BitAND = "and" + BitOR = "or" + BitXOR = "xor" + BitNot = "not" +) diff --git a/vendor/github.com/siddontang/ledisdb/ledis/doc.go b/vendor/github.com/siddontang/ledisdb/ledis/doc.go new file mode 100644 index 000000000000..c6bfe7807bef --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/doc.go @@ -0,0 +1,58 @@ +// Package ledis is a high performance embedded NoSQL. +// +// Ledis supports various data structure like kv, list, hash and zset like redis. +// +// Other features include replication, data with a limited time-to-live. +// +// Usage +// +// First create a ledis instance before use: +// +// l := ledis.Open(cfg) +// +// cfg is a Config instance which contains configuration for ledis use, +// like DataDir (root directory for ledis working to store data). +// +// After you create a ledis instance, you can select a DB to store you data: +// +// db, _ := l.Select(0) +// +// DB must be selected by a index, ledis supports only 16 databases, so the index range is [0-15]. +// +// KV +// +// KV is the most basic ledis type like any other key-value database. +// +// err := db.Set(key, value) +// value, err := db.Get(key) +// +// List +// +// List is simply lists of values, sorted by insertion order. +// You can push or pop value on the list head (left) or tail (right). +// +// err := db.LPush(key, value1) +// err := db.RPush(key, value2) +// value1, err := db.LPop(key) +// value2, err := db.RPop(key) +// +// Hash +// +// Hash is a map between fields and values. +// +// n, err := db.HSet(key, field1, value1) +// n, err := db.HSet(key, field2, value2) +// value1, err := db.HGet(key, field1) +// value2, err := db.HGet(key, field2) +// +// ZSet +// +// ZSet is a sorted collections of values. +// Every member of zset is associated with score, a int64 value which used to sort, from smallest to greatest score. +// Members are unique, but score may be same. +// +// n, err := db.ZAdd(key, ScorePair{score1, member1}, ScorePair{score2, member2}) +// ay, err := db.ZRangeByScore(key, minScore, maxScore, 0, -1) +// +// +package ledis diff --git a/vendor/github.com/siddontang/ledisdb/ledis/dump.go b/vendor/github.com/siddontang/ledisdb/ledis/dump.go new file mode 100644 index 000000000000..9784575046d8 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/dump.go @@ -0,0 +1,220 @@ +package ledis + +import ( + "bufio" + "bytes" + "encoding/binary" + "io" + "os" + + "github.com/siddontang/go/snappy" + "github.com/siddontang/ledisdb/store" +) + +// DumpHead is the head of a dump. +type DumpHead struct { + CommitID uint64 +} + +// Read reads meta from the Reader. +func (h *DumpHead) Read(r io.Reader) error { + return binary.Read(r, binary.BigEndian, &h.CommitID) +} + +// Write writes meta to the Writer +func (h *DumpHead) Write(w io.Writer) error { + return binary.Write(w, binary.BigEndian, h.CommitID) +} + +// DumpFile dumps data to the file +func (l *Ledis) DumpFile(path string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + return l.Dump(f) +} + +// Dump dumps data to the Writer. +func (l *Ledis) Dump(w io.Writer) error { + var err error + + var commitID uint64 + var snap *store.Snapshot + + l.wLock.Lock() + + if l.r != nil { + if commitID, err = l.r.LastCommitID(); err != nil { + l.wLock.Unlock() + return err + } + } + + if snap, err = l.ldb.NewSnapshot(); err != nil { + l.wLock.Unlock() + return err + } + defer snap.Close() + + l.wLock.Unlock() + + wb := bufio.NewWriterSize(w, 4096) + + h := &DumpHead{commitID} + + if err = h.Write(wb); err != nil { + return err + } + + it := snap.NewIterator() + defer it.Close() + it.SeekToFirst() + + compressBuf := make([]byte, 4096) + + var key []byte + var value []byte + for ; it.Valid(); it.Next() { + key = it.RawKey() + value = it.RawValue() + + if key, err = snappy.Encode(compressBuf, key); err != nil { + return err + } + + if err = binary.Write(wb, binary.BigEndian, uint16(len(key))); err != nil { + return err + } + + if _, err = wb.Write(key); err != nil { + return err + } + + if value, err = snappy.Encode(compressBuf, value); err != nil { + return err + } + + if err = binary.Write(wb, binary.BigEndian, uint32(len(value))); err != nil { + return err + } + + if _, err = wb.Write(value); err != nil { + return err + } + } + + if err = wb.Flush(); err != nil { + return err + } + + compressBuf = nil + + return nil +} + +// LoadDumpFile clears all data and loads dump file to db +func (l *Ledis) LoadDumpFile(path string) (*DumpHead, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + return l.LoadDump(f) +} + +// LoadDump clears all data and loads dump file to db +func (l *Ledis) LoadDump(r io.Reader) (*DumpHead, error) { + l.wLock.Lock() + defer l.wLock.Unlock() + + var err error + if err = l.flushAll(); err != nil { + return nil, err + } + + rb := bufio.NewReaderSize(r, 4096) + + h := new(DumpHead) + + if err = h.Read(rb); err != nil { + return nil, err + } + + var keyLen uint16 + var valueLen uint32 + + var keyBuf bytes.Buffer + var valueBuf bytes.Buffer + + deKeyBuf := make([]byte, 4096) + deValueBuf := make([]byte, 4096) + + var key, value []byte + + wb := l.ldb.NewWriteBatch() + defer wb.Close() + + n := 0 + + for { + if err = binary.Read(rb, binary.BigEndian, &keyLen); err != nil && err != io.EOF { + return nil, err + } else if err == io.EOF { + break + } + + if _, err = io.CopyN(&keyBuf, rb, int64(keyLen)); err != nil { + return nil, err + } + + if key, err = snappy.Decode(deKeyBuf, keyBuf.Bytes()); err != nil { + return nil, err + } + + if err = binary.Read(rb, binary.BigEndian, &valueLen); err != nil { + return nil, err + } + + if _, err = io.CopyN(&valueBuf, rb, int64(valueLen)); err != nil { + return nil, err + } + + if value, err = snappy.Decode(deValueBuf, valueBuf.Bytes()); err != nil { + return nil, err + } + + wb.Put(key, value) + n++ + if n%1024 == 0 { + if err = wb.Commit(); err != nil { + return nil, err + } + } + + // if err = l.ldb.Put(key, value); err != nil { + // return nil, err + // } + + keyBuf.Reset() + valueBuf.Reset() + } + + if err = wb.Commit(); err != nil { + return nil, err + } + + deKeyBuf = nil + deValueBuf = nil + + if l.r != nil { + if err := l.r.UpdateCommitID(h.CommitID); err != nil { + return nil, err + } + } + + return h, nil +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/event.go b/vendor/github.com/siddontang/ledisdb/ledis/event.go new file mode 100644 index 000000000000..f35c893fe114 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/event.go @@ -0,0 +1,136 @@ +package ledis + +import ( + "errors" + "fmt" + "strconv" + + "github.com/siddontang/go/hack" +) + +var errInvalidEvent = errors.New("invalid event") + +func formatEventKey(buf []byte, k []byte) ([]byte, error) { + if len(k) < 2 { + return nil, errInvalidEvent + } + + buf = append(buf, fmt.Sprintf("DB:%2d ", k[0])...) + buf = append(buf, fmt.Sprintf("%s ", TypeName[k[1]])...) + + db := new(DB) + index, _, err := decodeDBIndex(k) + if err != nil { + return nil, err + } + db.setIndex(index) + + //to do format at respective place + + switch k[1] { + case KVType: + key, err := db.decodeKVKey(k) + if err != nil { + return nil, err + } + buf = strconv.AppendQuote(buf, hack.String(key)) + case HashType: + key, field, err := db.hDecodeHashKey(k) + if err != nil { + return nil, err + } + + buf = strconv.AppendQuote(buf, hack.String(key)) + buf = append(buf, ' ') + buf = strconv.AppendQuote(buf, hack.String(field)) + case HSizeType: + key, err := db.hDecodeSizeKey(k) + if err != nil { + return nil, err + } + + buf = strconv.AppendQuote(buf, hack.String(key)) + case ListType: + key, seq, err := db.lDecodeListKey(k) + if err != nil { + return nil, err + } + + buf = strconv.AppendQuote(buf, hack.String(key)) + buf = append(buf, ' ') + buf = strconv.AppendInt(buf, int64(seq), 10) + case LMetaType: + key, err := db.lDecodeMetaKey(k) + if err != nil { + return nil, err + } + + buf = strconv.AppendQuote(buf, hack.String(key)) + case ZSetType: + key, m, err := db.zDecodeSetKey(k) + if err != nil { + return nil, err + } + + buf = strconv.AppendQuote(buf, hack.String(key)) + buf = append(buf, ' ') + buf = strconv.AppendQuote(buf, hack.String(m)) + case ZSizeType: + key, err := db.zDecodeSizeKey(k) + if err != nil { + return nil, err + } + + buf = strconv.AppendQuote(buf, hack.String(key)) + case ZScoreType: + key, m, score, err := db.zDecodeScoreKey(k) + if err != nil { + return nil, err + } + buf = strconv.AppendQuote(buf, hack.String(key)) + buf = append(buf, ' ') + buf = strconv.AppendQuote(buf, hack.String(m)) + buf = append(buf, ' ') + buf = strconv.AppendInt(buf, score, 10) + case SetType: + key, member, err := db.sDecodeSetKey(k) + if err != nil { + return nil, err + } + + buf = strconv.AppendQuote(buf, hack.String(key)) + buf = append(buf, ' ') + buf = strconv.AppendQuote(buf, hack.String(member)) + case SSizeType: + key, err := db.sDecodeSizeKey(k) + if err != nil { + return nil, err + } + + buf = strconv.AppendQuote(buf, hack.String(key)) + case ExpTimeType: + tp, key, t, err := db.expDecodeTimeKey(k) + if err != nil { + return nil, err + } + + buf = append(buf, TypeName[tp]...) + buf = append(buf, ' ') + buf = strconv.AppendQuote(buf, hack.String(key)) + buf = append(buf, ' ') + buf = strconv.AppendInt(buf, t, 10) + case ExpMetaType: + tp, key, err := db.expDecodeMetaKey(k) + if err != nil { + return nil, err + } + + buf = append(buf, TypeName[tp]...) + buf = append(buf, ' ') + buf = strconv.AppendQuote(buf, hack.String(key)) + default: + return nil, errInvalidEvent + } + + return buf, nil +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/ledis.go b/vendor/github.com/siddontang/ledisdb/ledis/ledis.go new file mode 100644 index 000000000000..ab8543c00950 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/ledis.go @@ -0,0 +1,248 @@ +package ledis + +import ( + "fmt" + "io" + "os" + "path" + "sync" + "time" + + "github.com/siddontang/go/filelock" + "github.com/siddontang/go/log" + "github.com/siddontang/ledisdb/config" + "github.com/siddontang/ledisdb/rpl" + "github.com/siddontang/ledisdb/store" +) + +// Ledis is the core structure to handle the database. +type Ledis struct { + cfg *config.Config + + ldb *store.DB + + dbLock sync.Mutex + dbs map[int]*DB + + quit chan struct{} + wg sync.WaitGroup + + //for replication + r *rpl.Replication + rc chan struct{} + rbatch *store.WriteBatch + rDoneCh chan struct{} + rhs []NewLogEventHandler + + wLock sync.RWMutex //allow one write at same time + commitLock sync.Mutex //allow one write commit at same time + + lock io.Closer + + ttlCheckers []*ttlChecker + ttlCheckerCh chan *ttlChecker +} + +// Open opens the Ledis with a config. +func Open(cfg *config.Config) (*Ledis, error) { + if len(cfg.DataDir) == 0 { + cfg.DataDir = config.DefaultDataDir + } + + if cfg.Databases == 0 { + cfg.Databases = 16 + } else if cfg.Databases > MaxDatabases { + cfg.Databases = MaxDatabases + } + + os.MkdirAll(cfg.DataDir, 0755) + + var err error + + l := new(Ledis) + l.cfg = cfg + + if l.lock, err = filelock.Lock(path.Join(cfg.DataDir, "LOCK")); err != nil { + return nil, err + } + + l.quit = make(chan struct{}) + + if l.ldb, err = store.Open(cfg); err != nil { + return nil, err + } + + if cfg.UseReplication { + if l.r, err = rpl.NewReplication(cfg); err != nil { + return nil, err + } + + l.rc = make(chan struct{}, 1) + l.rbatch = l.ldb.NewWriteBatch() + l.rDoneCh = make(chan struct{}, 1) + + l.wg.Add(1) + go l.onReplication() + + //first we must try wait all replication ok + //maybe some logs are not committed + l.WaitReplication() + } else { + l.r = nil + } + + l.dbs = make(map[int]*DB, 16) + + l.checkTTL() + + return l, nil +} + +// Close closes the Ledis. +func (l *Ledis) Close() { + close(l.quit) + l.wg.Wait() + + l.ldb.Close() + + if l.r != nil { + l.r.Close() + //l.r = nil + } + + if l.lock != nil { + l.lock.Close() + //l.lock = nil + } +} + +// Select chooses a database. +func (l *Ledis) Select(index int) (*DB, error) { + if index < 0 || index >= l.cfg.Databases { + return nil, fmt.Errorf("invalid db index %d, must in [0, %d]", index, l.cfg.Databases-1) + } + + l.dbLock.Lock() + defer l.dbLock.Unlock() + + db, ok := l.dbs[index] + if ok { + return db, nil + } + + db = l.newDB(index) + l.dbs[index] = db + + go func(db *DB) { + l.ttlCheckerCh <- db.ttlChecker + }(db) + + return db, nil +} + +// FlushAll will clear all data and replication logs +func (l *Ledis) FlushAll() error { + l.wLock.Lock() + defer l.wLock.Unlock() + + return l.flushAll() +} + +func (l *Ledis) flushAll() error { + it := l.ldb.NewIterator() + defer it.Close() + + it.SeekToFirst() + + w := l.ldb.NewWriteBatch() + defer w.Rollback() + + n := 0 + for ; it.Valid(); it.Next() { + n++ + if n == 10000 { + if err := w.Commit(); err != nil { + log.Fatalf("flush all commit error: %s", err.Error()) + return err + } + n = 0 + } + w.Delete(it.RawKey()) + } + + if err := w.Commit(); err != nil { + log.Fatalf("flush all commit error: %s", err.Error()) + return err + } + + if l.r != nil { + if err := l.r.Clear(); err != nil { + log.Fatalf("flush all replication clear error: %s", err.Error()) + return err + } + } + + return nil +} + +// IsReadOnly returns whether Ledis is read only or not. +func (l *Ledis) IsReadOnly() bool { + if l.cfg.GetReadonly() { + return true + } else if l.r != nil { + if b, _ := l.r.CommitIDBehind(); b { + return true + } + } + return false +} + +func (l *Ledis) checkTTL() { + l.ttlCheckers = make([]*ttlChecker, 0, 16) + l.ttlCheckerCh = make(chan *ttlChecker, 16) + + if l.cfg.TTLCheckInterval == 0 { + l.cfg.TTLCheckInterval = 1 + } + + l.wg.Add(1) + go func() { + defer l.wg.Done() + + tick := time.NewTicker(time.Duration(l.cfg.TTLCheckInterval) * time.Second) + defer tick.Stop() + + for { + select { + case <-tick.C: + if l.IsReadOnly() { + break + } + + for _, c := range l.ttlCheckers { + c.check() + } + case c := <-l.ttlCheckerCh: + l.ttlCheckers = append(l.ttlCheckers, c) + c.check() + case <-l.quit: + return + } + } + + }() + +} + +// StoreStat returns the statistics. +func (l *Ledis) StoreStat() *store.Stat { + return l.ldb.Stat() +} + +// CompactStore compacts the backend storage. +func (l *Ledis) CompactStore() error { + l.wLock.Lock() + defer l.wLock.Unlock() + + return l.ldb.Compact() +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/ledis_db.go b/vendor/github.com/siddontang/ledisdb/ledis/ledis_db.go new file mode 100644 index 000000000000..4d250b50fb98 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/ledis_db.go @@ -0,0 +1,208 @@ +package ledis + +import ( + "bytes" + "encoding/binary" + "fmt" + "sync" + + "github.com/siddontang/ledisdb/store" +) + +type ibucket interface { + Get(key []byte) ([]byte, error) + GetSlice(key []byte) (store.Slice, error) + + Put(key []byte, value []byte) error + Delete(key []byte) error + + NewIterator() *store.Iterator + + NewWriteBatch() *store.WriteBatch + + RangeIterator(min []byte, max []byte, rangeType uint8) *store.RangeLimitIterator + RevRangeIterator(min []byte, max []byte, rangeType uint8) *store.RangeLimitIterator + RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *store.RangeLimitIterator + RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *store.RangeLimitIterator +} + +// DB is the database. +type DB struct { + l *Ledis + + sdb *store.DB + + bucket ibucket + + index int + + // buffer to store index varint + indexVarBuf []byte + + kvBatch *batch + listBatch *batch + hashBatch *batch + zsetBatch *batch + // binBatch *batch + setBatch *batch + + // status uint8 + + ttlChecker *ttlChecker + + lbkeys *lBlockKeys +} + +func (l *Ledis) newDB(index int) *DB { + d := new(DB) + + d.l = l + + d.sdb = l.ldb + + d.bucket = d.sdb + + // d.status = DBAutoCommit + d.setIndex(index) + + d.kvBatch = d.newBatch() + d.listBatch = d.newBatch() + d.hashBatch = d.newBatch() + d.zsetBatch = d.newBatch() + // d.binBatch = d.newBatch() + d.setBatch = d.newBatch() + + d.lbkeys = newLBlockKeys() + + d.ttlChecker = d.newTTLChecker() + + return d +} + +func decodeDBIndex(buf []byte) (int, int, error) { + index, n := binary.Uvarint(buf) + if n == 0 { + return 0, 0, fmt.Errorf("buf is too small to save index") + } else if n < 0 { + return 0, 0, fmt.Errorf("value larger than 64 bits") + } else if index > uint64(MaxDatabases) { + return 0, 0, fmt.Errorf("value %d is larger than max databases %d", index, MaxDatabases) + } + return int(index), n, nil +} + +func (db *DB) setIndex(index int) { + db.index = index + // the most size for varint is 10 bytes + buf := make([]byte, 10) + n := binary.PutUvarint(buf, uint64(index)) + + db.indexVarBuf = buf[0:n] +} + +func (db *DB) checkKeyIndex(buf []byte) (int, error) { + if len(buf) < len(db.indexVarBuf) { + return 0, fmt.Errorf("key is too small") + } else if !bytes.Equal(db.indexVarBuf, buf[0:len(db.indexVarBuf)]) { + return 0, fmt.Errorf("invalid db index") + } + + return len(db.indexVarBuf), nil +} + +func (db *DB) newTTLChecker() *ttlChecker { + c := new(ttlChecker) + c.db = db + c.txs = make([]*batch, maxDataType) + c.cbs = make([]onExpired, maxDataType) + c.nc = 0 + + c.register(KVType, db.kvBatch, db.delete) + c.register(ListType, db.listBatch, db.lDelete) + c.register(HashType, db.hashBatch, db.hDelete) + c.register(ZSetType, db.zsetBatch, db.zDelete) + // c.register(BitType, db.binBatch, db.bDelete) + c.register(SetType, db.setBatch, db.sDelete) + + return c +} + +func (db *DB) newBatch() *batch { + return db.l.newBatch(db.bucket.NewWriteBatch(), &dbBatchLocker{l: &sync.Mutex{}, wrLock: &db.l.wLock}) +} + +// Index gets the index of database. +func (db *DB) Index() int { + return int(db.index) +} + +// func (db *DB) IsAutoCommit() bool { +// return db.status == DBAutoCommit +// } + +// FlushAll flushes the data. +func (db *DB) FlushAll() (drop int64, err error) { + all := [...](func() (int64, error)){ + db.flush, + db.lFlush, + db.hFlush, + db.zFlush, + db.sFlush} + + for _, flush := range all { + n, e := flush() + if e != nil { + err = e + return + } + + drop += n + } + + return +} + +func (db *DB) flushType(t *batch, dataType byte) (drop int64, err error) { + var deleteFunc func(t *batch, key []byte) int64 + var metaDataType byte + switch dataType { + case KVType: + deleteFunc = db.delete + metaDataType = KVType + case ListType: + deleteFunc = db.lDelete + metaDataType = LMetaType + case HashType: + deleteFunc = db.hDelete + metaDataType = HSizeType + case ZSetType: + deleteFunc = db.zDelete + metaDataType = ZSizeType + // case BitType: + // deleteFunc = db.bDelete + // metaDataType = BitMetaType + case SetType: + deleteFunc = db.sDelete + metaDataType = SSizeType + default: + return 0, fmt.Errorf("invalid data type: %s", TypeName[dataType]) + } + + var keys [][]byte + keys, err = db.scanGeneric(metaDataType, nil, 1024, false, "", false) + for len(keys) != 0 || err != nil { + for _, key := range keys { + deleteFunc(t, key) + db.rmExpire(t, dataType, key) + + } + + if err = t.Commit(); err != nil { + return + } + + drop += int64(len(keys)) + keys, err = db.scanGeneric(metaDataType, nil, 1024, false, "", false) + } + return +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/migrate.go b/vendor/github.com/siddontang/ledisdb/ledis/migrate.go new file mode 100644 index 000000000000..eea5b102b0eb --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/migrate.go @@ -0,0 +1,195 @@ +package ledis + +import ( + "fmt" + + "github.com/siddontang/rdb" +) + +/* + To support redis <-> ledisdb, the dump value format is the same as redis. + We will not support bitmap, and may add bit operations for kv later. + + But you must know that we use int64 for zset score, not double. + Only support rdb version 6. +*/ + +// Dump dumps the KV value of key +func (db *DB) Dump(key []byte) ([]byte, error) { + v, err := db.Get(key) + if err != nil { + return nil, err + } else if v == nil { + return nil, err + } + + return rdb.Dump(rdb.String(v)) +} + +// LDump dumps the list value of key +func (db *DB) LDump(key []byte) ([]byte, error) { + v, err := db.LRange(key, 0, -1) + if err != nil { + return nil, err + } else if len(v) == 0 { + return nil, err + } + + return rdb.Dump(rdb.List(v)) +} + +// HDump dumps the hash value of key +func (db *DB) HDump(key []byte) ([]byte, error) { + v, err := db.HGetAll(key) + if err != nil { + return nil, err + } else if len(v) == 0 { + return nil, err + } + + o := make(rdb.Hash, len(v)) + for i := 0; i < len(v); i++ { + o[i].Field = v[i].Field + o[i].Value = v[i].Value + } + + return rdb.Dump(o) +} + +// SDump dumps the set value of key +func (db *DB) SDump(key []byte) ([]byte, error) { + v, err := db.SMembers(key) + if err != nil { + return nil, err + } else if len(v) == 0 { + return nil, err + } + + return rdb.Dump(rdb.Set(v)) +} + +// ZDump dumps the zset value of key +func (db *DB) ZDump(key []byte) ([]byte, error) { + v, err := db.ZRangeByScore(key, MinScore, MaxScore, 0, -1) + if err != nil { + return nil, err + } else if len(v) == 0 { + return nil, err + } + + o := make(rdb.ZSet, len(v)) + for i := 0; i < len(v); i++ { + o[i].Member = v[i].Member + o[i].Score = float64(v[i].Score) + } + + return rdb.Dump(o) +} + +// Restore restores a key into database. +func (db *DB) Restore(key []byte, ttl int64, data []byte) error { + d, err := rdb.DecodeDump(data) + if err != nil { + return err + } + + //ttl is milliseconds, but we only support seconds + //later may support milliseconds + if ttl > 0 { + ttl = ttl / 1e3 + if ttl == 0 { + ttl = 1 + } + } + + switch value := d.(type) { + case rdb.String: + if _, err = db.Del(key); err != nil { + return err + } + + if err = db.Set(key, value); err != nil { + return err + } + + if ttl > 0 { + if _, err = db.Expire(key, ttl); err != nil { + return err + } + } + case rdb.Hash: + //first clear old key + if _, err = db.HClear(key); err != nil { + return err + } + + fv := make([]FVPair, len(value)) + for i := 0; i < len(value); i++ { + fv[i] = FVPair{Field: value[i].Field, Value: value[i].Value} + } + + if err = db.HMset(key, fv...); err != nil { + return err + } + + if ttl > 0 { + if _, err = db.HExpire(key, ttl); err != nil { + return err + } + } + case rdb.List: + //first clear old key + if _, err = db.LClear(key); err != nil { + return err + } + + if _, err = db.RPush(key, value...); err != nil { + return err + } + + if ttl > 0 { + if _, err = db.LExpire(key, ttl); err != nil { + return err + } + } + case rdb.ZSet: + //first clear old key + if _, err = db.ZClear(key); err != nil { + return err + } + + sp := make([]ScorePair, len(value)) + for i := 0; i < len(value); i++ { + sp[i] = ScorePair{int64(value[i].Score), value[i].Member} + } + + if _, err = db.ZAdd(key, sp...); err != nil { + return err + } + + if ttl > 0 { + if _, err = db.ZExpire(key, ttl); err != nil { + return err + } + } + case rdb.Set: + //first clear old key + if _, err = db.SClear(key); err != nil { + return err + } + + if _, err = db.SAdd(key, value...); err != nil { + return err + } + + if ttl > 0 { + if _, err = db.SExpire(key, ttl); err != nil { + return err + } + } + default: + return fmt.Errorf("invalid data type %T", d) + } + + return nil +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/replication.go b/vendor/github.com/siddontang/ledisdb/ledis/replication.go new file mode 100644 index 000000000000..2d0c9763f1e1 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/replication.go @@ -0,0 +1,259 @@ +package ledis + +import ( + "bytes" + "errors" + "io" + "time" + + "github.com/siddontang/go/log" + "github.com/siddontang/go/snappy" + "github.com/siddontang/ledisdb/rpl" + "github.com/siddontang/ledisdb/store" +) + +const ( + maxReplLogSize = 1 * 1024 * 1024 +) + +// For replication error. +var ( + ErrLogMissed = errors.New("log is pured in server") +) + +// ReplicationUsed returns whether replication is used or not. +func (l *Ledis) ReplicationUsed() bool { + return l.r != nil +} + +func (l *Ledis) handleReplication() error { + l.wLock.Lock() + defer l.wLock.Unlock() + + defer AsyncNotify(l.rDoneCh) + + rl := &rpl.Log{} + + var err error + for { + if err = l.r.NextNeedCommitLog(rl); err != nil { + if err != rpl.ErrNoBehindLog { + log.Errorf("get next commit log err, %s", err.Error()) + return err + } + + return nil + } + + l.rbatch.Rollback() + + if rl.Compression == 1 { + //todo optimize + if rl.Data, err = snappy.Decode(nil, rl.Data); err != nil { + log.Errorf("decode log error %s", err.Error()) + return err + } + } + + if bd, err := store.NewBatchData(rl.Data); err != nil { + log.Errorf("decode batch log error %s", err.Error()) + return err + } else if err = bd.Replay(l.rbatch); err != nil { + log.Errorf("replay batch log error %s", err.Error()) + } + + l.commitLock.Lock() + if err = l.rbatch.Commit(); err != nil { + log.Errorf("commit log error %s", err.Error()) + } else if err = l.r.UpdateCommitID(rl.ID); err != nil { + log.Errorf("update commit id error %s", err.Error()) + } + + l.commitLock.Unlock() + if err != nil { + return err + } + } +} + +func (l *Ledis) onReplication() { + defer l.wg.Done() + + l.noticeReplication() + + for { + select { + case <-l.rc: + l.handleReplication() + case <-l.quit: + return + } + } +} + +// WaitReplication waits replication done +func (l *Ledis) WaitReplication() error { + if !l.ReplicationUsed() { + return ErrRplNotSupport + + } + + for i := 0; i < 100; i++ { + l.noticeReplication() + + select { + case <-l.rDoneCh: + case <-l.quit: + return nil + } + time.Sleep(100 * time.Millisecond) + + b, err := l.r.CommitIDBehind() + if err != nil { + return err + } else if !b { + return nil + } + } + + return errors.New("wait replication too many times") +} + +// StoreLogsFromReader stores logs from the Reader +func (l *Ledis) StoreLogsFromReader(rb io.Reader) error { + if !l.ReplicationUsed() { + return ErrRplNotSupport + } else if !l.cfg.Readonly { + return ErrRplInRDWR + } + + log := &rpl.Log{} + + for { + if err := log.Decode(rb); err != nil { + if err == io.EOF { + break + } else { + return err + } + } + + if err := l.r.StoreLog(log); err != nil { + return err + } + + } + + l.noticeReplication() + + return nil +} + +func (l *Ledis) noticeReplication() { + AsyncNotify(l.rc) +} + +// StoreLogsFromData stores logs from data. +func (l *Ledis) StoreLogsFromData(data []byte) error { + rb := bytes.NewReader(data) + + return l.StoreLogsFromReader(rb) +} + +// ReadLogsTo reads logs and write to the Writer. +func (l *Ledis) ReadLogsTo(startLogID uint64, w io.Writer) (n int, nextLogID uint64, err error) { + if !l.ReplicationUsed() { + // no replication log + nextLogID = 0 + err = ErrRplNotSupport + return + } + + var firtID, lastID uint64 + + firtID, err = l.r.FirstLogID() + if err != nil { + return + } + + if startLogID < firtID { + err = ErrLogMissed + return + } + + lastID, err = l.r.LastLogID() + if err != nil { + return + } + + nextLogID = startLogID + + log := &rpl.Log{} + for i := startLogID; i <= lastID; i++ { + if err = l.r.GetLog(i, log); err != nil { + return + } + + if err = log.Encode(w); err != nil { + return + } + + nextLogID = i + 1 + + n += log.Size() + + if n > maxReplLogSize { + break + } + } + + return +} + +// ReadLogsToTimeout tries to read events, if no events read, +// tres to wait the new event singal until timeout seconds +func (l *Ledis) ReadLogsToTimeout(startLogID uint64, w io.Writer, timeout int, quitCh chan struct{}) (n int, nextLogID uint64, err error) { + n, nextLogID, err = l.ReadLogsTo(startLogID, w) + if err != nil { + return + } else if n != 0 { + return + } + //no events read + select { + case <-l.r.WaitLog(): + case <-time.After(time.Duration(timeout) * time.Second): + case <-quitCh: + return + } + return l.ReadLogsTo(startLogID, w) +} + +func (l *Ledis) propagate(rl *rpl.Log) { + for _, h := range l.rhs { + h(rl) + } +} + +// NewLogEventHandler is the handler to handle new log event. +type NewLogEventHandler func(rl *rpl.Log) + +// AddNewLogEventHandler adds the handler for the new log event +func (l *Ledis) AddNewLogEventHandler(h NewLogEventHandler) error { + if !l.ReplicationUsed() { + return ErrRplNotSupport + } + + l.rhs = append(l.rhs, h) + + return nil +} + +// ReplicationStat returns the statistics of repliaciton. +func (l *Ledis) ReplicationStat() (*rpl.Stat, error) { + if !l.ReplicationUsed() { + return nil, ErrRplNotSupport + } + + return l.r.Stat() +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/scan.go b/vendor/github.com/siddontang/ledisdb/ledis/scan.go new file mode 100644 index 000000000000..b91640f754f4 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/scan.go @@ -0,0 +1,402 @@ +package ledis + +import ( + "errors" + "regexp" + + "github.com/siddontang/ledisdb/store" +) + +var errDataType = errors.New("error data type") +var errMetaKey = errors.New("error meta key") + +//Scan scans the data. If inclusive is true, scan range [cursor, inf) else (cursor, inf) +func (db *DB) Scan(dataType DataType, cursor []byte, count int, inclusive bool, match string) ([][]byte, error) { + storeDataType, err := getDataStoreType(dataType) + if err != nil { + return nil, err + } + + return db.scanGeneric(storeDataType, cursor, count, inclusive, match, false) +} + +// RevScan scans the data reversed. if inclusive is true, revscan range (-inf, cursor] else (inf, cursor) +func (db *DB) RevScan(dataType DataType, cursor []byte, count int, inclusive bool, match string) ([][]byte, error) { + storeDataType, err := getDataStoreType(dataType) + if err != nil { + return nil, err + } + + return db.scanGeneric(storeDataType, cursor, count, inclusive, match, true) +} + +func getDataStoreType(dataType DataType) (byte, error) { + var storeDataType byte + switch dataType { + case KV: + storeDataType = KVType + case LIST: + storeDataType = LMetaType + case HASH: + storeDataType = HSizeType + case SET: + storeDataType = SSizeType + case ZSET: + storeDataType = ZSizeType + default: + return 0, errDataType + } + return storeDataType, nil +} + +func buildMatchRegexp(match string) (*regexp.Regexp, error) { + var err error + var r *regexp.Regexp + + if len(match) > 0 { + if r, err = regexp.Compile(match); err != nil { + return nil, err + } + } + + return r, nil +} + +func (db *DB) buildScanIterator(minKey []byte, maxKey []byte, inclusive bool, reverse bool) *store.RangeLimitIterator { + tp := store.RangeOpen + + if !reverse { + if inclusive { + tp = store.RangeROpen + } + } else { + if inclusive { + tp = store.RangeLOpen + } + } + + var it *store.RangeLimitIterator + if !reverse { + it = db.bucket.RangeIterator(minKey, maxKey, tp) + } else { + it = db.bucket.RevRangeIterator(minKey, maxKey, tp) + } + + return it +} + +func (db *DB) buildScanKeyRange(storeDataType byte, key []byte, reverse bool) (minKey []byte, maxKey []byte, err error) { + if !reverse { + if minKey, err = db.encodeScanMinKey(storeDataType, key); err != nil { + return + } + if maxKey, err = db.encodeScanMaxKey(storeDataType, nil); err != nil { + return + } + } else { + if minKey, err = db.encodeScanMinKey(storeDataType, nil); err != nil { + return + } + if maxKey, err = db.encodeScanMaxKey(storeDataType, key); err != nil { + return + } + } + return +} + +func checkScanCount(count int) int { + if count <= 0 { + count = defaultScanCount + } + + return count +} + +func (db *DB) scanGeneric(storeDataType byte, key []byte, count int, + inclusive bool, match string, reverse bool) ([][]byte, error) { + + r, err := buildMatchRegexp(match) + if err != nil { + return nil, err + } + + minKey, maxKey, err := db.buildScanKeyRange(storeDataType, key, reverse) + if err != nil { + return nil, err + } + + count = checkScanCount(count) + + it := db.buildScanIterator(minKey, maxKey, inclusive, reverse) + + v := make([][]byte, 0, count) + + for i := 0; it.Valid() && i < count; it.Next() { + if k, err := db.decodeScanKey(storeDataType, it.Key()); err != nil { + continue + } else if r != nil && !r.Match(k) { + continue + } else { + v = append(v, k) + i++ + } + } + it.Close() + return v, nil +} + +func (db *DB) encodeScanMinKey(storeDataType byte, key []byte) ([]byte, error) { + return db.encodeScanKey(storeDataType, key) +} + +func (db *DB) encodeScanMaxKey(storeDataType byte, key []byte) ([]byte, error) { + if len(key) > 0 { + return db.encodeScanKey(storeDataType, key) + } + + k, err := db.encodeScanKey(storeDataType, nil) + if err != nil { + return nil, err + } + k[len(k)-1] = storeDataType + 1 + return k, nil +} + +func (db *DB) encodeScanKey(storeDataType byte, key []byte) ([]byte, error) { + switch storeDataType { + case KVType: + return db.encodeKVKey(key), nil + case LMetaType: + return db.lEncodeMetaKey(key), nil + case HSizeType: + return db.hEncodeSizeKey(key), nil + case ZSizeType: + return db.zEncodeSizeKey(key), nil + case SSizeType: + return db.sEncodeSizeKey(key), nil + default: + return nil, errDataType + } +} + +func (db *DB) decodeScanKey(storeDataType byte, ek []byte) (key []byte, err error) { + switch storeDataType { + case KVType: + key, err = db.decodeKVKey(ek) + case LMetaType: + key, err = db.lDecodeMetaKey(ek) + case HSizeType: + key, err = db.hDecodeSizeKey(ek) + case ZSizeType: + key, err = db.zDecodeSizeKey(ek) + case SSizeType: + key, err = db.sDecodeSizeKey(ek) + default: + err = errDataType + } + return +} + +// for specail data scan + +func (db *DB) buildDataScanKeyRange(storeDataType byte, key []byte, cursor []byte, reverse bool) (minKey []byte, maxKey []byte, err error) { + if !reverse { + if minKey, err = db.encodeDataScanMinKey(storeDataType, key, cursor); err != nil { + return + } + if maxKey, err = db.encodeDataScanMaxKey(storeDataType, key, nil); err != nil { + return + } + } else { + if minKey, err = db.encodeDataScanMinKey(storeDataType, key, nil); err != nil { + return + } + if maxKey, err = db.encodeDataScanMaxKey(storeDataType, key, cursor); err != nil { + return + } + } + return +} + +func (db *DB) encodeDataScanMinKey(storeDataType byte, key []byte, cursor []byte) ([]byte, error) { + return db.encodeDataScanKey(storeDataType, key, cursor) +} + +func (db *DB) encodeDataScanMaxKey(storeDataType byte, key []byte, cursor []byte) ([]byte, error) { + if len(cursor) > 0 { + return db.encodeDataScanKey(storeDataType, key, cursor) + } + + k, err := db.encodeDataScanKey(storeDataType, key, nil) + if err != nil { + return nil, err + } + + // here, the last byte is the start seperator, set it to stop seperator + k[len(k)-1] = k[len(k)-1] + 1 + return k, nil +} + +func (db *DB) encodeDataScanKey(storeDataType byte, key []byte, cursor []byte) ([]byte, error) { + switch storeDataType { + case HashType: + return db.hEncodeHashKey(key, cursor), nil + case ZSetType: + return db.zEncodeSetKey(key, cursor), nil + case SetType: + return db.sEncodeSetKey(key, cursor), nil + default: + return nil, errDataType + } +} + +func (db *DB) buildDataScanIterator(storeDataType byte, key []byte, cursor []byte, count int, + inclusive bool, reverse bool) (*store.RangeLimitIterator, error) { + + if err := checkKeySize(key); err != nil { + return nil, err + } + + minKey, maxKey, err := db.buildDataScanKeyRange(storeDataType, key, cursor, reverse) + if err != nil { + return nil, err + } + + it := db.buildScanIterator(minKey, maxKey, inclusive, reverse) + + return it, nil +} + +func (db *DB) hScanGeneric(key []byte, cursor []byte, count int, inclusive bool, match string, reverse bool) ([]FVPair, error) { + count = checkScanCount(count) + + r, err := buildMatchRegexp(match) + if err != nil { + return nil, err + } + + v := make([]FVPair, 0, count) + + it, err := db.buildDataScanIterator(HashType, key, cursor, count, inclusive, reverse) + if err != nil { + return nil, err + } + + defer it.Close() + + for i := 0; it.Valid() && i < count; it.Next() { + _, f, err := db.hDecodeHashKey(it.Key()) + if err != nil { + return nil, err + } else if r != nil && !r.Match(f) { + continue + } + + v = append(v, FVPair{Field: f, Value: it.Value()}) + + i++ + } + + return v, nil +} + +// HScan scans data for hash. +func (db *DB) HScan(key []byte, cursor []byte, count int, inclusive bool, match string) ([]FVPair, error) { + return db.hScanGeneric(key, cursor, count, inclusive, match, false) +} + +// HRevScan reversed scans data for hash. +func (db *DB) HRevScan(key []byte, cursor []byte, count int, inclusive bool, match string) ([]FVPair, error) { + return db.hScanGeneric(key, cursor, count, inclusive, match, true) +} + +func (db *DB) sScanGeneric(key []byte, cursor []byte, count int, inclusive bool, match string, reverse bool) ([][]byte, error) { + count = checkScanCount(count) + + r, err := buildMatchRegexp(match) + if err != nil { + return nil, err + } + + v := make([][]byte, 0, count) + + it, err := db.buildDataScanIterator(SetType, key, cursor, count, inclusive, reverse) + if err != nil { + return nil, err + } + + defer it.Close() + + for i := 0; it.Valid() && i < count; it.Next() { + _, m, err := db.sDecodeSetKey(it.Key()) + if err != nil { + return nil, err + } else if r != nil && !r.Match(m) { + continue + } + + v = append(v, m) + + i++ + } + + return v, nil +} + +// SScan scans data for set. +func (db *DB) SScan(key []byte, cursor []byte, count int, inclusive bool, match string) ([][]byte, error) { + return db.sScanGeneric(key, cursor, count, inclusive, match, false) +} + +// SRevScan scans data reversed for set. +func (db *DB) SRevScan(key []byte, cursor []byte, count int, inclusive bool, match string) ([][]byte, error) { + return db.sScanGeneric(key, cursor, count, inclusive, match, true) +} + +func (db *DB) zScanGeneric(key []byte, cursor []byte, count int, inclusive bool, match string, reverse bool) ([]ScorePair, error) { + count = checkScanCount(count) + + r, err := buildMatchRegexp(match) + if err != nil { + return nil, err + } + + v := make([]ScorePair, 0, count) + + it, err := db.buildDataScanIterator(ZSetType, key, cursor, count, inclusive, reverse) + if err != nil { + return nil, err + } + + defer it.Close() + + for i := 0; it.Valid() && i < count; it.Next() { + _, m, err := db.zDecodeSetKey(it.Key()) + if err != nil { + return nil, err + } else if r != nil && !r.Match(m) { + continue + } + + score, err := Int64(it.Value(), nil) + if err != nil { + return nil, err + } + + v = append(v, ScorePair{Score: score, Member: m}) + + i++ + } + + return v, nil +} + +// ZScan scans data for zset. +func (db *DB) ZScan(key []byte, cursor []byte, count int, inclusive bool, match string) ([]ScorePair, error) { + return db.zScanGeneric(key, cursor, count, inclusive, match, false) +} + +// ZRevScan scans data reversed for zset. +func (db *DB) ZRevScan(key []byte, cursor []byte, count int, inclusive bool, match string) ([]ScorePair, error) { + return db.zScanGeneric(key, cursor, count, inclusive, match, true) +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/sort.go b/vendor/github.com/siddontang/ledisdb/ledis/sort.go new file mode 100644 index 000000000000..413db10f2790 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/sort.go @@ -0,0 +1,235 @@ +package ledis + +import ( + "bytes" + "fmt" + "sort" + "strconv" + + "github.com/siddontang/ledisdb/store" +) + +// Limit is for sort. +type Limit struct { + Offset int + Size int +} + +func getSortRange(values [][]byte, offset int, size int) (int, int) { + var start = 0 + if offset > 0 { + start = offset + } + + valueLen := len(values) + var end = valueLen - 1 + if size > 0 { + end = start + size - 1 + } + + if start >= valueLen { + start = valueLen - 1 + end = valueLen - 2 + } + + if end >= valueLen { + end = valueLen - 1 + } + + return start, end +} + +var hashPattern = []byte("*->") + +func (db *DB) lookupKeyByPattern(pattern []byte, subKey []byte) []byte { + // If the pattern is #, return the substitution key itself + if bytes.Equal(pattern, []byte{'#'}) { + return subKey + } + + // If we can't find '*' in the pattern, return nil + if !bytes.Contains(pattern, []byte{'*'}) { + return nil + } + + key := pattern + var field []byte + + // Find out if we're dealing with a hash dereference + if n := bytes.Index(pattern, hashPattern); n > 0 && n+3 < len(pattern) { + key = pattern[0 : n+1] + field = pattern[n+3:] + } + + // Perform the '*' substitution + key = bytes.Replace(key, []byte{'*'}, subKey, 1) + + var value []byte + if field == nil { + value, _ = db.Get(key) + } else { + value, _ = db.HGet(key, field) + } + + return value +} + +type sortItem struct { + value []byte + cmpValue []byte + score float64 +} + +type sortItemSlice struct { + alpha bool + sortByPattern bool + items []sortItem +} + +func (s *sortItemSlice) Len() int { + return len(s.items) +} + +func (s *sortItemSlice) Swap(i, j int) { + s.items[i], s.items[j] = s.items[j], s.items[i] +} + +func (s *sortItemSlice) Less(i, j int) bool { + s1 := s.items[i] + s2 := s.items[j] + if !s.alpha { + if s1.score < s2.score { + return true + } else if s1.score > s2.score { + return false + } else { + return bytes.Compare(s1.value, s2.value) < 0 + } + } else { + if s.sortByPattern { + if s1.cmpValue == nil || s2.cmpValue == nil { + if s1.cmpValue == nil { + return true + } + return false + } + // Unlike redis, we only use bytes compare + return bytes.Compare(s1.cmpValue, s2.cmpValue) < 0 + } + + // Unlike redis, we only use bytes compare + return bytes.Compare(s1.value, s2.value) < 0 + } +} + +func (db *DB) xsort(values [][]byte, offset int, size int, alpha bool, desc bool, sortBy []byte, sortGet [][]byte) ([][]byte, error) { + if len(values) == 0 { + return [][]byte{}, nil + } + + start, end := getSortRange(values, offset, size) + + dontsort := 0 + + if sortBy != nil { + if !bytes.Contains(sortBy, []byte{'*'}) { + dontsort = 1 + } + } + + items := &sortItemSlice{ + alpha: alpha, + sortByPattern: sortBy != nil, + items: make([]sortItem, len(values)), + } + + for i, value := range values { + items.items[i].value = value + items.items[i].score = 0 + items.items[i].cmpValue = nil + + if dontsort == 0 { + var cmpValue []byte + if sortBy != nil { + cmpValue = db.lookupKeyByPattern(sortBy, value) + } else { + // use value iteself to sort by + cmpValue = value + } + + if cmpValue == nil { + continue + } + + if alpha { + if sortBy != nil { + items.items[i].cmpValue = cmpValue + } + } else { + score, err := strconv.ParseFloat(string(cmpValue), 64) + if err != nil { + return nil, fmt.Errorf("%s scores can't be converted into double", cmpValue) + } + items.items[i].score = score + } + } + } + + if dontsort == 0 { + if !desc { + sort.Sort(items) + } else { + sort.Sort(sort.Reverse(items)) + } + } + + resLen := end - start + 1 + if len(sortGet) > 0 { + resLen = len(sortGet) * (end - start + 1) + } + + res := make([][]byte, 0, resLen) + for i := start; i <= end; i++ { + if len(sortGet) == 0 { + res = append(res, items.items[i].value) + } else { + for _, getPattern := range sortGet { + v := db.lookupKeyByPattern(getPattern, items.items[i].value) + res = append(res, v) + } + } + } + + return res, nil +} + +// XLSort sorts list. +func (db *DB) XLSort(key []byte, offset int, size int, alpha bool, desc bool, sortBy []byte, sortGet [][]byte) ([][]byte, error) { + values, err := db.LRange(key, 0, -1) + + if err != nil { + return nil, err + } + + return db.xsort(values, offset, size, alpha, desc, sortBy, sortGet) +} + +// XSSort sorts set. +func (db *DB) XSSort(key []byte, offset int, size int, alpha bool, desc bool, sortBy []byte, sortGet [][]byte) ([][]byte, error) { + values, err := db.SMembers(key) + if err != nil { + return nil, err + } + + return db.xsort(values, offset, size, alpha, desc, sortBy, sortGet) +} + +// XZSort sorts zset. +func (db *DB) XZSort(key []byte, offset int, size int, alpha bool, desc bool, sortBy []byte, sortGet [][]byte) ([][]byte, error) { + values, err := db.ZRangeByLex(key, nil, nil, store.RangeClose, 0, -1) + if err != nil { + return nil, err + } + + return db.xsort(values, offset, size, alpha, desc, sortBy, sortGet) +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/t_hash.go b/vendor/github.com/siddontang/ledisdb/ledis/t_hash.go new file mode 100644 index 000000000000..b9a8595fe1d4 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/t_hash.go @@ -0,0 +1,556 @@ +package ledis + +import ( + "encoding/binary" + "errors" + "time" + + "github.com/siddontang/go/num" + "github.com/siddontang/ledisdb/store" +) + +// FVPair is the pair of field and value. +type FVPair struct { + Field []byte + Value []byte +} + +var errHashKey = errors.New("invalid hash key") +var errHSizeKey = errors.New("invalid hsize key") + +const ( + hashStartSep byte = ':' + hashStopSep byte = hashStartSep + 1 +) + +func checkHashKFSize(key []byte, field []byte) error { + if len(key) > MaxKeySize || len(key) == 0 { + return errKeySize + } else if len(field) > MaxHashFieldSize || len(field) == 0 { + return errHashFieldSize + } + return nil +} + +func (db *DB) hEncodeSizeKey(key []byte) []byte { + buf := make([]byte, len(key)+1+len(db.indexVarBuf)) + + pos := 0 + n := copy(buf, db.indexVarBuf) + + pos += n + buf[pos] = HSizeType + + pos++ + copy(buf[pos:], key) + + return buf +} + +func (db *DB) hDecodeSizeKey(ek []byte) ([]byte, error) { + pos, err := db.checkKeyIndex(ek) + if err != nil { + return nil, err + } + + if pos+1 > len(ek) || ek[pos] != HSizeType { + return nil, errHSizeKey + } + pos++ + + return ek[pos:], nil +} + +func (db *DB) hEncodeHashKey(key []byte, field []byte) []byte { + buf := make([]byte, len(key)+len(field)+1+1+2+len(db.indexVarBuf)) + + pos := 0 + n := copy(buf, db.indexVarBuf) + pos += n + + buf[pos] = HashType + pos++ + + binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) + pos += 2 + + copy(buf[pos:], key) + pos += len(key) + + buf[pos] = hashStartSep + pos++ + copy(buf[pos:], field) + + return buf +} + +func (db *DB) hDecodeHashKey(ek []byte) ([]byte, []byte, error) { + pos, err := db.checkKeyIndex(ek) + if err != nil { + return nil, nil, err + } + + if pos+1 > len(ek) || ek[pos] != HashType { + return nil, nil, errHashKey + } + pos++ + + if pos+2 > len(ek) { + return nil, nil, errHashKey + } + + keyLen := int(binary.BigEndian.Uint16(ek[pos:])) + pos += 2 + + if keyLen+pos > len(ek) { + return nil, nil, errHashKey + } + + key := ek[pos : pos+keyLen] + pos += keyLen + + if ek[pos] != hashStartSep { + return nil, nil, errHashKey + } + + pos++ + field := ek[pos:] + return key, field, nil +} + +func (db *DB) hEncodeStartKey(key []byte) []byte { + return db.hEncodeHashKey(key, nil) +} + +func (db *DB) hEncodeStopKey(key []byte) []byte { + k := db.hEncodeHashKey(key, nil) + + k[len(k)-1] = hashStopSep + + return k +} + +func (db *DB) hSetItem(key []byte, field []byte, value []byte) (int64, error) { + t := db.hashBatch + + ek := db.hEncodeHashKey(key, field) + + var n int64 = 1 + if v, _ := db.bucket.Get(ek); v != nil { + n = 0 + } else { + if _, err := db.hIncrSize(key, 1); err != nil { + return 0, err + } + } + + t.Put(ek, value) + return n, nil +} + +// ps : here just focus on deleting the hash data, +// any other likes expire is ignore. +func (db *DB) hDelete(t *batch, key []byte) int64 { + sk := db.hEncodeSizeKey(key) + start := db.hEncodeStartKey(key) + stop := db.hEncodeStopKey(key) + + var num int64 + it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) + for ; it.Valid(); it.Next() { + t.Delete(it.Key()) + num++ + } + it.Close() + + t.Delete(sk) + return num +} + +func (db *DB) hExpireAt(key []byte, when int64) (int64, error) { + t := db.hashBatch + t.Lock() + defer t.Unlock() + + if hlen, err := db.HLen(key); err != nil || hlen == 0 { + return 0, err + } + + db.expireAt(t, HashType, key, when) + if err := t.Commit(); err != nil { + return 0, err + } + + return 1, nil +} + +// HLen returns the lengh of hash. +func (db *DB) HLen(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + return Int64(db.bucket.Get(db.hEncodeSizeKey(key))) +} + +// HSet sets the field with value of key. +func (db *DB) HSet(key []byte, field []byte, value []byte) (int64, error) { + if err := checkHashKFSize(key, field); err != nil { + return 0, err + } else if err := checkValueSize(value); err != nil { + return 0, err + } + + t := db.hashBatch + t.Lock() + defer t.Unlock() + + n, err := db.hSetItem(key, field, value) + if err != nil { + return 0, err + } + + err = t.Commit() + return n, err +} + +// HGet gets the value of the field. +func (db *DB) HGet(key []byte, field []byte) ([]byte, error) { + if err := checkHashKFSize(key, field); err != nil { + return nil, err + } + + return db.bucket.Get(db.hEncodeHashKey(key, field)) +} + +// HMset sets multi field-values. +func (db *DB) HMset(key []byte, args ...FVPair) error { + t := db.hashBatch + t.Lock() + defer t.Unlock() + + var err error + var ek []byte + var num int64 + for i := 0; i < len(args); i++ { + if err := checkHashKFSize(key, args[i].Field); err != nil { + return err + } else if err := checkValueSize(args[i].Value); err != nil { + return err + } + + ek = db.hEncodeHashKey(key, args[i].Field) + + if v, err := db.bucket.Get(ek); err != nil { + return err + } else if v == nil { + num++ + } + + t.Put(ek, args[i].Value) + } + + if _, err = db.hIncrSize(key, num); err != nil { + return err + } + + //todo add binglog + err = t.Commit() + return err +} + +// HMget gets multi values of fields +func (db *DB) HMget(key []byte, args ...[]byte) ([][]byte, error) { + var ek []byte + + it := db.bucket.NewIterator() + defer it.Close() + + r := make([][]byte, len(args)) + for i := 0; i < len(args); i++ { + if err := checkHashKFSize(key, args[i]); err != nil { + return nil, err + } + + ek = db.hEncodeHashKey(key, args[i]) + + r[i] = it.Find(ek) + } + + return r, nil +} + +// HDel deletes the fields. +func (db *DB) HDel(key []byte, args ...[]byte) (int64, error) { + t := db.hashBatch + + var ek []byte + var v []byte + var err error + + t.Lock() + defer t.Unlock() + + it := db.bucket.NewIterator() + defer it.Close() + + var num int64 + for i := 0; i < len(args); i++ { + if err := checkHashKFSize(key, args[i]); err != nil { + return 0, err + } + + ek = db.hEncodeHashKey(key, args[i]) + + v = it.RawFind(ek) + if v == nil { + continue + } else { + num++ + t.Delete(ek) + } + } + + if _, err = db.hIncrSize(key, -num); err != nil { + return 0, err + } + + err = t.Commit() + + return num, err +} + +func (db *DB) hIncrSize(key []byte, delta int64) (int64, error) { + t := db.hashBatch + sk := db.hEncodeSizeKey(key) + + var err error + var size int64 + if size, err = Int64(db.bucket.Get(sk)); err != nil { + return 0, err + } + + size += delta + if size <= 0 { + size = 0 + t.Delete(sk) + db.rmExpire(t, HashType, key) + } else { + t.Put(sk, PutInt64(size)) + } + + return size, nil +} + +// HIncrBy increases the value of field by delta. +func (db *DB) HIncrBy(key []byte, field []byte, delta int64) (int64, error) { + if err := checkHashKFSize(key, field); err != nil { + return 0, err + } + + t := db.hashBatch + var ek []byte + var err error + + t.Lock() + defer t.Unlock() + + ek = db.hEncodeHashKey(key, field) + + var n int64 + if n, err = StrInt64(db.bucket.Get(ek)); err != nil { + return 0, err + } + + n += delta + + _, err = db.hSetItem(key, field, num.FormatInt64ToSlice(n)) + if err != nil { + return 0, err + } + + err = t.Commit() + + return n, err +} + +// HGetAll returns all field-values. +func (db *DB) HGetAll(key []byte) ([]FVPair, error) { + if err := checkKeySize(key); err != nil { + return nil, err + } + + start := db.hEncodeStartKey(key) + stop := db.hEncodeStopKey(key) + + v := make([]FVPair, 0, 16) + + it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) + defer it.Close() + + for ; it.Valid(); it.Next() { + _, f, err := db.hDecodeHashKey(it.Key()) + if err != nil { + return nil, err + } + + v = append(v, FVPair{Field: f, Value: it.Value()}) + } + + return v, nil +} + +// HKeys returns the all fields. +func (db *DB) HKeys(key []byte) ([][]byte, error) { + if err := checkKeySize(key); err != nil { + return nil, err + } + + start := db.hEncodeStartKey(key) + stop := db.hEncodeStopKey(key) + + v := make([][]byte, 0, 16) + + it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) + defer it.Close() + + for ; it.Valid(); it.Next() { + _, f, err := db.hDecodeHashKey(it.Key()) + if err != nil { + return nil, err + } + v = append(v, f) + } + + return v, nil +} + +// HValues returns all values +func (db *DB) HValues(key []byte) ([][]byte, error) { + if err := checkKeySize(key); err != nil { + return nil, err + } + + start := db.hEncodeStartKey(key) + stop := db.hEncodeStopKey(key) + + v := make([][]byte, 0, 16) + + it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) + defer it.Close() + + for ; it.Valid(); it.Next() { + _, _, err := db.hDecodeHashKey(it.Key()) + if err != nil { + return nil, err + } + + v = append(v, it.Value()) + } + + return v, nil +} + +// HClear clears the data. +func (db *DB) HClear(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + t := db.hashBatch + t.Lock() + defer t.Unlock() + + num := db.hDelete(t, key) + db.rmExpire(t, HashType, key) + + err := t.Commit() + return num, err +} + +// HMclear cleans multi data. +func (db *DB) HMclear(keys ...[]byte) (int64, error) { + t := db.hashBatch + t.Lock() + defer t.Unlock() + + for _, key := range keys { + if err := checkKeySize(key); err != nil { + return 0, err + } + + db.hDelete(t, key) + db.rmExpire(t, HashType, key) + } + + err := t.Commit() + return int64(len(keys)), err +} + +func (db *DB) hFlush() (drop int64, err error) { + t := db.hashBatch + + t.Lock() + defer t.Unlock() + + return db.flushType(t, HashType) +} + +// HExpire expires the data with duration. +func (db *DB) HExpire(key []byte, duration int64) (int64, error) { + if duration <= 0 { + return 0, errExpireValue + } + + return db.hExpireAt(key, time.Now().Unix()+duration) +} + +// HExpireAt expires the data at time when. +func (db *DB) HExpireAt(key []byte, when int64) (int64, error) { + if when <= time.Now().Unix() { + return 0, errExpireValue + } + + return db.hExpireAt(key, when) +} + +// HTTL gets the TTL of data. +func (db *DB) HTTL(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return -1, err + } + + return db.ttl(HashType, key) +} + +// HPersist removes the TTL of data. +func (db *DB) HPersist(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + t := db.hashBatch + t.Lock() + defer t.Unlock() + + n, err := db.rmExpire(t, HashType, key) + if err != nil { + return 0, err + } + + err = t.Commit() + return n, err +} + +// HKeyExists checks whether data exists or not. +func (db *DB) HKeyExists(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + sk := db.hEncodeSizeKey(key) + v, err := db.bucket.Get(sk) + if v != nil && err == nil { + return 1, nil + } + return 0, err +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/t_kv.go b/vendor/github.com/siddontang/ledisdb/ledis/t_kv.go new file mode 100644 index 000000000000..9abec2af8097 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/t_kv.go @@ -0,0 +1,794 @@ +package ledis + +import ( + "encoding/binary" + "errors" + "fmt" + "strings" + "time" + + "github.com/siddontang/go/num" + "github.com/siddontang/ledisdb/store" +) + +// KVPair is the pair of key-value. +type KVPair struct { + Key []byte + Value []byte +} + +var errKVKey = errors.New("invalid encode kv key") + +func checkKeySize(key []byte) error { + if len(key) > MaxKeySize || len(key) == 0 { + return errKeySize + } + return nil +} + +func checkValueSize(value []byte) error { + if len(value) > MaxValueSize { + return errValueSize + } + + return nil +} + +func (db *DB) encodeKVKey(key []byte) []byte { + ek := make([]byte, len(key)+1+len(db.indexVarBuf)) + pos := copy(ek, db.indexVarBuf) + ek[pos] = KVType + pos++ + copy(ek[pos:], key) + return ek +} + +func (db *DB) decodeKVKey(ek []byte) ([]byte, error) { + pos, err := db.checkKeyIndex(ek) + if err != nil { + return nil, err + } + if pos+1 > len(ek) || ek[pos] != KVType { + return nil, errKVKey + } + + pos++ + + return ek[pos:], nil +} + +func (db *DB) encodeKVMinKey() []byte { + ek := db.encodeKVKey(nil) + return ek +} + +func (db *DB) encodeKVMaxKey() []byte { + ek := db.encodeKVKey(nil) + ek[len(ek)-1] = KVType + 1 + return ek +} + +func (db *DB) incr(key []byte, delta int64) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + var err error + key = db.encodeKVKey(key) + + t := db.kvBatch + + t.Lock() + defer t.Unlock() + + var n int64 + n, err = StrInt64(db.bucket.Get(key)) + if err != nil { + return 0, err + } + + n += delta + + t.Put(key, num.FormatInt64ToSlice(n)) + + err = t.Commit() + return n, err +} + +// ps : here just focus on deleting the key-value data, +// any other likes expire is ignore. +func (db *DB) delete(t *batch, key []byte) int64 { + key = db.encodeKVKey(key) + t.Delete(key) + return 1 +} + +func (db *DB) setExpireAt(key []byte, when int64) (int64, error) { + t := db.kvBatch + t.Lock() + defer t.Unlock() + + if exist, err := db.Exists(key); err != nil || exist == 0 { + return 0, err + } + + db.expireAt(t, KVType, key, when) + if err := t.Commit(); err != nil { + return 0, err + } + + return 1, nil +} + +// Decr decreases the data. +func (db *DB) Decr(key []byte) (int64, error) { + return db.incr(key, -1) +} + +// DecrBy decreases the data by decrement. +func (db *DB) DecrBy(key []byte, decrement int64) (int64, error) { + return db.incr(key, -decrement) +} + +// Del deletes the data. +func (db *DB) Del(keys ...[]byte) (int64, error) { + if len(keys) == 0 { + return 0, nil + } + + codedKeys := make([][]byte, len(keys)) + for i, k := range keys { + codedKeys[i] = db.encodeKVKey(k) + } + + t := db.kvBatch + t.Lock() + defer t.Unlock() + + for i, k := range keys { + t.Delete(codedKeys[i]) + db.rmExpire(t, KVType, k) + } + + err := t.Commit() + return int64(len(keys)), err +} + +// Exists check data exists or not. +func (db *DB) Exists(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + var err error + key = db.encodeKVKey(key) + + var v []byte + v, err = db.bucket.Get(key) + if v != nil && err == nil { + return 1, nil + } + + return 0, err +} + +// Get gets the value. +func (db *DB) Get(key []byte) ([]byte, error) { + if err := checkKeySize(key); err != nil { + return nil, err + } + + key = db.encodeKVKey(key) + + return db.bucket.Get(key) +} + +// GetSlice gets the slice of the data. +func (db *DB) GetSlice(key []byte) (store.Slice, error) { + if err := checkKeySize(key); err != nil { + return nil, err + } + + key = db.encodeKVKey(key) + + return db.bucket.GetSlice(key) +} + +// GetSet gets the value and sets new value. +func (db *DB) GetSet(key []byte, value []byte) ([]byte, error) { + if err := checkKeySize(key); err != nil { + return nil, err + } else if err := checkValueSize(value); err != nil { + return nil, err + } + + key = db.encodeKVKey(key) + + t := db.kvBatch + + t.Lock() + defer t.Unlock() + + oldValue, err := db.bucket.Get(key) + if err != nil { + return nil, err + } + + t.Put(key, value) + + err = t.Commit() + + return oldValue, err +} + +// Incr increases the data. +func (db *DB) Incr(key []byte) (int64, error) { + return db.incr(key, 1) +} + +// IncrBy increases the data by increment. +func (db *DB) IncrBy(key []byte, increment int64) (int64, error) { + return db.incr(key, increment) +} + +// MGet gets multi data. +func (db *DB) MGet(keys ...[]byte) ([][]byte, error) { + values := make([][]byte, len(keys)) + + it := db.bucket.NewIterator() + defer it.Close() + + for i := range keys { + if err := checkKeySize(keys[i]); err != nil { + return nil, err + } + + values[i] = it.Find(db.encodeKVKey(keys[i])) + } + + return values, nil +} + +// MSet sets multi data. +func (db *DB) MSet(args ...KVPair) error { + if len(args) == 0 { + return nil + } + + t := db.kvBatch + + var err error + var key []byte + var value []byte + + t.Lock() + defer t.Unlock() + + for i := 0; i < len(args); i++ { + if err := checkKeySize(args[i].Key); err != nil { + return err + } else if err := checkValueSize(args[i].Value); err != nil { + return err + } + + key = db.encodeKVKey(args[i].Key) + + value = args[i].Value + + t.Put(key, value) + + } + + err = t.Commit() + return err +} + +// Set sets the data. +func (db *DB) Set(key []byte, value []byte) error { + if err := checkKeySize(key); err != nil { + return err + } else if err := checkValueSize(value); err != nil { + return err + } + + var err error + key = db.encodeKVKey(key) + + t := db.kvBatch + + t.Lock() + defer t.Unlock() + + t.Put(key, value) + + err = t.Commit() + + return err +} + +// SetNX sets the data if not existed. +func (db *DB) SetNX(key []byte, value []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } else if err := checkValueSize(value); err != nil { + return 0, err + } + + var err error + key = db.encodeKVKey(key) + + var n int64 = 1 + + t := db.kvBatch + + t.Lock() + defer t.Unlock() + + if v, err := db.bucket.Get(key); err != nil { + return 0, err + } else if v != nil { + n = 0 + } else { + t.Put(key, value) + + err = t.Commit() + } + + return n, err +} + +// SetEX sets the data with a TTL. +func (db *DB) SetEX(key []byte, duration int64, value []byte) error { + if err := checkKeySize(key); err != nil { + return err + } else if err := checkValueSize(value); err != nil { + return err + } else if duration <= 0 { + return errExpireValue + } + + ek := db.encodeKVKey(key) + + t := db.kvBatch + + t.Lock() + defer t.Unlock() + + t.Put(ek, value) + db.expireAt(t, KVType, key, time.Now().Unix()+duration) + + return t.Commit() +} + +func (db *DB) flush() (drop int64, err error) { + t := db.kvBatch + t.Lock() + defer t.Unlock() + return db.flushType(t, KVType) +} + +// Expire expires the data. +func (db *DB) Expire(key []byte, duration int64) (int64, error) { + if duration <= 0 { + return 0, errExpireValue + } + + return db.setExpireAt(key, time.Now().Unix()+duration) +} + +// ExpireAt expires the data at when. +func (db *DB) ExpireAt(key []byte, when int64) (int64, error) { + if when <= time.Now().Unix() { + return 0, errExpireValue + } + + return db.setExpireAt(key, when) +} + +// TTL returns the TTL of the data. +func (db *DB) TTL(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return -1, err + } + + return db.ttl(KVType, key) +} + +// Persist removes the TTL of the data. +func (db *DB) Persist(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + t := db.kvBatch + t.Lock() + defer t.Unlock() + n, err := db.rmExpire(t, KVType, key) + if err != nil { + return 0, err + } + + err = t.Commit() + return n, err +} + +// SetRange sets the data with new value from offset. +func (db *DB) SetRange(key []byte, offset int, value []byte) (int64, error) { + if len(value) == 0 { + return 0, nil + } + + if err := checkKeySize(key); err != nil { + return 0, err + } else if len(value)+offset > MaxValueSize { + return 0, errValueSize + } + + key = db.encodeKVKey(key) + + t := db.kvBatch + + t.Lock() + defer t.Unlock() + + oldValue, err := db.bucket.Get(key) + if err != nil { + return 0, err + } + + extra := offset + len(value) - len(oldValue) + if extra > 0 { + oldValue = append(oldValue, make([]byte, extra)...) + } + + copy(oldValue[offset:], value) + + t.Put(key, oldValue) + + if err := t.Commit(); err != nil { + return 0, err + } + + return int64(len(oldValue)), nil +} + +func getRange(start int, end int, valLen int) (int, int) { + if start < 0 { + start = valLen + start + } + + if end < 0 { + end = valLen + end + } + + if start < 0 { + start = 0 + } + + if end < 0 { + end = 0 + } + + if end >= valLen { + end = valLen - 1 + } + return start, end +} + +// GetRange gets the range of the data. +func (db *DB) GetRange(key []byte, start int, end int) ([]byte, error) { + if err := checkKeySize(key); err != nil { + return nil, err + } + key = db.encodeKVKey(key) + + value, err := db.bucket.Get(key) + if err != nil { + return nil, err + } + + valLen := len(value) + + start, end = getRange(start, end, valLen) + + if start > end { + return nil, nil + } + + return value[start : end+1], nil +} + +// StrLen returns the length of the data. +func (db *DB) StrLen(key []byte) (int64, error) { + s, err := db.GetSlice(key) + if err != nil { + return 0, err + } + + n := s.Size() + s.Free() + return int64(n), nil +} + +// Append appends the value to the data. +func (db *DB) Append(key []byte, value []byte) (int64, error) { + if len(value) == 0 { + return 0, nil + } + + if err := checkKeySize(key); err != nil { + return 0, err + } + key = db.encodeKVKey(key) + + t := db.kvBatch + + t.Lock() + defer t.Unlock() + + oldValue, err := db.bucket.Get(key) + if err != nil { + return 0, err + } + + if len(oldValue)+len(value) > MaxValueSize { + return 0, errValueSize + } + + oldValue = append(oldValue, value...) + + t.Put(key, oldValue) + + if err := t.Commit(); err != nil { + return 0, nil + } + + return int64(len(oldValue)), nil +} + +// BitOP does the bit operations in data. +func (db *DB) BitOP(op string, destKey []byte, srcKeys ...[]byte) (int64, error) { + if err := checkKeySize(destKey); err != nil { + return 0, err + } + + op = strings.ToLower(op) + if len(srcKeys) == 0 { + return 0, nil + } else if op == BitNot && len(srcKeys) > 1 { + return 0, fmt.Errorf("BITOP NOT has only one srckey") + } else if len(srcKeys) < 2 { + return 0, nil + } + + key := db.encodeKVKey(srcKeys[0]) + + value, err := db.bucket.Get(key) + if err != nil { + return 0, err + } + + if op == BitNot { + for i := 0; i < len(value); i++ { + value[i] = ^value[i] + } + } else { + for j := 1; j < len(srcKeys); j++ { + if err := checkKeySize(srcKeys[j]); err != nil { + return 0, err + } + + key = db.encodeKVKey(srcKeys[j]) + ovalue, err := db.bucket.Get(key) + if err != nil { + return 0, err + } + + if len(value) < len(ovalue) { + value, ovalue = ovalue, value + } + + for i := 0; i < len(ovalue); i++ { + switch op { + case BitAND: + value[i] &= ovalue[i] + case BitOR: + value[i] |= ovalue[i] + case BitXOR: + value[i] ^= ovalue[i] + default: + return 0, fmt.Errorf("invalid op type: %s", op) + } + } + + for i := len(ovalue); i < len(value); i++ { + switch op { + case BitAND: + value[i] &= 0 + case BitOR: + value[i] |= 0 + case BitXOR: + value[i] ^= 0 + } + } + } + } + + key = db.encodeKVKey(destKey) + + t := db.kvBatch + + t.Lock() + defer t.Unlock() + + t.Put(key, value) + + if err := t.Commit(); err != nil { + return 0, err + } + + return int64(len(value)), nil +} + +var bitsInByte = [256]int32{0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, + 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, + 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, + 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, + 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, + 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, + 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, + 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, + 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, + 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, + 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8} + +func numberBitCount(i uint32) uint32 { + i = i - ((i >> 1) & 0x55555555) + i = (i & 0x33333333) + ((i >> 2) & 0x33333333) + return (((i + (i >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24 +} + +// BitCount returns the bit count of data. +func (db *DB) BitCount(key []byte, start int, end int) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + key = db.encodeKVKey(key) + value, err := db.bucket.Get(key) + if err != nil { + return 0, err + } + + start, end = getRange(start, end, len(value)) + value = value[start : end+1] + + var n int64 + + pos := 0 + for ; pos+4 <= len(value); pos = pos + 4 { + n += int64(numberBitCount(binary.BigEndian.Uint32(value[pos : pos+4]))) + } + + for ; pos < len(value); pos++ { + n += int64(bitsInByte[value[pos]]) + } + + return n, nil +} + +// BitPos returns the pos of the data. +func (db *DB) BitPos(key []byte, on int, start int, end int) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + if (on & ^1) != 0 { + return 0, fmt.Errorf("bit must be 0 or 1, not %d", on) + } + + var skipValue uint8 + if on == 0 { + skipValue = 0xFF + } + + key = db.encodeKVKey(key) + value, err := db.bucket.Get(key) + if err != nil { + return 0, err + } + + start, end = getRange(start, end, len(value)) + value = value[start : end+1] + + for i, v := range value { + if uint8(v) != skipValue { + for j := 0; j < 8; j++ { + isNull := uint8(v)&(1<> 3) + extra := byteOffset + 1 - len(value) + if extra > 0 { + value = append(value, make([]byte, extra)...) + } + + byteVal := value[byteOffset] + bit := 7 - uint8(uint32(offset)&0x7) + bitVal := byteVal & (1 << bit) + + byteVal &= ^(1 << bit) + byteVal |= (uint8(on&0x1) << bit) + + value[byteOffset] = byteVal + + t.Put(key, value) + if err := t.Commit(); err != nil { + return 0, err + } + + if bitVal > 0 { + return 1, nil + } + + return 0, nil +} + +// GetBit gets the bit of data at offset. +func (db *DB) GetBit(key []byte, offset int) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + key = db.encodeKVKey(key) + + value, err := db.bucket.Get(key) + if err != nil { + return 0, err + } + + byteOffset := uint32(offset) >> 3 + bit := 7 - uint8(uint32(offset)&0x7) + + if byteOffset >= uint32(len(value)) { + return 0, nil + } + + bitVal := value[byteOffset] & (1 << bit) + if bitVal > 0 { + return 1, nil + } + + return 0, nil +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/t_list.go b/vendor/github.com/siddontang/ledisdb/ledis/t_list.go new file mode 100644 index 000000000000..c177cd9a9276 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/t_list.go @@ -0,0 +1,808 @@ +package ledis + +import ( + "container/list" + "encoding/binary" + "errors" + "sync" + "time" + + "github.com/siddontang/go/hack" + "github.com/siddontang/go/log" + "github.com/siddontang/go/num" + "github.com/siddontang/ledisdb/store" + "golang.org/x/net/context" +) + +const ( + listHeadSeq int32 = 1 + listTailSeq int32 = 2 + + listMinSeq int32 = 1000 + listMaxSeq int32 = 1<<31 - 1000 + listInitialSeq int32 = listMinSeq + (listMaxSeq-listMinSeq)/2 +) + +var errLMetaKey = errors.New("invalid lmeta key") +var errListKey = errors.New("invalid list key") +var errListSeq = errors.New("invalid list sequence, overflow") + +func (db *DB) lEncodeMetaKey(key []byte) []byte { + buf := make([]byte, len(key)+1+len(db.indexVarBuf)) + pos := copy(buf, db.indexVarBuf) + buf[pos] = LMetaType + pos++ + + copy(buf[pos:], key) + return buf +} + +func (db *DB) lDecodeMetaKey(ek []byte) ([]byte, error) { + pos, err := db.checkKeyIndex(ek) + if err != nil { + return nil, err + } + + if pos+1 > len(ek) || ek[pos] != LMetaType { + return nil, errLMetaKey + } + + pos++ + return ek[pos:], nil +} + +func (db *DB) lEncodeListKey(key []byte, seq int32) []byte { + buf := make([]byte, len(key)+7+len(db.indexVarBuf)) + + pos := copy(buf, db.indexVarBuf) + + buf[pos] = ListType + pos++ + + binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) + pos += 2 + + copy(buf[pos:], key) + pos += len(key) + + binary.BigEndian.PutUint32(buf[pos:], uint32(seq)) + + return buf +} + +func (db *DB) lDecodeListKey(ek []byte) (key []byte, seq int32, err error) { + pos := 0 + pos, err = db.checkKeyIndex(ek) + if err != nil { + return + } + + if pos+1 > len(ek) || ek[pos] != ListType { + err = errListKey + return + } + + pos++ + + if pos+2 > len(ek) { + err = errListKey + return + } + + keyLen := int(binary.BigEndian.Uint16(ek[pos:])) + pos += 2 + if keyLen+pos+4 != len(ek) { + err = errListKey + return + } + + key = ek[pos : pos+keyLen] + seq = int32(binary.BigEndian.Uint32(ek[pos+keyLen:])) + return +} + +func (db *DB) lpush(key []byte, whereSeq int32, args ...[]byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + var headSeq int32 + var tailSeq int32 + var size int32 + var err error + + t := db.listBatch + t.Lock() + defer t.Unlock() + + metaKey := db.lEncodeMetaKey(key) + headSeq, tailSeq, size, err = db.lGetMeta(nil, metaKey) + if err != nil { + return 0, err + } + + pushCnt := len(args) + if pushCnt == 0 { + return int64(size), nil + } + + seq := headSeq + var delta int32 = -1 + if whereSeq == listTailSeq { + seq = tailSeq + delta = 1 + } + + // append elements + if size > 0 { + seq += delta + } + + for i := 0; i < pushCnt; i++ { + ek := db.lEncodeListKey(key, seq+int32(i)*delta) + t.Put(ek, args[i]) + } + + seq += int32(pushCnt-1) * delta + if seq <= listMinSeq || seq >= listMaxSeq { + return 0, errListSeq + } + + // set meta info + if whereSeq == listHeadSeq { + headSeq = seq + } else { + tailSeq = seq + } + + db.lSetMeta(metaKey, headSeq, tailSeq) + + err = t.Commit() + + if err == nil { + db.lSignalAsReady(key) + } + + return int64(size) + int64(pushCnt), err +} + +func (db *DB) lpop(key []byte, whereSeq int32) ([]byte, error) { + if err := checkKeySize(key); err != nil { + return nil, err + } + + t := db.listBatch + t.Lock() + defer t.Unlock() + + var headSeq int32 + var tailSeq int32 + var size int32 + var err error + + metaKey := db.lEncodeMetaKey(key) + headSeq, tailSeq, size, err = db.lGetMeta(nil, metaKey) + if err != nil { + return nil, err + } else if size == 0 { + return nil, nil + } + + var value []byte + + seq := headSeq + if whereSeq == listTailSeq { + seq = tailSeq + } + + itemKey := db.lEncodeListKey(key, seq) + value, err = db.bucket.Get(itemKey) + if err != nil { + return nil, err + } + + if whereSeq == listHeadSeq { + headSeq++ + } else { + tailSeq-- + } + + t.Delete(itemKey) + size = db.lSetMeta(metaKey, headSeq, tailSeq) + if size == 0 { + db.rmExpire(t, ListType, key) + } + + err = t.Commit() + return value, err +} + +func (db *DB) ltrim2(key []byte, startP, stopP int64) (err error) { + if err := checkKeySize(key); err != nil { + return err + } + + t := db.listBatch + t.Lock() + defer t.Unlock() + + var headSeq int32 + var llen int32 + start := int32(startP) + stop := int32(stopP) + + ek := db.lEncodeMetaKey(key) + if headSeq, _, llen, err = db.lGetMeta(nil, ek); err != nil { + return err + } + + if start < 0 { + start = llen + start + } + if stop < 0 { + stop = llen + stop + } + if start >= llen || start > stop { + db.lDelete(t, key) + db.rmExpire(t, ListType, key) + return t.Commit() + } + + if start < 0 { + start = 0 + } + if stop >= llen { + stop = llen - 1 + } + + if start > 0 { + for i := int32(0); i < start; i++ { + t.Delete(db.lEncodeListKey(key, headSeq+i)) + } + } + if stop < int32(llen-1) { + for i := int32(stop + 1); i < llen; i++ { + t.Delete(db.lEncodeListKey(key, headSeq+i)) + } + } + + db.lSetMeta(ek, headSeq+start, headSeq+stop) + + return t.Commit() +} + +func (db *DB) ltrim(key []byte, trimSize, whereSeq int32) (int32, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + if trimSize == 0 { + return 0, nil + } + + t := db.listBatch + t.Lock() + defer t.Unlock() + + var headSeq int32 + var tailSeq int32 + var size int32 + var err error + + metaKey := db.lEncodeMetaKey(key) + headSeq, tailSeq, size, err = db.lGetMeta(nil, metaKey) + if err != nil { + return 0, err + } else if size == 0 { + return 0, nil + } + + var ( + trimStartSeq int32 + trimEndSeq int32 + ) + + if whereSeq == listHeadSeq { + trimStartSeq = headSeq + trimEndSeq = num.MinInt32(trimStartSeq+trimSize-1, tailSeq) + headSeq = trimEndSeq + 1 + } else { + trimEndSeq = tailSeq + trimStartSeq = num.MaxInt32(trimEndSeq-trimSize+1, headSeq) + tailSeq = trimStartSeq - 1 + } + + for trimSeq := trimStartSeq; trimSeq <= trimEndSeq; trimSeq++ { + itemKey := db.lEncodeListKey(key, trimSeq) + t.Delete(itemKey) + } + + size = db.lSetMeta(metaKey, headSeq, tailSeq) + if size == 0 { + db.rmExpire(t, ListType, key) + } + + err = t.Commit() + return trimEndSeq - trimStartSeq + 1, err +} + +// ps : here just focus on deleting the list data, +// any other likes expire is ignore. +func (db *DB) lDelete(t *batch, key []byte) int64 { + mk := db.lEncodeMetaKey(key) + + var headSeq int32 + var tailSeq int32 + var err error + + it := db.bucket.NewIterator() + defer it.Close() + + headSeq, tailSeq, _, err = db.lGetMeta(it, mk) + if err != nil { + return 0 + } + + var num int64 + startKey := db.lEncodeListKey(key, headSeq) + stopKey := db.lEncodeListKey(key, tailSeq) + + rit := store.NewRangeIterator(it, &store.Range{ + Min: startKey, + Max: stopKey, + Type: store.RangeClose}) + for ; rit.Valid(); rit.Next() { + t.Delete(rit.RawKey()) + num++ + } + + t.Delete(mk) + + return num +} + +func (db *DB) lGetMeta(it *store.Iterator, ek []byte) (headSeq int32, tailSeq int32, size int32, err error) { + var v []byte + if it != nil { + v = it.Find(ek) + } else { + v, err = db.bucket.Get(ek) + } + if err != nil { + return + } else if v == nil { + headSeq = listInitialSeq + tailSeq = listInitialSeq + size = 0 + return + } else { + headSeq = int32(binary.LittleEndian.Uint32(v[0:4])) + tailSeq = int32(binary.LittleEndian.Uint32(v[4:8])) + size = tailSeq - headSeq + 1 + } + return +} + +func (db *DB) lSetMeta(ek []byte, headSeq int32, tailSeq int32) int32 { + t := db.listBatch + + size := tailSeq - headSeq + 1 + if size < 0 { + // todo : log error + panic + log.Fatalf("invalid meta sequence range [%d, %d]", headSeq, tailSeq) + } else if size == 0 { + t.Delete(ek) + } else { + buf := make([]byte, 8) + + binary.LittleEndian.PutUint32(buf[0:4], uint32(headSeq)) + binary.LittleEndian.PutUint32(buf[4:8], uint32(tailSeq)) + + t.Put(ek, buf) + } + + return size +} + +func (db *DB) lExpireAt(key []byte, when int64) (int64, error) { + t := db.listBatch + t.Lock() + defer t.Unlock() + + if llen, err := db.LLen(key); err != nil || llen == 0 { + return 0, err + } + + db.expireAt(t, ListType, key, when) + if err := t.Commit(); err != nil { + return 0, err + } + + return 1, nil +} + +// LIndex returns the value at index. +func (db *DB) LIndex(key []byte, index int32) ([]byte, error) { + if err := checkKeySize(key); err != nil { + return nil, err + } + + var seq int32 + var headSeq int32 + var tailSeq int32 + var err error + + metaKey := db.lEncodeMetaKey(key) + + it := db.bucket.NewIterator() + defer it.Close() + + headSeq, tailSeq, _, err = db.lGetMeta(it, metaKey) + if err != nil { + return nil, err + } + + if index >= 0 { + seq = headSeq + index + } else { + seq = tailSeq + index + 1 + } + + sk := db.lEncodeListKey(key, seq) + v := it.Find(sk) + + return v, nil +} + +// LLen gets the length of the list. +func (db *DB) LLen(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + ek := db.lEncodeMetaKey(key) + _, _, size, err := db.lGetMeta(nil, ek) + return int64(size), err +} + +// LPop pops the value. +func (db *DB) LPop(key []byte) ([]byte, error) { + return db.lpop(key, listHeadSeq) +} + +// LTrim trims the value from start to stop. +func (db *DB) LTrim(key []byte, start, stop int64) error { + return db.ltrim2(key, start, stop) +} + +// LTrimFront trims the value from top. +func (db *DB) LTrimFront(key []byte, trimSize int32) (int32, error) { + return db.ltrim(key, trimSize, listHeadSeq) +} + +// LTrimBack trims the value from back. +func (db *DB) LTrimBack(key []byte, trimSize int32) (int32, error) { + return db.ltrim(key, trimSize, listTailSeq) +} + +// LPush push the value to the list. +func (db *DB) LPush(key []byte, args ...[]byte) (int64, error) { + return db.lpush(key, listHeadSeq, args...) +} + +// LSet sets the value at index. +func (db *DB) LSet(key []byte, index int32, value []byte) error { + if err := checkKeySize(key); err != nil { + return err + } + + var seq int32 + var headSeq int32 + var tailSeq int32 + //var size int32 + var err error + t := db.listBatch + t.Lock() + defer t.Unlock() + metaKey := db.lEncodeMetaKey(key) + + headSeq, tailSeq, _, err = db.lGetMeta(nil, metaKey) + if err != nil { + return err + } + + if index >= 0 { + seq = headSeq + index + } else { + seq = tailSeq + index + 1 + } + if seq < headSeq || seq > tailSeq { + return errListIndex + } + sk := db.lEncodeListKey(key, seq) + t.Put(sk, value) + err = t.Commit() + return err +} + +// LRange gets the value of list at range. +func (db *DB) LRange(key []byte, start int32, stop int32) ([][]byte, error) { + if err := checkKeySize(key); err != nil { + return nil, err + } + + var headSeq int32 + var llen int32 + var err error + + metaKey := db.lEncodeMetaKey(key) + + it := db.bucket.NewIterator() + defer it.Close() + + if headSeq, _, llen, err = db.lGetMeta(it, metaKey); err != nil { + return nil, err + } + + if start < 0 { + start = llen + start + } + if stop < 0 { + stop = llen + stop + } + if start < 0 { + start = 0 + } + + if start > stop || start >= llen { + return [][]byte{}, nil + } + + if stop >= llen { + stop = llen - 1 + } + + limit := (stop - start) + 1 + headSeq += start + + v := make([][]byte, 0, limit) + + startKey := db.lEncodeListKey(key, headSeq) + rit := store.NewRangeLimitIterator(it, + &store.Range{ + Min: startKey, + Max: nil, + Type: store.RangeClose}, + &store.Limit{ + Offset: 0, + Count: int(limit)}) + + for ; rit.Valid(); rit.Next() { + v = append(v, rit.Value()) + } + + return v, nil +} + +// RPop rpops the value. +func (db *DB) RPop(key []byte) ([]byte, error) { + return db.lpop(key, listTailSeq) +} + +// RPush rpushs the value . +func (db *DB) RPush(key []byte, args ...[]byte) (int64, error) { + return db.lpush(key, listTailSeq, args...) +} + +// LClear clears the list. +func (db *DB) LClear(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + t := db.listBatch + t.Lock() + defer t.Unlock() + + num := db.lDelete(t, key) + db.rmExpire(t, ListType, key) + + err := t.Commit() + return num, err +} + +// LMclear clears multi lists. +func (db *DB) LMclear(keys ...[]byte) (int64, error) { + t := db.listBatch + t.Lock() + defer t.Unlock() + + for _, key := range keys { + if err := checkKeySize(key); err != nil { + return 0, err + } + + db.lDelete(t, key) + db.rmExpire(t, ListType, key) + + } + + err := t.Commit() + return int64(len(keys)), err +} + +func (db *DB) lFlush() (drop int64, err error) { + t := db.listBatch + t.Lock() + defer t.Unlock() + return db.flushType(t, ListType) +} + +// LExpire expires the list. +func (db *DB) LExpire(key []byte, duration int64) (int64, error) { + if duration <= 0 { + return 0, errExpireValue + } + + return db.lExpireAt(key, time.Now().Unix()+duration) +} + +// LExpireAt expires the list at when. +func (db *DB) LExpireAt(key []byte, when int64) (int64, error) { + if when <= time.Now().Unix() { + return 0, errExpireValue + } + + return db.lExpireAt(key, when) +} + +// LTTL gets the TTL of list. +func (db *DB) LTTL(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return -1, err + } + + return db.ttl(ListType, key) +} + +// LPersist removes the TTL of list. +func (db *DB) LPersist(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + t := db.listBatch + t.Lock() + defer t.Unlock() + + n, err := db.rmExpire(t, ListType, key) + if err != nil { + return 0, err + } + + err = t.Commit() + return n, err +} + +func (db *DB) lEncodeMinKey() []byte { + return db.lEncodeMetaKey(nil) +} + +func (db *DB) lEncodeMaxKey() []byte { + ek := db.lEncodeMetaKey(nil) + ek[len(ek)-1] = LMetaType + 1 + return ek +} + +// BLPop pops the list with block way. +func (db *DB) BLPop(keys [][]byte, timeout time.Duration) ([]interface{}, error) { + return db.lblockPop(keys, listHeadSeq, timeout) +} + +// BRPop bpops the list with block way. +func (db *DB) BRPop(keys [][]byte, timeout time.Duration) ([]interface{}, error) { + return db.lblockPop(keys, listTailSeq, timeout) +} + +// LKeyExists check list existed or not. +func (db *DB) LKeyExists(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + sk := db.lEncodeMetaKey(key) + v, err := db.bucket.Get(sk) + if v != nil && err == nil { + return 1, nil + } + return 0, err +} + +func (db *DB) lblockPop(keys [][]byte, whereSeq int32, timeout time.Duration) ([]interface{}, error) { + for { + var ctx context.Context + var cancel context.CancelFunc + if timeout > 0 { + ctx, cancel = context.WithTimeout(context.Background(), timeout) + } else { + ctx, cancel = context.WithCancel(context.Background()) + } + + for _, key := range keys { + v, err := db.lbkeys.popOrWait(db, key, whereSeq, cancel) + + if err != nil { + cancel() + return nil, err + } else if v != nil { + cancel() + return []interface{}{key, v}, nil + } + } + + //blocking wait + <-ctx.Done() + cancel() + + //if ctx.Err() is a deadline exceeded (timeout) we return + //otherwise we try to pop one of the keys again. + if ctx.Err() == context.DeadlineExceeded { + return nil, nil + } + } +} + +func (db *DB) lSignalAsReady(key []byte) { + db.lbkeys.signal(key) +} + +type lBlockKeys struct { + sync.Mutex + + keys map[string]*list.List +} + +func newLBlockKeys() *lBlockKeys { + l := new(lBlockKeys) + + l.keys = make(map[string]*list.List) + return l +} + +func (l *lBlockKeys) signal(key []byte) { + l.Lock() + defer l.Unlock() + + s := hack.String(key) + fns, ok := l.keys[s] + if !ok { + return + } + for e := fns.Front(); e != nil; e = e.Next() { + fn := e.Value.(context.CancelFunc) + fn() + } + + delete(l.keys, s) +} + +func (l *lBlockKeys) popOrWait(db *DB, key []byte, whereSeq int32, fn context.CancelFunc) ([]interface{}, error) { + v, err := db.lpop(key, whereSeq) + if err != nil { + return nil, err + } else if v != nil { + return []interface{}{key, v}, nil + } + + l.Lock() + + s := hack.String(key) + chs, ok := l.keys[s] + if !ok { + chs = list.New() + l.keys[s] = chs + } + + chs.PushBack(fn) + l.Unlock() + return nil, nil +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/t_set.go b/vendor/github.com/siddontang/ledisdb/ledis/t_set.go new file mode 100644 index 000000000000..926d4eaf4d6e --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/t_set.go @@ -0,0 +1,644 @@ +package ledis + +import ( + "encoding/binary" + "errors" + "time" + + "github.com/siddontang/go/hack" + "github.com/siddontang/ledisdb/store" +) + +var errSetKey = errors.New("invalid set key") +var errSSizeKey = errors.New("invalid ssize key") + +// For set operation type. +const ( + setStartSep byte = ':' + setStopSep byte = setStartSep + 1 + UnionType byte = 51 + DiffType byte = 52 + InterType byte = 53 +) + +func checkSetKMSize(key []byte, member []byte) error { + if len(key) > MaxKeySize || len(key) == 0 { + return errKeySize + } else if len(member) > MaxSetMemberSize || len(member) == 0 { + return errSetMemberSize + } + return nil +} + +func (db *DB) sEncodeSizeKey(key []byte) []byte { + buf := make([]byte, len(key)+1+len(db.indexVarBuf)) + + pos := copy(buf, db.indexVarBuf) + buf[pos] = SSizeType + + pos++ + + copy(buf[pos:], key) + return buf +} + +func (db *DB) sDecodeSizeKey(ek []byte) ([]byte, error) { + pos, err := db.checkKeyIndex(ek) + if err != nil { + return nil, err + } + + if pos+1 > len(ek) || ek[pos] != SSizeType { + return nil, errSSizeKey + } + pos++ + + return ek[pos:], nil +} + +func (db *DB) sEncodeSetKey(key []byte, member []byte) []byte { + buf := make([]byte, len(key)+len(member)+1+1+2+len(db.indexVarBuf)) + + pos := copy(buf, db.indexVarBuf) + + buf[pos] = SetType + pos++ + + binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) + pos += 2 + + copy(buf[pos:], key) + pos += len(key) + + buf[pos] = setStartSep + pos++ + copy(buf[pos:], member) + + return buf +} + +func (db *DB) sDecodeSetKey(ek []byte) ([]byte, []byte, error) { + pos, err := db.checkKeyIndex(ek) + if err != nil { + return nil, nil, err + } + + if pos+1 > len(ek) || ek[pos] != SetType { + return nil, nil, errSetKey + } + + pos++ + + if pos+2 > len(ek) { + return nil, nil, errSetKey + } + + keyLen := int(binary.BigEndian.Uint16(ek[pos:])) + pos += 2 + + if keyLen+pos > len(ek) { + return nil, nil, errSetKey + } + + key := ek[pos : pos+keyLen] + pos += keyLen + + if ek[pos] != hashStartSep { + return nil, nil, errSetKey + } + + pos++ + member := ek[pos:] + return key, member, nil +} + +func (db *DB) sEncodeStartKey(key []byte) []byte { + return db.sEncodeSetKey(key, nil) +} + +func (db *DB) sEncodeStopKey(key []byte) []byte { + k := db.sEncodeSetKey(key, nil) + + k[len(k)-1] = setStopSep + + return k +} + +func (db *DB) sFlush() (drop int64, err error) { + + t := db.setBatch + t.Lock() + defer t.Unlock() + + return db.flushType(t, SetType) +} + +func (db *DB) sDelete(t *batch, key []byte) int64 { + sk := db.sEncodeSizeKey(key) + start := db.sEncodeStartKey(key) + stop := db.sEncodeStopKey(key) + + var num int64 + it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) + for ; it.Valid(); it.Next() { + t.Delete(it.RawKey()) + num++ + } + + it.Close() + + t.Delete(sk) + return num +} + +func (db *DB) sIncrSize(key []byte, delta int64) (int64, error) { + t := db.setBatch + sk := db.sEncodeSizeKey(key) + + var err error + var size int64 + if size, err = Int64(db.bucket.Get(sk)); err != nil { + return 0, err + } + + size += delta + if size <= 0 { + size = 0 + t.Delete(sk) + db.rmExpire(t, SetType, key) + } else { + t.Put(sk, PutInt64(size)) + } + + return size, nil +} + +func (db *DB) sExpireAt(key []byte, when int64) (int64, error) { + t := db.setBatch + t.Lock() + defer t.Unlock() + + if scnt, err := db.SCard(key); err != nil || scnt == 0 { + return 0, err + } + db.expireAt(t, SetType, key, when) + if err := t.Commit(); err != nil { + return 0, err + } + + return 1, nil +} + +func (db *DB) sSetItem(key []byte, member []byte) (int64, error) { + t := db.setBatch + ek := db.sEncodeSetKey(key, member) + + var n int64 = 1 + if v, _ := db.bucket.Get(ek); v != nil { + n = 0 + } else { + if _, err := db.sIncrSize(key, 1); err != nil { + return 0, err + } + } + + t.Put(ek, nil) + return n, nil +} + +// SAdd adds the value to the set. +func (db *DB) SAdd(key []byte, args ...[]byte) (int64, error) { + t := db.setBatch + t.Lock() + defer t.Unlock() + + var err error + var ek []byte + var num int64 + for i := 0; i < len(args); i++ { + if err := checkSetKMSize(key, args[i]); err != nil { + return 0, err + } + + ek = db.sEncodeSetKey(key, args[i]) + + if v, err := db.bucket.Get(ek); err != nil { + return 0, err + } else if v == nil { + num++ + } + + t.Put(ek, nil) + } + + if _, err = db.sIncrSize(key, num); err != nil { + return 0, err + } + + err = t.Commit() + return num, err + +} + +// SCard gets the size of set. +func (db *DB) SCard(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + sk := db.sEncodeSizeKey(key) + + return Int64(db.bucket.Get(sk)) +} + +func (db *DB) sDiffGeneric(keys ...[]byte) ([][]byte, error) { + destMap := make(map[string]bool) + + members, err := db.SMembers(keys[0]) + if err != nil { + return nil, err + } + + for _, m := range members { + destMap[hack.String(m)] = true + } + + for _, k := range keys[1:] { + members, err := db.SMembers(k) + if err != nil { + return nil, err + } + + for _, m := range members { + if _, ok := destMap[hack.String(m)]; !ok { + continue + } else if ok { + delete(destMap, hack.String(m)) + } + } + // O - A = O, O is zero set. + if len(destMap) == 0 { + return nil, nil + } + } + + slice := make([][]byte, len(destMap)) + idx := 0 + for k, v := range destMap { + if !v { + continue + } + slice[idx] = []byte(k) + idx++ + } + + return slice, nil +} + +// SDiff gets the different of sets. +func (db *DB) SDiff(keys ...[]byte) ([][]byte, error) { + v, err := db.sDiffGeneric(keys...) + return v, err +} + +// SDiffStore gets the different of sets and stores to dest set. +func (db *DB) SDiffStore(dstKey []byte, keys ...[]byte) (int64, error) { + n, err := db.sStoreGeneric(dstKey, DiffType, keys...) + return n, err +} + +// SKeyExists checks whether set existed or not. +func (db *DB) SKeyExists(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + sk := db.sEncodeSizeKey(key) + v, err := db.bucket.Get(sk) + if v != nil && err == nil { + return 1, nil + } + return 0, err +} + +func (db *DB) sInterGeneric(keys ...[]byte) ([][]byte, error) { + destMap := make(map[string]bool) + + members, err := db.SMembers(keys[0]) + if err != nil { + return nil, err + } + + for _, m := range members { + destMap[hack.String(m)] = true + } + + for _, key := range keys[1:] { + if err := checkKeySize(key); err != nil { + return nil, err + } + + members, err := db.SMembers(key) + if err != nil { + return nil, err + } else if len(members) == 0 { + return nil, err + } + + tempMap := make(map[string]bool) + for _, member := range members { + if err := checkKeySize(member); err != nil { + return nil, err + } + if _, ok := destMap[hack.String(member)]; ok { + tempMap[hack.String(member)] = true //mark this item as selected + } + } + destMap = tempMap //reduce the size of the result set + if len(destMap) == 0 { + return nil, nil + } + } + + slice := make([][]byte, len(destMap)) + idx := 0 + for k, v := range destMap { + if !v { + continue + } + + slice[idx] = []byte(k) + idx++ + } + + return slice, nil + +} + +// SInter intersects the sets. +func (db *DB) SInter(keys ...[]byte) ([][]byte, error) { + v, err := db.sInterGeneric(keys...) + return v, err + +} + +// SInterStore intersects the sets and stores to dest set. +func (db *DB) SInterStore(dstKey []byte, keys ...[]byte) (int64, error) { + n, err := db.sStoreGeneric(dstKey, InterType, keys...) + return n, err +} + +// SIsMember checks member in set. +func (db *DB) SIsMember(key []byte, member []byte) (int64, error) { + ek := db.sEncodeSetKey(key, member) + + var n int64 = 1 + if v, err := db.bucket.Get(ek); err != nil { + return 0, err + } else if v == nil { + n = 0 + } + return n, nil +} + +// SMembers gets members of set. +func (db *DB) SMembers(key []byte) ([][]byte, error) { + if err := checkKeySize(key); err != nil { + return nil, err + } + + start := db.sEncodeStartKey(key) + stop := db.sEncodeStopKey(key) + + v := make([][]byte, 0, 16) + + it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) + defer it.Close() + + for ; it.Valid(); it.Next() { + _, m, err := db.sDecodeSetKey(it.Key()) + if err != nil { + return nil, err + } + + v = append(v, m) + } + + return v, nil +} + +// SRem removes the members of set. +func (db *DB) SRem(key []byte, args ...[]byte) (int64, error) { + t := db.setBatch + t.Lock() + defer t.Unlock() + + var ek []byte + var v []byte + var err error + + it := db.bucket.NewIterator() + defer it.Close() + + var num int64 + for i := 0; i < len(args); i++ { + if err := checkSetKMSize(key, args[i]); err != nil { + return 0, err + } + + ek = db.sEncodeSetKey(key, args[i]) + + v = it.RawFind(ek) + if v == nil { + continue + } else { + num++ + t.Delete(ek) + } + } + + if _, err = db.sIncrSize(key, -num); err != nil { + return 0, err + } + + err = t.Commit() + return num, err + +} + +func (db *DB) sUnionGeneric(keys ...[]byte) ([][]byte, error) { + dstMap := make(map[string]bool) + + for _, key := range keys { + if err := checkKeySize(key); err != nil { + return nil, err + } + + members, err := db.SMembers(key) + if err != nil { + return nil, err + } + + for _, member := range members { + dstMap[hack.String(member)] = true + } + } + + slice := make([][]byte, len(dstMap)) + idx := 0 + for k, v := range dstMap { + if !v { + continue + } + slice[idx] = []byte(k) + idx++ + } + + return slice, nil +} + +// SUnion unions the sets. +func (db *DB) SUnion(keys ...[]byte) ([][]byte, error) { + v, err := db.sUnionGeneric(keys...) + return v, err +} + +// SUnionStore unions the sets and stores to the dest set. +func (db *DB) SUnionStore(dstKey []byte, keys ...[]byte) (int64, error) { + n, err := db.sStoreGeneric(dstKey, UnionType, keys...) + return n, err +} + +func (db *DB) sStoreGeneric(dstKey []byte, optType byte, keys ...[]byte) (int64, error) { + if err := checkKeySize(dstKey); err != nil { + return 0, err + } + + t := db.setBatch + t.Lock() + defer t.Unlock() + + db.sDelete(t, dstKey) + + var err error + var ek []byte + var v [][]byte + + switch optType { + case UnionType: + v, err = db.sUnionGeneric(keys...) + case DiffType: + v, err = db.sDiffGeneric(keys...) + case InterType: + v, err = db.sInterGeneric(keys...) + } + + if err != nil { + return 0, err + } + + for _, m := range v { + if err := checkSetKMSize(dstKey, m); err != nil { + return 0, err + } + + ek = db.sEncodeSetKey(dstKey, m) + + if _, err := db.bucket.Get(ek); err != nil { + return 0, err + } + + t.Put(ek, nil) + } + + var n = int64(len(v)) + sk := db.sEncodeSizeKey(dstKey) + t.Put(sk, PutInt64(n)) + + if err = t.Commit(); err != nil { + return 0, err + } + return n, nil +} + +// SClear clears the set. +func (db *DB) SClear(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + t := db.setBatch + t.Lock() + defer t.Unlock() + + num := db.sDelete(t, key) + db.rmExpire(t, SetType, key) + + err := t.Commit() + return num, err +} + +// SMclear clears multi sets. +func (db *DB) SMclear(keys ...[]byte) (int64, error) { + t := db.setBatch + t.Lock() + defer t.Unlock() + + for _, key := range keys { + if err := checkKeySize(key); err != nil { + return 0, err + } + + db.sDelete(t, key) + db.rmExpire(t, SetType, key) + } + + err := t.Commit() + return int64(len(keys)), err +} + +// SExpire expries the set. +func (db *DB) SExpire(key []byte, duration int64) (int64, error) { + if duration <= 0 { + return 0, errExpireValue + } + + return db.sExpireAt(key, time.Now().Unix()+duration) + +} + +// SExpireAt expires the set at when. +func (db *DB) SExpireAt(key []byte, when int64) (int64, error) { + if when <= time.Now().Unix() { + return 0, errExpireValue + } + + return db.sExpireAt(key, when) + +} + +// STTL gets the TTL of set. +func (db *DB) STTL(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return -1, err + } + + return db.ttl(SetType, key) +} + +// SPersist removes the TTL of set. +func (db *DB) SPersist(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + t := db.setBatch + t.Lock() + defer t.Unlock() + + n, err := db.rmExpire(t, SetType, key) + if err != nil { + return 0, err + } + err = t.Commit() + return n, err +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/t_ttl.go b/vendor/github.com/siddontang/ledisdb/ledis/t_ttl.go new file mode 100644 index 000000000000..126c8f62b4fa --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/t_ttl.go @@ -0,0 +1,217 @@ +package ledis + +import ( + "encoding/binary" + "errors" + "sync" + "time" + + "github.com/siddontang/ledisdb/store" +) + +var ( + errExpMetaKey = errors.New("invalid expire meta key") + errExpTimeKey = errors.New("invalid expire time key") +) + +type onExpired func(*batch, []byte) int64 + +type ttlChecker struct { + sync.Mutex + db *DB + txs []*batch + cbs []onExpired + + //next check time + nc int64 +} + +var errExpType = errors.New("invalid expire type") + +func (db *DB) expEncodeTimeKey(dataType byte, key []byte, when int64) []byte { + buf := make([]byte, len(key)+10+len(db.indexVarBuf)) + + pos := copy(buf, db.indexVarBuf) + + buf[pos] = ExpTimeType + pos++ + + binary.BigEndian.PutUint64(buf[pos:], uint64(when)) + pos += 8 + + buf[pos] = dataType + pos++ + + copy(buf[pos:], key) + + return buf +} + +func (db *DB) expEncodeMetaKey(dataType byte, key []byte) []byte { + buf := make([]byte, len(key)+2+len(db.indexVarBuf)) + + pos := copy(buf, db.indexVarBuf) + buf[pos] = ExpMetaType + pos++ + buf[pos] = dataType + pos++ + + copy(buf[pos:], key) + + return buf +} + +func (db *DB) expDecodeMetaKey(mk []byte) (byte, []byte, error) { + pos, err := db.checkKeyIndex(mk) + if err != nil { + return 0, nil, err + } + + if pos+2 > len(mk) || mk[pos] != ExpMetaType { + return 0, nil, errExpMetaKey + } + + return mk[pos+1], mk[pos+2:], nil +} + +func (db *DB) expDecodeTimeKey(tk []byte) (byte, []byte, int64, error) { + pos, err := db.checkKeyIndex(tk) + if err != nil { + return 0, nil, 0, err + } + + if pos+10 > len(tk) || tk[pos] != ExpTimeType { + return 0, nil, 0, errExpTimeKey + } + + return tk[pos+9], tk[pos+10:], int64(binary.BigEndian.Uint64(tk[pos+1:])), nil +} + +func (db *DB) expire(t *batch, dataType byte, key []byte, duration int64) { + db.expireAt(t, dataType, key, time.Now().Unix()+duration) +} + +func (db *DB) expireAt(t *batch, dataType byte, key []byte, when int64) { + mk := db.expEncodeMetaKey(dataType, key) + tk := db.expEncodeTimeKey(dataType, key, when) + + t.Put(tk, mk) + t.Put(mk, PutInt64(when)) + + db.ttlChecker.setNextCheckTime(when, false) +} + +func (db *DB) ttl(dataType byte, key []byte) (t int64, err error) { + mk := db.expEncodeMetaKey(dataType, key) + + if t, err = Int64(db.bucket.Get(mk)); err != nil || t == 0 { + t = -1 + } else { + t -= time.Now().Unix() + if t <= 0 { + t = -1 + } + // if t == -1 : to remove ???? + } + + return t, err +} + +func (db *DB) rmExpire(t *batch, dataType byte, key []byte) (int64, error) { + mk := db.expEncodeMetaKey(dataType, key) + v, err := db.bucket.Get(mk) + if err != nil { + return 0, err + } else if v == nil { + return 0, nil + } + + when, err2 := Int64(v, nil) + if err2 != nil { + return 0, err2 + } + + tk := db.expEncodeTimeKey(dataType, key, when) + t.Delete(mk) + t.Delete(tk) + return 1, nil +} + +func (c *ttlChecker) register(dataType byte, t *batch, f onExpired) { + c.txs[dataType] = t + c.cbs[dataType] = f +} + +func (c *ttlChecker) setNextCheckTime(when int64, force bool) { + c.Lock() + if force { + c.nc = when + } else if c.nc > when { + c.nc = when + } + c.Unlock() +} + +func (c *ttlChecker) check() { + now := time.Now().Unix() + + c.Lock() + nc := c.nc + c.Unlock() + + if now < nc { + return + } + + nc = now + 3600 + + db := c.db + dbGet := db.bucket.Get + + minKey := db.expEncodeTimeKey(NoneType, nil, 0) + maxKey := db.expEncodeTimeKey(maxDataType, nil, nc) + + it := db.bucket.RangeLimitIterator(minKey, maxKey, store.RangeROpen, 0, -1) + for ; it.Valid(); it.Next() { + tk := it.RawKey() + mk := it.RawValue() + + dt, k, nt, err := db.expDecodeTimeKey(tk) + if err != nil { + continue + } + + if nt > now { + //the next ttl check time is nt! + nc = nt + break + } + + t := c.txs[dt] + cb := c.cbs[dt] + if tk == nil || cb == nil { + continue + } + + t.Lock() + + if exp, err := Int64(dbGet(mk)); err == nil { + // check expire again + if exp <= now { + cb(t, k) + t.Delete(tk) + t.Delete(mk) + + t.Commit() + } + + } + + t.Unlock() + } + it.Close() + + c.setNextCheckTime(nc, true) + + return +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/t_zset.go b/vendor/github.com/siddontang/ledisdb/ledis/t_zset.go new file mode 100644 index 000000000000..030b2976f6d8 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/t_zset.go @@ -0,0 +1,1093 @@ +package ledis + +import ( + "bytes" + "encoding/binary" + "errors" + "time" + + "github.com/siddontang/go/hack" + "github.com/siddontang/ledisdb/store" +) + +// For zset const. +const ( + MinScore int64 = -1<<63 + 1 + MaxScore int64 = 1<<63 - 1 + InvalidScore int64 = -1 << 63 + + AggregateSum byte = 0 + AggregateMin byte = 1 + AggregateMax byte = 2 +) + +// ScorePair is the pair of score and member. +type ScorePair struct { + Score int64 + Member []byte +} + +var errZSizeKey = errors.New("invalid zsize key") +var errZSetKey = errors.New("invalid zset key") +var errZScoreKey = errors.New("invalid zscore key") +var errScoreOverflow = errors.New("zset score overflow") +var errInvalidAggregate = errors.New("invalid aggregate") +var errInvalidWeightNum = errors.New("invalid weight number") +var errInvalidSrcKeyNum = errors.New("invalid src key number") + +const ( + zsetNScoreSep byte = '<' + zsetPScoreSep byte = zsetNScoreSep + 1 + zsetStopScoreSep byte = zsetPScoreSep + 1 + + zsetStartMemSep byte = ':' + zsetStopMemSep byte = zsetStartMemSep + 1 +) + +func checkZSetKMSize(key []byte, member []byte) error { + if len(key) > MaxKeySize || len(key) == 0 { + return errKeySize + } else if len(member) > MaxZSetMemberSize || len(member) == 0 { + return errZSetMemberSize + } + return nil +} + +func (db *DB) zEncodeSizeKey(key []byte) []byte { + buf := make([]byte, len(key)+1+len(db.indexVarBuf)) + pos := copy(buf, db.indexVarBuf) + buf[pos] = ZSizeType + pos++ + copy(buf[pos:], key) + return buf +} + +func (db *DB) zDecodeSizeKey(ek []byte) ([]byte, error) { + pos, err := db.checkKeyIndex(ek) + if err != nil { + return nil, err + } + + if pos+1 > len(ek) || ek[pos] != ZSizeType { + return nil, errZSizeKey + } + pos++ + return ek[pos:], nil +} + +func (db *DB) zEncodeSetKey(key []byte, member []byte) []byte { + buf := make([]byte, len(key)+len(member)+4+len(db.indexVarBuf)) + + pos := copy(buf, db.indexVarBuf) + + buf[pos] = ZSetType + pos++ + + binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) + pos += 2 + + copy(buf[pos:], key) + pos += len(key) + + buf[pos] = zsetStartMemSep + pos++ + + copy(buf[pos:], member) + + return buf +} + +func (db *DB) zDecodeSetKey(ek []byte) ([]byte, []byte, error) { + pos, err := db.checkKeyIndex(ek) + if err != nil { + return nil, nil, err + } + + if pos+1 > len(ek) || ek[pos] != ZSetType { + return nil, nil, errZSetKey + } + + pos++ + + if pos+2 > len(ek) { + return nil, nil, errZSetKey + } + + keyLen := int(binary.BigEndian.Uint16(ek[pos:])) + if keyLen+pos > len(ek) { + return nil, nil, errZSetKey + } + + pos += 2 + key := ek[pos : pos+keyLen] + + if ek[pos+keyLen] != zsetStartMemSep { + return nil, nil, errZSetKey + } + pos++ + + member := ek[pos+keyLen:] + return key, member, nil +} + +func (db *DB) zEncodeStartSetKey(key []byte) []byte { + k := db.zEncodeSetKey(key, nil) + return k +} + +func (db *DB) zEncodeStopSetKey(key []byte) []byte { + k := db.zEncodeSetKey(key, nil) + k[len(k)-1] = zsetStartMemSep + 1 + return k +} + +func (db *DB) zEncodeScoreKey(key []byte, member []byte, score int64) []byte { + buf := make([]byte, len(key)+len(member)+13+len(db.indexVarBuf)) + + pos := copy(buf, db.indexVarBuf) + + buf[pos] = ZScoreType + pos++ + + binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) + pos += 2 + + copy(buf[pos:], key) + pos += len(key) + + if score < 0 { + buf[pos] = zsetNScoreSep + } else { + buf[pos] = zsetPScoreSep + } + + pos++ + binary.BigEndian.PutUint64(buf[pos:], uint64(score)) + pos += 8 + + buf[pos] = zsetStartMemSep + pos++ + + copy(buf[pos:], member) + return buf +} + +func (db *DB) zEncodeStartScoreKey(key []byte, score int64) []byte { + return db.zEncodeScoreKey(key, nil, score) +} + +func (db *DB) zEncodeStopScoreKey(key []byte, score int64) []byte { + k := db.zEncodeScoreKey(key, nil, score) + k[len(k)-1] = zsetStopMemSep + return k +} + +func (db *DB) zDecodeScoreKey(ek []byte) (key []byte, member []byte, score int64, err error) { + pos := 0 + pos, err = db.checkKeyIndex(ek) + if err != nil { + return + } + + if pos+1 > len(ek) || ek[pos] != ZScoreType { + err = errZScoreKey + return + } + pos++ + + if pos+2 > len(ek) { + err = errZScoreKey + return + } + keyLen := int(binary.BigEndian.Uint16(ek[pos:])) + pos += 2 + + if keyLen+pos > len(ek) { + err = errZScoreKey + return + } + + key = ek[pos : pos+keyLen] + pos += keyLen + + if pos+10 > len(ek) { + err = errZScoreKey + return + } + + if (ek[pos] != zsetNScoreSep) && (ek[pos] != zsetPScoreSep) { + err = errZScoreKey + return + } + pos++ + + score = int64(binary.BigEndian.Uint64(ek[pos:])) + pos += 8 + + if ek[pos] != zsetStartMemSep { + err = errZScoreKey + return + } + + pos++ + + member = ek[pos:] + return +} + +func (db *DB) zSetItem(t *batch, key []byte, score int64, member []byte) (int64, error) { + if score <= MinScore || score >= MaxScore { + return 0, errScoreOverflow + } + + var exists int64 + ek := db.zEncodeSetKey(key, member) + + if v, err := db.bucket.Get(ek); err != nil { + return 0, err + } else if v != nil { + exists = 1 + + s, err := Int64(v, err) + if err != nil { + return 0, err + } + + sk := db.zEncodeScoreKey(key, member, s) + t.Delete(sk) + } + + t.Put(ek, PutInt64(score)) + + sk := db.zEncodeScoreKey(key, member, score) + t.Put(sk, []byte{}) + + return exists, nil +} + +func (db *DB) zDelItem(t *batch, key []byte, member []byte, skipDelScore bool) (int64, error) { + ek := db.zEncodeSetKey(key, member) + if v, err := db.bucket.Get(ek); err != nil { + return 0, err + } else if v == nil { + //not exists + return 0, nil + } else { + //exists + if !skipDelScore { + //we must del score + s, err := Int64(v, err) + if err != nil { + return 0, err + } + sk := db.zEncodeScoreKey(key, member, s) + t.Delete(sk) + } + } + + t.Delete(ek) + + return 1, nil +} + +func (db *DB) zDelete(t *batch, key []byte) int64 { + delMembCnt, _ := db.zRemRange(t, key, MinScore, MaxScore, 0, -1) + // todo : log err + return delMembCnt +} + +func (db *DB) zExpireAt(key []byte, when int64) (int64, error) { + t := db.zsetBatch + t.Lock() + defer t.Unlock() + + if zcnt, err := db.ZCard(key); err != nil || zcnt == 0 { + return 0, err + } + + db.expireAt(t, ZSetType, key, when) + if err := t.Commit(); err != nil { + return 0, err + } + + return 1, nil +} + +// ZAdd add the members. +func (db *DB) ZAdd(key []byte, args ...ScorePair) (int64, error) { + if len(args) == 0 { + return 0, nil + } + + t := db.zsetBatch + t.Lock() + defer t.Unlock() + + var num int64 + for i := 0; i < len(args); i++ { + score := args[i].Score + member := args[i].Member + + if err := checkZSetKMSize(key, member); err != nil { + return 0, err + } + + if n, err := db.zSetItem(t, key, score, member); err != nil { + return 0, err + } else if n == 0 { + //add new + num++ + } + } + + if _, err := db.zIncrSize(t, key, num); err != nil { + return 0, err + } + + err := t.Commit() + return num, err +} + +func (db *DB) zIncrSize(t *batch, key []byte, delta int64) (int64, error) { + sk := db.zEncodeSizeKey(key) + + size, err := Int64(db.bucket.Get(sk)) + if err != nil { + return 0, err + } + size += delta + if size <= 0 { + size = 0 + t.Delete(sk) + db.rmExpire(t, ZSetType, key) + } else { + t.Put(sk, PutInt64(size)) + } + + return size, nil +} + +// ZCard gets the size of the zset. +func (db *DB) ZCard(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + sk := db.zEncodeSizeKey(key) + return Int64(db.bucket.Get(sk)) +} + +// ZScore gets the score of member. +func (db *DB) ZScore(key []byte, member []byte) (int64, error) { + if err := checkZSetKMSize(key, member); err != nil { + return InvalidScore, err + } + + score := InvalidScore + + k := db.zEncodeSetKey(key, member) + if v, err := db.bucket.Get(k); err != nil { + return InvalidScore, err + } else if v == nil { + return InvalidScore, ErrScoreMiss + } else { + if score, err = Int64(v, nil); err != nil { + return InvalidScore, err + } + } + + return score, nil +} + +// ZRem removes members +func (db *DB) ZRem(key []byte, members ...[]byte) (int64, error) { + if len(members) == 0 { + return 0, nil + } + + t := db.zsetBatch + t.Lock() + defer t.Unlock() + + var num int64 + for i := 0; i < len(members); i++ { + if err := checkZSetKMSize(key, members[i]); err != nil { + return 0, err + } + + if n, err := db.zDelItem(t, key, members[i], false); err != nil { + return 0, err + } else if n == 1 { + num++ + } + } + + if _, err := db.zIncrSize(t, key, -num); err != nil { + return 0, err + } + + err := t.Commit() + return num, err +} + +// ZIncrBy increases the score of member with delta. +func (db *DB) ZIncrBy(key []byte, delta int64, member []byte) (int64, error) { + if err := checkZSetKMSize(key, member); err != nil { + return InvalidScore, err + } + + t := db.zsetBatch + t.Lock() + defer t.Unlock() + + ek := db.zEncodeSetKey(key, member) + + var oldScore int64 + v, err := db.bucket.Get(ek) + if err != nil { + return InvalidScore, err + } else if v == nil { + db.zIncrSize(t, key, 1) + } else { + if oldScore, err = Int64(v, err); err != nil { + return InvalidScore, err + } + } + + newScore := oldScore + delta + if newScore >= MaxScore || newScore <= MinScore { + return InvalidScore, errScoreOverflow + } + + sk := db.zEncodeScoreKey(key, member, newScore) + t.Put(sk, []byte{}) + t.Put(ek, PutInt64(newScore)) + + if v != nil { + // so as to update score, we must delete the old one + oldSk := db.zEncodeScoreKey(key, member, oldScore) + t.Delete(oldSk) + } + + err = t.Commit() + return newScore, err +} + +// ZCount gets the number of score in [min, max] +func (db *DB) ZCount(key []byte, min int64, max int64) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + minKey := db.zEncodeStartScoreKey(key, min) + maxKey := db.zEncodeStopScoreKey(key, max) + + rangeType := store.RangeROpen + + it := db.bucket.RangeLimitIterator(minKey, maxKey, rangeType, 0, -1) + var n int64 + for ; it.Valid(); it.Next() { + n++ + } + it.Close() + + return n, nil +} + +func (db *DB) zrank(key []byte, member []byte, reverse bool) (int64, error) { + if err := checkZSetKMSize(key, member); err != nil { + return 0, err + } + + k := db.zEncodeSetKey(key, member) + + it := db.bucket.NewIterator() + defer it.Close() + + v := it.Find(k) + if v == nil { + return -1, nil + } + + s, err := Int64(v, nil) + if err != nil { + return 0, err + } + var rit *store.RangeLimitIterator + + sk := db.zEncodeScoreKey(key, member, s) + + if !reverse { + minKey := db.zEncodeStartScoreKey(key, MinScore) + + rit = store.NewRangeIterator(it, &store.Range{Min: minKey, Max: sk, Type: store.RangeClose}) + } else { + maxKey := db.zEncodeStopScoreKey(key, MaxScore) + rit = store.NewRevRangeIterator(it, &store.Range{Min: sk, Max: maxKey, Type: store.RangeClose}) + } + + var lastKey []byte + var n int64 + + for ; rit.Valid(); rit.Next() { + n++ + + lastKey = rit.BufKey(lastKey) + } + + if _, m, _, err := db.zDecodeScoreKey(lastKey); err == nil && bytes.Equal(m, member) { + n-- + return n, nil + } + + return -1, nil +} + +func (db *DB) zIterator(key []byte, min int64, max int64, offset int, count int, reverse bool) *store.RangeLimitIterator { + minKey := db.zEncodeStartScoreKey(key, min) + maxKey := db.zEncodeStopScoreKey(key, max) + + if !reverse { + return db.bucket.RangeLimitIterator(minKey, maxKey, store.RangeClose, offset, count) + } + return db.bucket.RevRangeLimitIterator(minKey, maxKey, store.RangeClose, offset, count) +} + +func (db *DB) zRemRange(t *batch, key []byte, min int64, max int64, offset int, count int) (int64, error) { + if len(key) > MaxKeySize { + return 0, errKeySize + } + + it := db.zIterator(key, min, max, offset, count, false) + var num int64 + for ; it.Valid(); it.Next() { + sk := it.RawKey() + _, m, _, err := db.zDecodeScoreKey(sk) + if err != nil { + continue + } + + if n, err := db.zDelItem(t, key, m, true); err != nil { + return 0, err + } else if n == 1 { + num++ + } + + t.Delete(sk) + } + it.Close() + + if _, err := db.zIncrSize(t, key, -num); err != nil { + return 0, err + } + + return num, nil +} + +func (db *DB) zRange(key []byte, min int64, max int64, offset int, count int, reverse bool) ([]ScorePair, error) { + if len(key) > MaxKeySize { + return nil, errKeySize + } + + if offset < 0 { + return []ScorePair{}, nil + } + + nv := count + // count may be very large, so we must limit it for below mem make. + if nv <= 0 || nv > 1024 { + nv = 64 + } + + v := make([]ScorePair, 0, nv) + + var it *store.RangeLimitIterator + + //if reverse and offset is 0, count < 0, we may use forward iterator then reverse + //because store iterator prev is slower than next + if !reverse || (offset == 0 && count < 0) { + it = db.zIterator(key, min, max, offset, count, false) + } else { + it = db.zIterator(key, min, max, offset, count, true) + } + + for ; it.Valid(); it.Next() { + _, m, s, err := db.zDecodeScoreKey(it.Key()) + //may be we will check key equal? + if err != nil { + continue + } + + v = append(v, ScorePair{Member: m, Score: s}) + } + it.Close() + + if reverse && (offset == 0 && count < 0) { + for i, j := 0, len(v)-1; i < j; i, j = i+1, j-1 { + v[i], v[j] = v[j], v[i] + } + } + + return v, nil +} + +func (db *DB) zParseLimit(key []byte, start int, stop int) (offset int, count int, err error) { + if start < 0 || stop < 0 { + //refer redis implementation + var size int64 + size, err = db.ZCard(key) + if err != nil { + return + } + + llen := int(size) + + if start < 0 { + start = llen + start + } + if stop < 0 { + stop = llen + stop + } + + if start < 0 { + start = 0 + } + + if start >= llen { + offset = -1 + return + } + } + + if start > stop { + offset = -1 + return + } + + offset = start + count = (stop - start) + 1 + return +} + +// ZClear clears the zset. +func (db *DB) ZClear(key []byte) (int64, error) { + t := db.zsetBatch + t.Lock() + defer t.Unlock() + + rmCnt, err := db.zRemRange(t, key, MinScore, MaxScore, 0, -1) + if err == nil { + err = t.Commit() + } + + return rmCnt, err +} + +// ZMclear clears multi zsets. +func (db *DB) ZMclear(keys ...[]byte) (int64, error) { + t := db.zsetBatch + t.Lock() + defer t.Unlock() + + for _, key := range keys { + if _, err := db.zRemRange(t, key, MinScore, MaxScore, 0, -1); err != nil { + return 0, err + } + } + + err := t.Commit() + + return int64(len(keys)), err +} + +// ZRange gets the members from start to stop. +func (db *DB) ZRange(key []byte, start int, stop int) ([]ScorePair, error) { + return db.ZRangeGeneric(key, start, stop, false) +} + +// ZRangeByScore gets the data with score in min and max. +// min and max must be inclusive +// if no limit, set offset = 0 and count = -1 +func (db *DB) ZRangeByScore(key []byte, min int64, max int64, + offset int, count int) ([]ScorePair, error) { + return db.ZRangeByScoreGeneric(key, min, max, offset, count, false) +} + +// ZRank gets the rank of member. +func (db *DB) ZRank(key []byte, member []byte) (int64, error) { + return db.zrank(key, member, false) +} + +// ZRemRangeByRank removes the member at range from start to stop. +func (db *DB) ZRemRangeByRank(key []byte, start int, stop int) (int64, error) { + offset, count, err := db.zParseLimit(key, start, stop) + if err != nil { + return 0, err + } + + var rmCnt int64 + + t := db.zsetBatch + t.Lock() + defer t.Unlock() + + rmCnt, err = db.zRemRange(t, key, MinScore, MaxScore, offset, count) + if err == nil { + err = t.Commit() + } + + return rmCnt, err +} + +// ZRemRangeByScore removes the data with score at [min, max] +func (db *DB) ZRemRangeByScore(key []byte, min int64, max int64) (int64, error) { + t := db.zsetBatch + t.Lock() + defer t.Unlock() + + rmCnt, err := db.zRemRange(t, key, min, max, 0, -1) + if err == nil { + err = t.Commit() + } + + return rmCnt, err +} + +// ZRevRange gets the data reversed. +func (db *DB) ZRevRange(key []byte, start int, stop int) ([]ScorePair, error) { + return db.ZRangeGeneric(key, start, stop, true) +} + +// ZRevRank gets the rank of member reversed. +func (db *DB) ZRevRank(key []byte, member []byte) (int64, error) { + return db.zrank(key, member, true) +} + +// ZRevRangeByScore gets the data with score at [min, max] +// min and max must be inclusive +// if no limit, set offset = 0 and count = -1 +func (db *DB) ZRevRangeByScore(key []byte, min int64, max int64, offset int, count int) ([]ScorePair, error) { + return db.ZRangeByScoreGeneric(key, min, max, offset, count, true) +} + +// ZRangeGeneric is a generic function for scan zset. +func (db *DB) ZRangeGeneric(key []byte, start int, stop int, reverse bool) ([]ScorePair, error) { + offset, count, err := db.zParseLimit(key, start, stop) + if err != nil { + return nil, err + } + + return db.zRange(key, MinScore, MaxScore, offset, count, reverse) +} + +// ZRangeByScoreGeneric is a generic function to scan zset with score. +// min and max must be inclusive +// if no limit, set offset = 0 and count = -1 +func (db *DB) ZRangeByScoreGeneric(key []byte, min int64, max int64, + offset int, count int, reverse bool) ([]ScorePair, error) { + + return db.zRange(key, min, max, offset, count, reverse) +} + +func (db *DB) zFlush() (drop int64, err error) { + t := db.zsetBatch + t.Lock() + defer t.Unlock() + return db.flushType(t, ZSetType) +} + +// ZExpire expires the zset. +func (db *DB) ZExpire(key []byte, duration int64) (int64, error) { + if duration <= 0 { + return 0, errExpireValue + } + + return db.zExpireAt(key, time.Now().Unix()+duration) +} + +// ZExpireAt expires the zset at when. +func (db *DB) ZExpireAt(key []byte, when int64) (int64, error) { + if when <= time.Now().Unix() { + return 0, errExpireValue + } + + return db.zExpireAt(key, when) +} + +// ZTTL gets the TTL of zset. +func (db *DB) ZTTL(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return -1, err + } + + return db.ttl(ZSetType, key) +} + +// ZPersist removes the TTL of zset. +func (db *DB) ZPersist(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + t := db.zsetBatch + t.Lock() + defer t.Unlock() + + n, err := db.rmExpire(t, ZSetType, key) + if err != nil { + return 0, err + } + + err = t.Commit() + return n, err +} + +func getAggregateFunc(aggregate byte) func(int64, int64) int64 { + switch aggregate { + case AggregateSum: + return func(a int64, b int64) int64 { + return a + b + } + case AggregateMax: + return func(a int64, b int64) int64 { + if a > b { + return a + } + return b + } + case AggregateMin: + return func(a int64, b int64) int64 { + if a > b { + return b + } + return a + } + } + return nil +} + +// ZUnionStore unions the zsets and stores to dest zset. +func (db *DB) ZUnionStore(destKey []byte, srcKeys [][]byte, weights []int64, aggregate byte) (int64, error) { + + var destMap = map[string]int64{} + aggregateFunc := getAggregateFunc(aggregate) + if aggregateFunc == nil { + return 0, errInvalidAggregate + } + if len(srcKeys) < 1 { + return 0, errInvalidSrcKeyNum + } + if weights != nil { + if len(srcKeys) != len(weights) { + return 0, errInvalidWeightNum + } + } else { + weights = make([]int64, len(srcKeys)) + for i := 0; i < len(weights); i++ { + weights[i] = 1 + } + } + + for i, key := range srcKeys { + scorePairs, err := db.ZRange(key, 0, -1) + if err != nil { + return 0, err + } + for _, pair := range scorePairs { + if score, ok := destMap[hack.String(pair.Member)]; !ok { + destMap[hack.String(pair.Member)] = pair.Score * weights[i] + } else { + destMap[hack.String(pair.Member)] = aggregateFunc(score, pair.Score*weights[i]) + } + } + } + + t := db.zsetBatch + t.Lock() + defer t.Unlock() + + db.zDelete(t, destKey) + + for member, score := range destMap { + if err := checkZSetKMSize(destKey, []byte(member)); err != nil { + return 0, err + } + + if _, err := db.zSetItem(t, destKey, score, []byte(member)); err != nil { + return 0, err + } + } + + var n = int64(len(destMap)) + sk := db.zEncodeSizeKey(destKey) + t.Put(sk, PutInt64(n)) + + if err := t.Commit(); err != nil { + return 0, err + } + return n, nil +} + +// ZInterStore intersects the zsets and stores to dest zset. +func (db *DB) ZInterStore(destKey []byte, srcKeys [][]byte, weights []int64, aggregate byte) (int64, error) { + + aggregateFunc := getAggregateFunc(aggregate) + if aggregateFunc == nil { + return 0, errInvalidAggregate + } + if len(srcKeys) < 1 { + return 0, errInvalidSrcKeyNum + } + if weights != nil { + if len(srcKeys) != len(weights) { + return 0, errInvalidWeightNum + } + } else { + weights = make([]int64, len(srcKeys)) + for i := 0; i < len(weights); i++ { + weights[i] = 1 + } + } + + var destMap = map[string]int64{} + scorePairs, err := db.ZRange(srcKeys[0], 0, -1) + if err != nil { + return 0, err + } + for _, pair := range scorePairs { + destMap[hack.String(pair.Member)] = pair.Score * weights[0] + } + + for i, key := range srcKeys[1:] { + scorePairs, err := db.ZRange(key, 0, -1) + if err != nil { + return 0, err + } + tmpMap := map[string]int64{} + for _, pair := range scorePairs { + if score, ok := destMap[hack.String(pair.Member)]; ok { + tmpMap[hack.String(pair.Member)] = aggregateFunc(score, pair.Score*weights[i+1]) + } + } + destMap = tmpMap + } + + t := db.zsetBatch + t.Lock() + defer t.Unlock() + + db.zDelete(t, destKey) + + for member, score := range destMap { + if err := checkZSetKMSize(destKey, []byte(member)); err != nil { + return 0, err + } + if _, err := db.zSetItem(t, destKey, score, []byte(member)); err != nil { + return 0, err + } + } + + n := int64(len(destMap)) + sk := db.zEncodeSizeKey(destKey) + t.Put(sk, PutInt64(n)) + + if err := t.Commit(); err != nil { + return 0, err + } + return n, nil +} + +// ZRangeByLex scans the zset lexicographically +func (db *DB) ZRangeByLex(key []byte, min []byte, max []byte, rangeType uint8, offset int, count int) ([][]byte, error) { + if min == nil { + min = db.zEncodeStartSetKey(key) + } else { + min = db.zEncodeSetKey(key, min) + } + if max == nil { + max = db.zEncodeStopSetKey(key) + } else { + max = db.zEncodeSetKey(key, max) + } + + it := db.bucket.RangeLimitIterator(min, max, rangeType, offset, count) + defer it.Close() + + ay := make([][]byte, 0, 16) + for ; it.Valid(); it.Next() { + if _, m, err := db.zDecodeSetKey(it.Key()); err == nil { + ay = append(ay, m) + } + } + + return ay, nil +} + +// ZRemRangeByLex remvoes members in [min, max] lexicographically +func (db *DB) ZRemRangeByLex(key []byte, min []byte, max []byte, rangeType uint8) (int64, error) { + if min == nil { + min = db.zEncodeStartSetKey(key) + } else { + min = db.zEncodeSetKey(key, min) + } + if max == nil { + max = db.zEncodeStopSetKey(key) + } else { + max = db.zEncodeSetKey(key, max) + } + + t := db.zsetBatch + t.Lock() + defer t.Unlock() + + it := db.bucket.RangeIterator(min, max, rangeType) + defer it.Close() + + var n int64 + for ; it.Valid(); it.Next() { + t.Delete(it.RawKey()) + n++ + } + + if err := t.Commit(); err != nil { + return 0, err + } + + return n, nil +} + +// ZLexCount gets the count of zset lexicographically. +func (db *DB) ZLexCount(key []byte, min []byte, max []byte, rangeType uint8) (int64, error) { + if min == nil { + min = db.zEncodeStartSetKey(key) + } else { + min = db.zEncodeSetKey(key, min) + } + if max == nil { + max = db.zEncodeStopSetKey(key) + } else { + max = db.zEncodeSetKey(key, max) + } + + it := db.bucket.RangeIterator(min, max, rangeType) + defer it.Close() + + var n int64 + for ; it.Valid(); it.Next() { + n++ + } + + return n, nil +} + +// ZKeyExists checks zset existed or not. +func (db *DB) ZKeyExists(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + sk := db.zEncodeSizeKey(key) + v, err := db.bucket.Get(sk) + if v != nil && err == nil { + return 1, nil + } + return 0, err +} diff --git a/vendor/github.com/siddontang/ledisdb/ledis/util.go b/vendor/github.com/siddontang/ledisdb/ledis/util.go new file mode 100644 index 000000000000..5a054c419dbd --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/ledis/util.go @@ -0,0 +1,103 @@ +package ledis + +import ( + "encoding/binary" + "errors" + "strconv" + + "github.com/siddontang/go/hack" +) + +var errIntNumber = errors.New("invalid integer") + +/* + Below I forget why I use little endian to store int. + Maybe I was foolish at that time. +*/ + +// Int64 gets 64 integer with the little endian format. +func Int64(v []byte, err error) (int64, error) { + if err != nil { + return 0, err + } else if v == nil || len(v) == 0 { + return 0, nil + } else if len(v) != 8 { + return 0, errIntNumber + } + + return int64(binary.LittleEndian.Uint64(v)), nil +} + +// Uint64 gets unsigned 64 integer. +func Uint64(v []byte, err error) (uint64, error) { + if err != nil { + return 0, err + } else if v == nil || len(v) == 0 { + return 0, nil + } else if len(v) != 8 { + return 0, errIntNumber + } + + return binary.LittleEndian.Uint64(v), nil +} + +// PutInt64 puts the 64 integer. +func PutInt64(v int64) []byte { + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, uint64(v)) + return b +} + +// StrInt64 gets the 64 integer with string format. +func StrInt64(v []byte, err error) (int64, error) { + if err != nil { + return 0, err + } else if v == nil { + return 0, nil + } else { + return strconv.ParseInt(hack.String(v), 10, 64) + } +} + +// StrUint64 gets the unsigned 64 integer with string format. +func StrUint64(v []byte, err error) (uint64, error) { + if err != nil { + return 0, err + } else if v == nil { + return 0, nil + } else { + return strconv.ParseUint(hack.String(v), 10, 64) + } +} + +// StrInt32 gets the 32 integer with string format. +func StrInt32(v []byte, err error) (int32, error) { + if err != nil { + return 0, err + } else if v == nil { + return 0, nil + } else { + res, err := strconv.ParseInt(hack.String(v), 10, 32) + return int32(res), err + } +} + +// StrInt8 ets the 8 integer with string format. +func StrInt8(v []byte, err error) (int8, error) { + if err != nil { + return 0, err + } else if v == nil { + return 0, nil + } else { + res, err := strconv.ParseInt(hack.String(v), 10, 8) + return int8(res), err + } +} + +// AsyncNotify notices the channel. +func AsyncNotify(ch chan struct{}) { + select { + case ch <- struct{}{}: + default: + } +} diff --git a/vendor/github.com/siddontang/ledisdb/rpl/file_io.go b/vendor/github.com/siddontang/ledisdb/rpl/file_io.go new file mode 100644 index 000000000000..6eac87890907 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/rpl/file_io.go @@ -0,0 +1,363 @@ +package rpl + +import ( + "fmt" + "io" + "os" + + "github.com/edsrzf/mmap-go" + "github.com/siddontang/go/log" +) + +//like leveldb or rocksdb file interface, haha! + +type writeFile interface { + Sync() error + Write(b []byte) (n int, err error) + Close() error + ReadAt(buf []byte, offset int64) (int, error) + Truncate(size int64) error + SetOffset(o int64) + Name() string + Size() int + Offset() int64 +} + +type readFile interface { + ReadAt(buf []byte, offset int64) (int, error) + Close() error + Size() int + Name() string +} + +type rawWriteFile struct { + writeFile + f *os.File + offset int64 + name string +} + +func newRawWriteFile(name string, size int64) (writeFile, error) { + m := new(rawWriteFile) + var err error + + m.name = name + + m.f, err = os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + return nil, err + } + + return m, nil +} + +func (m *rawWriteFile) Close() error { + if err := m.f.Truncate(m.offset); err != nil { + return fmt.Errorf("close truncate %s error %s", m.name, err.Error()) + } + + if err := m.f.Close(); err != nil { + return fmt.Errorf("close %s error %s", m.name, err.Error()) + } + + return nil +} + +func (m *rawWriteFile) Sync() error { + return m.f.Sync() +} + +func (m *rawWriteFile) Write(b []byte) (n int, err error) { + n, err = m.f.WriteAt(b, m.offset) + if err != nil { + return + } else if n != len(b) { + err = io.ErrShortWrite + return + } + + m.offset += int64(n) + return +} + +func (m *rawWriteFile) ReadAt(buf []byte, offset int64) (int, error) { + return m.f.ReadAt(buf, offset) +} + +func (m *rawWriteFile) Truncate(size int64) error { + var err error + if err = m.f.Truncate(size); err != nil { + return err + } + + if m.offset > size { + m.offset = size + } + return nil +} + +func (m *rawWriteFile) SetOffset(o int64) { + m.offset = o +} + +func (m *rawWriteFile) Offset() int64 { + return m.offset +} + +func (m *rawWriteFile) Name() string { + return m.name +} + +func (m *rawWriteFile) Size() int { + st, _ := m.f.Stat() + return int(st.Size()) +} + +type rawReadFile struct { + readFile + + f *os.File + name string +} + +func newRawReadFile(name string) (readFile, error) { + m := new(rawReadFile) + + var err error + m.f, err = os.Open(name) + m.name = name + + if err != nil { + return nil, err + } + + return m, err +} + +func (m *rawReadFile) Close() error { + return m.f.Close() +} + +func (m *rawReadFile) Size() int { + st, _ := m.f.Stat() + return int(st.Size()) +} + +func (m *rawReadFile) ReadAt(b []byte, offset int64) (int, error) { + return m.f.ReadAt(b, offset) +} + +func (m *rawReadFile) Name() string { + return m.name +} + +///////////////////////////////////////////////// + +type mmapWriteFile struct { + writeFile + + f *os.File + m mmap.MMap + name string + size int64 + offset int64 +} + +func newMmapWriteFile(name string, size int64) (writeFile, error) { + m := new(mmapWriteFile) + + m.name = name + + var err error + + m.f, err = os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + return nil, err + } + + if size == 0 { + st, _ := m.f.Stat() + size = st.Size() + } + + if err = m.f.Truncate(size); err != nil { + return nil, err + } + + if m.m, err = mmap.Map(m.f, mmap.RDWR, 0); err != nil { + return nil, err + } + + m.size = size + m.offset = 0 + return m, nil +} + +func (m *mmapWriteFile) Size() int { + return int(m.size) +} + +func (m *mmapWriteFile) Sync() error { + return m.m.Flush() +} + +func (m *mmapWriteFile) Close() error { + if err := m.m.Unmap(); err != nil { + return fmt.Errorf("unmap %s error %s", m.name, err.Error()) + } + + if err := m.f.Truncate(m.offset); err != nil { + return fmt.Errorf("close truncate %s error %s", m.name, err.Error()) + } + + if err := m.f.Close(); err != nil { + return fmt.Errorf("close %s error %s", m.name, err.Error()) + } + + return nil +} + +func (m *mmapWriteFile) Write(b []byte) (n int, err error) { + extra := int64(len(b)) - (m.size - m.offset) + if extra > 0 { + newSize := m.size + extra + m.size/10 + if err = m.Truncate(newSize); err != nil { + return + } + m.size = newSize + } + + n = copy(m.m[m.offset:], b) + if n != len(b) { + return 0, io.ErrShortWrite + } + + m.offset += int64(len(b)) + return len(b), nil +} + +func (m *mmapWriteFile) ReadAt(buf []byte, offset int64) (int, error) { + if offset > m.offset { + return 0, fmt.Errorf("invalid offset %d", offset) + } + + n := copy(buf, m.m[offset:m.offset]) + if n != len(buf) { + return n, io.ErrUnexpectedEOF + } + + return n, nil +} + +func (m *mmapWriteFile) Truncate(size int64) error { + var err error + if err = m.m.Unmap(); err != nil { + return err + } + + if err = m.f.Truncate(size); err != nil { + return err + } + + if m.m, err = mmap.Map(m.f, mmap.RDWR, 0); err != nil { + return err + } + + m.size = size + if m.offset > m.size { + m.offset = m.size + } + return nil +} + +func (m *mmapWriteFile) SetOffset(o int64) { + m.offset = o +} + +func (m *mmapWriteFile) Offset() int64 { + return m.offset +} + +func (m *mmapWriteFile) Name() string { + return m.name +} + +type mmapReadFile struct { + readFile + + f *os.File + m mmap.MMap + name string +} + +func newMmapReadFile(name string) (readFile, error) { + m := new(mmapReadFile) + + m.name = name + + var err error + m.f, err = os.Open(name) + if err != nil { + return nil, err + } + + m.m, err = mmap.Map(m.f, mmap.RDONLY, 0) + return m, err +} + +func (m *mmapReadFile) ReadAt(buf []byte, offset int64) (int, error) { + if int64(offset) > int64(len(m.m)) { + return 0, fmt.Errorf("invalid offset %d", offset) + } + + n := copy(buf, m.m[offset:]) + if n != len(buf) { + return n, io.ErrUnexpectedEOF + } + + return n, nil +} + +func (m *mmapReadFile) Close() error { + if m.m != nil { + if err := m.m.Unmap(); err != nil { + log.Errorf("unmap %s error %s", m.name, err.Error()) + } + m.m = nil + } + + if m.f != nil { + if err := m.f.Close(); err != nil { + log.Errorf("close %s error %s", m.name, err.Error()) + } + m.f = nil + } + + return nil +} + +func (m *mmapReadFile) Size() int { + return len(m.m) +} + +func (m *mmapReadFile) Name() string { + return m.name +} + +///////////////////////////////////// + +func newWriteFile(useMmap bool, name string, size int64) (writeFile, error) { + if useMmap { + return newMmapWriteFile(name, size) + } else { + return newRawWriteFile(name, size) + } +} + +func newReadFile(useMmap bool, name string) (readFile, error) { + if useMmap { + return newMmapReadFile(name) + } else { + return newRawReadFile(name) + } +} diff --git a/vendor/github.com/siddontang/ledisdb/rpl/file_store.go b/vendor/github.com/siddontang/ledisdb/rpl/file_store.go new file mode 100644 index 000000000000..f6f708b7c60f --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/rpl/file_store.go @@ -0,0 +1,416 @@ +package rpl + +import ( + "fmt" + "io/ioutil" + "os" + "sort" + "sync" + "time" + + "github.com/siddontang/go/log" + "github.com/siddontang/go/num" + "github.com/siddontang/ledisdb/config" +) + +const ( + defaultMaxLogFileSize = int64(256 * 1024 * 1024) + + maxLogFileSize = int64(1024 * 1024 * 1024) + + defaultLogNumInFile = int64(1024 * 1024) +) + +/* + File Store: + 00000001.data + 00000001.meta + 00000002.data + 00000002.meta + + data: log1 data | log2 data | magic data + + if data has no magic data, it means that we don't close replication gracefully. + so we must repair the log data + log data: id (bigendian uint64), create time (bigendian uint32), compression (byte), data len(bigendian uint32), data + split data = log0 data + [padding 0] -> file % pagesize() == 0 + + meta: log1 offset | log2 offset + log offset: bigendian uint32 | bigendian uint32 + + //sha1 of github.com/siddontang/ledisdb 20 bytes + magic data = "\x1c\x1d\xb8\x88\xff\x9e\x45\x55\x40\xf0\x4c\xda\xe0\xce\x47\xde\x65\x48\x71\x17" + + we must guarantee that the log id is monotonic increment strictly. + if log1's id is 1, log2 must be 2 +*/ + +type FileStore struct { + LogStore + + cfg *config.Config + + base string + + rm sync.RWMutex + wm sync.Mutex + + rs tableReaders + w *tableWriter + + quit chan struct{} +} + +func NewFileStore(base string, cfg *config.Config) (*FileStore, error) { + s := new(FileStore) + + s.quit = make(chan struct{}) + + var err error + + if err = os.MkdirAll(base, 0755); err != nil { + return nil, err + } + + s.base = base + + if cfg.Replication.MaxLogFileSize == 0 { + cfg.Replication.MaxLogFileSize = defaultMaxLogFileSize + } + + cfg.Replication.MaxLogFileSize = num.MinInt64(cfg.Replication.MaxLogFileSize, maxLogFileSize) + + s.cfg = cfg + + if err = s.load(); err != nil { + return nil, err + } + + index := int64(1) + if len(s.rs) != 0 { + index = s.rs[len(s.rs)-1].index + 1 + } + + s.w = newTableWriter(s.base, index, cfg.Replication.MaxLogFileSize, cfg.Replication.UseMmap) + s.w.SetSyncType(cfg.Replication.SyncLog) + + go s.checkTableReaders() + + return s, nil +} + +func (s *FileStore) GetLog(id uint64, l *Log) error { + //first search in table writer + if err := s.w.GetLog(id, l); err == nil { + return nil + } else if err != ErrLogNotFound { + return err + } + + s.rm.RLock() + t := s.rs.Search(id) + + if t == nil { + s.rm.RUnlock() + + return ErrLogNotFound + } + + err := t.GetLog(id, l) + s.rm.RUnlock() + + return err +} + +func (s *FileStore) FirstID() (uint64, error) { + id := uint64(0) + + s.rm.RLock() + if len(s.rs) > 0 { + id = s.rs[0].first + } else { + id = 0 + } + s.rm.RUnlock() + + if id > 0 { + return id, nil + } + + //if id = 0, + + return s.w.First(), nil +} + +func (s *FileStore) LastID() (uint64, error) { + id := s.w.Last() + if id > 0 { + return id, nil + } + + //if table writer has no last id, we may find in the last table reader + + s.rm.RLock() + if len(s.rs) > 0 { + id = s.rs[len(s.rs)-1].last + } + s.rm.RUnlock() + + return id, nil +} + +func (s *FileStore) StoreLog(l *Log) error { + s.wm.Lock() + err := s.storeLog(l) + s.wm.Unlock() + return err +} + +func (s *FileStore) storeLog(l *Log) error { + err := s.w.StoreLog(l) + if err == nil { + return nil + } else if err != errTableNeedFlush { + return err + } + + var r *tableReader + r, err = s.w.Flush() + + if err != nil { + log.Fatalf("write table flush error %s, can not store!!!", err.Error()) + + s.w.Close() + + return err + } + + s.rm.Lock() + s.rs = append(s.rs, r) + s.rm.Unlock() + + err = s.w.StoreLog(l) + + return err +} + +func (s *FileStore) PurgeExpired(n int64) error { + s.rm.Lock() + + var purges []*tableReader + + t := uint32(time.Now().Unix() - int64(n)) + + for i, r := range s.rs { + if r.lastTime > t { + purges = append([]*tableReader{}, s.rs[0:i]...) + n := copy(s.rs, s.rs[i:]) + s.rs = s.rs[0:n] + break + } + } + + s.rm.Unlock() + + s.purgeTableReaders(purges) + + return nil +} + +func (s *FileStore) Sync() error { + return s.w.Sync() +} + +func (s *FileStore) Clear() error { + s.wm.Lock() + s.rm.Lock() + + defer func() { + s.rm.Unlock() + s.wm.Unlock() + }() + + s.w.Close() + + for i := range s.rs { + s.rs[i].Close() + } + + s.rs = tableReaders{} + + if err := os.RemoveAll(s.base); err != nil { + return err + } + + if err := os.MkdirAll(s.base, 0755); err != nil { + return err + } + + s.w = newTableWriter(s.base, 1, s.cfg.Replication.MaxLogFileSize, s.cfg.Replication.UseMmap) + + return nil +} + +func (s *FileStore) Close() error { + close(s.quit) + + s.wm.Lock() + s.rm.Lock() + + if r, err := s.w.Flush(); err != nil { + if err != errNilHandler { + log.Errorf("close err: %s", err.Error()) + } + } else { + r.Close() + s.w.Close() + } + + for i := range s.rs { + s.rs[i].Close() + } + + s.rs = tableReaders{} + + s.rm.Unlock() + s.wm.Unlock() + + return nil +} + +func (s *FileStore) checkTableReaders() { + t := time.NewTicker(60 * time.Second) + defer t.Stop() + for { + select { + case <-t.C: + s.rm.Lock() + + for _, r := range s.rs { + if !r.Keepalived() { + r.Close() + } + } + + purges := []*tableReader{} + maxNum := s.cfg.Replication.MaxLogFileNum + num := len(s.rs) + if num > maxNum { + purges = s.rs[:num-maxNum] + s.rs = s.rs[num-maxNum:] + } + + s.rm.Unlock() + + s.purgeTableReaders(purges) + + case <-s.quit: + return + } + } +} + +func (s *FileStore) purgeTableReaders(purges []*tableReader) { + for _, r := range purges { + dataName := fmtTableDataName(r.base, r.index) + metaName := fmtTableMetaName(r.base, r.index) + r.Close() + if err := os.Remove(dataName); err != nil { + log.Errorf("purge table data %s err: %s", dataName, err.Error()) + } + if err := os.Remove(metaName); err != nil { + log.Errorf("purge table meta %s err: %s", metaName, err.Error()) + } + + } +} + +func (s *FileStore) load() error { + fs, err := ioutil.ReadDir(s.base) + if err != nil { + return err + } + + s.rs = make(tableReaders, 0, len(fs)) + + var r *tableReader + var index int64 + for _, f := range fs { + if _, err := fmt.Sscanf(f.Name(), "%08d.data", &index); err == nil { + if r, err = newTableReader(s.base, index, s.cfg.Replication.UseMmap); err != nil { + log.Errorf("load table %s err: %s", f.Name(), err.Error()) + } else { + s.rs = append(s.rs, r) + } + } + } + + if err := s.rs.check(); err != nil { + return err + } + + return nil +} + +type tableReaders []*tableReader + +func (ts tableReaders) Len() int { + return len(ts) +} + +func (ts tableReaders) Swap(i, j int) { + ts[i], ts[j] = ts[j], ts[i] +} + +func (ts tableReaders) Less(i, j int) bool { + return ts[i].first < ts[j].first +} + +func (ts tableReaders) Search(id uint64) *tableReader { + i, j := 0, len(ts)-1 + + for i <= j { + h := i + (j-i)/2 + + if ts[h].first <= id && id <= ts[h].last { + return ts[h] + } else if ts[h].last < id { + i = h + 1 + } else { + j = h - 1 + } + } + + return nil +} + +func (ts tableReaders) check() error { + if len(ts) == 0 { + return nil + } + + sort.Sort(ts) + + first := ts[0].first + last := ts[0].last + index := ts[0].index + + if first == 0 || first > last { + return fmt.Errorf("invalid log in table %s", ts[0]) + } + + for i := 1; i < len(ts); i++ { + if ts[i].first <= last { + return fmt.Errorf("invalid first log id %d in table %s", ts[i].first, ts[i]) + } + + if ts[i].index <= index { + return fmt.Errorf("invalid index %d in table %s", ts[i].index, ts[i]) + } + + first = ts[i].first + last = ts[i].last + index = ts[i].index + } + return nil +} diff --git a/vendor/github.com/siddontang/ledisdb/rpl/file_table.go b/vendor/github.com/siddontang/ledisdb/rpl/file_table.go new file mode 100644 index 000000000000..9658102ff65c --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/rpl/file_table.go @@ -0,0 +1,571 @@ +package rpl + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "path" + "sync" + "time" + + "github.com/siddontang/go/log" + "github.com/siddontang/go/sync2" +) + +var ( + magic = []byte("\x1c\x1d\xb8\x88\xff\x9e\x45\x55\x40\xf0\x4c\xda\xe0\xce\x47\xde\x65\x48\x71\x17") + errTableNeedFlush = errors.New("write table need flush") + errNilHandler = errors.New("nil write handler") +) + +const tableReaderKeepaliveInterval int64 = 30 + +func fmtTableDataName(base string, index int64) string { + return path.Join(base, fmt.Sprintf("%08d.data", index)) +} + +func fmtTableMetaName(base string, index int64) string { + return path.Join(base, fmt.Sprintf("%08d.meta", index)) +} + +type tableReader struct { + sync.Mutex + + base string + index int64 + + data readFile + meta readFile + + first uint64 + last uint64 + + lastTime uint32 + + lastReadTime sync2.AtomicInt64 + + useMmap bool +} + +func newTableReader(base string, index int64, useMmap bool) (*tableReader, error) { + if index <= 0 { + return nil, fmt.Errorf("invalid index %d", index) + } + t := new(tableReader) + t.base = base + t.index = index + + t.useMmap = useMmap + + var err error + + if err = t.check(); err != nil { + log.Errorf("check %d error: %s, try to repair", t.index, err.Error()) + + if err = t.repair(); err != nil { + log.Errorf("repair %d error: %s", t.index, err.Error()) + return nil, err + } + } + + t.close() + + return t, nil +} + +func (t *tableReader) String() string { + return fmt.Sprintf("%d", t.index) +} + +func (t *tableReader) Close() { + t.Lock() + + t.close() + + t.Unlock() +} + +func (t *tableReader) close() { + if t.data != nil { + t.data.Close() + t.data = nil + } + + if t.meta != nil { + t.meta.Close() + t.meta = nil + } +} + +func (t *tableReader) Keepalived() bool { + l := t.lastReadTime.Get() + if l > 0 && time.Now().Unix()-l > tableReaderKeepaliveInterval { + return false + } + + return true +} + +func (t *tableReader) getLogPos(index int) (uint32, error) { + var buf [4]byte + if _, err := t.meta.ReadAt(buf[0:4], int64(index)*4); err != nil { + return 0, err + } + + return binary.BigEndian.Uint32(buf[0:4]), nil +} + +func (t *tableReader) checkData() error { + var err error + //check will use raw file mode + if t.data, err = newReadFile(false, fmtTableDataName(t.base, t.index)); err != nil { + return err + } + + if t.data.Size() < len(magic) { + return fmt.Errorf("data file %s size %d too short", t.data.Name(), t.data.Size()) + } + + buf := make([]byte, len(magic)) + if _, err := t.data.ReadAt(buf, int64(t.data.Size()-len(magic))); err != nil { + return err + } + + if !bytes.Equal(magic, buf) { + return fmt.Errorf("data file %s invalid magic data %q", t.data.Name(), buf) + } + + return nil +} + +func (t *tableReader) checkMeta() error { + var err error + //check will use raw file mode + if t.meta, err = newReadFile(false, fmtTableMetaName(t.base, t.index)); err != nil { + return err + } + + if t.meta.Size()%4 != 0 || t.meta.Size() == 0 { + return fmt.Errorf("meta file %s invalid offset len %d, must 4 multiple and not 0", t.meta.Name(), t.meta.Size()) + } + + return nil +} + +func (t *tableReader) check() error { + var err error + + if err := t.checkMeta(); err != nil { + return err + } + + if err := t.checkData(); err != nil { + return err + } + + firstLogPos, _ := t.getLogPos(0) + lastLogPos, _ := t.getLogPos(t.meta.Size()/4 - 1) + + if firstLogPos != 0 { + return fmt.Errorf("invalid first log pos %d, must 0", firstLogPos) + } + + var l Log + if _, err = t.decodeLogHead(&l, t.data, int64(firstLogPos)); err != nil { + return fmt.Errorf("decode first log err %s", err.Error()) + } + + t.first = l.ID + var n int64 + if n, err = t.decodeLogHead(&l, t.data, int64(lastLogPos)); err != nil { + return fmt.Errorf("decode last log err %s", err.Error()) + } else if n+int64(len(magic)) != int64(t.data.Size()) { + return fmt.Errorf("extra log data at offset %d", n) + } + + t.last = l.ID + t.lastTime = l.CreateTime + + if t.first > t.last { + return fmt.Errorf("invalid log table first %d > last %d", t.first, t.last) + } else if (t.last - t.first + 1) != uint64(t.meta.Size()/4) { + return fmt.Errorf("invalid log table, first %d, last %d, and log num %d", t.first, t.last, t.meta.Size()/4) + } + + return nil +} + +func (t *tableReader) repair() error { + t.close() + + var err error + var data writeFile + var meta writeFile + + //repair will use raw file mode + data, err = newWriteFile(false, fmtTableDataName(t.base, t.index), 0) + data.SetOffset(int64(data.Size())) + + meta, err = newWriteFile(false, fmtTableMetaName(t.base, t.index), int64(defaultLogNumInFile*4)) + + var l Log + var pos int64 = 0 + var nextPos int64 = 0 + b := make([]byte, 4) + + t.first = 0 + t.last = 0 + + for { + nextPos, err = t.decodeLogHead(&l, data, pos) + if err != nil { + //if error, we may lost all logs from pos + log.Errorf("%s may lost logs from %d", data.Name(), pos) + break + } + + if l.ID == 0 { + log.Errorf("%s may lost logs from %d, invalid log 0", data.Name(), pos) + break + } + + if t.first == 0 { + t.first = l.ID + } + + if t.last == 0 { + t.last = l.ID + } else if l.ID <= t.last { + log.Errorf("%s may lost logs from %d, invalid logid %d", t.data.Name(), pos, l.ID) + break + } + + t.last = l.ID + t.lastTime = l.CreateTime + + binary.BigEndian.PutUint32(b, uint32(pos)) + meta.Write(b) + + pos = nextPos + + t.lastTime = l.CreateTime + } + + var e error + if err := meta.Close(); err != nil { + e = err + } + + data.SetOffset(pos) + + if _, err = data.Write(magic); err != nil { + log.Errorf("write magic error %s", err.Error()) + } + + if err = data.Close(); err != nil { + return err + } + + return e +} + +func (t *tableReader) decodeLogHead(l *Log, r io.ReaderAt, pos int64) (int64, error) { + dataLen, err := l.DecodeHeadAt(r, pos) + if err != nil { + return 0, err + } + + return pos + int64(l.HeadSize()) + int64(dataLen), nil +} + +func (t *tableReader) GetLog(id uint64, l *Log) error { + if id < t.first || id > t.last { + return ErrLogNotFound + } + + t.lastReadTime.Set(time.Now().Unix()) + + t.Lock() + + if err := t.openTable(); err != nil { + t.close() + t.Unlock() + return err + } + t.Unlock() + + pos, err := t.getLogPos(int(id - t.first)) + if err != nil { + return err + } + + if err := l.DecodeAt(t.data, int64(pos)); err != nil { + return err + } else if l.ID != id { + return fmt.Errorf("invalid log id %d != %d", l.ID, id) + } + + return nil +} + +func (t *tableReader) openTable() error { + var err error + if t.data == nil { + if t.data, err = newReadFile(t.useMmap, fmtTableDataName(t.base, t.index)); err != nil { + return err + } + } + + if t.meta == nil { + if t.meta, err = newReadFile(t.useMmap, fmtTableMetaName(t.base, t.index)); err != nil { + return err + } + + } + + return nil +} + +type tableWriter struct { + sync.RWMutex + + data writeFile + meta writeFile + + base string + index int64 + + first uint64 + last uint64 + lastTime uint32 + + maxLogSize int64 + + closed bool + + syncType int + + posBuf []byte + + useMmap bool +} + +func newTableWriter(base string, index int64, maxLogSize int64, useMmap bool) *tableWriter { + if index <= 0 { + panic(fmt.Errorf("invalid index %d", index)) + } + + t := new(tableWriter) + + t.base = base + t.index = index + + t.maxLogSize = maxLogSize + + t.closed = false + + t.posBuf = make([]byte, 4) + + t.useMmap = useMmap + + return t +} + +func (t *tableWriter) String() string { + return fmt.Sprintf("%d", t.index) +} + +func (t *tableWriter) SetMaxLogSize(s int64) { + t.maxLogSize = s +} + +func (t *tableWriter) SetSyncType(tp int) { + t.syncType = tp +} + +func (t *tableWriter) close() { + if t.meta != nil { + if err := t.meta.Close(); err != nil { + log.Fatalf("close log meta error %s", err.Error()) + } + t.meta = nil + } + + if t.data != nil { + if _, err := t.data.Write(magic); err != nil { + log.Fatalf("write magic error %s", err.Error()) + } + + if err := t.data.Close(); err != nil { + log.Fatalf("close log data error %s", err.Error()) + } + t.data = nil + } +} + +func (t *tableWriter) Close() { + t.Lock() + t.closed = true + + t.close() + t.Unlock() +} + +func (t *tableWriter) First() uint64 { + t.Lock() + id := t.first + t.Unlock() + return id +} + +func (t *tableWriter) Last() uint64 { + t.Lock() + id := t.last + t.Unlock() + return id +} + +func (t *tableWriter) Flush() (*tableReader, error) { + t.Lock() + + if t.data == nil || t.meta == nil { + t.Unlock() + return nil, errNilHandler + } + + tr := new(tableReader) + tr.base = t.base + tr.index = t.index + + tr.first = t.first + tr.last = t.last + tr.lastTime = t.lastTime + tr.useMmap = t.useMmap + + t.close() + + t.first = 0 + t.last = 0 + t.index = t.index + 1 + + t.Unlock() + + return tr, nil +} + +func (t *tableWriter) StoreLog(l *Log) error { + t.Lock() + err := t.storeLog(l) + t.Unlock() + + return err +} + +func (t *tableWriter) openFile() error { + var err error + if t.data == nil { + if t.data, err = newWriteFile(t.useMmap, fmtTableDataName(t.base, t.index), t.maxLogSize+t.maxLogSize/10+int64(len(magic))); err != nil { + return err + } + } + + if t.meta == nil { + if t.meta, err = newWriteFile(t.useMmap, fmtTableMetaName(t.base, t.index), int64(defaultLogNumInFile*4)); err != nil { + return err + } + } + return err +} + +func (t *tableWriter) storeLog(l *Log) error { + if l.ID == 0 { + return ErrStoreLogID + } + + if t.closed { + return fmt.Errorf("table writer is closed") + } + + if t.last > 0 && l.ID != t.last+1 { + return ErrStoreLogID + } + + if t.data != nil && t.data.Offset() > t.maxLogSize { + return errTableNeedFlush + } + + var err error + if err = t.openFile(); err != nil { + return err + } + + offsetPos := t.data.Offset() + if err = l.Encode(t.data); err != nil { + return err + } + + binary.BigEndian.PutUint32(t.posBuf, uint32(offsetPos)) + if _, err = t.meta.Write(t.posBuf); err != nil { + return err + } + + if t.first == 0 { + t.first = l.ID + } + + t.last = l.ID + t.lastTime = l.CreateTime + + if t.syncType == 2 { + if err := t.data.Sync(); err != nil { + log.Errorf("sync table error %s", err.Error()) + } + } + + return nil +} + +func (t *tableWriter) GetLog(id uint64, l *Log) error { + t.RLock() + defer t.RUnlock() + + if id < t.first || id > t.last { + return ErrLogNotFound + } + + var buf [4]byte + if _, err := t.meta.ReadAt(buf[0:4], int64((id-t.first)*4)); err != nil { + return err + } + + offset := binary.BigEndian.Uint32(buf[0:4]) + + if err := l.DecodeAt(t.data, int64(offset)); err != nil { + return err + } else if l.ID != id { + return fmt.Errorf("invalid log id %d != %d", id, l.ID) + } + + return nil +} + +func (t *tableWriter) Sync() error { + t.Lock() + + var err error + if t.data != nil { + err = t.data.Sync() + t.Unlock() + return err + } + + if t.meta != nil { + err = t.meta.Sync() + } + + t.Unlock() + + return err +} diff --git a/vendor/github.com/siddontang/ledisdb/rpl/goleveldb_store.go b/vendor/github.com/siddontang/ledisdb/rpl/goleveldb_store.go new file mode 100644 index 000000000000..445c17cfcf68 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/rpl/goleveldb_store.go @@ -0,0 +1,225 @@ +package rpl + +import ( + "bytes" + "fmt" + "os" + "sync" + "time" + + "github.com/siddontang/go/num" + "github.com/siddontang/ledisdb/config" + "github.com/siddontang/ledisdb/store" +) + +type GoLevelDBStore struct { + LogStore + + m sync.Mutex + db *store.DB + + cfg *config.Config + + first uint64 + last uint64 + + buf bytes.Buffer +} + +func (s *GoLevelDBStore) FirstID() (uint64, error) { + s.m.Lock() + id, err := s.firstID() + s.m.Unlock() + + return id, err +} + +func (s *GoLevelDBStore) LastID() (uint64, error) { + s.m.Lock() + id, err := s.lastID() + s.m.Unlock() + + return id, err +} + +func (s *GoLevelDBStore) firstID() (uint64, error) { + if s.first != InvalidLogID { + return s.first, nil + } + + it := s.db.NewIterator() + defer it.Close() + + it.SeekToFirst() + + if it.Valid() { + s.first = num.BytesToUint64(it.RawKey()) + } + + return s.first, nil +} + +func (s *GoLevelDBStore) lastID() (uint64, error) { + if s.last != InvalidLogID { + return s.last, nil + } + + it := s.db.NewIterator() + defer it.Close() + + it.SeekToLast() + + if it.Valid() { + s.last = num.BytesToUint64(it.RawKey()) + } + + return s.last, nil +} + +func (s *GoLevelDBStore) GetLog(id uint64, log *Log) error { + v, err := s.db.Get(num.Uint64ToBytes(id)) + if err != nil { + return err + } else if v == nil { + return ErrLogNotFound + } else { + return log.Decode(bytes.NewBuffer(v)) + } +} + +func (s *GoLevelDBStore) StoreLog(log *Log) error { + s.m.Lock() + defer s.m.Unlock() + + last, err := s.lastID() + if err != nil { + return err + } + + s.last = InvalidLogID + + s.buf.Reset() + + if log.ID != last+1 { + return ErrStoreLogID + } + + last = log.ID + key := num.Uint64ToBytes(log.ID) + + if err := log.Encode(&s.buf); err != nil { + return err + } + + if err = s.db.Put(key, s.buf.Bytes()); err != nil { + return err + } + + s.last = last + return nil +} + +func (s *GoLevelDBStore) PurgeExpired(n int64) error { + if n <= 0 { + return fmt.Errorf("invalid expired time %d", n) + } + + t := uint32(time.Now().Unix() - int64(n)) + + s.m.Lock() + defer s.m.Unlock() + + s.reset() + + it := s.db.NewIterator() + it.SeekToFirst() + + w := s.db.NewWriteBatch() + defer w.Rollback() + + l := new(Log) + for ; it.Valid(); it.Next() { + v := it.RawValue() + + if err := l.Unmarshal(v); err != nil { + return err + } else if l.CreateTime > t { + break + } else { + w.Delete(it.RawKey()) + } + } + + if err := w.Commit(); err != nil { + return err + } + + return nil +} + +func (s *GoLevelDBStore) Sync() error { + //no other way for sync, so ignore here + return nil +} + +func (s *GoLevelDBStore) reset() { + s.first = InvalidLogID + s.last = InvalidLogID +} + +func (s *GoLevelDBStore) Clear() error { + s.m.Lock() + defer s.m.Unlock() + + if s.db != nil { + s.db.Close() + } + + s.reset() + os.RemoveAll(s.cfg.DBPath) + + return s.open() +} + +func (s *GoLevelDBStore) Close() error { + s.m.Lock() + defer s.m.Unlock() + + if s.db == nil { + return nil + } + + err := s.db.Close() + s.db = nil + return err +} + +func (s *GoLevelDBStore) open() error { + var err error + + s.first = InvalidLogID + s.last = InvalidLogID + + s.db, err = store.Open(s.cfg) + return err +} + +func NewGoLevelDBStore(base string, syncLog int) (*GoLevelDBStore, error) { + cfg := config.NewConfigDefault() + cfg.DBName = "goleveldb" + cfg.DBPath = base + cfg.LevelDB.BlockSize = 16 * 1024 * 1024 + cfg.LevelDB.CacheSize = 64 * 1024 * 1024 + cfg.LevelDB.WriteBufferSize = 64 * 1024 * 1024 + cfg.LevelDB.Compression = false + cfg.DBSyncCommit = syncLog + + s := new(GoLevelDBStore) + s.cfg = cfg + + if err := s.open(); err != nil { + return nil, err + } + + return s, nil +} diff --git a/vendor/github.com/siddontang/ledisdb/rpl/log.go b/vendor/github.com/siddontang/ledisdb/rpl/log.go new file mode 100644 index 000000000000..ad0b48cd4f79 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/rpl/log.go @@ -0,0 +1,167 @@ +package rpl + +import ( + "bytes" + "encoding/binary" + "io" + "sync" +) + +const LogHeadSize = 17 + +type Log struct { + ID uint64 + CreateTime uint32 + Compression uint8 + + Data []byte +} + +func (l *Log) HeadSize() int { + return LogHeadSize +} + +func (l *Log) Size() int { + return l.HeadSize() + len(l.Data) +} + +func (l *Log) Marshal() ([]byte, error) { + buf := bytes.NewBuffer(make([]byte, l.Size())) + buf.Reset() + + if err := l.Encode(buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (l *Log) Unmarshal(b []byte) error { + buf := bytes.NewBuffer(b) + + return l.Decode(buf) +} + +var headPool = sync.Pool{ + New: func() interface{} { return make([]byte, LogHeadSize) }, +} + +func (l *Log) Encode(w io.Writer) error { + b := headPool.Get().([]byte) + pos := 0 + + binary.BigEndian.PutUint64(b[pos:], l.ID) + pos += 8 + binary.BigEndian.PutUint32(b[pos:], uint32(l.CreateTime)) + pos += 4 + b[pos] = l.Compression + pos++ + binary.BigEndian.PutUint32(b[pos:], uint32(len(l.Data))) + + n, err := w.Write(b) + headPool.Put(b) + + if err != nil { + return err + } else if n != LogHeadSize { + return io.ErrShortWrite + } + + if n, err = w.Write(l.Data); err != nil { + return err + } else if n != len(l.Data) { + return io.ErrShortWrite + } + return nil +} + +func (l *Log) Decode(r io.Reader) error { + length, err := l.DecodeHead(r) + if err != nil { + return err + } + + l.growData(int(length)) + + if _, err := io.ReadFull(r, l.Data); err != nil { + return err + } + + return nil +} + +func (l *Log) DecodeHead(r io.Reader) (uint32, error) { + buf := headPool.Get().([]byte) + + if _, err := io.ReadFull(r, buf); err != nil { + headPool.Put(buf) + return 0, err + } + + length := l.decodeHeadBuf(buf) + + headPool.Put(buf) + + return length, nil +} + +func (l *Log) DecodeAt(r io.ReaderAt, pos int64) error { + length, err := l.DecodeHeadAt(r, pos) + if err != nil { + return err + } + + l.growData(int(length)) + var n int + n, err = r.ReadAt(l.Data, pos+int64(LogHeadSize)) + if err == io.EOF && n == len(l.Data) { + err = nil + } + + return err +} + +func (l *Log) growData(length int) { + l.Data = l.Data[0:0] + + if cap(l.Data) >= length { + l.Data = l.Data[0:length] + } else { + l.Data = make([]byte, length) + } +} + +func (l *Log) DecodeHeadAt(r io.ReaderAt, pos int64) (uint32, error) { + buf := headPool.Get().([]byte) + + n, err := r.ReadAt(buf, pos) + if err != nil && err != io.EOF { + headPool.Put(buf) + + return 0, err + } + + length := l.decodeHeadBuf(buf) + headPool.Put(buf) + + if err == io.EOF && (length != 0 || n != len(buf)) { + return 0, err + } + + return length, nil +} + +func (l *Log) decodeHeadBuf(buf []byte) uint32 { + pos := 0 + l.ID = binary.BigEndian.Uint64(buf[pos:]) + pos += 8 + + l.CreateTime = binary.BigEndian.Uint32(buf[pos:]) + pos += 4 + + l.Compression = uint8(buf[pos]) + pos++ + + length := binary.BigEndian.Uint32(buf[pos:]) + return length +} diff --git a/vendor/github.com/siddontang/ledisdb/rpl/rpl.go b/vendor/github.com/siddontang/ledisdb/rpl/rpl.go new file mode 100644 index 000000000000..0ebf66de413c --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/rpl/rpl.go @@ -0,0 +1,336 @@ +package rpl + +import ( + "encoding/binary" + "os" + "path" + "sync" + "time" + + "github.com/siddontang/go/log" + "github.com/siddontang/go/snappy" + "github.com/siddontang/ledisdb/config" +) + +type Stat struct { + FirstID uint64 + LastID uint64 + CommitID uint64 +} + +type Replication struct { + m sync.Mutex + + cfg *config.Config + + s LogStore + + commitID uint64 + commitLog *os.File + + quit chan struct{} + + wg sync.WaitGroup + + nc chan struct{} + + ncm sync.Mutex +} + +func NewReplication(cfg *config.Config) (*Replication, error) { + if len(cfg.Replication.Path) == 0 { + cfg.Replication.Path = path.Join(cfg.DataDir, "rpl") + } + + base := cfg.Replication.Path + + r := new(Replication) + + r.quit = make(chan struct{}) + r.nc = make(chan struct{}) + + r.cfg = cfg + + var err error + + switch cfg.Replication.StoreName { + case "goleveldb": + if r.s, err = NewGoLevelDBStore(path.Join(base, "wal"), cfg.Replication.SyncLog); err != nil { + return nil, err + } + default: + if r.s, err = NewFileStore(path.Join(base, "ldb"), cfg); err != nil { + return nil, err + } + } + + if r.commitLog, err = os.OpenFile(path.Join(base, "commit.log"), os.O_RDWR|os.O_CREATE, 0644); err != nil { + return nil, err + } + + if s, _ := r.commitLog.Stat(); s.Size() == 0 { + r.commitID = 0 + } else if err = binary.Read(r.commitLog, binary.BigEndian, &r.commitID); err != nil { + return nil, err + } + + log.Infof("staring replication with commit ID %d", r.commitID) + + r.wg.Add(1) + go r.run() + + return r, nil +} + +func (r *Replication) Close() error { + close(r.quit) + + r.wg.Wait() + + r.m.Lock() + defer r.m.Unlock() + + log.Infof("closing replication with commit ID %d", r.commitID) + + if r.s != nil { + r.s.Close() + r.s = nil + } + + if err := r.updateCommitID(r.commitID, true); err != nil { + log.Errorf("update commit id err %s", err.Error()) + } + + if r.commitLog != nil { + r.commitLog.Close() + r.commitLog = nil + } + + return nil +} + +func (r *Replication) Log(data []byte) (*Log, error) { + if r.cfg.Replication.Compression { + //todo optimize + var err error + if data, err = snappy.Encode(nil, data); err != nil { + return nil, err + } + } + + r.m.Lock() + + lastID, err := r.s.LastID() + if err != nil { + r.m.Unlock() + return nil, err + } + + commitId := r.commitID + if lastID < commitId { + lastID = commitId + } else if lastID > commitId { + r.m.Unlock() + return nil, ErrCommitIDBehind + } + + l := new(Log) + l.ID = lastID + 1 + l.CreateTime = uint32(time.Now().Unix()) + + if r.cfg.Replication.Compression { + l.Compression = 1 + } else { + l.Compression = 0 + } + + l.Data = data + + if err = r.s.StoreLog(l); err != nil { + r.m.Unlock() + return nil, err + } + + r.m.Unlock() + + r.ncm.Lock() + close(r.nc) + r.nc = make(chan struct{}) + r.ncm.Unlock() + + return l, nil +} + +func (r *Replication) WaitLog() <-chan struct{} { + r.ncm.Lock() + ch := r.nc + r.ncm.Unlock() + return ch +} + +func (r *Replication) StoreLog(log *Log) error { + r.m.Lock() + err := r.s.StoreLog(log) + r.m.Unlock() + + return err +} + +func (r *Replication) FirstLogID() (uint64, error) { + r.m.Lock() + id, err := r.s.FirstID() + r.m.Unlock() + + return id, err +} + +func (r *Replication) LastLogID() (uint64, error) { + r.m.Lock() + id, err := r.s.LastID() + r.m.Unlock() + return id, err +} + +func (r *Replication) LastCommitID() (uint64, error) { + r.m.Lock() + id := r.commitID + r.m.Unlock() + return id, nil +} + +func (r *Replication) UpdateCommitID(id uint64) error { + r.m.Lock() + err := r.updateCommitID(id, r.cfg.Replication.SyncLog == 2) + r.m.Unlock() + + return err +} + +func (r *Replication) Stat() (*Stat, error) { + r.m.Lock() + defer r.m.Unlock() + + s := &Stat{} + var err error + + if s.FirstID, err = r.s.FirstID(); err != nil { + return nil, err + } + + if s.LastID, err = r.s.LastID(); err != nil { + return nil, err + } + + s.CommitID = r.commitID + return s, nil +} + +func (r *Replication) updateCommitID(id uint64, force bool) error { + if force { + if _, err := r.commitLog.Seek(0, os.SEEK_SET); err != nil { + return err + } + + if err := binary.Write(r.commitLog, binary.BigEndian, id); err != nil { + return err + } + } + + r.commitID = id + + return nil +} + +func (r *Replication) CommitIDBehind() (bool, error) { + r.m.Lock() + + id, err := r.s.LastID() + if err != nil { + r.m.Unlock() + return false, err + } + + behind := id > r.commitID + r.m.Unlock() + + return behind, nil +} + +func (r *Replication) GetLog(id uint64, log *Log) error { + return r.s.GetLog(id, log) +} + +func (r *Replication) NextNeedCommitLog(log *Log) error { + r.m.Lock() + defer r.m.Unlock() + + id, err := r.s.LastID() + if err != nil { + return err + } + + if id <= r.commitID { + return ErrNoBehindLog + } + + return r.s.GetLog(r.commitID+1, log) + +} + +func (r *Replication) Clear() error { + return r.ClearWithCommitID(0) +} + +func (r *Replication) ClearWithCommitID(id uint64) error { + r.m.Lock() + defer r.m.Unlock() + + if err := r.s.Clear(); err != nil { + return err + } + + return r.updateCommitID(id, true) +} + +func (r *Replication) run() { + defer r.wg.Done() + + syncTc := time.NewTicker(1 * time.Second) + purgeTc := time.NewTicker(1 * time.Hour) + + for { + select { + case <-purgeTc.C: + n := (r.cfg.Replication.ExpiredLogDays * 24 * 3600) + r.m.Lock() + err := r.s.PurgeExpired(int64(n)) + r.m.Unlock() + if err != nil { + log.Errorf("purge expired log error %s", err.Error()) + } + case <-syncTc.C: + if r.cfg.Replication.SyncLog == 1 { + r.m.Lock() + err := r.s.Sync() + r.m.Unlock() + if err != nil { + log.Errorf("sync store error %s", err.Error()) + } + } + if r.cfg.Replication.SyncLog != 2 { + //we will sync commit id every 1 second + r.m.Lock() + err := r.updateCommitID(r.commitID, true) + r.m.Unlock() + + if err != nil { + log.Errorf("sync commitid error %s", err.Error()) + } + } + case <-r.quit: + syncTc.Stop() + purgeTc.Stop() + return + } + } +} diff --git a/vendor/github.com/siddontang/ledisdb/rpl/store.go b/vendor/github.com/siddontang/ledisdb/rpl/store.go new file mode 100644 index 000000000000..9f985ec6be0f --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/rpl/store.go @@ -0,0 +1,36 @@ +package rpl + +import ( + "errors" +) + +const ( + InvalidLogID uint64 = 0 +) + +var ( + ErrLogNotFound = errors.New("log not found") + ErrStoreLogID = errors.New("log id is less") + ErrNoBehindLog = errors.New("no behind commit log") + ErrCommitIDBehind = errors.New("commit id is behind last log id") +) + +type LogStore interface { + GetLog(id uint64, log *Log) error + + FirstID() (uint64, error) + LastID() (uint64, error) + + // if log id is less than current last id, return error + StoreLog(log *Log) error + + // Delete logs before n seconds + PurgeExpired(n int64) error + + Sync() error + + // Clear all logs + Clear() error + + Close() error +} diff --git a/vendor/github.com/siddontang/ledisdb/store/db.go b/vendor/github.com/siddontang/ledisdb/store/db.go new file mode 100644 index 000000000000..bfc0418298b8 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/db.go @@ -0,0 +1,169 @@ +package store + +import ( + "sync" + "time" + + "github.com/siddontang/ledisdb/config" + "github.com/siddontang/ledisdb/store/driver" +) + +type DB struct { + db driver.IDB + name string + + st *Stat + + cfg *config.Config + + lastCommit time.Time + + m sync.Mutex +} + +func (db *DB) Close() error { + return db.db.Close() +} + +func (db *DB) String() string { + return db.name +} + +func (db *DB) NewIterator() *Iterator { + db.st.IterNum.Add(1) + + it := new(Iterator) + it.it = db.db.NewIterator() + it.st = db.st + + return it +} + +func (db *DB) Get(key []byte) ([]byte, error) { + t := time.Now() + v, err := db.db.Get(key) + db.st.statGet(v, err) + db.st.GetTotalTime.Add(time.Now().Sub(t)) + return v, err +} + +func (db *DB) Put(key []byte, value []byte) error { + db.st.PutNum.Add(1) + + if db.needSyncCommit() { + return db.db.SyncPut(key, value) + + } else { + return db.db.Put(key, value) + + } +} + +func (db *DB) Delete(key []byte) error { + db.st.DeleteNum.Add(1) + + if db.needSyncCommit() { + return db.db.SyncDelete(key) + } else { + return db.db.Delete(key) + } +} + +func (db *DB) NewWriteBatch() *WriteBatch { + db.st.BatchNum.Add(1) + wb := new(WriteBatch) + wb.wb = db.db.NewWriteBatch() + wb.st = db.st + wb.db = db + return wb +} + +func (db *DB) NewSnapshot() (*Snapshot, error) { + db.st.SnapshotNum.Add(1) + + var err error + s := &Snapshot{} + if s.ISnapshot, err = db.db.NewSnapshot(); err != nil { + return nil, err + } + s.st = db.st + + return s, nil +} + +func (db *DB) Compact() error { + db.st.CompactNum.Add(1) + + t := time.Now() + err := db.db.Compact() + + db.st.CompactTotalTime.Add(time.Now().Sub(t)) + + return err +} + +func (db *DB) RangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator { + return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1}) +} + +func (db *DB) RevRangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator { + return NewRevRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1}) +} + +//count < 0, unlimit. +// +//offset must >= 0, if < 0, will get nothing. +func (db *DB) RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator { + return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count}) +} + +//count < 0, unlimit. +// +//offset must >= 0, if < 0, will get nothing. +func (db *DB) RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator { + return NewRevRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count}) +} + +func (db *DB) Stat() *Stat { + return db.st +} + +func (db *DB) needSyncCommit() bool { + if db.cfg.DBSyncCommit == 0 { + return false + } else if db.cfg.DBSyncCommit == 2 { + return true + } else { + n := time.Now() + need := false + db.m.Lock() + + if n.Sub(db.lastCommit) > time.Second { + need = true + } + db.lastCommit = n + + db.m.Unlock() + return need + } + +} + +func (db *DB) GetSlice(key []byte) (Slice, error) { + if d, ok := db.db.(driver.ISliceGeter); ok { + t := time.Now() + v, err := d.GetSlice(key) + db.st.statGet(v, err) + db.st.GetTotalTime.Add(time.Now().Sub(t)) + return v, err + } else { + v, err := db.Get(key) + if err != nil { + return nil, err + } else if v == nil { + return nil, nil + } else { + return driver.GoSlice(v), nil + } + } +} diff --git a/vendor/github.com/siddontang/ledisdb/store/driver/driver.go b/vendor/github.com/siddontang/ledisdb/store/driver/driver.go new file mode 100644 index 000000000000..afa549cd258d --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/driver/driver.go @@ -0,0 +1,57 @@ +package driver + +type IDB interface { + Close() error + + Get(key []byte) ([]byte, error) + + Put(key []byte, value []byte) error + Delete(key []byte) error + + SyncPut(key []byte, value []byte) error + SyncDelete(key []byte) error + + NewIterator() IIterator + + NewWriteBatch() IWriteBatch + + NewSnapshot() (ISnapshot, error) + + Compact() error +} + +type ISnapshot interface { + Get(key []byte) ([]byte, error) + NewIterator() IIterator + Close() +} + +type IIterator interface { + Close() error + + First() + Last() + Seek(key []byte) + + Next() + Prev() + + Valid() bool + + Key() []byte + Value() []byte +} + +type IWriteBatch interface { + Put(key []byte, value []byte) + Delete(key []byte) + Commit() error + SyncCommit() error + Rollback() error + Data() []byte + Close() +} + +type ISliceGeter interface { + GetSlice(key []byte) (ISlice, error) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/driver/slice.go b/vendor/github.com/siddontang/ledisdb/store/driver/slice.go new file mode 100644 index 000000000000..d0c80e0b8fcf --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/driver/slice.go @@ -0,0 +1,21 @@ +package driver + +type ISlice interface { + Data() []byte + Size() int + Free() +} + +type GoSlice []byte + +func (s GoSlice) Data() []byte { + return []byte(s) +} + +func (s GoSlice) Size() int { + return len(s) +} + +func (s GoSlice) Free() { + +} diff --git a/vendor/github.com/siddontang/ledisdb/store/driver/store.go b/vendor/github.com/siddontang/ledisdb/store/driver/store.go new file mode 100644 index 000000000000..fbaebfc98b05 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/driver/store.go @@ -0,0 +1,46 @@ +package driver + +import ( + "fmt" + + "github.com/siddontang/ledisdb/config" +) + +type Store interface { + String() string + Open(path string, cfg *config.Config) (IDB, error) + Repair(path string, cfg *config.Config) error +} + +var dbs = map[string]Store{} + +func Register(s Store) { + name := s.String() + if _, ok := dbs[name]; ok { + panic(fmt.Errorf("store %s is registered", s)) + } + + dbs[name] = s +} + +func ListStores() []string { + s := []string{} + for k := range dbs { + s = append(s, k) + } + + return s +} + +func GetStore(cfg *config.Config) (Store, error) { + if len(cfg.DBName) == 0 { + cfg.DBName = config.DefaultDBName + } + + s, ok := dbs[cfg.DBName] + if !ok { + return nil, fmt.Errorf("store %s is not registered", cfg.DBName) + } + + return s, nil +} diff --git a/vendor/github.com/siddontang/ledisdb/store/goleveldb/batch.go b/vendor/github.com/siddontang/ledisdb/store/goleveldb/batch.go new file mode 100644 index 000000000000..2032279a2a8e --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/goleveldb/batch.go @@ -0,0 +1,39 @@ +package goleveldb + +import ( + "github.com/syndtr/goleveldb/leveldb" +) + +type WriteBatch struct { + db *DB + wbatch *leveldb.Batch +} + +func (w *WriteBatch) Put(key, value []byte) { + w.wbatch.Put(key, value) +} + +func (w *WriteBatch) Delete(key []byte) { + w.wbatch.Delete(key) +} + +func (w *WriteBatch) Commit() error { + return w.db.db.Write(w.wbatch, nil) +} + +func (w *WriteBatch) SyncCommit() error { + return w.db.db.Write(w.wbatch, w.db.syncOpts) +} + +func (w *WriteBatch) Rollback() error { + w.wbatch.Reset() + return nil +} + +func (w *WriteBatch) Close() { + w.wbatch.Reset() +} + +func (w *WriteBatch) Data() []byte { + return w.wbatch.Dump() +} diff --git a/vendor/github.com/siddontang/ledisdb/store/goleveldb/const.go b/vendor/github.com/siddontang/ledisdb/store/goleveldb/const.go new file mode 100644 index 000000000000..2fffa7c82bb4 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/goleveldb/const.go @@ -0,0 +1,4 @@ +package goleveldb + +const DBName = "goleveldb" +const MemDBName = "memory" diff --git a/vendor/github.com/siddontang/ledisdb/store/goleveldb/db.go b/vendor/github.com/siddontang/ledisdb/store/goleveldb/db.go new file mode 100644 index 000000000000..1afc32fb75b5 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/goleveldb/db.go @@ -0,0 +1,204 @@ +package goleveldb + +import ( + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/cache" + "github.com/syndtr/goleveldb/leveldb/filter" + "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/storage" + "github.com/syndtr/goleveldb/leveldb/util" + + "github.com/siddontang/ledisdb/config" + "github.com/siddontang/ledisdb/store/driver" + + "os" +) + +const defaultFilterBits int = 10 + +type Store struct { +} + +func (s Store) String() string { + return DBName +} + +type MemStore struct { +} + +func (s MemStore) String() string { + return MemDBName +} + +type DB struct { + path string + + cfg *config.LevelDBConfig + + db *leveldb.DB + + opts *opt.Options + + iteratorOpts *opt.ReadOptions + + syncOpts *opt.WriteOptions + + cache cache.Cache + + filter filter.Filter +} + +func (s Store) Open(path string, cfg *config.Config) (driver.IDB, error) { + if err := os.MkdirAll(path, 0755); err != nil { + return nil, err + } + + db := new(DB) + db.path = path + db.cfg = &cfg.LevelDB + + db.initOpts() + + var err error + db.db, err = leveldb.OpenFile(db.path, db.opts) + + if err != nil { + return nil, err + } + + return db, nil +} + +func (s Store) Repair(path string, cfg *config.Config) error { + db, err := leveldb.RecoverFile(path, newOptions(&cfg.LevelDB)) + if err != nil { + return err + } + + db.Close() + return nil +} + +func (s MemStore) Open(path string, cfg *config.Config) (driver.IDB, error) { + db := new(DB) + db.path = path + db.cfg = &cfg.LevelDB + + db.initOpts() + + var err error + db.db, err = leveldb.Open(storage.NewMemStorage(), db.opts) + if err != nil { + return nil, err + } + + return db, nil +} + +func (s MemStore) Repair(path string, cfg *config.Config) error { + return nil +} + +func (db *DB) initOpts() { + db.opts = newOptions(db.cfg) + + db.iteratorOpts = &opt.ReadOptions{} + db.iteratorOpts.DontFillCache = true + + db.syncOpts = &opt.WriteOptions{} + db.syncOpts.Sync = true +} + +func newOptions(cfg *config.LevelDBConfig) *opt.Options { + opts := &opt.Options{} + opts.ErrorIfMissing = false + + opts.BlockCacheCapacity = cfg.CacheSize + + //we must use bloomfilter + opts.Filter = filter.NewBloomFilter(defaultFilterBits) + + if !cfg.Compression { + opts.Compression = opt.NoCompression + } else { + opts.Compression = opt.SnappyCompression + } + + opts.BlockSize = cfg.BlockSize + opts.WriteBuffer = cfg.WriteBufferSize + opts.OpenFilesCacheCapacity = cfg.MaxOpenFiles + + //here we use default value, later add config support + opts.CompactionTableSize = 32 * 1024 * 1024 + opts.WriteL0SlowdownTrigger = 16 + opts.WriteL0PauseTrigger = 64 + + return opts +} + +func (db *DB) Close() error { + return db.db.Close() +} + +func (db *DB) Put(key, value []byte) error { + return db.db.Put(key, value, nil) +} + +func (db *DB) Get(key []byte) ([]byte, error) { + v, err := db.db.Get(key, nil) + if err == leveldb.ErrNotFound { + return nil, nil + } + return v, nil +} + +func (db *DB) Delete(key []byte) error { + return db.db.Delete(key, nil) +} + +func (db *DB) SyncPut(key []byte, value []byte) error { + return db.db.Put(key, value, db.syncOpts) +} + +func (db *DB) SyncDelete(key []byte) error { + return db.db.Delete(key, db.syncOpts) +} + +func (db *DB) NewWriteBatch() driver.IWriteBatch { + wb := &WriteBatch{ + db: db, + wbatch: new(leveldb.Batch), + } + return wb +} + +func (db *DB) NewIterator() driver.IIterator { + it := &Iterator{ + db.db.NewIterator(nil, db.iteratorOpts), + } + + return it +} + +func (db *DB) NewSnapshot() (driver.ISnapshot, error) { + snapshot, err := db.db.GetSnapshot() + if err != nil { + return nil, err + } + + s := &Snapshot{ + db: db, + snp: snapshot, + } + + return s, nil +} + +func (db *DB) Compact() error { + return db.db.CompactRange(util.Range{nil, nil}) +} + +func init() { + driver.Register(Store{}) + driver.Register(MemStore{}) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/goleveldb/iterator.go b/vendor/github.com/siddontang/ledisdb/store/goleveldb/iterator.go new file mode 100644 index 000000000000..c1fd8b5573bb --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/goleveldb/iterator.go @@ -0,0 +1,49 @@ +package goleveldb + +import ( + "github.com/syndtr/goleveldb/leveldb/iterator" +) + +type Iterator struct { + it iterator.Iterator +} + +func (it *Iterator) Key() []byte { + return it.it.Key() +} + +func (it *Iterator) Value() []byte { + return it.it.Value() +} + +func (it *Iterator) Close() error { + if it.it != nil { + it.it.Release() + it.it = nil + } + return nil +} + +func (it *Iterator) Valid() bool { + return it.it.Valid() +} + +func (it *Iterator) Next() { + it.it.Next() +} + +func (it *Iterator) Prev() { + it.it.Prev() +} + +func (it *Iterator) First() { + it.it.First() +} + +func (it *Iterator) Last() { + it.it.Last() +} + +func (it *Iterator) Seek(key []byte) { + it.it.Seek(key) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/goleveldb/snapshot.go b/vendor/github.com/siddontang/ledisdb/store/goleveldb/snapshot.go new file mode 100644 index 000000000000..c615579bb89d --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/goleveldb/snapshot.go @@ -0,0 +1,26 @@ +package goleveldb + +import ( + "github.com/siddontang/ledisdb/store/driver" + "github.com/syndtr/goleveldb/leveldb" +) + +type Snapshot struct { + db *DB + snp *leveldb.Snapshot +} + +func (s *Snapshot) Get(key []byte) ([]byte, error) { + return s.snp.Get(key, s.db.iteratorOpts) +} + +func (s *Snapshot) NewIterator() driver.IIterator { + it := &Iterator{ + s.snp.NewIterator(nil, s.db.iteratorOpts), + } + return it +} + +func (s *Snapshot) Close() { + s.snp.Release() +} diff --git a/vendor/github.com/siddontang/ledisdb/store/iterator.go b/vendor/github.com/siddontang/ledisdb/store/iterator.go new file mode 100644 index 000000000000..12a03b6cd402 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/iterator.go @@ -0,0 +1,334 @@ +package store + +import ( + "bytes" + + "github.com/siddontang/ledisdb/store/driver" +) + +const ( + IteratorForward uint8 = 0 + IteratorBackward uint8 = 1 +) + +const ( + RangeClose uint8 = 0x00 + RangeLOpen uint8 = 0x01 + RangeROpen uint8 = 0x10 + RangeOpen uint8 = 0x11 +) + +// min must less or equal than max +// +// range type: +// +// close: [min, max] +// open: (min, max) +// lopen: (min, max] +// ropen: [min, max) +// +type Range struct { + Min []byte + Max []byte + + Type uint8 +} + +type Limit struct { + Offset int + Count int +} + +type Iterator struct { + it driver.IIterator + st *Stat +} + +// Returns a copy of key. +func (it *Iterator) Key() []byte { + k := it.it.Key() + if k == nil { + return nil + } + + return append([]byte{}, k...) +} + +// Returns a copy of value. +func (it *Iterator) Value() []byte { + v := it.it.Value() + if v == nil { + return nil + } + + return append([]byte{}, v...) +} + +// Returns a reference of key. +// you must be careful that it will be changed after next iterate. +func (it *Iterator) RawKey() []byte { + return it.it.Key() +} + +// Returns a reference of value. +// you must be careful that it will be changed after next iterate. +func (it *Iterator) RawValue() []byte { + return it.it.Value() +} + +// Copy key to b, if b len is small or nil, returns a new one. +func (it *Iterator) BufKey(b []byte) []byte { + k := it.RawKey() + if k == nil { + return nil + } + if b == nil { + b = []byte{} + } + + b = b[0:0] + return append(b, k...) +} + +// Copy value to b, if b len is small or nil, returns a new one. +func (it *Iterator) BufValue(b []byte) []byte { + v := it.RawValue() + if v == nil { + return nil + } + + if b == nil { + b = []byte{} + } + + b = b[0:0] + return append(b, v...) +} + +func (it *Iterator) Close() { + if it.it != nil { + it.st.IterCloseNum.Add(1) + it.it.Close() + it.it = nil + } +} + +func (it *Iterator) Valid() bool { + return it.it.Valid() +} + +func (it *Iterator) Next() { + it.st.IterSeekNum.Add(1) + it.it.Next() +} + +func (it *Iterator) Prev() { + it.st.IterSeekNum.Add(1) + it.it.Prev() +} + +func (it *Iterator) SeekToFirst() { + it.st.IterSeekNum.Add(1) + it.it.First() +} + +func (it *Iterator) SeekToLast() { + it.st.IterSeekNum.Add(1) + it.it.Last() +} + +func (it *Iterator) Seek(key []byte) { + it.st.IterSeekNum.Add(1) + it.it.Seek(key) +} + +// Finds by key, if not found, nil returns. +func (it *Iterator) Find(key []byte) []byte { + it.Seek(key) + if it.Valid() { + k := it.RawKey() + if k == nil { + return nil + } else if bytes.Equal(k, key) { + return it.Value() + } + } + + return nil +} + +// Finds by key, if not found, nil returns, else a reference of value returns. +// you must be careful that it will be changed after next iterate. +func (it *Iterator) RawFind(key []byte) []byte { + it.Seek(key) + if it.Valid() { + k := it.RawKey() + if k == nil { + return nil + } else if bytes.Equal(k, key) { + return it.RawValue() + } + } + + return nil +} + +type RangeLimitIterator struct { + it *Iterator + + r *Range + l *Limit + + step int + + //0 for IteratorForward, 1 for IteratorBackward + direction uint8 +} + +func (it *RangeLimitIterator) Key() []byte { + return it.it.Key() +} + +func (it *RangeLimitIterator) Value() []byte { + return it.it.Value() +} + +func (it *RangeLimitIterator) RawKey() []byte { + return it.it.RawKey() +} + +func (it *RangeLimitIterator) RawValue() []byte { + return it.it.RawValue() +} + +func (it *RangeLimitIterator) BufKey(b []byte) []byte { + return it.it.BufKey(b) +} + +func (it *RangeLimitIterator) BufValue(b []byte) []byte { + return it.it.BufValue(b) +} + +func (it *RangeLimitIterator) Valid() bool { + if it.l.Offset < 0 { + return false + } else if !it.it.Valid() { + return false + } else if it.l.Count >= 0 && it.step >= it.l.Count { + return false + } + + if it.direction == IteratorForward { + if it.r.Max != nil { + r := bytes.Compare(it.it.RawKey(), it.r.Max) + if it.r.Type&RangeROpen > 0 { + return !(r >= 0) + } else { + return !(r > 0) + } + } + } else { + if it.r.Min != nil { + r := bytes.Compare(it.it.RawKey(), it.r.Min) + if it.r.Type&RangeLOpen > 0 { + return !(r <= 0) + } else { + return !(r < 0) + } + } + } + + return true +} + +func (it *RangeLimitIterator) Next() { + it.step++ + + if it.direction == IteratorForward { + it.it.Next() + } else { + it.it.Prev() + } +} + +func (it *RangeLimitIterator) Close() { + it.it.Close() +} + +func NewRangeLimitIterator(i *Iterator, r *Range, l *Limit) *RangeLimitIterator { + return rangeLimitIterator(i, r, l, IteratorForward) +} + +func NewRevRangeLimitIterator(i *Iterator, r *Range, l *Limit) *RangeLimitIterator { + return rangeLimitIterator(i, r, l, IteratorBackward) +} + +func NewRangeIterator(i *Iterator, r *Range) *RangeLimitIterator { + return rangeLimitIterator(i, r, &Limit{0, -1}, IteratorForward) +} + +func NewRevRangeIterator(i *Iterator, r *Range) *RangeLimitIterator { + return rangeLimitIterator(i, r, &Limit{0, -1}, IteratorBackward) +} + +func rangeLimitIterator(i *Iterator, r *Range, l *Limit, direction uint8) *RangeLimitIterator { + it := new(RangeLimitIterator) + + it.it = i + + it.r = r + it.l = l + it.direction = direction + + it.step = 0 + + if l.Offset < 0 { + return it + } + + if direction == IteratorForward { + if r.Min == nil { + it.it.SeekToFirst() + } else { + it.it.Seek(r.Min) + + if r.Type&RangeLOpen > 0 { + if it.it.Valid() && bytes.Equal(it.it.RawKey(), r.Min) { + it.it.Next() + } + } + } + } else { + if r.Max == nil { + it.it.SeekToLast() + } else { + it.it.Seek(r.Max) + + if !it.it.Valid() { + it.it.SeekToLast() + } else { + if !bytes.Equal(it.it.RawKey(), r.Max) { + it.it.Prev() + } + } + + if r.Type&RangeROpen > 0 { + if it.it.Valid() && bytes.Equal(it.it.RawKey(), r.Max) { + it.it.Prev() + } + } + } + } + + for i := 0; i < l.Offset; i++ { + if it.it.Valid() { + if it.direction == IteratorForward { + it.it.Next() + } else { + it.it.Prev() + } + } + } + + return it +} diff --git a/vendor/github.com/siddontang/ledisdb/store/leveldb/batch.go b/vendor/github.com/siddontang/ledisdb/store/leveldb/batch.go new file mode 100644 index 000000000000..cc1b02448896 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/leveldb/batch.go @@ -0,0 +1,99 @@ +// +build leveldb + +package leveldb + +// #cgo LDFLAGS: -lleveldb +// #include "leveldb/c.h" +// #include "leveldb_ext.h" +import "C" + +import ( + "unsafe" + + "github.com/syndtr/goleveldb/leveldb" +) + +type WriteBatch struct { + db *DB + wbatch *C.leveldb_writebatch_t +} + +func newWriteBatch(db *DB) *WriteBatch { + w := new(WriteBatch) + w.db = db + w.wbatch = C.leveldb_writebatch_create() + + return w +} + +func (w *WriteBatch) Close() { + if w.wbatch != nil { + C.leveldb_writebatch_destroy(w.wbatch) + w.wbatch = nil + } +} + +func (w *WriteBatch) Put(key, value []byte) { + var k, v *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + if len(value) != 0 { + v = (*C.char)(unsafe.Pointer(&value[0])) + } + + lenk := len(key) + lenv := len(value) + + C.leveldb_writebatch_put(w.wbatch, k, C.size_t(lenk), v, C.size_t(lenv)) +} + +func (w *WriteBatch) Delete(key []byte) { + C.leveldb_writebatch_delete(w.wbatch, + (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key))) +} + +func (w *WriteBatch) Commit() error { + return w.commit(w.db.writeOpts) +} + +func (w *WriteBatch) SyncCommit() error { + return w.commit(w.db.syncOpts) +} + +func (w *WriteBatch) Rollback() error { + C.leveldb_writebatch_clear(w.wbatch) + + return nil +} + +func (w *WriteBatch) commit(wb *WriteOptions) error { + var errStr *C.char + C.leveldb_write(w.db.db, wb.Opt, w.wbatch, &errStr) + if errStr != nil { + return saveError(errStr) + } + return nil +} + +//export leveldb_writebatch_iterate_put +func leveldb_writebatch_iterate_put(p unsafe.Pointer, k *C.char, klen C.size_t, v *C.char, vlen C.size_t) { + b := (*leveldb.Batch)(p) + key := slice(unsafe.Pointer(k), int(klen)) + value := slice(unsafe.Pointer(v), int(vlen)) + b.Put(key, value) +} + +//export leveldb_writebatch_iterate_delete +func leveldb_writebatch_iterate_delete(p unsafe.Pointer, k *C.char, klen C.size_t) { + b := (*leveldb.Batch)(p) + key := slice(unsafe.Pointer(k), int(klen)) + b.Delete(key) +} + +func (w *WriteBatch) Data() []byte { + gbatch := leveldb.Batch{} + C.leveldb_writebatch_iterate_ext(w.wbatch, + unsafe.Pointer(&gbatch)) + return gbatch.Dump() +} diff --git a/vendor/github.com/siddontang/ledisdb/store/leveldb/cache.go b/vendor/github.com/siddontang/ledisdb/store/leveldb/cache.go new file mode 100644 index 000000000000..e5587cbf89e2 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/leveldb/cache.go @@ -0,0 +1,20 @@ +// +build leveldb + +package leveldb + +// #cgo LDFLAGS: -lleveldb +// #include +// #include "leveldb/c.h" +import "C" + +type Cache struct { + Cache *C.leveldb_cache_t +} + +func NewLRUCache(capacity int) *Cache { + return &Cache{C.leveldb_cache_create_lru(C.size_t(capacity))} +} + +func (c *Cache) Close() { + C.leveldb_cache_destroy(c.Cache) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/leveldb/const.go b/vendor/github.com/siddontang/ledisdb/store/leveldb/const.go new file mode 100644 index 000000000000..df5b3c7a83b5 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/leveldb/const.go @@ -0,0 +1,3 @@ +package leveldb + +const DBName = "leveldb" diff --git a/vendor/github.com/siddontang/ledisdb/store/leveldb/db.go b/vendor/github.com/siddontang/ledisdb/store/leveldb/db.go new file mode 100644 index 000000000000..7f1ee676ec39 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/leveldb/db.go @@ -0,0 +1,314 @@ +// +build leveldb + +// Package leveldb is a wrapper for c++ leveldb +package leveldb + +/* +#cgo LDFLAGS: -lleveldb +#include +#include "leveldb_ext.h" +*/ +import "C" + +import ( + "os" + "runtime" + "unsafe" + + "github.com/siddontang/ledisdb/config" + "github.com/siddontang/ledisdb/store/driver" +) + +const defaultFilterBits int = 10 + +type Store struct { +} + +func (s Store) String() string { + return DBName +} + +func (s Store) Open(path string, cfg *config.Config) (driver.IDB, error) { + if err := os.MkdirAll(path, 0755); err != nil { + return nil, err + } + + db := new(DB) + db.path = path + db.cfg = &cfg.LevelDB + + if err := db.open(); err != nil { + return nil, err + } + + return db, nil +} + +func (s Store) Repair(path string, cfg *config.Config) error { + db := new(DB) + db.cfg = &cfg.LevelDB + db.path = path + + err := db.open() + defer db.Close() + + //open ok, do not need repair + if err == nil { + return nil + } + + var errStr *C.char + ldbname := C.CString(path) + defer C.leveldb_free(unsafe.Pointer(ldbname)) + + C.leveldb_repair_db(db.opts.Opt, ldbname, &errStr) + if errStr != nil { + return saveError(errStr) + } + return nil +} + +type DB struct { + path string + + cfg *config.LevelDBConfig + + db *C.leveldb_t + + opts *Options + + //for default read and write options + readOpts *ReadOptions + writeOpts *WriteOptions + iteratorOpts *ReadOptions + + syncOpts *WriteOptions + + cache *Cache + + filter *FilterPolicy +} + +func (db *DB) open() error { + db.initOptions(db.cfg) + + var errStr *C.char + ldbname := C.CString(db.path) + defer C.leveldb_free(unsafe.Pointer(ldbname)) + + db.db = C.leveldb_open(db.opts.Opt, ldbname, &errStr) + if errStr != nil { + db.db = nil + return saveError(errStr) + } + return nil +} + +func (db *DB) initOptions(cfg *config.LevelDBConfig) { + opts := NewOptions() + + opts.SetCreateIfMissing(true) + + db.cache = NewLRUCache(cfg.CacheSize) + opts.SetCache(db.cache) + + //we must use bloomfilter + db.filter = NewBloomFilter(defaultFilterBits) + opts.SetFilterPolicy(db.filter) + + if !cfg.Compression { + opts.SetCompression(NoCompression) + } else { + opts.SetCompression(SnappyCompression) + } + + opts.SetBlockSize(cfg.BlockSize) + + opts.SetWriteBufferSize(cfg.WriteBufferSize) + + opts.SetMaxOpenFiles(cfg.MaxOpenFiles) + + opts.SetMaxFileSize(cfg.MaxFileSize) + + db.opts = opts + + db.readOpts = NewReadOptions() + db.writeOpts = NewWriteOptions() + + db.syncOpts = NewWriteOptions() + db.syncOpts.SetSync(true) + + db.iteratorOpts = NewReadOptions() + db.iteratorOpts.SetFillCache(false) +} + +func (db *DB) Close() error { + if db.db != nil { + C.leveldb_close(db.db) + db.db = nil + } + + db.opts.Close() + + if db.cache != nil { + db.cache.Close() + } + + if db.filter != nil { + db.filter.Close() + } + + db.readOpts.Close() + db.writeOpts.Close() + db.iteratorOpts.Close() + + return nil +} + +func (db *DB) Put(key, value []byte) error { + return db.put(db.writeOpts, key, value) +} + +func (db *DB) Get(key []byte) ([]byte, error) { + return db.get(db.readOpts, key) +} + +func (db *DB) Delete(key []byte) error { + return db.delete(db.writeOpts, key) +} + +func (db *DB) SyncPut(key []byte, value []byte) error { + return db.put(db.syncOpts, key, value) +} + +func (db *DB) SyncDelete(key []byte) error { + return db.delete(db.syncOpts, key) +} + +func (db *DB) NewWriteBatch() driver.IWriteBatch { + wb := newWriteBatch(db) + + runtime.SetFinalizer(wb, func(w *WriteBatch) { + w.Close() + }) + + return wb +} + +func (db *DB) NewIterator() driver.IIterator { + it := new(Iterator) + + it.it = C.leveldb_create_iterator(db.db, db.iteratorOpts.Opt) + + return it +} + +func (db *DB) NewSnapshot() (driver.ISnapshot, error) { + snap := &Snapshot{ + db: db, + snap: C.leveldb_create_snapshot(db.db), + readOpts: NewReadOptions(), + iteratorOpts: NewReadOptions(), + } + snap.readOpts.SetSnapshot(snap) + snap.iteratorOpts.SetSnapshot(snap) + snap.iteratorOpts.SetFillCache(false) + + return snap, nil +} + +func (db *DB) put(wo *WriteOptions, key, value []byte) error { + var errStr *C.char + var k, v *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + if len(value) != 0 { + v = (*C.char)(unsafe.Pointer(&value[0])) + } + + lenk := len(key) + lenv := len(value) + C.leveldb_put( + db.db, wo.Opt, k, C.size_t(lenk), v, C.size_t(lenv), &errStr) + + if errStr != nil { + return saveError(errStr) + } + return nil +} + +func (db *DB) get(ro *ReadOptions, key []byte) ([]byte, error) { + var errStr *C.char + var vallen C.size_t + var k *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + + value := C.leveldb_get( + db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr) + + if errStr != nil { + return nil, saveError(errStr) + } + + if value == nil { + return nil, nil + } + + defer C.leveldb_free(unsafe.Pointer(value)) + + return C.GoBytes(unsafe.Pointer(value), C.int(vallen)), nil +} + +func (db *DB) getSlice(ro *ReadOptions, key []byte) (driver.ISlice, error) { + var errStr *C.char + var vallen C.size_t + var k *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + + value := C.leveldb_get( + db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr) + + if errStr != nil { + return nil, saveError(errStr) + } + + if value == nil { + return nil, nil + } + + return NewCSlice(unsafe.Pointer(value), int(vallen)), nil +} + +func (db *DB) delete(wo *WriteOptions, key []byte) error { + var errStr *C.char + var k *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + + C.leveldb_delete( + db.db, wo.Opt, k, C.size_t(len(key)), &errStr) + + if errStr != nil { + return saveError(errStr) + } + return nil +} + +func (db *DB) Compact() error { + C.leveldb_compact_range(db.db, nil, 0, nil, 0) + return nil +} + +func (db *DB) GetSlice(key []byte) (driver.ISlice, error) { + return db.getSlice(db.readOpts, key) +} + +func init() { + driver.Register(Store{}) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/leveldb/filterpolicy.go b/vendor/github.com/siddontang/ledisdb/store/leveldb/filterpolicy.go new file mode 100644 index 000000000000..640139fb8b8e --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/leveldb/filterpolicy.go @@ -0,0 +1,21 @@ +// +build leveldb + +package leveldb + +// #cgo LDFLAGS: -lleveldb +// #include +// #include "leveldb/c.h" +import "C" + +type FilterPolicy struct { + Policy *C.leveldb_filterpolicy_t +} + +func NewBloomFilter(bitsPerKey int) *FilterPolicy { + policy := C.leveldb_filterpolicy_create_bloom(C.int(bitsPerKey)) + return &FilterPolicy{policy} +} + +func (fp *FilterPolicy) Close() { + C.leveldb_filterpolicy_destroy(fp.Policy) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/leveldb/iterator.go b/vendor/github.com/siddontang/ledisdb/store/leveldb/iterator.go new file mode 100644 index 000000000000..49cfd7db18ea --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/leveldb/iterator.go @@ -0,0 +1,70 @@ +// +build leveldb + +package leveldb + +// #cgo LDFLAGS: -lleveldb +// #include +// #include "leveldb/c.h" +// #include "leveldb_ext.h" +import "C" + +import ( + "unsafe" +) + +type Iterator struct { + it *C.leveldb_iterator_t + isValid C.uchar +} + +func (it *Iterator) Key() []byte { + var klen C.size_t + kdata := C.leveldb_iter_key(it.it, &klen) + if kdata == nil { + return nil + } + + return slice(unsafe.Pointer(kdata), int(C.int(klen))) +} + +func (it *Iterator) Value() []byte { + var vlen C.size_t + vdata := C.leveldb_iter_value(it.it, &vlen) + if vdata == nil { + return nil + } + + return slice(unsafe.Pointer(vdata), int(C.int(vlen))) +} + +func (it *Iterator) Close() error { + if it.it != nil { + C.leveldb_iter_destroy(it.it) + it.it = nil + } + return nil +} + +func (it *Iterator) Valid() bool { + return ucharToBool(it.isValid) +} + +func (it *Iterator) Next() { + it.isValid = C.leveldb_iter_next_ext(it.it) +} + +func (it *Iterator) Prev() { + it.isValid = C.leveldb_iter_prev_ext(it.it) +} + +func (it *Iterator) First() { + it.isValid = C.leveldb_iter_seek_to_first_ext(it.it) +} + +func (it *Iterator) Last() { + it.isValid = C.leveldb_iter_seek_to_last_ext(it.it) +} + +func (it *Iterator) Seek(key []byte) { + it.isValid = C.leveldb_iter_seek_ext(it.it, (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key))) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/leveldb/leveldb_ext.cc b/vendor/github.com/siddontang/ledisdb/store/leveldb/leveldb_ext.cc new file mode 100644 index 000000000000..540b7397b01b --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/leveldb/leveldb_ext.cc @@ -0,0 +1,95 @@ +// +build leveldb + +#include "leveldb_ext.h" + +#include +//#include + +//#include "leveldb/db.h" + +//using namespace leveldb; + +extern "C" { + +// static bool SaveError(char** errptr, const Status& s) { +// assert(errptr != NULL); +// if (s.ok()) { +// return false; +// } else if (*errptr == NULL) { +// *errptr = strdup(s.ToString().c_str()); +// } else { +// free(*errptr); +// *errptr = strdup(s.ToString().c_str()); +// } +// return true; +// } + +// void* leveldb_get_ext( +// leveldb_t* db, +// const leveldb_readoptions_t* options, +// const char* key, size_t keylen, +// char** valptr, +// size_t* vallen, +// char** errptr) { + +// std::string *tmp = new(std::string); + +// //very tricky, maybe changed with c++ leveldb upgrade +// Status s = (*(DB**)db)->Get(*(ReadOptions*)options, Slice(key, keylen), tmp); + +// if (s.ok()) { +// *valptr = (char*)tmp->data(); +// *vallen = tmp->size(); +// } else { +// delete(tmp); +// tmp = NULL; +// *valptr = NULL; +// *vallen = 0; +// if (!s.IsNotFound()) { +// SaveError(errptr, s); +// } +// } +// return tmp; +// } + +// void leveldb_get_free_ext(void* context) { +// std::string* s = (std::string*)context; + +// delete(s); +// } + + +unsigned char leveldb_iter_seek_to_first_ext(leveldb_iterator_t* iter) { + leveldb_iter_seek_to_first(iter); + return leveldb_iter_valid(iter); +} + +unsigned char leveldb_iter_seek_to_last_ext(leveldb_iterator_t* iter) { + leveldb_iter_seek_to_last(iter); + return leveldb_iter_valid(iter); +} + +unsigned char leveldb_iter_seek_ext(leveldb_iterator_t* iter, const char* k, size_t klen) { + leveldb_iter_seek(iter, k, klen); + return leveldb_iter_valid(iter); +} + +unsigned char leveldb_iter_next_ext(leveldb_iterator_t* iter) { + leveldb_iter_next(iter); + return leveldb_iter_valid(iter); +} + +unsigned char leveldb_iter_prev_ext(leveldb_iterator_t* iter) { + leveldb_iter_prev(iter); + return leveldb_iter_valid(iter); +} + +extern void leveldb_writebatch_iterate_put(void*, const char* k, size_t klen, const char* v, size_t vlen); +extern void leveldb_writebatch_iterate_delete(void*, const char* k, size_t klen); + +void leveldb_writebatch_iterate_ext(leveldb_writebatch_t* w, void *p) { + leveldb_writebatch_iterate(w, p, + leveldb_writebatch_iterate_put, leveldb_writebatch_iterate_delete); +} + +} \ No newline at end of file diff --git a/vendor/github.com/siddontang/ledisdb/store/leveldb/leveldb_ext.h b/vendor/github.com/siddontang/ledisdb/store/leveldb/leveldb_ext.h new file mode 100644 index 000000000000..3eed41bdf9b0 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/leveldb/leveldb_ext.h @@ -0,0 +1,41 @@ +// +build leveldb + +#ifndef LEVELDB_EXT_H +#define LEVELDB_EXT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "leveldb/c.h" + + +// /* Returns NULL if not found. Otherwise stores the value in **valptr. +// Stores the length of the value in *vallen. +// Returns a context must be later to free*/ +// extern void* leveldb_get_ext( +// leveldb_t* db, +// const leveldb_readoptions_t* options, +// const char* key, size_t keylen, +// char** valptr, +// size_t* vallen, +// char** errptr); + +// // Free context returns by leveldb_get_ext +// extern void leveldb_get_free_ext(void* context); + + +// Below iterator functions like leveldb iterator but returns valid status for iterator +extern unsigned char leveldb_iter_seek_to_first_ext(leveldb_iterator_t*); +extern unsigned char leveldb_iter_seek_to_last_ext(leveldb_iterator_t*); +extern unsigned char leveldb_iter_seek_ext(leveldb_iterator_t*, const char* k, size_t klen); +extern unsigned char leveldb_iter_next_ext(leveldb_iterator_t*); +extern unsigned char leveldb_iter_prev_ext(leveldb_iterator_t*); + +extern void leveldb_writebatch_iterate_ext(leveldb_writebatch_t*, void* p); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/vendor/github.com/siddontang/ledisdb/store/leveldb/levigo-license b/vendor/github.com/siddontang/ledisdb/store/leveldb/levigo-license new file mode 100644 index 000000000000..c7c73befc3b5 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/leveldb/levigo-license @@ -0,0 +1,7 @@ +Copyright (c) 2012 Jeffrey M Hodges + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/siddontang/ledisdb/store/leveldb/options.go b/vendor/github.com/siddontang/ledisdb/store/leveldb/options.go new file mode 100644 index 000000000000..68733bb612be --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/leveldb/options.go @@ -0,0 +1,126 @@ +// +build leveldb + +package leveldb + +// #cgo LDFLAGS: -lleveldb +// #include "leveldb/c.h" +import "C" + +type CompressionOpt int + +const ( + NoCompression = CompressionOpt(0) + SnappyCompression = CompressionOpt(1) +) + +type Options struct { + Opt *C.leveldb_options_t +} + +type ReadOptions struct { + Opt *C.leveldb_readoptions_t +} + +type WriteOptions struct { + Opt *C.leveldb_writeoptions_t +} + +func NewOptions() *Options { + opt := C.leveldb_options_create() + return &Options{opt} +} + +func NewReadOptions() *ReadOptions { + opt := C.leveldb_readoptions_create() + return &ReadOptions{opt} +} + +func NewWriteOptions() *WriteOptions { + opt := C.leveldb_writeoptions_create() + return &WriteOptions{opt} +} + +func (o *Options) Close() { + C.leveldb_options_destroy(o.Opt) +} + +func (o *Options) SetComparator(cmp *C.leveldb_comparator_t) { + C.leveldb_options_set_comparator(o.Opt, cmp) +} + +func (o *Options) SetErrorIfExists(error_if_exists bool) { + eie := boolToUchar(error_if_exists) + C.leveldb_options_set_error_if_exists(o.Opt, eie) +} + +func (o *Options) SetCache(cache *Cache) { + C.leveldb_options_set_cache(o.Opt, cache.Cache) +} + +func (o *Options) SetWriteBufferSize(s int) { + C.leveldb_options_set_write_buffer_size(o.Opt, C.size_t(s)) +} + +func (o *Options) SetParanoidChecks(pc bool) { + C.leveldb_options_set_paranoid_checks(o.Opt, boolToUchar(pc)) +} + +func (o *Options) SetMaxOpenFiles(n int) { + C.leveldb_options_set_max_open_files(o.Opt, C.int(n)) +} + +func (o *Options) SetMaxFileSize(n int) { + C.leveldb_options_set_max_file_size(o.Opt, C.size_t(n)) +} + +func (o *Options) SetBlockSize(s int) { + C.leveldb_options_set_block_size(o.Opt, C.size_t(s)) +} + +func (o *Options) SetBlockRestartInterval(n int) { + C.leveldb_options_set_block_restart_interval(o.Opt, C.int(n)) +} + +func (o *Options) SetCompression(t CompressionOpt) { + C.leveldb_options_set_compression(o.Opt, C.int(t)) +} + +func (o *Options) SetCreateIfMissing(b bool) { + C.leveldb_options_set_create_if_missing(o.Opt, boolToUchar(b)) +} + +func (o *Options) SetFilterPolicy(fp *FilterPolicy) { + var policy *C.leveldb_filterpolicy_t + if fp != nil { + policy = fp.Policy + } + C.leveldb_options_set_filter_policy(o.Opt, policy) +} + +func (ro *ReadOptions) Close() { + C.leveldb_readoptions_destroy(ro.Opt) +} + +func (ro *ReadOptions) SetVerifyChecksums(b bool) { + C.leveldb_readoptions_set_verify_checksums(ro.Opt, boolToUchar(b)) +} + +func (ro *ReadOptions) SetFillCache(b bool) { + C.leveldb_readoptions_set_fill_cache(ro.Opt, boolToUchar(b)) +} + +func (ro *ReadOptions) SetSnapshot(snap *Snapshot) { + var s *C.leveldb_snapshot_t + if snap != nil { + s = snap.snap + } + C.leveldb_readoptions_set_snapshot(ro.Opt, s) +} + +func (wo *WriteOptions) Close() { + C.leveldb_writeoptions_destroy(wo.Opt) +} + +func (wo *WriteOptions) SetSync(b bool) { + C.leveldb_writeoptions_set_sync(wo.Opt, boolToUchar(b)) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/leveldb/slice.go b/vendor/github.com/siddontang/ledisdb/store/leveldb/slice.go new file mode 100644 index 000000000000..83ebf55c0369 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/leveldb/slice.go @@ -0,0 +1,40 @@ +// +build leveldb + +package leveldb + +// #cgo LDFLAGS: -lleveldb +// #include "leveldb/c.h" +import "C" + +import ( + "reflect" + "unsafe" +) + +type CSlice struct { + data unsafe.Pointer + size int +} + +func NewCSlice(p unsafe.Pointer, n int) *CSlice { + return &CSlice{p, n} +} + +func (s *CSlice) Data() []byte { + var value []byte + + sH := (*reflect.SliceHeader)(unsafe.Pointer(&value)) + sH.Cap = int(s.size) + sH.Len = int(s.size) + sH.Data = uintptr(s.data) + + return value +} + +func (s *CSlice) Size() int { + return int(s.size) +} + +func (s *CSlice) Free() { + C.leveldb_free(s.data) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/leveldb/snapshot.go b/vendor/github.com/siddontang/ledisdb/store/leveldb/snapshot.go new file mode 100644 index 000000000000..bdc8d514da8f --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/leveldb/snapshot.go @@ -0,0 +1,39 @@ +// +build leveldb + +package leveldb + +// #cgo LDFLAGS: -lleveldb +// #include "leveldb/c.h" +import "C" + +import ( + "github.com/siddontang/ledisdb/store/driver" +) + +type Snapshot struct { + db *DB + snap *C.leveldb_snapshot_t + readOpts *ReadOptions + iteratorOpts *ReadOptions +} + +func (s *Snapshot) Get(key []byte) ([]byte, error) { + return s.db.get(s.readOpts, key) +} + +func (s *Snapshot) GetSlice(key []byte) (driver.ISlice, error) { + return s.db.getSlice(s.readOpts, key) +} + +func (s *Snapshot) NewIterator() driver.IIterator { + it := new(Iterator) + it.it = C.leveldb_create_iterator(s.db.db, s.db.iteratorOpts.Opt) + return it + +} + +func (s *Snapshot) Close() { + C.leveldb_release_snapshot(s.db.db, s.snap) + s.iteratorOpts.Close() + s.readOpts.Close() +} diff --git a/vendor/github.com/siddontang/ledisdb/store/leveldb/util.go b/vendor/github.com/siddontang/ledisdb/store/leveldb/util.go new file mode 100644 index 000000000000..6efe33b661d6 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/leveldb/util.go @@ -0,0 +1,45 @@ +// +build leveldb + +package leveldb + +// #include "leveldb/c.h" +import "C" + +import ( + "fmt" + "reflect" + "unsafe" +) + +func boolToUchar(b bool) C.uchar { + uc := C.uchar(0) + if b { + uc = C.uchar(1) + } + return uc +} + +func ucharToBool(uc C.uchar) bool { + if uc == C.uchar(0) { + return false + } + return true +} + +func saveError(errStr *C.char) error { + if errStr != nil { + gs := C.GoString(errStr) + C.leveldb_free(unsafe.Pointer(errStr)) + return fmt.Errorf(gs) + } + return nil +} + +func slice(p unsafe.Pointer, n int) []byte { + var b []byte + pbyte := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + pbyte.Data = uintptr(p) + pbyte.Len = n + pbyte.Cap = n + return b +} diff --git a/vendor/github.com/siddontang/ledisdb/store/rocksdb/batch.go b/vendor/github.com/siddontang/ledisdb/store/rocksdb/batch.go new file mode 100644 index 000000000000..bb727e7015ef --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/rocksdb/batch.go @@ -0,0 +1,83 @@ +// +build rocksdb + +package rocksdb + +// #cgo LDFLAGS: -lrocksdb +// #include "rocksdb/c.h" +// #include "rocksdb_ext.h" +import "C" + +import ( + "unsafe" +) + +type WriteBatch struct { + db *DB + wbatch *C.rocksdb_writebatch_t + commitOk bool +} + +func (w *WriteBatch) Close() { + if w.wbatch != nil { + C.rocksdb_writebatch_destroy(w.wbatch) + w.wbatch = nil + } +} + +func (w *WriteBatch) Put(key, value []byte) { + w.commitOk = false + + var k, v *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + if len(value) != 0 { + v = (*C.char)(unsafe.Pointer(&value[0])) + } + + lenk := len(key) + lenv := len(value) + + C.rocksdb_writebatch_put(w.wbatch, k, C.size_t(lenk), v, C.size_t(lenv)) +} + +func (w *WriteBatch) Delete(key []byte) { + w.commitOk = false + + C.rocksdb_writebatch_delete(w.wbatch, + (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key))) +} + +func (w *WriteBatch) Commit() error { + return w.commit(w.db.writeOpts) +} + +func (w *WriteBatch) SyncCommit() error { + return w.commit(w.db.syncOpts) +} + +func (w *WriteBatch) Rollback() error { + if !w.commitOk { + C.rocksdb_writebatch_clear(w.wbatch) + } + return nil +} + +func (w *WriteBatch) commit(wb *WriteOptions) error { + w.commitOk = true + + var errStr *C.char + C.rocksdb_write_ext(w.db.db, wb.Opt, w.wbatch, &errStr) + if errStr != nil { + w.commitOk = false + return saveError(errStr) + } + return nil +} + +func (w *WriteBatch) Data() []byte { + var vallen C.size_t + value := C.rocksdb_writebatch_data(w.wbatch, &vallen) + + return slice(unsafe.Pointer(value), int(vallen)) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/rocksdb/cache.go b/vendor/github.com/siddontang/ledisdb/store/rocksdb/cache.go new file mode 100644 index 000000000000..931998ba4af4 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/rocksdb/cache.go @@ -0,0 +1,20 @@ +// +build rocksdb + +package rocksdb + +// #cgo LDFLAGS: -lrocksdb +// #include +// #include "rocksdb/c.h" +import "C" + +type Cache struct { + Cache *C.rocksdb_cache_t +} + +func NewLRUCache(capacity int) *Cache { + return &Cache{C.rocksdb_cache_create_lru(C.size_t(capacity))} +} + +func (c *Cache) Close() { + C.rocksdb_cache_destroy(c.Cache) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/rocksdb/const.go b/vendor/github.com/siddontang/ledisdb/store/rocksdb/const.go new file mode 100644 index 000000000000..f4155bbe2018 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/rocksdb/const.go @@ -0,0 +1,3 @@ +package rocksdb + +const DBName = "rocksdb" diff --git a/vendor/github.com/siddontang/ledisdb/store/rocksdb/db.go b/vendor/github.com/siddontang/ledisdb/store/rocksdb/db.go new file mode 100644 index 000000000000..d5b708043ec6 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/rocksdb/db.go @@ -0,0 +1,342 @@ +// +build rocksdb + +// Package rocksdb is a wrapper for c++ rocksdb +package rocksdb + +/* +#cgo LDFLAGS: -lrocksdb +#include +#include +#include "rocksdb_ext.h" +*/ +import "C" + +import ( + "os" + "runtime" + "unsafe" + + "github.com/siddontang/ledisdb/config" + "github.com/siddontang/ledisdb/store/driver" +) + +const defaultFilterBits int = 10 + +type Store struct { +} + +func (s Store) String() string { + return DBName +} + +func (s Store) Open(path string, cfg *config.Config) (driver.IDB, error) { + if err := os.MkdirAll(path, 0755); err != nil { + return nil, err + } + + db := new(DB) + db.path = path + db.cfg = &cfg.RocksDB + + if err := db.open(); err != nil { + return nil, err + } + + return db, nil +} + +func (s Store) Repair(path string, cfg *config.Config) error { + db := new(DB) + db.path = path + db.cfg = &cfg.RocksDB + + err := db.open() + defer db.Close() + + //open ok, do not need repair + if err == nil { + return nil + } + + var errStr *C.char + ldbname := C.CString(path) + defer C.free(unsafe.Pointer(ldbname)) + + C.rocksdb_repair_db(db.opts.Opt, ldbname, &errStr) + if errStr != nil { + return saveError(errStr) + } + return nil +} + +type DB struct { + path string + + cfg *config.RocksDBConfig + + db *C.rocksdb_t + + env *Env + + opts *Options + blockOpts *BlockBasedTableOptions + + //for default read and write options + readOpts *ReadOptions + writeOpts *WriteOptions + iteratorOpts *ReadOptions + + syncOpts *WriteOptions + + cache *Cache + + filter *FilterPolicy +} + +func (db *DB) open() error { + db.initOptions(db.cfg) + + var errStr *C.char + ldbname := C.CString(db.path) + defer C.free(unsafe.Pointer(ldbname)) + + db.db = C.rocksdb_open(db.opts.Opt, ldbname, &errStr) + if errStr != nil { + db.db = nil + return saveError(errStr) + } + return nil +} + +func (db *DB) initOptions(cfg *config.RocksDBConfig) { + opts := NewOptions() + blockOpts := NewBlockBasedTableOptions() + + opts.SetCreateIfMissing(true) + + db.env = NewDefaultEnv() + db.env.SetBackgroundThreads(cfg.BackgroundThreads) + db.env.SetHighPriorityBackgroundThreads(cfg.HighPriorityBackgroundThreads) + opts.SetEnv(db.env) + + db.cache = NewLRUCache(cfg.CacheSize) + blockOpts.SetCache(db.cache) + + //we must use bloomfilter + db.filter = NewBloomFilter(defaultFilterBits) + blockOpts.SetFilterPolicy(db.filter) + blockOpts.SetBlockSize(cfg.BlockSize) + opts.SetBlockBasedTableFactory(blockOpts) + + opts.SetCompression(CompressionOpt(cfg.Compression)) + opts.SetWriteBufferSize(cfg.WriteBufferSize) + opts.SetMaxOpenFiles(cfg.MaxOpenFiles) + opts.SetMaxBackgroundCompactions(cfg.MaxBackgroundCompactions) + opts.SetMaxBackgroundFlushes(cfg.MaxBackgroundFlushes) + opts.SetLevel0FileNumCompactionTrigger(cfg.Level0FileNumCompactionTrigger) + opts.SetLevel0SlowdownWritesTrigger(cfg.Level0SlowdownWritesTrigger) + opts.SetLevel0StopWritesTrigger(cfg.Level0StopWritesTrigger) + opts.SetTargetFileSizeBase(cfg.TargetFileSizeBase) + opts.SetTargetFileSizeMultiplier(cfg.TargetFileSizeMultiplier) + opts.SetMaxBytesForLevelBase(cfg.MaxBytesForLevelBase) + opts.SetMaxBytesForLevelMultiplier(cfg.MaxBytesForLevelMultiplier) + opts.SetMinWriteBufferNumberToMerge(cfg.MinWriteBufferNumberToMerge) + opts.DisableAutoCompactions(cfg.DisableAutoCompactions) + opts.EnableStatistics(cfg.EnableStatistics) + opts.UseFsync(cfg.UseFsync) + opts.SetStatsDumpPeriodSec(cfg.StatsDumpPeriodSec) + opts.SetMaxManifestFileSize(cfg.MaxManifestFileSize) + + db.opts = opts + db.blockOpts = blockOpts + + db.readOpts = NewReadOptions() + db.writeOpts = NewWriteOptions() + db.writeOpts.DisableWAL(cfg.DisableWAL) + + db.syncOpts = NewWriteOptions() + db.syncOpts.SetSync(true) + db.syncOpts.DisableWAL(cfg.DisableWAL) + + db.iteratorOpts = NewReadOptions() + db.iteratorOpts.SetFillCache(false) +} + +func (db *DB) Close() error { + if db.db != nil { + C.rocksdb_close(db.db) + db.db = nil + } + + if db.filter != nil { + db.filter.Close() + } + + if db.cache != nil { + db.cache.Close() + } + + if db.env != nil { + db.env.Close() + } + + //db.blockOpts.Close() + + db.opts.Close() + + db.readOpts.Close() + db.writeOpts.Close() + db.iteratorOpts.Close() + + return nil +} + +func (db *DB) Put(key, value []byte) error { + return db.put(db.writeOpts, key, value) +} + +func (db *DB) Get(key []byte) ([]byte, error) { + return db.get(db.readOpts, key) +} + +func (db *DB) Delete(key []byte) error { + return db.delete(db.writeOpts, key) +} + +func (db *DB) SyncPut(key []byte, value []byte) error { + return db.put(db.syncOpts, key, value) +} + +func (db *DB) SyncDelete(key []byte) error { + return db.delete(db.syncOpts, key) +} + +func (db *DB) NewWriteBatch() driver.IWriteBatch { + wb := &WriteBatch{ + db: db, + wbatch: C.rocksdb_writebatch_create(), + } + + runtime.SetFinalizer(wb, func(w *WriteBatch) { + w.Close() + }) + + return wb +} + +func (db *DB) NewIterator() driver.IIterator { + it := new(Iterator) + + it.it = C.rocksdb_create_iterator(db.db, db.iteratorOpts.Opt) + + return it +} + +func (db *DB) NewSnapshot() (driver.ISnapshot, error) { + snap := &Snapshot{ + db: db, + snap: C.rocksdb_create_snapshot(db.db), + readOpts: NewReadOptions(), + iteratorOpts: NewReadOptions(), + } + snap.readOpts.SetSnapshot(snap) + snap.iteratorOpts.SetSnapshot(snap) + snap.iteratorOpts.SetFillCache(false) + + return snap, nil +} + +func (db *DB) put(wo *WriteOptions, key, value []byte) error { + var errStr *C.char + var k, v *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + if len(value) != 0 { + v = (*C.char)(unsafe.Pointer(&value[0])) + } + + lenk := len(key) + lenv := len(value) + C.rocksdb_put( + db.db, wo.Opt, k, C.size_t(lenk), v, C.size_t(lenv), &errStr) + + if errStr != nil { + return saveError(errStr) + } + return nil +} + +func (db *DB) get(ro *ReadOptions, key []byte) ([]byte, error) { + var errStr *C.char + var vallen C.size_t + var k *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + + value := C.rocksdb_get( + db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr) + + if errStr != nil { + return nil, saveError(errStr) + } + + if value == nil { + return nil, nil + } + + defer C.free(unsafe.Pointer(value)) + return C.GoBytes(unsafe.Pointer(value), C.int(vallen)), nil +} + +func (db *DB) getSlice(ro *ReadOptions, key []byte) (driver.ISlice, error) { + var errStr *C.char + var vallen C.size_t + var k *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + + value := C.rocksdb_get( + db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr) + + if errStr != nil { + return nil, saveError(errStr) + } + + if value == nil { + return nil, nil + } + + return NewCSlice(unsafe.Pointer(value), int(vallen)), nil +} + +func (db *DB) delete(wo *WriteOptions, key []byte) error { + var errStr *C.char + var k *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + + C.rocksdb_delete( + db.db, wo.Opt, k, C.size_t(len(key)), &errStr) + + if errStr != nil { + return saveError(errStr) + } + return nil +} + +func (db *DB) Compact() error { + C.rocksdb_compact_range(db.db, nil, 0, nil, 0) + return nil +} + +func (db *DB) GetSlice(key []byte) (driver.ISlice, error) { + return db.getSlice(db.readOpts, key) +} + +func init() { + driver.Register(Store{}) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/rocksdb/env.go b/vendor/github.com/siddontang/ledisdb/store/rocksdb/env.go new file mode 100644 index 000000000000..e239c1b6c0e8 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/rocksdb/env.go @@ -0,0 +1,27 @@ +// +build rocksdb + +package rocksdb + +// #cgo LDFLAGS: -lrocksdb +// #include "rocksdb/c.h" +import "C" + +type Env struct { + Env *C.rocksdb_env_t +} + +func NewDefaultEnv() *Env { + return &Env{C.rocksdb_create_default_env()} +} + +func (env *Env) SetHighPriorityBackgroundThreads(n int) { + C.rocksdb_env_set_high_priority_background_threads(env.Env, C.int(n)) +} + +func (env *Env) SetBackgroundThreads(n int) { + C.rocksdb_env_set_background_threads(env.Env, C.int(n)) +} + +func (env *Env) Close() { + C.rocksdb_env_destroy(env.Env) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/rocksdb/filterpolicy.go b/vendor/github.com/siddontang/ledisdb/store/rocksdb/filterpolicy.go new file mode 100644 index 000000000000..3be4ef6acac7 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/rocksdb/filterpolicy.go @@ -0,0 +1,21 @@ +// +build rocksdb + +package rocksdb + +// #cgo LDFLAGS: -lrocksdb +// #include +// #include "rocksdb/c.h" +import "C" + +type FilterPolicy struct { + Policy *C.rocksdb_filterpolicy_t +} + +func NewBloomFilter(bitsPerKey int) *FilterPolicy { + policy := C.rocksdb_filterpolicy_create_bloom(C.int(bitsPerKey)) + return &FilterPolicy{policy} +} + +func (fp *FilterPolicy) Close() { + C.rocksdb_filterpolicy_destroy(fp.Policy) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/rocksdb/iterator.go b/vendor/github.com/siddontang/ledisdb/store/rocksdb/iterator.go new file mode 100644 index 000000000000..046c5e9dfaa4 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/rocksdb/iterator.go @@ -0,0 +1,70 @@ +// +build rocksdb + +package rocksdb + +// #cgo LDFLAGS: -lrocksdb +// #include +// #include "rocksdb/c.h" +// #include "rocksdb_ext.h" +import "C" + +import ( + "unsafe" +) + +type Iterator struct { + it *C.rocksdb_iterator_t + isValid C.uchar +} + +func (it *Iterator) Key() []byte { + var klen C.size_t + kdata := C.rocksdb_iter_key(it.it, &klen) + if kdata == nil { + return nil + } + + return slice(unsafe.Pointer(kdata), int(C.int(klen))) +} + +func (it *Iterator) Value() []byte { + var vlen C.size_t + vdata := C.rocksdb_iter_value(it.it, &vlen) + if vdata == nil { + return nil + } + + return slice(unsafe.Pointer(vdata), int(C.int(vlen))) +} + +func (it *Iterator) Close() error { + if it.it != nil { + C.rocksdb_iter_destroy(it.it) + it.it = nil + } + return nil +} + +func (it *Iterator) Valid() bool { + return ucharToBool(it.isValid) +} + +func (it *Iterator) Next() { + it.isValid = C.rocksdb_iter_next_ext(it.it) +} + +func (it *Iterator) Prev() { + it.isValid = C.rocksdb_iter_prev_ext(it.it) +} + +func (it *Iterator) First() { + it.isValid = C.rocksdb_iter_seek_to_first_ext(it.it) +} + +func (it *Iterator) Last() { + it.isValid = C.rocksdb_iter_seek_to_last_ext(it.it) +} + +func (it *Iterator) Seek(key []byte) { + it.isValid = C.rocksdb_iter_seek_ext(it.it, (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key))) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/rocksdb/options.go b/vendor/github.com/siddontang/ledisdb/store/rocksdb/options.go new file mode 100644 index 000000000000..48ca2301a49a --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/rocksdb/options.go @@ -0,0 +1,229 @@ +// +build rocksdb + +package rocksdb + +// #cgo LDFLAGS: -lrocksdb +// #include "rocksdb/c.h" +import "C" + +type CompressionOpt int + +const ( + NoCompression = CompressionOpt(0) + SnappyCompression = CompressionOpt(1) + ZlibCompression = CompressionOpt(2) + Bz2Compression = CompressionOpt(3) + Lz4Compression = CompressionOpt(4) + Lz4hcCompression = CompressionOpt(5) +) + +type Options struct { + Opt *C.rocksdb_options_t +} + +type ReadOptions struct { + Opt *C.rocksdb_readoptions_t +} + +type WriteOptions struct { + Opt *C.rocksdb_writeoptions_t +} + +type BlockBasedTableOptions struct { + Opt *C.rocksdb_block_based_table_options_t +} + +func NewOptions() *Options { + opt := C.rocksdb_options_create() + return &Options{opt} +} + +func NewReadOptions() *ReadOptions { + opt := C.rocksdb_readoptions_create() + return &ReadOptions{opt} +} + +func NewWriteOptions() *WriteOptions { + opt := C.rocksdb_writeoptions_create() + return &WriteOptions{opt} +} + +func NewBlockBasedTableOptions() *BlockBasedTableOptions { + opt := C.rocksdb_block_based_options_create() + return &BlockBasedTableOptions{opt} +} + +func (o *Options) Close() { + C.rocksdb_options_destroy(o.Opt) +} + +func (o *Options) IncreaseParallelism(n int) { + C.rocksdb_options_increase_parallelism(o.Opt, C.int(n)) +} + +func (o *Options) OptimizeLevelStyleCompaction(n int) { + C.rocksdb_options_optimize_level_style_compaction(o.Opt, C.uint64_t(n)) +} + +func (o *Options) SetComparator(cmp *C.rocksdb_comparator_t) { + C.rocksdb_options_set_comparator(o.Opt, cmp) +} + +func (o *Options) SetErrorIfExists(error_if_exists bool) { + eie := boolToUchar(error_if_exists) + C.rocksdb_options_set_error_if_exists(o.Opt, eie) +} + +func (o *Options) SetEnv(env *Env) { + C.rocksdb_options_set_env(o.Opt, env.Env) +} + +func (o *Options) SetWriteBufferSize(s int) { + C.rocksdb_options_set_write_buffer_size(o.Opt, C.size_t(s)) +} + +func (o *Options) SetParanoidChecks(pc bool) { + C.rocksdb_options_set_paranoid_checks(o.Opt, boolToUchar(pc)) +} + +func (o *Options) SetMaxOpenFiles(n int) { + C.rocksdb_options_set_max_open_files(o.Opt, C.int(n)) +} + +func (o *Options) SetCompression(t CompressionOpt) { + C.rocksdb_options_set_compression(o.Opt, C.int(t)) +} + +func (o *Options) SetCreateIfMissing(b bool) { + C.rocksdb_options_set_create_if_missing(o.Opt, boolToUchar(b)) +} + +func (o *Options) SetMaxWriteBufferNumber(n int) { + C.rocksdb_options_set_max_write_buffer_number(o.Opt, C.int(n)) +} + +func (o *Options) SetMaxBackgroundCompactions(n int) { + C.rocksdb_options_set_max_background_compactions(o.Opt, C.int(n)) +} + +func (o *Options) SetMaxBackgroundFlushes(n int) { + C.rocksdb_options_set_max_background_flushes(o.Opt, C.int(n)) +} + +func (o *Options) SetNumLevels(n int) { + C.rocksdb_options_set_num_levels(o.Opt, C.int(n)) +} + +func (o *Options) SetLevel0FileNumCompactionTrigger(n int) { + C.rocksdb_options_set_level0_file_num_compaction_trigger(o.Opt, C.int(n)) +} + +func (o *Options) SetLevel0SlowdownWritesTrigger(n int) { + C.rocksdb_options_set_level0_slowdown_writes_trigger(o.Opt, C.int(n)) +} + +func (o *Options) SetLevel0StopWritesTrigger(n int) { + C.rocksdb_options_set_level0_stop_writes_trigger(o.Opt, C.int(n)) +} + +func (o *Options) SetTargetFileSizeBase(n int) { + C.rocksdb_options_set_target_file_size_base(o.Opt, C.uint64_t(uint64(n))) +} + +func (o *Options) SetTargetFileSizeMultiplier(n int) { + C.rocksdb_options_set_target_file_size_multiplier(o.Opt, C.int(n)) +} + +func (o *Options) SetMaxBytesForLevelBase(n int) { + C.rocksdb_options_set_max_bytes_for_level_base(o.Opt, C.uint64_t(uint64(n))) +} + +func (o *Options) SetMaxBytesForLevelMultiplier(n int) { + C.rocksdb_options_set_max_bytes_for_level_multiplier(o.Opt, C.double(n)) +} + +func (o *Options) SetBlockBasedTableFactory(opt *BlockBasedTableOptions) { + C.rocksdb_options_set_block_based_table_factory(o.Opt, opt.Opt) +} + +func (o *Options) SetMinWriteBufferNumberToMerge(n int) { + C.rocksdb_options_set_min_write_buffer_number_to_merge(o.Opt, C.int(n)) +} + +func (o *Options) DisableAutoCompactions(b bool) { + C.rocksdb_options_set_disable_auto_compactions(o.Opt, boolToInt(b)) +} + +func (o *Options) UseFsync(b bool) { + C.rocksdb_options_set_use_fsync(o.Opt, boolToInt(b)) +} + +func (o *Options) EnableStatistics(b bool) { + if b { + C.rocksdb_options_enable_statistics(o.Opt) + } +} + +func (o *Options) SetStatsDumpPeriodSec(n int) { + C.rocksdb_options_set_stats_dump_period_sec(o.Opt, C.uint(n)) +} + +func (o *Options) SetMaxManifestFileSize(n int) { + C.rocksdb_options_set_max_manifest_file_size(o.Opt, C.size_t(n)) +} + +func (o *BlockBasedTableOptions) Close() { + C.rocksdb_block_based_options_destroy(o.Opt) +} + +func (o *BlockBasedTableOptions) SetFilterPolicy(fp *FilterPolicy) { + var policy *C.rocksdb_filterpolicy_t + if fp != nil { + policy = fp.Policy + } + C.rocksdb_block_based_options_set_filter_policy(o.Opt, policy) +} + +func (o *BlockBasedTableOptions) SetBlockSize(s int) { + C.rocksdb_block_based_options_set_block_size(o.Opt, C.size_t(s)) +} + +func (o *BlockBasedTableOptions) SetBlockRestartInterval(n int) { + C.rocksdb_block_based_options_set_block_restart_interval(o.Opt, C.int(n)) +} + +func (o *BlockBasedTableOptions) SetCache(cache *Cache) { + C.rocksdb_block_based_options_set_block_cache(o.Opt, cache.Cache) +} + +func (ro *ReadOptions) Close() { + C.rocksdb_readoptions_destroy(ro.Opt) +} + +func (ro *ReadOptions) SetVerifyChecksums(b bool) { + C.rocksdb_readoptions_set_verify_checksums(ro.Opt, boolToUchar(b)) +} + +func (ro *ReadOptions) SetFillCache(b bool) { + C.rocksdb_readoptions_set_fill_cache(ro.Opt, boolToUchar(b)) +} + +func (ro *ReadOptions) SetSnapshot(snap *Snapshot) { + var s *C.rocksdb_snapshot_t + if snap != nil { + s = snap.snap + } + C.rocksdb_readoptions_set_snapshot(ro.Opt, s) +} + +func (wo *WriteOptions) Close() { + C.rocksdb_writeoptions_destroy(wo.Opt) +} + +func (wo *WriteOptions) SetSync(b bool) { + C.rocksdb_writeoptions_set_sync(wo.Opt, boolToUchar(b)) +} + +func (wo *WriteOptions) DisableWAL(b bool) { + C.rocksdb_writeoptions_disable_WAL(wo.Opt, boolToInt(b)) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/rocksdb/rocksdb_ext.cc b/vendor/github.com/siddontang/ledisdb/store/rocksdb/rocksdb_ext.cc new file mode 100644 index 000000000000..39036ab96cc2 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/rocksdb/rocksdb_ext.cc @@ -0,0 +1,44 @@ +// +build rocksdb + +#include "rocksdb_ext.h" + +#include +#include + +extern "C" { + +unsigned char rocksdb_iter_seek_to_first_ext(rocksdb_iterator_t* iter) { + rocksdb_iter_seek_to_first(iter); + return rocksdb_iter_valid(iter); +} + +unsigned char rocksdb_iter_seek_to_last_ext(rocksdb_iterator_t* iter) { + rocksdb_iter_seek_to_last(iter); + return rocksdb_iter_valid(iter); +} + +unsigned char rocksdb_iter_seek_ext(rocksdb_iterator_t* iter, const char* k, size_t klen) { + rocksdb_iter_seek(iter, k, klen); + return rocksdb_iter_valid(iter); +} + +unsigned char rocksdb_iter_next_ext(rocksdb_iterator_t* iter) { + rocksdb_iter_next(iter); + return rocksdb_iter_valid(iter); +} + +unsigned char rocksdb_iter_prev_ext(rocksdb_iterator_t* iter) { + rocksdb_iter_prev(iter); + return rocksdb_iter_valid(iter); +} + +void rocksdb_write_ext(rocksdb_t* db, + const rocksdb_writeoptions_t* options, + rocksdb_writebatch_t* batch, char** errptr) { + rocksdb_write(db, options, batch, errptr); + if(*errptr == NULL) { + rocksdb_writebatch_clear(batch); + } +} + +} \ No newline at end of file diff --git a/vendor/github.com/siddontang/ledisdb/store/rocksdb/rocksdb_ext.h b/vendor/github.com/siddontang/ledisdb/store/rocksdb/rocksdb_ext.h new file mode 100644 index 000000000000..11cb65304105 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/rocksdb/rocksdb_ext.h @@ -0,0 +1,24 @@ +// +build rocksdb + +#ifndef ROCKSDB_EXT_H +#define ROCKSDB_EXT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "rocksdb/c.h" + +// Below iterator functions like rocksdb iterator but returns valid status for iterator +extern unsigned char rocksdb_iter_seek_to_first_ext(rocksdb_iterator_t*); +extern unsigned char rocksdb_iter_seek_to_last_ext(rocksdb_iterator_t*); +extern unsigned char rocksdb_iter_seek_ext(rocksdb_iterator_t*, const char* k, size_t klen); +extern unsigned char rocksdb_iter_next_ext(rocksdb_iterator_t*); +extern unsigned char rocksdb_iter_prev_ext(rocksdb_iterator_t*); +extern void rocksdb_write_ext(rocksdb_t* db, const rocksdb_writeoptions_t* options, rocksdb_writebatch_t* batch, char** errptr); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/vendor/github.com/siddontang/ledisdb/store/rocksdb/slice.go b/vendor/github.com/siddontang/ledisdb/store/rocksdb/slice.go new file mode 100644 index 000000000000..bbaa65bd7652 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/rocksdb/slice.go @@ -0,0 +1,41 @@ +//+build rocksdb + +package rocksdb + +// #cgo LDFLAGS: -lrocksdb +// #include +// #include +import "C" + +import ( + "reflect" + "unsafe" +) + +type CSlice struct { + data unsafe.Pointer + size int +} + +func NewCSlice(p unsafe.Pointer, n int) *CSlice { + return &CSlice{p, n} +} + +func (s *CSlice) Data() []byte { + var value []byte + + sH := (*reflect.SliceHeader)(unsafe.Pointer(&value)) + sH.Cap = int(s.size) + sH.Len = int(s.size) + sH.Data = uintptr(s.data) + + return value +} + +func (s *CSlice) Size() int { + return int(s.size) +} + +func (s *CSlice) Free() { + C.free(s.data) +} diff --git a/vendor/github.com/siddontang/ledisdb/store/rocksdb/snapshot.go b/vendor/github.com/siddontang/ledisdb/store/rocksdb/snapshot.go new file mode 100644 index 000000000000..1ced60020027 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/rocksdb/snapshot.go @@ -0,0 +1,39 @@ +// +build rocksdb + +package rocksdb + +// #cgo LDFLAGS: -lrocksdb +// #include "rocksdb/c.h" +import "C" + +import ( + "github.com/siddontang/ledisdb/store/driver" +) + +type Snapshot struct { + db *DB + snap *C.rocksdb_snapshot_t + readOpts *ReadOptions + iteratorOpts *ReadOptions +} + +func (s *Snapshot) Get(key []byte) ([]byte, error) { + return s.db.get(s.readOpts, key) +} + +func (s *Snapshot) GetSlice(key []byte) (driver.ISlice, error) { + return s.db.getSlice(s.readOpts, key) +} + +func (s *Snapshot) NewIterator() driver.IIterator { + it := new(Iterator) + it.it = C.rocksdb_create_iterator(s.db.db, s.db.iteratorOpts.Opt) + return it + +} + +func (s *Snapshot) Close() { + C.rocksdb_release_snapshot(s.db.db, s.snap) + s.iteratorOpts.Close() + s.readOpts.Close() +} diff --git a/vendor/github.com/siddontang/ledisdb/store/rocksdb/util.go b/vendor/github.com/siddontang/ledisdb/store/rocksdb/util.go new file mode 100644 index 000000000000..22b73baf4ad4 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/rocksdb/util.go @@ -0,0 +1,54 @@ +// +build rocksdb + +package rocksdb + +// #include +// #include "rocksdb/c.h" +import "C" + +import ( + "fmt" + "reflect" + "unsafe" +) + +func boolToUchar(b bool) C.uchar { + uc := C.uchar(0) + if b { + uc = C.uchar(1) + } + return uc +} + +func ucharToBool(uc C.uchar) bool { + if uc == C.uchar(0) { + return false + } + return true +} + +func boolToInt(b bool) C.int { + uc := C.int(0) + if b { + uc = C.int(1) + } + return uc +} + +func saveError(errStr *C.char) error { + if errStr != nil { + gs := C.GoString(errStr) + C.free(unsafe.Pointer(errStr)) + return fmt.Errorf(gs) + } + return nil +} + +func slice(p unsafe.Pointer, n int) []byte { + var b []byte + pbyte := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + pbyte.Data = uintptr(p) + pbyte.Len = n + pbyte.Cap = n + return b +} diff --git a/vendor/github.com/siddontang/ledisdb/store/slice.go b/vendor/github.com/siddontang/ledisdb/store/slice.go new file mode 100644 index 000000000000..b027f4f28b8e --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/slice.go @@ -0,0 +1,9 @@ +package store + +import ( + "github.com/siddontang/ledisdb/store/driver" +) + +type Slice interface { + driver.ISlice +} diff --git a/vendor/github.com/siddontang/ledisdb/store/snapshot.go b/vendor/github.com/siddontang/ledisdb/store/snapshot.go new file mode 100644 index 000000000000..a1c9de9944e6 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/snapshot.go @@ -0,0 +1,48 @@ +package store + +import ( + "github.com/siddontang/ledisdb/store/driver" +) + +type Snapshot struct { + driver.ISnapshot + st *Stat +} + +func (s *Snapshot) NewIterator() *Iterator { + it := new(Iterator) + it.it = s.ISnapshot.NewIterator() + it.st = s.st + + s.st.IterNum.Add(1) + + return it +} + +func (s *Snapshot) Get(key []byte) ([]byte, error) { + v, err := s.ISnapshot.Get(key) + s.st.statGet(v, err) + return v, err +} + +func (s *Snapshot) GetSlice(key []byte) (Slice, error) { + if d, ok := s.ISnapshot.(driver.ISliceGeter); ok { + v, err := d.GetSlice(key) + s.st.statGet(v, err) + return v, err + } else { + v, err := s.Get(key) + if err != nil { + return nil, err + } else if v == nil { + return nil, nil + } else { + return driver.GoSlice(v), nil + } + } +} + +func (s *Snapshot) Close() { + s.st.SnapshotCloseNum.Add(1) + s.ISnapshot.Close() +} diff --git a/vendor/github.com/siddontang/ledisdb/store/stat.go b/vendor/github.com/siddontang/ledisdb/store/stat.go new file mode 100644 index 000000000000..e0a035ab8e18 --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/stat.go @@ -0,0 +1,37 @@ +package store + +import ( + "github.com/siddontang/go/sync2" +) + +type Stat struct { + GetNum sync2.AtomicInt64 + GetMissingNum sync2.AtomicInt64 + GetTotalTime sync2.AtomicDuration + PutNum sync2.AtomicInt64 + DeleteNum sync2.AtomicInt64 + IterNum sync2.AtomicInt64 + IterSeekNum sync2.AtomicInt64 + IterCloseNum sync2.AtomicInt64 + SnapshotNum sync2.AtomicInt64 + SnapshotCloseNum sync2.AtomicInt64 + BatchNum sync2.AtomicInt64 + BatchCommitNum sync2.AtomicInt64 + BatchCommitTotalTime sync2.AtomicDuration + TxNum sync2.AtomicInt64 + TxCommitNum sync2.AtomicInt64 + TxCloseNum sync2.AtomicInt64 + CompactNum sync2.AtomicInt64 + CompactTotalTime sync2.AtomicDuration +} + +func (st *Stat) statGet(v interface{}, err error) { + st.GetNum.Add(1) + if v == nil && err == nil { + st.GetMissingNum.Add(1) + } +} + +func (st *Stat) Reset() { + *st = Stat{} +} diff --git a/vendor/github.com/siddontang/ledisdb/store/store.go b/vendor/github.com/siddontang/ledisdb/store/store.go new file mode 100644 index 000000000000..1352491254dc --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/store.go @@ -0,0 +1,62 @@ +package store + +import ( + "fmt" + "os" + "path" + + "github.com/siddontang/ledisdb/config" + "github.com/siddontang/ledisdb/store/driver" + + _ "github.com/siddontang/ledisdb/store/goleveldb" + _ "github.com/siddontang/ledisdb/store/leveldb" + _ "github.com/siddontang/ledisdb/store/rocksdb" +) + +func getStorePath(cfg *config.Config) string { + if len(cfg.DBPath) > 0 { + return cfg.DBPath + } else { + return path.Join(cfg.DataDir, fmt.Sprintf("%s_data", cfg.DBName)) + } +} + +func Open(cfg *config.Config) (*DB, error) { + s, err := driver.GetStore(cfg) + if err != nil { + return nil, err + } + + path := getStorePath(cfg) + + if err := os.MkdirAll(path, 0755); err != nil { + return nil, err + } + + idb, err := s.Open(path, cfg) + if err != nil { + return nil, err + } + + db := new(DB) + db.db = idb + db.name = s.String() + db.st = &Stat{} + db.cfg = cfg + + return db, nil +} + +func Repair(cfg *config.Config) error { + s, err := driver.GetStore(cfg) + if err != nil { + return err + } + + path := getStorePath(cfg) + + return s.Repair(path, cfg) +} + +func init() { +} diff --git a/vendor/github.com/siddontang/ledisdb/store/writebatch.go b/vendor/github.com/siddontang/ledisdb/store/writebatch.go new file mode 100644 index 000000000000..73760d719c3c --- /dev/null +++ b/vendor/github.com/siddontang/ledisdb/store/writebatch.go @@ -0,0 +1,136 @@ +package store + +import ( + "time" + + "github.com/siddontang/ledisdb/store/driver" + "github.com/syndtr/goleveldb/leveldb" +) + +type WriteBatch struct { + wb driver.IWriteBatch + st *Stat + + putNum int64 + deleteNum int64 + db *DB + + data *BatchData +} + +func (wb *WriteBatch) Close() { + wb.wb.Close() +} + +func (wb *WriteBatch) Put(key []byte, value []byte) { + wb.putNum++ + wb.wb.Put(key, value) +} + +func (wb *WriteBatch) Delete(key []byte) { + wb.deleteNum++ + wb.wb.Delete(key) +} + +func (wb *WriteBatch) Commit() error { + wb.st.BatchCommitNum.Add(1) + wb.st.PutNum.Add(wb.putNum) + wb.st.DeleteNum.Add(wb.deleteNum) + wb.putNum = 0 + wb.deleteNum = 0 + + var err error + t := time.Now() + if wb.db == nil || !wb.db.needSyncCommit() { + err = wb.wb.Commit() + } else { + err = wb.wb.SyncCommit() + } + + wb.st.BatchCommitTotalTime.Add(time.Now().Sub(t)) + + return err +} + +func (wb *WriteBatch) Rollback() error { + wb.putNum = 0 + wb.deleteNum = 0 + + return wb.wb.Rollback() +} + +// the data will be undefined after commit or rollback +func (wb *WriteBatch) BatchData() *BatchData { + data := wb.wb.Data() + if wb.data == nil { + wb.data = new(BatchData) + } + + wb.data.Load(data) + return wb.data +} + +func (wb *WriteBatch) Data() []byte { + b := wb.BatchData() + return b.Data() +} + +/* + see leveldb batch data format for more information +*/ + +type BatchData struct { + leveldb.Batch +} + +func NewBatchData(data []byte) (*BatchData, error) { + b := new(BatchData) + + if err := b.Load(data); err != nil { + return nil, err + } + + return b, nil +} + +func (d *BatchData) Data() []byte { + return d.Dump() +} + +func (d *BatchData) Reset() { + d.Batch.Reset() +} + +type BatchDataReplay interface { + Put(key, value []byte) + Delete(key []byte) +} + +type BatchItem struct { + Key []byte + Value []byte +} + +type batchItems []BatchItem + +func (bs *batchItems) Put(key, value []byte) { + *bs = append(*bs, BatchItem{key, value}) +} + +func (bs *batchItems) Delete(key []byte) { + *bs = append(*bs, BatchItem{key, nil}) +} + +func (d *BatchData) Replay(r BatchDataReplay) error { + return d.Batch.Replay(r) +} + +func (d *BatchData) Items() ([]BatchItem, error) { + is := make(batchItems, 0, d.Len()) + + if err := d.Replay(&is); err != nil { + return nil, err + } + + return []BatchItem(is), nil +} diff --git a/vendor/github.com/siddontang/rdb/LICENSE b/vendor/github.com/siddontang/rdb/LICENSE new file mode 100644 index 000000000000..c16e3affbd3d --- /dev/null +++ b/vendor/github.com/siddontang/rdb/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 siddontang + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/siddontang/rdb/README.md b/vendor/github.com/siddontang/rdb/README.md new file mode 100644 index 000000000000..aa1582e76e11 --- /dev/null +++ b/vendor/github.com/siddontang/rdb/README.md @@ -0,0 +1,3 @@ +# rdb + +Handling Redis RDB format. diff --git a/vendor/github.com/siddontang/rdb/decode.go b/vendor/github.com/siddontang/rdb/decode.go new file mode 100644 index 000000000000..865d24198120 --- /dev/null +++ b/vendor/github.com/siddontang/rdb/decode.go @@ -0,0 +1,128 @@ +package rdb + +// Copyright 2014 Wandoujia Inc. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +import "fmt" + +import ( + "github.com/cupcake/rdb" + "github.com/cupcake/rdb/nopdecoder" +) + +func DecodeDump(p []byte) (interface{}, error) { + d := &decoder{} + if err := rdb.DecodeDump(p, 0, nil, 0, d); err != nil { + return nil, err + } + return d.obj, d.err +} + +type decoder struct { + nopdecoder.NopDecoder + obj interface{} + err error +} + +func (d *decoder) initObject(obj interface{}) { + if d.err != nil { + return + } + if d.obj != nil { + d.err = fmt.Errorf("invalid object, init again") + } else { + d.obj = obj + } +} + +func (d *decoder) Set(key, value []byte, expiry int64) { + d.initObject(String(value)) +} + +func (d *decoder) StartHash(key []byte, length, expiry int64) { + d.initObject(Hash(nil)) +} + +func (d *decoder) Hset(key, field, value []byte) { + if d.err != nil { + return + } + switch h := d.obj.(type) { + default: + d.err = fmt.Errorf("invalid object, not a hashmap") + case Hash: + v := struct { + Field, Value []byte + }{ + field, + value, + } + d.obj = append(h, v) + } +} + +func (d *decoder) StartSet(key []byte, cardinality, expiry int64) { + d.initObject(Set(nil)) +} + +func (d *decoder) Sadd(key, member []byte) { + if d.err != nil { + return + } + switch s := d.obj.(type) { + default: + d.err = fmt.Errorf("invalid object, not a set") + case Set: + d.obj = append(s, member) + } +} + +func (d *decoder) StartList(key []byte, length, expiry int64) { + d.initObject(List(nil)) +} + +func (d *decoder) Rpush(key, value []byte) { + if d.err != nil { + return + } + switch l := d.obj.(type) { + default: + d.err = fmt.Errorf("invalid object, not a list") + case List: + d.obj = append(l, value) + } +} + +func (d *decoder) StartZSet(key []byte, cardinality, expiry int64) { + d.initObject(ZSet(nil)) +} + +func (d *decoder) Zadd(key []byte, score float64, member []byte) { + if d.err != nil { + return + } + switch z := d.obj.(type) { + default: + d.err = fmt.Errorf("invalid object, not a zset") + case ZSet: + v := struct { + Member []byte + Score float64 + }{ + member, + score, + } + d.obj = append(z, v) + } +} + +type String []byte +type List [][]byte +type Hash []struct { + Field, Value []byte +} +type Set [][]byte +type ZSet []struct { + Member []byte + Score float64 +} diff --git a/vendor/github.com/siddontang/rdb/digest.go b/vendor/github.com/siddontang/rdb/digest.go new file mode 100644 index 000000000000..b59e4dfb7d23 --- /dev/null +++ b/vendor/github.com/siddontang/rdb/digest.go @@ -0,0 +1,106 @@ +// Copyright 2014 Wandoujia Inc. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "encoding/binary" + "hash" +) + +var crc64_table = [256]uint64{ + 0x0000000000000000, 0x7ad870c830358979, 0xf5b0e190606b12f2, 0x8f689158505e9b8b, + 0xc038e5739841b68f, 0xbae095bba8743ff6, 0x358804e3f82aa47d, 0x4f50742bc81f2d04, + 0xab28ecb46814fe75, 0xd1f09c7c5821770c, 0x5e980d24087fec87, 0x24407dec384a65fe, + 0x6b1009c7f05548fa, 0x11c8790fc060c183, 0x9ea0e857903e5a08, 0xe478989fa00bd371, + 0x7d08ff3b88be6f81, 0x07d08ff3b88be6f8, 0x88b81eabe8d57d73, 0xf2606e63d8e0f40a, + 0xbd301a4810ffd90e, 0xc7e86a8020ca5077, 0x4880fbd87094cbfc, 0x32588b1040a14285, + 0xd620138fe0aa91f4, 0xacf86347d09f188d, 0x2390f21f80c18306, 0x594882d7b0f40a7f, + 0x1618f6fc78eb277b, 0x6cc0863448deae02, 0xe3a8176c18803589, 0x997067a428b5bcf0, + 0xfa11fe77117cdf02, 0x80c98ebf2149567b, 0x0fa11fe77117cdf0, 0x75796f2f41224489, + 0x3a291b04893d698d, 0x40f16bccb908e0f4, 0xcf99fa94e9567b7f, 0xb5418a5cd963f206, + 0x513912c379682177, 0x2be1620b495da80e, 0xa489f35319033385, 0xde51839b2936bafc, + 0x9101f7b0e12997f8, 0xebd98778d11c1e81, 0x64b116208142850a, 0x1e6966e8b1770c73, + 0x8719014c99c2b083, 0xfdc17184a9f739fa, 0x72a9e0dcf9a9a271, 0x08719014c99c2b08, + 0x4721e43f0183060c, 0x3df994f731b68f75, 0xb29105af61e814fe, 0xc849756751dd9d87, + 0x2c31edf8f1d64ef6, 0x56e99d30c1e3c78f, 0xd9810c6891bd5c04, 0xa3597ca0a188d57d, + 0xec09088b6997f879, 0x96d1784359a27100, 0x19b9e91b09fcea8b, 0x636199d339c963f2, + 0xdf7adabd7a6e2d6f, 0xa5a2aa754a5ba416, 0x2aca3b2d1a053f9d, 0x50124be52a30b6e4, + 0x1f423fcee22f9be0, 0x659a4f06d21a1299, 0xeaf2de5e82448912, 0x902aae96b271006b, + 0x74523609127ad31a, 0x0e8a46c1224f5a63, 0x81e2d7997211c1e8, 0xfb3aa75142244891, + 0xb46ad37a8a3b6595, 0xceb2a3b2ba0eecec, 0x41da32eaea507767, 0x3b024222da65fe1e, + 0xa2722586f2d042ee, 0xd8aa554ec2e5cb97, 0x57c2c41692bb501c, 0x2d1ab4dea28ed965, + 0x624ac0f56a91f461, 0x1892b03d5aa47d18, 0x97fa21650afae693, 0xed2251ad3acf6fea, + 0x095ac9329ac4bc9b, 0x7382b9faaaf135e2, 0xfcea28a2faafae69, 0x8632586aca9a2710, + 0xc9622c4102850a14, 0xb3ba5c8932b0836d, 0x3cd2cdd162ee18e6, 0x460abd1952db919f, + 0x256b24ca6b12f26d, 0x5fb354025b277b14, 0xd0dbc55a0b79e09f, 0xaa03b5923b4c69e6, + 0xe553c1b9f35344e2, 0x9f8bb171c366cd9b, 0x10e3202993385610, 0x6a3b50e1a30ddf69, + 0x8e43c87e03060c18, 0xf49bb8b633338561, 0x7bf329ee636d1eea, 0x012b592653589793, + 0x4e7b2d0d9b47ba97, 0x34a35dc5ab7233ee, 0xbbcbcc9dfb2ca865, 0xc113bc55cb19211c, + 0x5863dbf1e3ac9dec, 0x22bbab39d3991495, 0xadd33a6183c78f1e, 0xd70b4aa9b3f20667, + 0x985b3e827bed2b63, 0xe2834e4a4bd8a21a, 0x6debdf121b863991, 0x1733afda2bb3b0e8, + 0xf34b37458bb86399, 0x8993478dbb8deae0, 0x06fbd6d5ebd3716b, 0x7c23a61ddbe6f812, + 0x3373d23613f9d516, 0x49aba2fe23cc5c6f, 0xc6c333a67392c7e4, 0xbc1b436e43a74e9d, + 0x95ac9329ac4bc9b5, 0xef74e3e19c7e40cc, 0x601c72b9cc20db47, 0x1ac40271fc15523e, + 0x5594765a340a7f3a, 0x2f4c0692043ff643, 0xa02497ca54616dc8, 0xdafce7026454e4b1, + 0x3e847f9dc45f37c0, 0x445c0f55f46abeb9, 0xcb349e0da4342532, 0xb1eceec59401ac4b, + 0xfebc9aee5c1e814f, 0x8464ea266c2b0836, 0x0b0c7b7e3c7593bd, 0x71d40bb60c401ac4, + 0xe8a46c1224f5a634, 0x927c1cda14c02f4d, 0x1d148d82449eb4c6, 0x67ccfd4a74ab3dbf, + 0x289c8961bcb410bb, 0x5244f9a98c8199c2, 0xdd2c68f1dcdf0249, 0xa7f41839ecea8b30, + 0x438c80a64ce15841, 0x3954f06e7cd4d138, 0xb63c61362c8a4ab3, 0xcce411fe1cbfc3ca, + 0x83b465d5d4a0eece, 0xf96c151de49567b7, 0x76048445b4cbfc3c, 0x0cdcf48d84fe7545, + 0x6fbd6d5ebd3716b7, 0x15651d968d029fce, 0x9a0d8ccedd5c0445, 0xe0d5fc06ed698d3c, + 0xaf85882d2576a038, 0xd55df8e515432941, 0x5a3569bd451db2ca, 0x20ed197575283bb3, + 0xc49581ead523e8c2, 0xbe4df122e51661bb, 0x3125607ab548fa30, 0x4bfd10b2857d7349, + 0x04ad64994d625e4d, 0x7e7514517d57d734, 0xf11d85092d094cbf, 0x8bc5f5c11d3cc5c6, + 0x12b5926535897936, 0x686de2ad05bcf04f, 0xe70573f555e26bc4, 0x9ddd033d65d7e2bd, + 0xd28d7716adc8cfb9, 0xa85507de9dfd46c0, 0x273d9686cda3dd4b, 0x5de5e64efd965432, + 0xb99d7ed15d9d8743, 0xc3450e196da80e3a, 0x4c2d9f413df695b1, 0x36f5ef890dc31cc8, + 0x79a59ba2c5dc31cc, 0x037deb6af5e9b8b5, 0x8c157a32a5b7233e, 0xf6cd0afa9582aa47, + 0x4ad64994d625e4da, 0x300e395ce6106da3, 0xbf66a804b64ef628, 0xc5bed8cc867b7f51, + 0x8aeeace74e645255, 0xf036dc2f7e51db2c, 0x7f5e4d772e0f40a7, 0x05863dbf1e3ac9de, + 0xe1fea520be311aaf, 0x9b26d5e88e0493d6, 0x144e44b0de5a085d, 0x6e963478ee6f8124, + 0x21c640532670ac20, 0x5b1e309b16452559, 0xd476a1c3461bbed2, 0xaeaed10b762e37ab, + 0x37deb6af5e9b8b5b, 0x4d06c6676eae0222, 0xc26e573f3ef099a9, 0xb8b627f70ec510d0, + 0xf7e653dcc6da3dd4, 0x8d3e2314f6efb4ad, 0x0256b24ca6b12f26, 0x788ec2849684a65f, + 0x9cf65a1b368f752e, 0xe62e2ad306bafc57, 0x6946bb8b56e467dc, 0x139ecb4366d1eea5, + 0x5ccebf68aecec3a1, 0x2616cfa09efb4ad8, 0xa97e5ef8cea5d153, 0xd3a62e30fe90582a, + 0xb0c7b7e3c7593bd8, 0xca1fc72bf76cb2a1, 0x45775673a732292a, 0x3faf26bb9707a053, + 0x70ff52905f188d57, 0x0a2722586f2d042e, 0x854fb3003f739fa5, 0xff97c3c80f4616dc, + 0x1bef5b57af4dc5ad, 0x61372b9f9f784cd4, 0xee5fbac7cf26d75f, 0x9487ca0fff135e26, + 0xdbd7be24370c7322, 0xa10fceec0739fa5b, 0x2e675fb4576761d0, 0x54bf2f7c6752e8a9, + 0xcdcf48d84fe75459, 0xb71738107fd2dd20, 0x387fa9482f8c46ab, 0x42a7d9801fb9cfd2, + 0x0df7adabd7a6e2d6, 0x772fdd63e7936baf, 0xf8474c3bb7cdf024, 0x829f3cf387f8795d, + 0x66e7a46c27f3aa2c, 0x1c3fd4a417c62355, 0x935745fc4798b8de, 0xe98f353477ad31a7, + 0xa6df411fbfb21ca3, 0xdc0731d78f8795da, 0x536fa08fdfd90e51, 0x29b7d047efec8728} + +type digest struct { + crc uint64 +} + +func (d *digest) update(p []byte) { + for _, b := range p { + d.crc = crc64_table[byte(d.crc)^b] ^ (d.crc >> 8) + } +} + +func newDigest() hash.Hash64 { + d := &digest{} + return d +} + +func (d *digest) Write(p []byte) (int, error) { + d.update(p) + return len(p), nil +} + +func (d *digest) Sum(in []byte) []byte { + buf := make([]byte, 8) + binary.LittleEndian.PutUint64(buf, d.crc) + return append(in, buf...) +} + +func (d *digest) Sum64() uint64 { return d.crc } +func (d *digest) BlockSize() int { return 1 } +func (d *digest) Size() int { return 8 } +func (d *digest) Reset() { d.crc = 0 } diff --git a/vendor/github.com/siddontang/rdb/encode.go b/vendor/github.com/siddontang/rdb/encode.go new file mode 100644 index 000000000000..19c48c331fab --- /dev/null +++ b/vendor/github.com/siddontang/rdb/encode.go @@ -0,0 +1,52 @@ +package rdb + +import ( + "bytes" + "fmt" + "github.com/cupcake/rdb" +) + +func Dump(obj interface{}) ([]byte, error) { + var buf bytes.Buffer + + e := rdb.NewEncoder(&buf) + + switch v := obj.(type) { + case String: + e.EncodeType(rdb.TypeString) + e.EncodeString(v) + case Hash: + e.EncodeType(rdb.TypeHash) + e.EncodeLength(uint32(len(v))) + + for i := 0; i < len(v); i++ { + e.EncodeString(v[i].Field) + e.EncodeString(v[i].Value) + } + case List: + e.EncodeType(rdb.TypeList) + e.EncodeLength(uint32(len(v))) + for i := 0; i < len(v); i++ { + e.EncodeString(v[i]) + } + case Set: + e.EncodeType(rdb.TypeSet) + e.EncodeLength(uint32(len(v))) + for i := 0; i < len(v); i++ { + e.EncodeString(v[i]) + } + case ZSet: + e.EncodeType(rdb.TypeZSet) + e.EncodeLength(uint32(len(v))) + for i := 0; i < len(v); i++ { + e.EncodeString(v[i].Member) + e.EncodeFloat(v[i].Score) + } + default: + return nil, fmt.Errorf("invalid dump type %T", obj) + } + + e.EncodeDumpFooter() + + return buf.Bytes(), nil +} diff --git a/vendor/github.com/siddontang/rdb/loader.go b/vendor/github.com/siddontang/rdb/loader.go new file mode 100644 index 000000000000..22743cbdebda --- /dev/null +++ b/vendor/github.com/siddontang/rdb/loader.go @@ -0,0 +1,112 @@ +// Copyright 2014 Wandoujia Inc. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "bytes" + "encoding/binary" + "fmt" + "hash" + "io" + "strconv" +) + +type Loader struct { + *rdbReader + crc hash.Hash64 + db uint32 +} + +func NewLoader(r io.Reader) *Loader { + l := &Loader{} + l.crc = newDigest() + l.rdbReader = newRdbReader(io.TeeReader(r, l.crc)) + return l +} + +func (l *Loader) LoadHeader() error { + header := make([]byte, 9) + if err := l.readFull(header); err != nil { + return err + } + if !bytes.Equal(header[:5], []byte("REDIS")) { + return fmt.Errorf("verify magic string, invalid file format") + } + if version, err := strconv.ParseInt(string(header[5:]), 10, 64); err != nil { + return err + } else if version <= 0 || version > Version { + return fmt.Errorf("verify version, invalid RDB version number %d", version) + } + return nil +} + +func (l *Loader) LoadChecksum() error { + crc1 := l.crc.Sum64() + if crc2, err := l.readUint64(); err != nil { + return err + } else if crc1 != crc2 { + return fmt.Errorf("checksum validation failed") + } + return nil +} + +type Entry struct { + DB uint32 + Key []byte + ValDump []byte + ExpireAt uint64 +} + +func (l *Loader) LoadEntry() (entry *Entry, err error) { + var expireat uint64 + for { + var otype byte + if otype, err = l.readByte(); err != nil { + return + } + switch otype { + case rdbFlagExpiryMS: + if expireat, err = l.readUint64(); err != nil { + return + } + case rdbFlagExpiry: + var sec uint32 + if sec, err = l.readUint32(); err != nil { + return + } + expireat = uint64(sec) * 1000 + case rdbFlagSelectDB: + if l.db, err = l.readLength(); err != nil { + return + } + case rdbFlagEOF: + return + default: + var key, obj []byte + if key, err = l.readString(); err != nil { + return + } + if obj, err = l.readObject(otype); err != nil { + return + } + entry = &Entry{} + entry.DB = l.db + entry.Key = key + entry.ValDump = createValDump(otype, obj) + entry.ExpireAt = expireat + return + } + } +} + +func createValDump(otype byte, obj []byte) []byte { + var b bytes.Buffer + c := newDigest() + w := io.MultiWriter(&b, c) + w.Write([]byte{otype}) + w.Write(obj) + binary.Write(w, binary.LittleEndian, uint16(Version)) + binary.Write(w, binary.LittleEndian, c.Sum64()) + return b.Bytes() +} diff --git a/vendor/github.com/siddontang/rdb/reader.go b/vendor/github.com/siddontang/rdb/reader.go new file mode 100644 index 000000000000..89ae9ed18121 --- /dev/null +++ b/vendor/github.com/siddontang/rdb/reader.go @@ -0,0 +1,332 @@ +// Copyright 2014 Wandoujia Inc. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math" + "strconv" +) + +const ( + Version = 6 +) + +const ( + rdbTypeString = 0 + rdbTypeList = 1 + rdbTypeSet = 2 + rdbTypeZSet = 3 + rdbTypeHash = 4 + + rdbTypeHashZipmap = 9 + rdbTypeListZiplist = 10 + rdbTypeSetIntset = 11 + rdbTypeZSetZiplist = 12 + rdbTypeHashZiplist = 13 + + rdbFlagExpiryMS = 0xfc + rdbFlagExpiry = 0xfd + rdbFlagSelectDB = 0xfe + rdbFlagEOF = 0xff +) + +const ( + rdb6bitLen = 0 + rdb14bitLen = 1 + rdb32bitLen = 2 + rdbEncVal = 3 + + rdbEncInt8 = 0 + rdbEncInt16 = 1 + rdbEncInt32 = 2 + rdbEncLZF = 3 + + rdbZiplist6bitlenString = 0 + rdbZiplist14bitlenString = 1 + rdbZiplist32bitlenString = 2 + + rdbZiplistInt16 = 0xc0 + rdbZiplistInt32 = 0xd0 + rdbZiplistInt64 = 0xe0 + rdbZiplistInt24 = 0xf0 + rdbZiplistInt8 = 0xfe + rdbZiplistInt4 = 15 +) + +type rdbReader struct { + raw io.Reader + buf [8]byte + nread int64 +} + +func newRdbReader(r io.Reader) *rdbReader { + return &rdbReader{raw: r} +} + +func (r *rdbReader) Read(p []byte) (int, error) { + n, err := r.raw.Read(p) + r.nread += int64(n) + return n, err +} + +func (r *rdbReader) offset() int64 { + return r.nread +} + +func (r *rdbReader) readObject(otype byte) ([]byte, error) { + var b bytes.Buffer + r = newRdbReader(io.TeeReader(r, &b)) + switch otype { + default: + return nil, fmt.Errorf("unknown object-type %02x", otype) + case rdbTypeHashZipmap: + fallthrough + case rdbTypeListZiplist: + fallthrough + case rdbTypeSetIntset: + fallthrough + case rdbTypeZSetZiplist: + fallthrough + case rdbTypeHashZiplist: + fallthrough + case rdbTypeString: + if _, err := r.readString(); err != nil { + return nil, err + } + case rdbTypeList, rdbTypeSet: + if n, err := r.readLength(); err != nil { + return nil, err + } else { + for i := 0; i < int(n); i++ { + if _, err := r.readString(); err != nil { + return nil, err + } + } + } + case rdbTypeZSet: + if n, err := r.readLength(); err != nil { + return nil, err + } else { + for i := 0; i < int(n); i++ { + if _, err := r.readString(); err != nil { + return nil, err + } + if _, err := r.readFloat(); err != nil { + return nil, err + } + } + } + case rdbTypeHash: + if n, err := r.readLength(); err != nil { + return nil, err + } else { + for i := 0; i < int(n); i++ { + if _, err := r.readString(); err != nil { + return nil, err + } + if _, err := r.readString(); err != nil { + return nil, err + } + } + } + } + return b.Bytes(), nil +} + +func (r *rdbReader) readString() ([]byte, error) { + length, encoded, err := r.readEncodedLength() + if err != nil { + return nil, err + } + if !encoded { + return r.readBytes(int(length)) + } + switch t := uint8(length); t { + default: + return nil, fmt.Errorf("invalid encoded-string %02x", t) + case rdbEncInt8: + i, err := r.readInt8() + return []byte(strconv.FormatInt(int64(i), 10)), err + case rdbEncInt16: + i, err := r.readInt16() + return []byte(strconv.FormatInt(int64(i), 10)), err + case rdbEncInt32: + i, err := r.readInt32() + return []byte(strconv.FormatInt(int64(i), 10)), err + case rdbEncLZF: + var inlen, outlen uint32 + if inlen, err = r.readLength(); err != nil { + return nil, err + } + if outlen, err = r.readLength(); err != nil { + return nil, err + } + if in, err := r.readBytes(int(inlen)); err != nil { + return nil, err + } else { + return lzfDecompress(in, int(outlen)) + } + } +} + +func (r *rdbReader) readEncodedLength() (length uint32, encoded bool, err error) { + var u uint8 + if u, err = r.readUint8(); err != nil { + return + } + length = uint32(u & 0x3f) + switch u >> 6 { + case rdb6bitLen: + case rdb14bitLen: + u, err = r.readUint8() + length = (length << 8) + uint32(u) + case rdbEncVal: + encoded = true + default: + length, err = r.readUint32BigEndian() + } + return +} + +func (r *rdbReader) readLength() (uint32, error) { + length, encoded, err := r.readEncodedLength() + if err == nil && encoded { + err = fmt.Errorf("encoded-length") + } + return length, err +} + +func (r *rdbReader) readFloat() (float64, error) { + u, err := r.readUint8() + if err != nil { + return 0, err + } + switch u { + case 253: + return math.NaN(), nil + case 254: + return math.Inf(0), nil + case 255: + return math.Inf(-1), nil + default: + if b, err := r.readBytes(int(u)); err != nil { + return 0, err + } else { + v, err := strconv.ParseFloat(string(b), 64) + return v, err + } + } +} + +func (r *rdbReader) readByte() (byte, error) { + b := r.buf[:1] + _, err := r.Read(b) + return b[0], err +} + +func (r *rdbReader) readFull(p []byte) error { + _, err := io.ReadFull(r, p) + return err +} + +func (r *rdbReader) readBytes(n int) ([]byte, error) { + p := make([]byte, n) + return p, r.readFull(p) +} + +func (r *rdbReader) readUint8() (uint8, error) { + b, err := r.readByte() + return uint8(b), err +} + +func (r *rdbReader) readUint16() (uint16, error) { + b := r.buf[:2] + err := r.readFull(b) + return binary.LittleEndian.Uint16(b), err +} + +func (r *rdbReader) readUint32() (uint32, error) { + b := r.buf[:4] + err := r.readFull(b) + return binary.LittleEndian.Uint32(b), err +} + +func (r *rdbReader) readUint64() (uint64, error) { + b := r.buf[:8] + err := r.readFull(b) + return binary.LittleEndian.Uint64(b), err +} + +func (r *rdbReader) readUint32BigEndian() (uint32, error) { + b := r.buf[:4] + err := r.readFull(b) + return binary.BigEndian.Uint32(b), err +} + +func (r *rdbReader) readInt8() (int8, error) { + u, err := r.readUint8() + return int8(u), err +} + +func (r *rdbReader) readInt16() (int16, error) { + u, err := r.readUint16() + return int16(u), err +} + +func (r *rdbReader) readInt32() (int32, error) { + u, err := r.readUint32() + return int32(u), err +} + +func (r *rdbReader) readInt64() (int64, error) { + u, err := r.readUint64() + return int64(u), err +} + +func (r *rdbReader) readInt32BigEndian() (int32, error) { + u, err := r.readUint32BigEndian() + return int32(u), err +} + +func lzfDecompress(in []byte, outlen int) (out []byte, err error) { + defer func() { + if x := recover(); x != nil { + err = fmt.Errorf("decompress exception: %v", x) + } + }() + out = make([]byte, outlen) + i, o := 0, 0 + for i < len(in) { + ctrl := int(in[i]) + i++ + if ctrl < 32 { + for x := 0; x <= ctrl; x++ { + out[o] = in[i] + i++ + o++ + } + } else { + length := ctrl >> 5 + if length == 7 { + length = length + int(in[i]) + i++ + } + ref := o - ((ctrl & 0x1f) << 8) - int(in[i]) - 1 + i++ + for x := 0; x <= length+1; x++ { + out[o] = out[ref] + ref++ + o++ + } + } + } + if o != outlen { + return nil, fmt.Errorf("decompress length is %d != expected %d", o, outlen) + } + return out, nil +} diff --git a/vendor/github.com/siddontang/rdb/wandoujia-license b/vendor/github.com/siddontang/rdb/wandoujia-license new file mode 100644 index 000000000000..23320dcb431c --- /dev/null +++ b/vendor/github.com/siddontang/rdb/wandoujia-license @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Wandoujia Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/modules.txt b/vendor/modules.txt index 136d828e944c..20a0e4617843 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -14,6 +14,7 @@ gitea.com/go-chi/binding # gitea.com/go-chi/cache v0.0.0-20210110083709-82c4c9ce2d5e ## explicit gitea.com/go-chi/cache +gitea.com/go-chi/cache/ledis gitea.com/go-chi/cache/memcache # gitea.com/go-chi/captcha v0.0.0-20210110083842-e7696c336a1e ## explicit @@ -215,6 +216,10 @@ github.com/couchbase/goutils/logging github.com/couchbase/goutils/scramsha # github.com/cpuguy83/go-md2man/v2 v2.0.0 github.com/cpuguy83/go-md2man/v2/md2man +# github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76 +github.com/cupcake/rdb +github.com/cupcake/rdb/crc64 +github.com/cupcake/rdb/nopdecoder # github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 github.com/danwakefield/fnmatch # github.com/davecgh/go-spew v1.1.1 @@ -254,6 +259,8 @@ github.com/dustin/go-humanize # github.com/editorconfig/editorconfig-core-go/v2 v2.4.2 ## explicit github.com/editorconfig/editorconfig-core-go/v2 +# github.com/edsrzf/mmap-go v1.0.0 +github.com/edsrzf/mmap-go # github.com/emirpasic/gods v1.12.0 ## explicit github.com/emirpasic/gods/containers @@ -723,6 +730,25 @@ github.com/shurcooL/httpfs/vfsutil # github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 ## explicit github.com/shurcooL/vfsgen +# github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 +github.com/siddontang/go/filelock +github.com/siddontang/go/hack +github.com/siddontang/go/ioutil2 +github.com/siddontang/go/log +github.com/siddontang/go/num +github.com/siddontang/go/snappy +github.com/siddontang/go/sync2 +# github.com/siddontang/ledisdb v0.0.0-20190202134119-8ceb77e66a92 +github.com/siddontang/ledisdb/config +github.com/siddontang/ledisdb/ledis +github.com/siddontang/ledisdb/rpl +github.com/siddontang/ledisdb/store +github.com/siddontang/ledisdb/store/driver +github.com/siddontang/ledisdb/store/goleveldb +github.com/siddontang/ledisdb/store/leveldb +github.com/siddontang/ledisdb/store/rocksdb +# github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d +github.com/siddontang/rdb # github.com/sirupsen/logrus v1.8.1 github.com/sirupsen/logrus # github.com/spf13/afero v1.6.0