From 84de0ba2fbc499c33f2d87cdd6e130b895af044e Mon Sep 17 00:00:00 2001 From: qiuyesuifeng Date: Wed, 9 Sep 2015 18:30:36 +0800 Subject: [PATCH 1/6] codec, mysqldef: add decimal codec support. support decimal type index. --- mysqldef/decimal.go | 134 +++++++++++++++++++++++++++++++++++++++ mysqldef/decimal_test.go | 32 ++++++++++ util/codec/codec.go | 11 ++++ util/codec/codec_test.go | 75 ++++++++++++++++++++++ 4 files changed, 252 insertions(+) diff --git a/mysqldef/decimal.go b/mysqldef/decimal.go index 535465a80f3c..d8b29838cde8 100644 --- a/mysqldef/decimal.go +++ b/mysqldef/decimal.go @@ -78,7 +78,9 @@ package mysqldef // after the decimal point. import ( + "bytes" "database/sql/driver" + "encoding/binary" "fmt" "math" "math/big" @@ -729,3 +731,135 @@ func fracDigitsDiv(x int32) int32 { func fracDigitsMul(a, b int32) int32 { return min(MaxFractionDigits, a+b) } + +func encodeExp(expValue int64, expSign int, valSign int) int64 { + if expSign == 0 { + expValue = -expValue + } + + if expSign != valSign { + expValue = ^expValue + } + + return expValue +} + +func decodeExp(expValue int64, expSign int, valSign int) int64 { + if expSign != valSign { + expValue = ^expValue + } + + if expSign == 0 { + expValue = -expValue + } + + return expValue +} + +func codecValue(value []byte, valSign int) []byte { + if valSign == 0 { + for i, _ := range value { + value[i] = ^value[i] + } + } + + return value +} + +// EncodeDecimal encodes a decimal to bytes. +func EncodeDecimal(d Decimal) []byte { + // Value -> 0.Value ^ exp. + // If exp is bigger, the value is bigger. + // If exp is the same, we can compare 0.Value bytes by bytes. + + // Decimal "0" or "0.00...0" will be encoded "0". + if d.Equals(ZeroDecimal) { + b := make([]byte, 13) + // Value sign is 1. + binary.BigEndian.PutUint16(b[:2], uint16(1)) + // Exp sign is 0. + binary.BigEndian.PutUint16(b[2:4], uint16(0)) + // Exp value is 0. + binary.BigEndian.PutUint64(b[4:12], uint64(0)) + // Value string is "0". + b[12] = '0' + return b + } + + valSign := 1 + if d.value.Sign() < 0 { + valSign = 0 + } + + absVal := new(big.Int) + absVal.Abs(d.value) + + // Trim right side "0", like "12.34000" -> "12.34". + value := []byte(strings.TrimRight(absVal.String(), "0")) + + // Get exp and value, format is "value":"exp". + // like "12.34" -> "0.1234":"2". + // like "-0.01234" -> "-0.1234":"-1". + exp := int64(0) + div := big.NewInt(10) + for ; ; exp++ { + if absVal.Sign() == 0 { + break + } + absVal = absVal.Div(absVal, div) + } + + expSign := 1 + expVal := exp + int64(d.exp) + if expVal < 0 { + expSign = 0 + } + + // For negtive exp, do bit reverse for exp. + // For negtive decimal, do bit reverse for exp and value. + encExp := encodeExp(expVal, expSign, valSign) + encValue := codecValue(value, valSign) + + // Decimal decoding. + // 2 bytes -> value sign + // 2 bytes -> exp sign + // 8 bytes -> exp value + // others -> abs value bytes + b := make([]byte, 12+len(encValue)) + binary.BigEndian.PutUint16(b[:2], uint16(valSign)) + binary.BigEndian.PutUint16(b[2:4], uint16(expSign)) + binary.BigEndian.PutUint64(b[4:12], uint64(encExp)) + copy(b[12:], []byte(encValue)) + return b +} + +// DecodeDecimal decodes bytes to decimal. +func DecodeDecimal(b []byte) (Decimal, error) { + valSign := int(binary.BigEndian.Uint16(b[:2])) + expSign := int(binary.BigEndian.Uint16(b[2:4])) + decExp := int64(binary.BigEndian.Uint64(b[4:12])) + expVal := decodeExp(decExp, expSign, valSign) + decValue := codecValue(b[12:], valSign) + + // Generate value sign. + var decimalStr []byte + if valSign == 0 { + decimalStr = append(decimalStr, '-') + } + + // Generate decimal string value. + if expVal <= 0 { + // Like decimal "0.1234" or "0.01234". + decimalStr = append(decimalStr, '0') + decimalStr = append(decimalStr, '.') + decimalStr = append(decimalStr, bytes.Repeat([]byte{'0'}, -int(expVal))...) + decimalStr = append(decimalStr, decValue...) + } else { + // Like decimal "12.34". + decimalStr = append(decimalStr, decValue[:expVal]...) + decimalStr = append(decimalStr, '.') + decimalStr = append(decimalStr, decValue[expVal:]...) + } + + return ParseDecimal(string(decimalStr)) +} diff --git a/mysqldef/decimal_test.go b/mysqldef/decimal_test.go index 232beb9d801d..2405b153ab41 100644 --- a/mysqldef/decimal_test.go +++ b/mysqldef/decimal_test.go @@ -65,6 +65,8 @@ import ( "math" "strings" "testing" + + . "github.com/pingcap/check" ) var testTable = map[float64]string{ @@ -99,6 +101,11 @@ func init() { } } +var _ = Suite(&testDecimalSuite{}) + +type testDecimalSuite struct { +} + func TestNewFromFloat(t *testing.T) { for f, s := range testTable { d := NewDecimalFromFloat(f) @@ -868,5 +875,30 @@ func didPanic(f func()) bool { }() return ret +} +func (s *testTimeSuite) TestDecimalCodec(c *C) { + inputs := []struct { + Input float64 + }{ + {float64(1234)}, + {float64(12.34)}, + {float64(0.1234)}, + {float64(0.01234)}, + {float64(-0.1234)}, + {float64(-0.01234)}, + {float64(-12.34)}, + {float64(0.00000)}, + {float64(0)}, + {float64(-0.0)}, + {float64(-0.000)}, + } + + for _, input := range inputs { + v := NewDecimalFromFloat(input.Input) + b := EncodeDecimal(v) + d, err := DecodeDecimal(b) + c.Assert(err, IsNil) + c.Assert(v.Equals(d), IsTrue) + } } diff --git a/util/codec/codec.go b/util/codec/codec.go index b0abe61616a9..5a7ead0c7060 100644 --- a/util/codec/codec.go +++ b/util/codec/codec.go @@ -39,6 +39,7 @@ const ( formatStringFlag = 's' formatBytesFlag = 'b' formatDurationFlag = 't' + formatDecimalFlag = 'c' ) var sepKey = []byte{0x00, 0x00} @@ -107,6 +108,10 @@ func EncodeKey(args ...interface{}) ([]byte, error) { // duration may have negative value, so we cannot use String to encode directly. b = EncodeInt(b, int64(v.Duration)) format = append(format, formatDurationFlag) + case mysql.Decimal: + encBytes := mysql.EncodeDecimal(v) + b = EncodeBytes(b, encBytes) + format = append(format, formatDecimalFlag) case nil: // We will 0x00, 0x00 for nil. // The []byte{} will be encoded as 0x00, 0x01. @@ -177,6 +182,12 @@ func DecodeKey(b []byte) ([]interface{}, error) { // use max fsp, let outer to do round manually. v[i] = mysql.Duration{Duration: time.Duration(r), Fsp: mysql.MaxFsp} } + case formatDecimalFlag: + var r []byte + b, r, err = DecodeBytes(b) + if err == nil { + v[i], err = mysql.DecodeDecimal(r) + } case formatNilFlag: if len(b) < 2 || (b[0] != 0x00 && b[1] != 0x00) { return nil, errors.Errorf("malformed encoded nil") diff --git a/util/codec/codec_test.go b/util/codec/codec_test.go index f1b42705ff52..c68a447ee317 100644 --- a/util/codec/codec_test.go +++ b/util/codec/codec_test.go @@ -513,3 +513,78 @@ func (s *testCodecSuite) TestDuration(c *C) { c.Assert(ret, Equals, t.Ret) } } + +func (s *testCodecSuite) TestDecimal(c *C) { + tbl := []string{ + "1234.00", + "1234", + "12.34", + "12.340", + "0.1234", + "0.0", + "0", + "-0.0", + "-0.0000", + "-1234.00", + "-1234", + "-12.34", + "-12.340", + "-0.1234"} + + for _, t := range tbl { + m, err := mysql.ParseDecimal(t) + c.Assert(err, IsNil) + b, err := EncodeKey(m) + c.Assert(err, IsNil) + v, err := DecodeKey(b) + c.Assert(err, IsNil) + c.Assert(v, HasLen, 1) + vv, ok := v[0].(mysql.Decimal) + c.Assert(ok, IsTrue) + c.Assert(vv.Equals(m), IsTrue) + } + + tblCmp := []struct { + Arg1 string + Arg2 string + Ret int + }{ + {"1234", "1234.5", -1}, + {"1234", "1234.0000", 0}, + {"1234", "12.34", 1}, + {"12.34", "12.35", -1}, + {"0.1234", "12.3400", -1}, + {"0.1234", "0.1235", -1}, + {"0.123400", "12.34", -1}, + {"12.34000", "12.34", 0}, + {"0.01234", "0.01235", -1}, + {"0.1234", "0", 1}, + {"0.0000", "0", 0}, + {"0.0001", "0", 1}, + {"0.0001", "0.0000", 1}, + {"0", "-0.0000", 0}, + {"-0.0001", "0", -1}, + {"-0.1234", "0", -1}, + {"-0.1234", "0.1234", -1}, + {"-1.234", "-12.34", 1}, + {"-0.1234", "-12.34", 1}, + {"-12.34", "1234", -1}, + {"-12.34", "-12.35", 1}, + {"-0.01234", "-0.01235", 1}, + } + + for _, t := range tblCmp { + m1, err := mysql.ParseDecimal(t.Arg1) + c.Assert(err, IsNil) + m2, err := mysql.ParseDecimal(t.Arg2) + c.Assert(err, IsNil) + + b1, err := EncodeKey(m1) + c.Assert(err, IsNil) + b2, err := EncodeKey(m2) + c.Assert(err, IsNil) + + ret := bytes.Compare(b1, b2) + c.Assert(ret, Equals, t.Ret) + } +} From eb2849ef49f0638ef847ab6d20bb0adee3de3c16 Mon Sep 17 00:00:00 2001 From: qiuyesuifeng Date: Thu, 10 Sep 2015 12:14:25 +0800 Subject: [PATCH 2/6] util/codec: move decimal codec to util/codec/decimal. --- mysqldef/decimal.go | 139 ++------------------------------------- mysqldef/decimal_test.go | 33 ---------- util/codec/codec.go | 4 +- 3 files changed, 7 insertions(+), 169 deletions(-) diff --git a/mysqldef/decimal.go b/mysqldef/decimal.go index d8b29838cde8..4276e52d52af 100644 --- a/mysqldef/decimal.go +++ b/mysqldef/decimal.go @@ -78,9 +78,7 @@ package mysqldef // after the decimal point. import ( - "bytes" "database/sql/driver" - "encoding/binary" "fmt" "math" "math/big" @@ -600,6 +598,11 @@ func (d Decimal) Value() (driver.Value, error) { return d.String(), nil } +// BigIntValue returns the *bit.Int value member of decimal. +func (d Decimal) BigIntValue() *big.Int { + return d.value +} + // UnmarshalText implements the encoding.TextUnmarshaler interface for XML // deserialization. func (d *Decimal) UnmarshalText(text []byte) error { @@ -731,135 +734,3 @@ func fracDigitsDiv(x int32) int32 { func fracDigitsMul(a, b int32) int32 { return min(MaxFractionDigits, a+b) } - -func encodeExp(expValue int64, expSign int, valSign int) int64 { - if expSign == 0 { - expValue = -expValue - } - - if expSign != valSign { - expValue = ^expValue - } - - return expValue -} - -func decodeExp(expValue int64, expSign int, valSign int) int64 { - if expSign != valSign { - expValue = ^expValue - } - - if expSign == 0 { - expValue = -expValue - } - - return expValue -} - -func codecValue(value []byte, valSign int) []byte { - if valSign == 0 { - for i, _ := range value { - value[i] = ^value[i] - } - } - - return value -} - -// EncodeDecimal encodes a decimal to bytes. -func EncodeDecimal(d Decimal) []byte { - // Value -> 0.Value ^ exp. - // If exp is bigger, the value is bigger. - // If exp is the same, we can compare 0.Value bytes by bytes. - - // Decimal "0" or "0.00...0" will be encoded "0". - if d.Equals(ZeroDecimal) { - b := make([]byte, 13) - // Value sign is 1. - binary.BigEndian.PutUint16(b[:2], uint16(1)) - // Exp sign is 0. - binary.BigEndian.PutUint16(b[2:4], uint16(0)) - // Exp value is 0. - binary.BigEndian.PutUint64(b[4:12], uint64(0)) - // Value string is "0". - b[12] = '0' - return b - } - - valSign := 1 - if d.value.Sign() < 0 { - valSign = 0 - } - - absVal := new(big.Int) - absVal.Abs(d.value) - - // Trim right side "0", like "12.34000" -> "12.34". - value := []byte(strings.TrimRight(absVal.String(), "0")) - - // Get exp and value, format is "value":"exp". - // like "12.34" -> "0.1234":"2". - // like "-0.01234" -> "-0.1234":"-1". - exp := int64(0) - div := big.NewInt(10) - for ; ; exp++ { - if absVal.Sign() == 0 { - break - } - absVal = absVal.Div(absVal, div) - } - - expSign := 1 - expVal := exp + int64(d.exp) - if expVal < 0 { - expSign = 0 - } - - // For negtive exp, do bit reverse for exp. - // For negtive decimal, do bit reverse for exp and value. - encExp := encodeExp(expVal, expSign, valSign) - encValue := codecValue(value, valSign) - - // Decimal decoding. - // 2 bytes -> value sign - // 2 bytes -> exp sign - // 8 bytes -> exp value - // others -> abs value bytes - b := make([]byte, 12+len(encValue)) - binary.BigEndian.PutUint16(b[:2], uint16(valSign)) - binary.BigEndian.PutUint16(b[2:4], uint16(expSign)) - binary.BigEndian.PutUint64(b[4:12], uint64(encExp)) - copy(b[12:], []byte(encValue)) - return b -} - -// DecodeDecimal decodes bytes to decimal. -func DecodeDecimal(b []byte) (Decimal, error) { - valSign := int(binary.BigEndian.Uint16(b[:2])) - expSign := int(binary.BigEndian.Uint16(b[2:4])) - decExp := int64(binary.BigEndian.Uint64(b[4:12])) - expVal := decodeExp(decExp, expSign, valSign) - decValue := codecValue(b[12:], valSign) - - // Generate value sign. - var decimalStr []byte - if valSign == 0 { - decimalStr = append(decimalStr, '-') - } - - // Generate decimal string value. - if expVal <= 0 { - // Like decimal "0.1234" or "0.01234". - decimalStr = append(decimalStr, '0') - decimalStr = append(decimalStr, '.') - decimalStr = append(decimalStr, bytes.Repeat([]byte{'0'}, -int(expVal))...) - decimalStr = append(decimalStr, decValue...) - } else { - // Like decimal "12.34". - decimalStr = append(decimalStr, decValue[:expVal]...) - decimalStr = append(decimalStr, '.') - decimalStr = append(decimalStr, decValue[expVal:]...) - } - - return ParseDecimal(string(decimalStr)) -} diff --git a/mysqldef/decimal_test.go b/mysqldef/decimal_test.go index 2405b153ab41..88448d1f7d2c 100644 --- a/mysqldef/decimal_test.go +++ b/mysqldef/decimal_test.go @@ -65,8 +65,6 @@ import ( "math" "strings" "testing" - - . "github.com/pingcap/check" ) var testTable = map[float64]string{ @@ -101,11 +99,6 @@ func init() { } } -var _ = Suite(&testDecimalSuite{}) - -type testDecimalSuite struct { -} - func TestNewFromFloat(t *testing.T) { for f, s := range testTable { d := NewDecimalFromFloat(f) @@ -876,29 +869,3 @@ func didPanic(f func()) bool { return ret } - -func (s *testTimeSuite) TestDecimalCodec(c *C) { - inputs := []struct { - Input float64 - }{ - {float64(1234)}, - {float64(12.34)}, - {float64(0.1234)}, - {float64(0.01234)}, - {float64(-0.1234)}, - {float64(-0.01234)}, - {float64(-12.34)}, - {float64(0.00000)}, - {float64(0)}, - {float64(-0.0)}, - {float64(-0.000)}, - } - - for _, input := range inputs { - v := NewDecimalFromFloat(input.Input) - b := EncodeDecimal(v) - d, err := DecodeDecimal(b) - c.Assert(err, IsNil) - c.Assert(v.Equals(d), IsTrue) - } -} diff --git a/util/codec/codec.go b/util/codec/codec.go index 5a7ead0c7060..c1c8ad468967 100644 --- a/util/codec/codec.go +++ b/util/codec/codec.go @@ -109,7 +109,7 @@ func EncodeKey(args ...interface{}) ([]byte, error) { b = EncodeInt(b, int64(v.Duration)) format = append(format, formatDurationFlag) case mysql.Decimal: - encBytes := mysql.EncodeDecimal(v) + encBytes := EncodeDecimal(v) b = EncodeBytes(b, encBytes) format = append(format, formatDecimalFlag) case nil: @@ -186,7 +186,7 @@ func DecodeKey(b []byte) ([]interface{}, error) { var r []byte b, r, err = DecodeBytes(b) if err == nil { - v[i], err = mysql.DecodeDecimal(r) + v[i], err = DecodeDecimal(r) } case formatNilFlag: if len(b) < 2 || (b[0] != 0x00 && b[1] != 0x00) { From 1b1de4ba69771ec78914842c8f88a1524934a31a Mon Sep 17 00:00:00 2001 From: qiuyesuifeng Date: Thu, 10 Sep 2015 13:31:55 +0800 Subject: [PATCH 3/6] util/codec: update decimal codec. --- util/codec/codec.go | 9 +- util/codec/decimal.go | 202 +++++++++++++++++++++++++++++++++++++ util/codec/decimal_test.go | 66 ++++++++++++ 3 files changed, 270 insertions(+), 7 deletions(-) create mode 100644 util/codec/decimal.go create mode 100644 util/codec/decimal_test.go diff --git a/util/codec/codec.go b/util/codec/codec.go index c1c8ad468967..2133ac27481a 100644 --- a/util/codec/codec.go +++ b/util/codec/codec.go @@ -109,8 +109,7 @@ func EncodeKey(args ...interface{}) ([]byte, error) { b = EncodeInt(b, int64(v.Duration)) format = append(format, formatDurationFlag) case mysql.Decimal: - encBytes := EncodeDecimal(v) - b = EncodeBytes(b, encBytes) + b = EncodeDecimal(b, v) format = append(format, formatDecimalFlag) case nil: // We will 0x00, 0x00 for nil. @@ -183,11 +182,7 @@ func DecodeKey(b []byte) ([]interface{}, error) { v[i] = mysql.Duration{Duration: time.Duration(r), Fsp: mysql.MaxFsp} } case formatDecimalFlag: - var r []byte - b, r, err = DecodeBytes(b) - if err == nil { - v[i], err = DecodeDecimal(r) - } + b, v[i], err = DecodeDecimal(b) case formatNilFlag: if len(b) < 2 || (b[0] != 0x00 && b[1] != 0x00) { return nil, errors.Errorf("malformed encoded nil") diff --git a/util/codec/decimal.go b/util/codec/decimal.go new file mode 100644 index 000000000000..27fb91774191 --- /dev/null +++ b/util/codec/decimal.go @@ -0,0 +1,202 @@ +// Copyright 2014 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. See the AUTHORS file +// for names of contributors. +// +// Author: Tobias Schottdorf (tobias.schottdorf@gmail.com) + +// Copyright 2015 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "bytes" + "math/big" + "strings" + + "github.com/juju/errors" + mysql "github.com/pingcap/tidb/mysqldef" +) + +const ( + negativeSign int64 = 8 + zeroSign int64 = 16 + positiveSign int64 = 24 +) + +func codecSign(value int64) int64 { + if value < 0 { + return negativeSign + } + + return positiveSign +} + +func encodeExp(expValue int64, expSign int64, valSign int64) int64 { + if expSign == negativeSign { + expValue = -expValue + } + + if expSign != valSign { + expValue = ^expValue + } + + return expValue +} + +func decodeExp(expValue int64, expSign int64, valSign int64) int64 { + if expSign != valSign { + expValue = ^expValue + } + + if expSign == negativeSign { + expValue = -expValue + } + + return expValue +} + +func codecValue(value []byte, valSign int64) { + if valSign == negativeSign { + reverseBytes(value) + } +} + +// EncodeDecimal encodes a decimal d into a byte slice which can be sorted lexicographically later. +// EncodeDecimal guarantees that the encoded value is in ascending order for comparison. +// Decimal encoding: +// EncodeInt -> value sign +// EncodeInt -> exp sign +// EncodeInt -> exp value +// EncodeBytes -> abs value bytes +func EncodeDecimal(b []byte, d mysql.Decimal) []byte { + if d.Equals(mysql.ZeroDecimal) { + return EncodeInt(b, zeroSign) + } + + valSign := codecSign(int64(d.BigIntValue().Sign())) + + absVal := new(big.Int) + absVal.Abs(d.BigIntValue()) + + // Trim right side "0", like "12.34000" -> "12.34". + value := []byte(strings.TrimRight(absVal.String(), "0")) + + // Get exp and value, format is "value":"exp". + // like "12.34" -> "0.1234":"2". + // like "-0.01234" -> "-0.1234":"-1". + exp := int64(0) + div := big.NewInt(10) + for ; ; exp++ { + if absVal.Sign() == 0 { + break + } + absVal = absVal.Div(absVal, div) + } + + expVal := exp + int64(d.Exponent()) + expSign := codecSign(expVal) + + // For negtive exp, do bit reverse for exp. + // For negtive decimal, do bit reverse for exp and value. + expVal = encodeExp(expVal, expSign, valSign) + codecValue(value, valSign) + + r := EncodeInt(b, valSign) + r = EncodeInt(r, expSign) + r = EncodeInt(r, expVal) + r = EncodeBytes(r, value) + return r +} + +// DecodeDecimal decodes bytes to decimal. +// DecodeFloat decodes a float from a byte slice +// Decimal decoding: +// DecodeInt -> value sign +// DecodeInt -> exp sign +// DecodeInt -> exp value +// DecodeBytes -> abs value bytes +func DecodeDecimal(b []byte) ([]byte, mysql.Decimal, error) { + var ( + r []byte + d mysql.Decimal + err error + ) + + // Decode value sign. + valSign := zeroSign + r, valSign, err = DecodeInt(b) + if err != nil { + return r, d, errors.Trace(err) + } + if valSign == zeroSign { + d, err = mysql.ParseDecimal("0") + return r, d, errors.Trace(err) + } + + // Decode exp sign. + expSign := zeroSign + r, expSign, err = DecodeInt(r) + if err != nil { + return r, d, errors.Trace(err) + } + + // Decode exp value. + expVal := int64(0) + r, expVal, err = DecodeInt(r) + if err != nil { + return r, d, errors.Trace(err) + } + expVal = decodeExp(expVal, expSign, valSign) + + // Decode abs value bytes. + value := []byte{} + r, value, err = DecodeBytes(r) + if err != nil { + return r, d, errors.Trace(err) + } + codecValue(value, valSign) + + // Generate decimal string value. + var decimalStr []byte + if valSign == negativeSign { + decimalStr = append(decimalStr, '-') + } + + if expVal <= 0 { + // Like decimal "0.1234" or "0.01234". + decimalStr = append(decimalStr, '0') + decimalStr = append(decimalStr, '.') + decimalStr = append(decimalStr, bytes.Repeat([]byte{'0'}, -int(expVal))...) + decimalStr = append(decimalStr, value...) + } else { + // Like decimal "12.34". + decimalStr = append(decimalStr, value[:expVal]...) + decimalStr = append(decimalStr, '.') + decimalStr = append(decimalStr, value[expVal:]...) + } + + d, err = mysql.ParseDecimal(string(decimalStr)) + return r, d, errors.Trace(err) +} diff --git a/util/codec/decimal_test.go b/util/codec/decimal_test.go new file mode 100644 index 000000000000..227e77897833 --- /dev/null +++ b/util/codec/decimal_test.go @@ -0,0 +1,66 @@ +// Copyright 2014 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. See the AUTHORS file +// for names of contributors. +// +// Author: Tobias Schottdorf (tobias.schottdorf@gmail.com) + +// Copyright 2015 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. +package codec + +import ( + . "github.com/pingcap/check" + mysql "github.com/pingcap/tidb/mysqldef" +) + +var _ = Suite(&testDecimalSuite{}) + +type testDecimalSuite struct { +} + +func (s *testDecimalSuite) TestDecimalCodec(c *C) { + inputs := []struct { + Input float64 + }{ + {float64(1234)}, + {float64(12.34)}, + {float64(0.1234)}, + {float64(0.01234)}, + {float64(-0.1234)}, + {float64(-0.01234)}, + {float64(-12.34)}, + {float64(0.00000)}, + {float64(0)}, + {float64(-0.0)}, + {float64(-0.000)}, + } + + for _, input := range inputs { + v := mysql.NewDecimalFromFloat(input.Input) + b := EncodeDecimal([]byte{}, v) + _, d, err := DecodeDecimal(b) + c.Assert(err, IsNil) + c.Assert(v.Equals(d), IsTrue) + } +} From 5d0d30d642ccdec2f91a7099320f2a7398ef25c9 Mon Sep 17 00:00:00 2001 From: qiuyesuifeng Date: Thu, 10 Sep 2015 14:06:28 +0800 Subject: [PATCH 4/6] util/codec: fix a bug for decimal encoding and add more tests. fix bug. --- util/codec/codec_test.go | 4 ++++ util/codec/decimal.go | 9 ++++++--- util/codec/decimal_test.go | 3 +++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/util/codec/codec_test.go b/util/codec/codec_test.go index c68a447ee317..1532aef98698 100644 --- a/util/codec/codec_test.go +++ b/util/codec/codec_test.go @@ -549,6 +549,8 @@ func (s *testCodecSuite) TestDecimal(c *C) { Arg2 string Ret int }{ + {"1234", "123400", -1}, + {"12340", "123400", -1}, {"1234", "1234.5", -1}, {"1234", "1234.0000", 0}, {"1234", "12.34", 1}, @@ -571,6 +573,8 @@ func (s *testCodecSuite) TestDecimal(c *C) { {"-12.34", "1234", -1}, {"-12.34", "-12.35", 1}, {"-0.01234", "-0.01235", 1}, + {"-1234", "-123400", 1}, + {"-12340", "-123400", 1}, } for _, t := range tblCmp { diff --git a/util/codec/decimal.go b/util/codec/decimal.go index 27fb91774191..c0ca607a6095 100644 --- a/util/codec/decimal.go +++ b/util/codec/decimal.go @@ -33,7 +33,6 @@ package codec import ( "bytes" "math/big" - "strings" "github.com/juju/errors" mysql "github.com/pingcap/tidb/mysqldef" @@ -100,8 +99,12 @@ func EncodeDecimal(b []byte, d mysql.Decimal) []byte { absVal := new(big.Int) absVal.Abs(d.BigIntValue()) - // Trim right side "0", like "12.34000" -> "12.34". - value := []byte(strings.TrimRight(absVal.String(), "0")) + value := []byte(absVal.String()) + + // Trim right side "0", like "12.34000" -> "12.34" or "0.1234000" -> "0.1234". + if d.Exponent() != 0 { + value = bytes.TrimRight(value, "0") + } // Get exp and value, format is "value":"exp". // like "12.34" -> "0.1234":"2". diff --git a/util/codec/decimal_test.go b/util/codec/decimal_test.go index 227e77897833..7004372d59a5 100644 --- a/util/codec/decimal_test.go +++ b/util/codec/decimal_test.go @@ -27,6 +27,7 @@ // distributed under the License is distributed on an "AS IS" BASIS, // See the License for the specific language governing permissions and // limitations under the License. + package codec import ( @@ -43,12 +44,14 @@ func (s *testDecimalSuite) TestDecimalCodec(c *C) { inputs := []struct { Input float64 }{ + {float64(123400)}, {float64(1234)}, {float64(12.34)}, {float64(0.1234)}, {float64(0.01234)}, {float64(-0.1234)}, {float64(-0.01234)}, + {float64(12.3400)}, {float64(-12.34)}, {float64(0.00000)}, {float64(0)}, From 2a72346857d3a5c0c723df767ca28ffc668e8e87 Mon Sep 17 00:00:00 2001 From: qiuyesuifeng Date: Thu, 10 Sep 2015 14:58:23 +0800 Subject: [PATCH 5/6] util/codec: address comments. --- util/codec/codec_test.go | 64 ++++++++++++++++++++++++++++++++++++++ util/codec/decimal.go | 20 ++---------- util/codec/decimal_test.go | 17 ---------- 3 files changed, 66 insertions(+), 35 deletions(-) diff --git a/util/codec/codec_test.go b/util/codec/codec_test.go index 1532aef98698..cdac6b962202 100644 --- a/util/codec/codec_test.go +++ b/util/codec/codec_test.go @@ -591,4 +591,68 @@ func (s *testCodecSuite) TestDecimal(c *C) { ret := bytes.Compare(b1, b2) c.Assert(ret, Equals, t.Ret) } + + tblInt64 := []struct { + Arg1 int64 + Arg2 int64 + Ret int + }{ + {-1, 1, -1}, + {math.MaxInt64, math.MinInt64, 1}, + {math.MaxInt64, math.MaxInt32, 1}, + {math.MinInt32, math.MaxInt16, -1}, + {math.MinInt64, math.MaxInt8, -1}, + {0, math.MaxInt8, -1}, + {math.MinInt8, 0, -1}, + {math.MinInt16, math.MaxInt16, -1}, + {1, -1, 1}, + {1, 0, 1}, + {-1, 0, -1}, + {0, 0, 0}, + {math.MaxInt16, math.MaxInt16, 0}, + } + + for _, t := range tblInt64 { + m1 := mysql.NewDecimalFromInt(t.Arg1, 0) + m2 := mysql.NewDecimalFromInt(t.Arg2, 0) + + b1, err := EncodeKey(m1) + c.Assert(err, IsNil) + b2, err := EncodeKey(m2) + c.Assert(err, IsNil) + + ret := bytes.Compare(b1, b2) + c.Assert(ret, Equals, t.Ret) + } + + tblUint64 := []struct { + Arg1 uint64 + Arg2 uint64 + Ret int + }{ + {0, 0, 0}, + {1, 0, 1}, + {0, 1, -1}, + {math.MaxInt8, math.MaxInt16, -1}, + {math.MaxUint32, math.MaxInt32, 1}, + {math.MaxUint8, math.MaxInt8, 1}, + {math.MaxUint16, math.MaxInt32, -1}, + {math.MaxUint64, math.MaxInt64, 1}, + {math.MaxInt64, math.MaxUint32, 1}, + {math.MaxUint64, 0, 1}, + {0, math.MaxUint64, -1}, + } + + for _, t := range tblUint64 { + m1 := mysql.NewDecimalFromUint(t.Arg1, 0) + m2 := mysql.NewDecimalFromUint(t.Arg2, 0) + + b1, err := EncodeKey(m1) + c.Assert(err, IsNil) + b2, err := EncodeKey(m2) + c.Assert(err, IsNil) + + ret := bytes.Compare(b1, b2) + c.Assert(ret, Equals, t.Ret) + } } diff --git a/util/codec/decimal.go b/util/codec/decimal.go index c0ca607a6095..5e616438a142 100644 --- a/util/codec/decimal.go +++ b/util/codec/decimal.go @@ -1,20 +1,3 @@ -// Copyright 2014 The Cockroach Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the License. See the AUTHORS file -// for names of contributors. -// -// Author: Tobias Schottdorf (tobias.schottdorf@gmail.com) - // Copyright 2015 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -94,7 +77,8 @@ func EncodeDecimal(b []byte, d mysql.Decimal) []byte { return EncodeInt(b, zeroSign) } - valSign := codecSign(int64(d.BigIntValue().Sign())) + v := d.BigIntValue() + valSign := codecSign(int64(v.Sign())) absVal := new(big.Int) absVal.Abs(d.BigIntValue()) diff --git a/util/codec/decimal_test.go b/util/codec/decimal_test.go index 7004372d59a5..28532e766e6f 100644 --- a/util/codec/decimal_test.go +++ b/util/codec/decimal_test.go @@ -1,20 +1,3 @@ -// Copyright 2014 The Cockroach Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the License. See the AUTHORS file -// for names of contributors. -// -// Author: Tobias Schottdorf (tobias.schottdorf@gmail.com) - // Copyright 2015 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); From 038ab131b10d475ad56219ee471db7946a02df1d Mon Sep 17 00:00:00 2001 From: qiuyesuifeng Date: Thu, 10 Sep 2015 15:18:02 +0800 Subject: [PATCH 6/6] util/codec: address comment. --- util/codec/codec_test.go | 75 +++++++++++----------------------------- util/codec/decimal.go | 2 +- 2 files changed, 21 insertions(+), 56 deletions(-) diff --git a/util/codec/codec_test.go b/util/codec/codec_test.go index cdac6b962202..837bd56ba877 100644 --- a/util/codec/codec_test.go +++ b/util/codec/codec_test.go @@ -545,10 +545,11 @@ func (s *testCodecSuite) TestDecimal(c *C) { } tblCmp := []struct { - Arg1 string - Arg2 string + Arg1 interface{} + Arg2 interface{} Ret int }{ + // Test for float type decimal. {"1234", "123400", -1}, {"12340", "123400", -1}, {"1234", "1234.5", -1}, @@ -575,28 +576,8 @@ func (s *testCodecSuite) TestDecimal(c *C) { {"-0.01234", "-0.01235", 1}, {"-1234", "-123400", 1}, {"-12340", "-123400", 1}, - } - - for _, t := range tblCmp { - m1, err := mysql.ParseDecimal(t.Arg1) - c.Assert(err, IsNil) - m2, err := mysql.ParseDecimal(t.Arg2) - c.Assert(err, IsNil) - - b1, err := EncodeKey(m1) - c.Assert(err, IsNil) - b2, err := EncodeKey(m2) - c.Assert(err, IsNil) - - ret := bytes.Compare(b1, b2) - c.Assert(ret, Equals, t.Ret) - } - tblInt64 := []struct { - Arg1 int64 - Arg2 int64 - Ret int - }{ + // Test for int type decimal. {-1, 1, -1}, {math.MaxInt64, math.MinInt64, 1}, {math.MaxInt64, math.MaxInt32, 1}, @@ -610,43 +591,27 @@ func (s *testCodecSuite) TestDecimal(c *C) { {-1, 0, -1}, {0, 0, 0}, {math.MaxInt16, math.MaxInt16, 0}, - } - for _, t := range tblInt64 { - m1 := mysql.NewDecimalFromInt(t.Arg1, 0) - m2 := mysql.NewDecimalFromInt(t.Arg2, 0) + // Test for uint type decimal. + {uint64(0), uint64(0), 0}, + {uint64(1), uint64(0), 1}, + {uint64(0), uint64(1), -1}, + {uint64(math.MaxInt8), uint64(math.MaxInt16), -1}, + {uint64(math.MaxUint32), uint64(math.MaxInt32), 1}, + {uint64(math.MaxUint8), uint64(math.MaxInt8), 1}, + {uint64(math.MaxUint16), uint64(math.MaxInt32), -1}, + {uint64(math.MaxUint64), uint64(math.MaxInt64), 1}, + {uint64(math.MaxInt64), uint64(math.MaxUint32), 1}, + {uint64(math.MaxUint64), uint64(0), 1}, + {uint64(0), uint64(math.MaxUint64), -1}, + } - b1, err := EncodeKey(m1) + for _, t := range tblCmp { + m1, err := mysql.ConvertToDecimal(t.Arg1) c.Assert(err, IsNil) - b2, err := EncodeKey(m2) + m2, err := mysql.ConvertToDecimal(t.Arg2) c.Assert(err, IsNil) - ret := bytes.Compare(b1, b2) - c.Assert(ret, Equals, t.Ret) - } - - tblUint64 := []struct { - Arg1 uint64 - Arg2 uint64 - Ret int - }{ - {0, 0, 0}, - {1, 0, 1}, - {0, 1, -1}, - {math.MaxInt8, math.MaxInt16, -1}, - {math.MaxUint32, math.MaxInt32, 1}, - {math.MaxUint8, math.MaxInt8, 1}, - {math.MaxUint16, math.MaxInt32, -1}, - {math.MaxUint64, math.MaxInt64, 1}, - {math.MaxInt64, math.MaxUint32, 1}, - {math.MaxUint64, 0, 1}, - {0, math.MaxUint64, -1}, - } - - for _, t := range tblUint64 { - m1 := mysql.NewDecimalFromUint(t.Arg1, 0) - m2 := mysql.NewDecimalFromUint(t.Arg2, 0) - b1, err := EncodeKey(m1) c.Assert(err, IsNil) b2, err := EncodeKey(m2) diff --git a/util/codec/decimal.go b/util/codec/decimal.go index 5e616438a142..f33b5ab39f8e 100644 --- a/util/codec/decimal.go +++ b/util/codec/decimal.go @@ -81,7 +81,7 @@ func EncodeDecimal(b []byte, d mysql.Decimal) []byte { valSign := codecSign(int64(v.Sign())) absVal := new(big.Int) - absVal.Abs(d.BigIntValue()) + absVal.Abs(v) value := []byte(absVal.String())