From 0fbb78c18bc8d2ff7f925b632ee35f4626b704a3 Mon Sep 17 00:00:00 2001 From: Bulat Gaifullin Date: Fri, 11 Jan 2019 00:06:38 +0300 Subject: [PATCH 1/4] Better Decimal support --- README.md | 3 +++ clickhouse_test.go | 12 ++++++++---- conn_go18_test.go | 1 + conn_test.go | 12 +++++++++++- dataparser.go | 6 +----- dataparser_test.go | 6 ++++++ typeparser_test.go | 8 ++++++++ types.go | 30 ++++++++++++++++++++++++++++++ types_test.go | 16 ++++++++++++++++ 9 files changed, 84 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c324ed3..13bf854 100644 --- a/README.md +++ b/README.md @@ -28,18 +28,21 @@ http://user:password@host:8123/clicks?read_timeout=10&write_timeout=20 * UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 * Float32, Float64 +* Decimal(P, S), Decimal32(S), Decimal64(S), Decimal128(S) * String * FixedString(N) * Date * DateTime * Enum * [Array(T) (one-dimensional)](https://clickhouse.yandex/reference_en.html#Array(T)) +* [Nested(Name1 Type1, Name2 Type2, ...)](https://clickhouse.yandex/docs/en/data_types/nested_data_structures/nested/) Notes: database/sql does not allow to use big uint64 values. It is recommended use type `UInt64` which is provided by driver for such kind of values. type `[]byte` are used as raw string (without quoting) for passing value of type `[]uint8` to driver as array - please use the wrapper `clickhouse.Array` +for passing decimal value please use the wrappers `clickhouse.Decimal*` ## Install ``` diff --git a/clickhouse_test.go b/clickhouse_test.go index 19da900..4340943 100644 --- a/clickhouse_test.go +++ b/clickhouse_test.go @@ -27,12 +27,16 @@ var ddls = []string{ a8 Array(UInt8), d Date, t DateTime, - e Enum8('one'=1, 'two'=2, 'three'=3) + e Enum8('one'=1, 'two'=2, 'three'=3), + d32 Decimal32(4), + d64 Decimal64(4), + d128 Decimal128(4), + d10 Decimal(10, 4) ) ENGINE = Memory`, `INSERT INTO data VALUES - (-1, 1, 1.0, '1', '1', [1], [10], '2011-03-06', '2011-03-06 06:20:00', 'one'), - (-2, 2, 2.0, '2', '2', [2], [20], '2012-05-31', '2012-05-31 11:20:00', 'two'), - (-3, 3, 3.0, '3', '2', [3], [30], '2016-04-04', '2016-04-04 11:30:00', 'three') + (-1, 1, 1.0, '1', '1', [1], [10], '2011-03-06', '2011-03-06 06:20:00', 'one', '10', '100', '1000', '1'), + (-2, 2, 2.0, '2', '2', [2], [20], '2012-05-31', '2012-05-31 11:20:00', 'two', '30', '300', '2000', '2'), + (-3, 3, 3.0, '3', '2', [3], [30], '2016-04-04', '2016-04-04 11:30:00', 'three', '40', '400', '3000', '3') `, } diff --git a/conn_go18_test.go b/conn_go18_test.go index b10f27e..38aac38 100644 --- a/conn_go18_test.go +++ b/conn_go18_test.go @@ -46,6 +46,7 @@ func (s *connSuite) TestColumnTypes() { expected := []string{ "Int64", "UInt64", "Float64", "String", "String", "Array(Int16)", "Array(UInt8)", "Date", "DateTime", "Enum8('one' = 1, 'two' = 2, 'three' = 3)", + "Decimal(9, 4)", "Decimal(18, 4)", "Decimal(38, 4)", "Decimal(10, 4)", } s.Require().Equal(len(expected), len(types)) for i, e := range expected { diff --git a/conn_test.go b/conn_test.go index 9146588..ed51dd7 100644 --- a/conn_test.go +++ b/conn_test.go @@ -31,11 +31,15 @@ func (s *connSuite) TestQuery() { {"SELECT i64 AS num FROM data WHERE i64 Date: Tue, 22 Jan 2019 17:34:05 +0300 Subject: [PATCH 2/4] Add nullable parser --- dataparser.go | 26 +++++++++++++++++++++++++- dataparser_test.go | 14 ++++++++++---- tokenizer.go | 16 ++++++++++++++++ 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/dataparser.go b/dataparser.go index dc19079..bc3ea3a 100644 --- a/dataparser.go +++ b/dataparser.go @@ -43,6 +43,23 @@ type dateTimeParser struct { location *time.Location } +type nullableParser struct { + DataParser +} + +func (p *nullableParser) Parse(s io.RuneScanner) (driver.Value, error) { + // Clickhouse returns `\N` string for `null` in tsv format. + // For checking this value we need to check first two runes in `io.RuneScanner`, but we can't reset `io.RuneScanner` after it. + // Copy io.RuneScanner to `bytes.Buffer` and use it twice (1st for casting to string and checking to null, 2nd for passing to original parser) + d := readRaw(s) + + if bytes.Equal(d.Bytes(), []byte(`\N`)) { + return nil, nil + } + + return p.DataParser.Parse(d) +} + func readNumber(s io.RuneScanner) (string, error) { var builder bytes.Buffer @@ -367,7 +384,14 @@ func newDataParser(t *TypeDesc, unquote bool) (DataParser, error) { case "Nothing": return ¬hingParser{}, nil case "Nullable": - return nil, fmt.Errorf("Nullable types are not supported") + if len(t.Args) == 0 { + return nil, fmt.Errorf("Nullable should pass original type") + } + p, err := newDataParser(t.Args[0], unquote) + if err != nil { + return nil, err + } + return &nullableParser{p}, nil case "Date": // FIXME: support custom default/override location return newDateTimeParser(dateFormat, "UTC", unquote) diff --git a/dataparser_test.go b/dataparser_test.go index 5512acd..5860785 100644 --- a/dataparser_test.go +++ b/dataparser_test.go @@ -27,10 +27,16 @@ func TestParseData(t *testing.T) { testCases := []*testCase{ { - name: "nullable not supported", - inputtype: "Nullable(String)", - inputdata: "NULL", - failNewParser: true, + name: "nullable string", + inputtype: "Nullable(String)", + inputdata: `\N`, + output: nil, + }, + { + name: "nullable int not null", + inputtype: "Nullable(UInt64)", + inputdata: "655", + output: uint64(655), }, { name: "string", diff --git a/tokenizer.go b/tokenizer.go index e1bc497..3418986 100644 --- a/tokenizer.go +++ b/tokenizer.go @@ -60,6 +60,22 @@ func readEscaped(s io.RuneScanner) (rune, error) { } } +func readRaw(s io.RuneScanner) *bytes.Buffer { + var data bytes.Buffer + + for { + r := read(s) + + if r == eof { + break + } + + data.WriteRune(r) + } + + return &data +} + func readQuoted(s io.RuneScanner) (*token, error) { var data bytes.Buffer From 6df7ca0f8a51dd3f95255cfc31d33f7f00231752 Mon Sep 17 00:00:00 2001 From: Denis Dorozhkin Date: Thu, 1 Aug 2019 18:52:31 +0300 Subject: [PATCH 3/4] Add Opts to Nullable Dataparser --- dataparser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataparser.go b/dataparser.go index 6078270..ed3565c 100644 --- a/dataparser.go +++ b/dataparser.go @@ -404,7 +404,7 @@ func newDataParser(t *TypeDesc, unquote bool, opt *DataParserOptions) (DataParse if len(t.Args) == 0 { return nil, fmt.Errorf("Nullable should pass original type") } - p, err := newDataParser(t.Args[0], unquote) + p, err := newDataParser(t.Args[0], unquote, opt) if err != nil { return nil, err } From 69bb82f4683b726604369f3dc0675a1b44cc7053 Mon Sep 17 00:00:00 2001 From: Denis Dorozhkin Date: Tue, 6 Aug 2019 19:08:37 +0300 Subject: [PATCH 4/4] add go1.12 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 8e4ac05..beb881f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ go: - '1.9' - '1.10' - '1.11' + - '1.12' services: - docker