Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #128 from mailru/tuple-querying-support
tuple querying support
  • Loading branch information
DoubleDi committed Jun 6, 2021
2 parents 0355cf7 + ce9b23a commit 5ee65c8
Show file tree
Hide file tree
Showing 10 changed files with 143 additions and 51 deletions.
3 changes: 2 additions & 1 deletion .travis.yml
Expand Up @@ -12,14 +12,15 @@ services:

before_install:
- echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
- travis_retry docker pull yandex/clickhouse-server:20.3.8.53
- travis_retry docker pull yandex/clickhouse-server:latest
- make up_docker_server

install:
- travis_retry go get -v github.com/mattn/goveralls/...
- make init

before_script:
- export COVERALLS_SERVICE_JOB_ID=$TRAVIS_JOB_ID
- export TEST_CLICKHOUSE_DSN="http://localhost:8123/default"

script:
Expand Down
5 changes: 2 additions & 3 deletions Makefile
Expand Up @@ -2,7 +2,7 @@ SHELL := /bin/bash

init:
GO111MODULE=on go mod vendor
GO111MODULE=off go get golang.org/x/lint/golint
GO111MODULE=on go get github.com/golangci/golangci-lint/...@v1.35.2

up_docker_server: stop_docker_server
docker run --rm=true -p 8123:8123 --name dbr-clickhouse-server -d yandex/clickhouse-server:latest;
Expand All @@ -11,8 +11,7 @@ stop_docker_server:
test -n "$$(docker ps --format {{.Names}} | grep dbr-clickhouse-server)" && docker stop dbr-clickhouse-server || true

test: up_docker_server
test -z "$$(golint ./... | grep -v vendor | tee /dev/stderr)"
go vet -v ./...
golangci-lint run -v ./...
test -z "$$(gofmt -d -s $$(find . -name \*.go -print | grep -v vendor) | tee /dev/stderr)"
go test -v -covermode=count -coverprofile=coverage.out .
$(MAKE) stop_docker_server
2 changes: 2 additions & 0 deletions README.md
Expand Up @@ -43,13 +43,15 @@ http://user:password@host:8123/clicks?read_timeout=10&write_timeout=20
* [Array(T) (one-dimensional)](https://clickhouse.yandex/reference_en.html#Array(T))
* [Nested(Name1 Type1, Name2 Type2, ...)](https://clickhouse.yandex/docs/en/data_types/nested_data_structures/nested/)
* IPv4, IPv6
* Tuple

Notes:
* database/sql does not allow to use big uint64 values. It is recommended use type `UInt64` which is provided by driver for such kind of values.
* type `[]byte` are used as raw string (without quoting)
* for passing value of type `[]uint8` to driver as array - please use the wrapper `clickhouse.Array`
* for passing decimal value please use the wrappers `clickhouse.Decimal*`
* for passing IPv4/IPv6 types use `clickhouse.IP`
* for passing Tuple types use `clickhouse.Tuple` or structs

## Supported request params

Expand Down
14 changes: 7 additions & 7 deletions conn.go
Expand Up @@ -213,7 +213,7 @@ func (c *conn) killQuery(req *http.Request, args []driver.Value) error {
}
if body != nil {
// Drain body to enable connection reuse
io.Copy(ioutil.Discard, body)
_, _ = io.Copy(ioutil.Discard, body)
body.Close()
}
return nil
Expand Down Expand Up @@ -252,7 +252,7 @@ func (c *conn) exec(ctx context.Context, query string, args []driver.Value) (dri
body, err := c.doRequest(ctx, req)
if body != nil {
// Drain body to enable connection reuse
io.Copy(ioutil.Discard, body)
_, _ = io.Copy(ioutil.Discard, body)
body.Close()
}
return emptyResult, err
Expand Down Expand Up @@ -297,12 +297,12 @@ func (c *conn) buildRequest(ctx context.Context, query string, params []driver.V
go func() {
if c.useGzipCompression {
gz := gzip.NewWriter(bodyWriter)
gz.Write([]byte(query))
gz.Close()
bodyWriter.Close()
_, _ = gz.Write([]byte(query))
_ = gz.Close()
_ = bodyWriter.Close()
} else {
bodyWriter.Write([]byte(query))
bodyWriter.Close()
_, _ = bodyWriter.Write([]byte(query))
_ = bodyWriter.Close()
}
}()
c.log("query: ", query)
Expand Down
8 changes: 4 additions & 4 deletions conn_test.go
Expand Up @@ -17,8 +17,8 @@ import (

var (
_ driver.Conn = new(conn)
_ driver.Execer = new(conn)
_ driver.Queryer = new(conn)
_ driver.Execer = new(conn) // nolint:staticcheck
_ driver.Queryer = new(conn) // nolint:staticcheck
_ driver.Tx = new(conn)
)

Expand Down Expand Up @@ -192,9 +192,9 @@ func (s *connSuite) TestServerKillQuery() {

// not kill query and check if it is not cancelled
queryID = uuid.New().String()
_, err = s.connWithKillQuery.QueryContext(context.WithValue(context.Background(), QueryID, queryID), "SELECT sleep(0.5)")
_, err = s.connWithKillQuery.QueryContext(context.WithValue(context.Background(), QueryID, queryID), "SELECT sleep(0.1)")
s.NoError(err)
rows = s.connWithKillQuery.QueryRow("SELECT count(query_id) FROM system.processes where query_id=? and is_cancelled=?", queryID, 0)
rows = s.connWithKillQuery.QueryRow("SELECT count(query_id) FROM system.processes where query_id=? and is_cancelled=?", queryID, 1)
err = rows.Scan(&amount)
s.NoError(err)
s.Equal(0, amount)
Expand Down
10 changes: 5 additions & 5 deletions dataparser.go
Expand Up @@ -37,7 +37,7 @@ loop:
case eof:
break loop
case ',', ']', ')':
s.UnreadRune()
_ = s.UnreadRune()
break loop
}

Expand Down Expand Up @@ -65,7 +65,7 @@ loop:
}
r = escaped
case '\'':
s.UnreadRune()
_ = s.UnreadRune()
break loop
}

Expand Down Expand Up @@ -141,7 +141,7 @@ func (p *nullableParser) Parse(s io.RuneScanner) (driver.Value, error) {
}

if r != '\'' && iter == 0 {
s.UnreadRune()
_ = s.UnreadRune()
d := readRaw(s)
dB = d
isNotString = true
Expand Down Expand Up @@ -319,7 +319,7 @@ func (p *arrayParser) Parse(s io.RuneScanner) (driver.Value, error) {
slice := reflect.MakeSlice(p.Type(), 0, 0)
for i := 0; ; i++ {
r := read(s)
s.UnreadRune()
_ = s.UnreadRune()
if r == ']' {
break
}
Expand All @@ -340,7 +340,7 @@ func (p *arrayParser) Parse(s io.RuneScanner) (driver.Value, error) {

r = read(s)
if r != ',' {
s.UnreadRune()
_ = s.UnreadRune()
}
}

Expand Down
57 changes: 57 additions & 0 deletions encoder.go
Expand Up @@ -27,8 +27,12 @@ func (e *textEncoder) Encode(value driver.Value) ([]byte, error) {
switch v := value.(type) {
case array:
return e.encodeArray(reflect.ValueOf(v.v))
case tuple:
return e.encodeTuple(reflect.ValueOf(v.v))
case []byte:
return v, nil
case time.Time:
return []byte(e.encode(v)), nil
}

vv := reflect.ValueOf(value)
Expand All @@ -40,6 +44,8 @@ func (e *textEncoder) Encode(value driver.Value) ([]byte, error) {
return e.Encode(vv.Elem().Interface())
case reflect.Slice, reflect.Array:
return e.encodeArray(vv)
case reflect.Struct:
return e.encodeTuple(vv)
}
return []byte(e.encode(value)), nil
}
Expand Down Expand Up @@ -107,3 +113,54 @@ func (e *textEncoder) encodeArray(value reflect.Value) ([]byte, error) {
}
return append(res, ']'), nil
}

// EncodeTuple encodes a go struct as Clickhouse Tuple
func (e *textEncoder) encodeTuple(value reflect.Value) ([]byte, error) {
res := make([]byte, 0)
res = append(res, '(')
b, err := e.encodeTuplePart(value)
if err != nil {
return nil, err
}
res = append(res, b...)
res = append(res, ')')
return res, nil
}

func (e *textEncoder) encodeTuplePart(value reflect.Value) ([]byte, error) {
if value.Kind() == reflect.Ptr {
value = value.Elem()
}
if value.Kind() != reflect.Struct {
return nil, fmt.Errorf("expected struct, got %s", value.Kind())
}
t := value.Type()
res := []byte{}
for i := 0; i < value.NumField(); i++ {
ft := t.Field(i)
fv := value.Field(i)
if ft.Anonymous {
b, err := e.encodeTuplePart(fv)
if err != nil {
return nil, err
}
if i > 0 {
res = append(res, ',')
}
res = append(res, b...)
continue
}
if !fv.CanInterface() {
continue
}
b, err := e.Encode(fv.Interface())
if err != nil {
return nil, err
}
if i > 0 {
res = append(res, ',')
}
res = append(res, b...)
}
return res, nil
}
19 changes: 19 additions & 0 deletions encoder_test.go
Expand Up @@ -7,6 +7,22 @@ import (
"github.com/stretchr/testify/assert"
)

type TestEmbedTuple struct {
C bool
private int
}

type TestTuple struct {
A int
B string
TestEmbedTuple
}

type TestNestedTuple struct {
A *TestTuple
D int
}

func TestTextEncoder(t *testing.T) {
dt := time.Date(2011, 3, 6, 6, 20, 0, 0, time.UTC)
d := time.Date(2012, 5, 31, 0, 0, 0, 0, time.UTC)
Expand Down Expand Up @@ -40,6 +56,9 @@ func TestTextEncoder(t *testing.T) {
{[][]int16{{1}}, "[[1]]"},
{[]int16(nil), "[]"},
{(*int16)(nil), "NULL"},
{Tuple(TestTuple{A: 1, B: "2", TestEmbedTuple: TestEmbedTuple{C: true, private: 5}}), "(1,'2',1)"},
{Tuple(TestNestedTuple{A: &TestTuple{A: 1, B: "2", TestEmbedTuple: TestEmbedTuple{C: true}}, D: 4}), "((1,'2',1),4)"},
{[]TestTuple{{A: 1, B: "2", TestEmbedTuple: TestEmbedTuple{C: true, private: 5}}}, "[(1,'2',1)]"},
}

enc := new(textEncoder)
Expand Down
6 changes: 3 additions & 3 deletions tokenizer.go
Expand Up @@ -25,7 +25,7 @@ func skipWhiteSpace(s io.RuneScanner) {
case eof:
return
}
s.UnreadRune()
_ = s.UnreadRune()
return
}
}
Expand Down Expand Up @@ -113,7 +113,7 @@ loop:
case eof, ' ', '\t', '\n':
break loop
case '(', ')', ',':
s.UnreadRune()
_ = s.UnreadRune()
break loop
default:
data.WriteRune(r)
Expand Down Expand Up @@ -149,7 +149,7 @@ loop:
return nil, err
}
default:
s.UnreadRune()
_ = s.UnreadRune()
t = readNumberOrID(s)
}

Expand Down
70 changes: 42 additions & 28 deletions types.go
Expand Up @@ -13,16 +13,39 @@ func Array(v interface{}) driver.Valuer {
return array{v: v}
}

type array struct {
v interface{}
}

// Value implements driver.Valuer
func (a array) Value() (driver.Value, error) {
return textEncode.Encode(a)
}

// Date returns date for t
func Date(t time.Time) driver.Valuer {
return date(t)
}

type date time.Time

// Value implements driver.Valuer
func (d date) Value() (driver.Value, error) {
return []byte(formatDate(time.Time(d))), nil
}

// UInt64 returns uint64
func UInt64(u uint64) driver.Valuer {
return bigUint64(u)
}

type bigUint64 uint64

// Value implements driver.Valuer
func (u bigUint64) Value() (driver.Value, error) {
return []byte(strconv.FormatUint(uint64(u), 10)), nil
}

// Decimal32 converts value to Decimal32 of precision S.
// The value can be a number or a string. The S (scale) parameter specifies the number of decimal places.
func Decimal32(v interface{}, s int32) driver.Valuer {
Expand All @@ -41,34 +64,6 @@ func Decimal128(v interface{}, s int32) driver.Valuer {
return decimal{128, s, v}
}

// IP returns compatible database format for net.IP
func IP(i net.IP) driver.Valuer {
return ip(i)
}

type array struct {
v interface{}
}

// Value implements driver.Valuer
func (a array) Value() (driver.Value, error) {
return textEncode.Encode(a)
}

type date time.Time

// Value implements driver.Valuer
func (d date) Value() (driver.Value, error) {
return []byte(formatDate(time.Time(d))), nil
}

type bigUint64 uint64

// Value implements driver.Valuer
func (u bigUint64) Value() (driver.Value, error) {
return []byte(strconv.FormatUint(uint64(u), 10)), nil
}

type decimal struct {
p int32
s int32
Expand All @@ -80,9 +75,28 @@ func (d decimal) Value() (driver.Value, error) {
return []byte(fmt.Sprintf("toDecimal%d(%v, %d)", d.p, d.v, d.s)), nil
}

// IP returns compatible database format for net.IP
func IP(i net.IP) driver.Valuer {
return ip(i)
}

type ip net.IP

// Value implements driver.Valuer
func (i ip) Value() (driver.Value, error) {
return net.IP(i).String(), nil
}

// Tuple converts a struct into a tuple
// struct{A string, B int}{"a", 1} -> ("a", 1)
func Tuple(v interface{}) driver.Valuer {
return tuple{v: v}
}

type tuple struct {
v interface{}
}

func (t tuple) Value() (driver.Value, error) {
return textEncode.Encode(t)
}

0 comments on commit 5ee65c8

Please sign in to comment.