From bf9ce5f390e735f8c95402d3c4944fe69eabd4a9 Mon Sep 17 00:00:00 2001 From: AlexisOMG Date: Wed, 10 Jan 2024 23:06:21 +0300 Subject: [PATCH 01/10] add ydb support --- Makefile | 2 +- database/ydb/helpers.go | 109 ++++++++++++++ database/ydb/helpers_test.go | 42 ++++++ database/ydb/ydb.go | 285 +++++++++++++++++++++++++++++++++++ database/ydb/ydb_test.go | 121 +++++++++++++++ go.mod | 6 + go.sum | 29 ++++ internal/cli/build_ydb.go | 8 + 8 files changed, 601 insertions(+), 1 deletion(-) create mode 100644 database/ydb/helpers.go create mode 100644 database/ydb/helpers_test.go create mode 100644 database/ydb/ydb.go create mode 100644 database/ydb/ydb_test.go create mode 100644 internal/cli/build_ydb.go diff --git a/Makefile b/Makefile index 8e23a43c7..7c9535327 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab -DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5 rqlite +DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5 rqlite ydb DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= diff --git a/database/ydb/helpers.go b/database/ydb/helpers.go new file mode 100644 index 000000000..608be9d83 --- /dev/null +++ b/database/ydb/helpers.go @@ -0,0 +1,109 @@ +package ydb + +import ( + "regexp" + "strings" +) + +const ( + createVersionTableQueryTemplate = ` + CREATE TABLE %s ( + version Int32, + dirty Bool, + applied_at Timestamp, + PRIMARY KEY (version) + ); + ` + + deleteVersionsQueryTemplate = ` + DELETE FROM %s;` + + setVersionQueryTemplate = ` + DECLARE $version AS Int32; + DECLARE $dirty AS Bool; + DECLARE $applied_at AS Timestamp; + UPSERT INTO %s (version, dirty, applied_at) + VALUES ($version, $dirty, $applied_at);` + + getVersionQueryTemplate = ` + SELECT version, dirty FROM %s + ORDER BY version DESC LIMIT 1;` + + dropTablesQueryTemplate = "DROP TABLE `%s`;" +) + +type queryMode int + +const ( + notSetMode queryMode = iota + ddlMode + dmlMode + unknownMode +) + +func skipComments(statements string) (string, error) { + type interval struct { + start int + end int + } + comments := make([]interval, 0) + inString := false + stringDelimiter := byte(0) + + for i := 0; i < len(statements); i++ { + // Check for string literal start/end + if (statements[i] == '\'' || statements[i] == '`') && (i == 0 || statements[i-1] != '\\') { + if inString { + if statements[i] == stringDelimiter { + inString = false + } + } else { + inString = true + stringDelimiter = statements[i] + } + } + + if !inString { + if statements[i] == '-' && i+1 < len(statements) && statements[i+1] == '-' { + start := i + for ; i < len(statements) && statements[i] != '\n'; i++ { + } + comments = append(comments, interval{start: start, end: i}) + } else if statements[i] == '/' && i+1 < len(statements) && statements[i+1] == '*' { + start := i + for ; i < len(statements)-1 && !(statements[i] == '*' && statements[i+1] == '/'); i++ { + } + comments = append(comments, interval{start: start, end: i + 1}) + } + } + } + + res := strings.Builder{} + curPos := 0 + + for _, comment := range comments { + _, err := res.WriteString(statements[curPos:comment.start]) + if err != nil { + return "", err + } + curPos = comment.end + 1 + } + + if curPos < len(statements) { + _, err := res.WriteString(statements[curPos:]) + if err != nil { + return "", err + } + } + + return res.String(), nil +} + +func detectQueryMode(statement string) queryMode { + ddlReg := regexp.MustCompile(`^(?i)(CREATE|ALTER|DECLARE|GRANT|REVOKE|DROP).*`) + if ddlReg.MatchString(statement) { + return ddlMode + } + + return dmlMode +} diff --git a/database/ydb/helpers_test.go b/database/ydb/helpers_test.go new file mode 100644 index 000000000..128d5ef00 --- /dev/null +++ b/database/ydb/helpers_test.go @@ -0,0 +1,42 @@ +package ydb + +import "testing" + +func TestSkipComments(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + {"empty input", "", ""}, + {"no comments", "SELECT * FROM table;", "SELECT * FROM table;"}, + {"single line comments", "-- This is a comment\nSELECT * FROM table;", "SELECT * FROM table;"}, + {"multi-line comments", "/* This is a comment */SELECT * FROM table;", "SELECT * FROM table;"}, + {"mixed comments", "-- Single line comment\n/* Multi\nLine\nComment */SELECT * FROM table;\nDROP TABLE table;", "SELECT * FROM table;\nDROP TABLE table;"}, + {"comments at the start", "-- Comment\nSELECT * FROM table;", "SELECT * FROM table;"}, + {"comments at the end", "SELECT * FROM table;-- Comment", "SELECT * FROM table;"}, + {"comments with special characters", "/* Com!ment */SELECT * FROM table;", "SELECT * FROM table;"}, + {"single line comment at end of line", "SELECT * FROM table; -- Comment", "SELECT * FROM table; "}, + {"multi-line comment in middle of line", "SELECT /* Comment */ * FROM table;", "SELECT * FROM table;"}, + {"multiple single line comments consecutively", "-- Comment 1\n-- Comment 2\nSELECT * FROM table;", "SELECT * FROM table;"}, + {"multiple multi-line comments consecutively", "/* Comment 1 *//* Comment 2 */SELECT * FROM table;", "SELECT * FROM table;"}, + {"mixed comments in single line", "SELECT * -- Comment\nFROM /* Comment */table;", "SELECT * FROM table;"}, + {"comments with sql keywords", "-- SELECT * FROM table\nSELECT name FROM users;", "SELECT name FROM users;"}, + {"sql commands with comment-like syntax", "SELECT * FROM `--table--` WHERE name = '/* John */';", "SELECT * FROM `--table--` WHERE name = '/* John */';"}, + {"whitespace handling", " -- Comment\nSELECT * FROM table; ", " SELECT * FROM table; "}, + {"comments with escape characters", "-- Comment \n\t SELECT * FROM table;", "\t SELECT * FROM table;"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := skipComments(tt.input) + if err != nil { + t.Errorf("skipComments() error = %v", err) + return + } + if result != tt.expected { + t.Errorf("skipComments() = %v, want %v", result, tt.expected) + } + }) + } +} diff --git a/database/ydb/ydb.go b/database/ydb/ydb.go new file mode 100644 index 000000000..7771d2832 --- /dev/null +++ b/database/ydb/ydb.go @@ -0,0 +1,285 @@ +package ydb + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io" + "net/url" + "strings" + "sync/atomic" + "time" + + "github.com/golang-migrate/migrate/v4/database" + + ydb "github.com/ydb-platform/ydb-go-sdk/v3" +) + +const ( + migrationsTableQueryParam = "x-migrations-table" + defaultMigrationsTable = "schema_migrations" +) + +var _ database.Driver = (*YDB)(nil) + +func init() { + database.Register("ydb", &YDB{}) +} + +type YDB struct { + db *sql.DB + locked atomic.Bool + migrationsTable string + prefix string +} + +func (y *YDB) tableWithPrefix(table string) string { + return fmt.Sprintf("`%s/%s`", y.prefix, table) +} + +func (y *YDB) Lock() error { + if !y.locked.CompareAndSwap(false, true) { + return database.ErrLocked + } + + return nil +} +func (y *YDB) Unlock() error { + if !y.locked.CompareAndSwap(true, false) { + return database.ErrNotLocked + } + + return nil +} + +func (y *YDB) Open(dsn string) (database.Driver, error) { + nativeDriver, err := ydb.Open(context.Background(), dsn) + if err != nil { + return nil, err + } + + connector, err := ydb.Connector(nativeDriver) + if err != nil { + return nil, err + } + + connUrl, err := url.Parse(dsn) + if err != nil { + return nil, err + } + + migrationsTable := connUrl.Query().Get(migrationsTableQueryParam) + + if migrationsTable == "" { + migrationsTable = defaultMigrationsTable + } + + ydbDriver := &YDB{ + db: sql.OpenDB(connector), + migrationsTable: migrationsTable, + prefix: nativeDriver.Name(), + } + + err = ydbDriver.createMigrationsTable(context.Background()) + if err != nil { + return nil, database.Error{ + OrigErr: err, + Err: "failed to create migrations table", + } + } + + return ydbDriver, nil +} + +func (y *YDB) createMigrationsTable(ctx context.Context) (err error) { + if err = y.Lock(); err != nil { + return err + } + + defer func() { + if ierr := y.Unlock(); ierr != nil { + err = errors.Join(err, ierr) + } + }() + + ctx = ydb.WithQueryMode(ctx, ydb.SchemeQueryMode) + + if _, err := y.db.ExecContext(ctx, fmt.Sprintf(createVersionTableQueryTemplate, y.tableWithPrefix(y.migrationsTable))); err != nil { + return err + } + + return nil +} + +func (y *YDB) Close() error { + return y.db.Close() +} + +func (y *YDB) Drop() (err error) { + tablesQuery := fmt.Sprintf( + "SELECT Path FROM `%s/.sys/partition_stats` WHERE Path NOT LIKE '%%/.sys%%'", + y.prefix, + ) + + rows, err := y.db.QueryContext(ydb.WithQueryMode(context.Background(), ydb.ScanQueryMode), tablesQuery) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(tablesQuery)} + } + defer func() { + if ierr := rows.Close(); ierr != nil { + err = errors.Join(err, ierr) + } + }() + + if !rows.NextResultSet() { + return nil + } + + for rows.Next() { + var table string + err = rows.Scan(&table) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(tablesQuery)} + } + + query := fmt.Sprintf(dropTablesQueryTemplate, table) + + if _, err = y.db.ExecContext(ydb.WithQueryMode(context.Background(), ydb.SchemeQueryMode), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err = rows.Err(); err != nil { + return &database.Error{OrigErr: err, Query: []byte(tablesQuery)} + } + + return nil +} + +func (y *YDB) Run(migration io.Reader) error { + data, err := io.ReadAll(migration) + if err != nil { + return err + } + + statements, err := skipComments(string(data)) + if err != nil { + return database.Error{ + OrigErr: err, + Err: "failed to skip comments", + } + } + + currentMode := notSetMode + + for _, statement := range strings.Split(statements, ";") { + statement = strings.TrimSpace(statement) + + if statement == "" { + continue + } + + mode := detectQueryMode(statement) + + if currentMode == notSetMode { + currentMode = mode + } else if currentMode != mode { + return database.Error{ + Err: "mixed query modes in one migration", + Query: []byte(statements), + } + } + } + + ctx := context.Background() + + switch currentMode { + case ddlMode: + ctx = ydb.WithQueryMode(ctx, ydb.SchemeQueryMode) + case dmlMode: + ctx = ydb.WithQueryMode(ctx, ydb.DataQueryMode) + } + + _, err = y.db.ExecContext(ctx, statements) + if err != nil { + return database.Error{ + OrigErr: err, + Err: "migration failed", + Query: []byte(statements), + } + } + + return nil +} + +func (y *YDB) SetVersion(version int, dirty bool) error { + tx, err := y.db.BeginTx(context.Background(), &sql.TxOptions{ + ReadOnly: false, + Isolation: sql.LevelDefault, + }) + if err != nil { + return err + } + + deleteVersions := fmt.Sprintf(deleteVersionsQueryTemplate, y.tableWithPrefix(y.migrationsTable)) + + if _, err = tx.Exec(deleteVersions); err != nil { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + err = errors.Join(err, rollbackErr) + } + + return database.Error{ + OrigErr: err, + Err: "failed to delete versions", + Query: []byte(deleteVersions), + } + } + + versionQuery := fmt.Sprintf(setVersionQueryTemplate, y.tableWithPrefix(y.migrationsTable)) + + _, err = tx.Exec( + versionQuery, + sql.Named("version", version), + sql.Named("dirty", dirty), + sql.Named("applied_at", time.Now()), + ) + if err != nil { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + err = errors.Join(err, rollbackErr) + } + + return database.Error{ + OrigErr: err, + Err: "failed to set version", + } + } + + return tx.Commit() +} + +func (y *YDB) Version() (version int, dirty bool, err error) { + versionQuery := fmt.Sprintf(getVersionQueryTemplate, y.tableWithPrefix(y.migrationsTable)) + + row, err := y.db.QueryContext(ydb.WithQueryMode(context.Background(), ydb.ScanQueryMode), versionQuery) + if err != nil { + return 0, false, database.Error{ + OrigErr: err, + Err: "failed to get version", + Query: []byte(versionQuery), + } + } + if !row.NextResultSet() || !row.Next() { + return database.NilVersion, false, nil + } + + if err = row.Scan(&version, &dirty); err != nil { + return 0, false, &database.Error{ + OrigErr: err, + Err: "failed to scan version", + Query: []byte(versionQuery), + } + } + + return version, dirty, err +} diff --git a/database/ydb/ydb_test.go b/database/ydb/ydb_test.go new file mode 100644 index 000000000..0416eb566 --- /dev/null +++ b/database/ydb/ydb_test.go @@ -0,0 +1,121 @@ +package ydb + +import ( + "context" + "database/sql" + sqldriver "database/sql/driver" + "fmt" + "io" + "log" + "os" + "testing" + "time" + + "github.com/docker/go-connections/nat" + ydb "github.com/ydb-platform/ydb-go-sdk/v3" + + "github.com/dhui/dktest" + dt "github.com/golang-migrate/migrate/v4/database/testing" + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +const ( + host = "localhost" + port = "2136" + testDB = "local" + dbPingTimeout = 5 * time.Second +) + +var ( + opts = dktest.Options{ + ReadyTimeout: 15 * time.Second, + Hostname: host, + Env: map[string]string{ + "YDB_USE_IN_MEMORY_PDISKS": "true", + }, + PortBindings: nat.PortMap{ + nat.Port(fmt.Sprintf("%s/tcp", port)): []nat.PortBinding{{ + HostIP: "0.0.0.0", + HostPort: port, + }}, + }, + ReadyFunc: isReady, + } + + image = "cr.yandex/yc/yandex-docker-local-ydb:latest" +) + +func init() { + _ = os.Setenv("YDB_ANONYMOUS_CREDENTIALS", "1") +} + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + nativeDriver, err := ydb.Open(context.Background(), fmt.Sprintf("grpc://localhost:%s/%s", port, testDB)) + if err != nil { + log.Println(err) + return false + } + + connector, err := ydb.Connector(nativeDriver) + if err != nil { + log.Println("close error:", err) + return false + } + + db := sql.OpenDB(connector) + defer func() { + if err := db.Close(); err != nil { + log.Println("close error:", err) + } + }() + + ctxWithTimeout, cancel := context.WithTimeout(ctx, dbPingTimeout) + defer cancel() + + if err = db.PingContext(ctxWithTimeout); err != nil { + switch err { + case sqldriver.ErrBadConn, io.EOF: + return false + default: + log.Println(err) + } + return false + } + + ctxWithTimeout = ydb.WithQueryMode(ctxWithTimeout, ydb.SchemeQueryMode) + + _, err = db.ExecContext(ctxWithTimeout, ` + CREATE TABLE test ( + id Int, + PRIMARY KEY(id) + ); + DROP TABLE test;`) + if err != nil { + log.Println(err) + return false + } + + return true +} + +func Test(t *testing.T) { + dktest.Run(t, image, opts, func(t *testing.T, c dktest.ContainerInfo) { + addr := fmt.Sprintf("grpc://localhost:%s/%s", port, testDB) + p := &YDB{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + dt.Test(t, d, []byte(` + CREATE TABLE test ( + id Int, + PRIMARY KEY(id) + ); + DROP TABLE test;`)) + }) +} diff --git a/go.mod b/go.mod index b5678692c..5706da935 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/snowflakedb/gosnowflake v1.6.19 github.com/stretchr/testify v1.8.3 github.com/xanzy/go-gitlab v0.15.0 + github.com/ydb-platform/ydb-go-sdk/v3 v3.54.3 go.mongodb.org/mongo-driver v1.7.5 go.uber.org/atomic v1.7.0 golang.org/x/oauth2 v0.14.0 @@ -43,6 +44,11 @@ require ( modernc.org/sqlite v1.18.1 ) +require ( + github.com/jonboulle/clockwork v0.3.0 // indirect + github.com/ydb-platform/ydb-go-genproto v0.0.0-20231215113745-46f6d30f974a // indirect +) + require ( cloud.google.com/go v0.110.10 // indirect cloud.google.com/go/compute v1.23.3 // indirect diff --git a/go.sum b/go.sum index f5d80e340..21591ea92 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,7 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/v10 v10.0.1 h1:n9dERvixoC/1JjDmBcs9FPaEryoANa2sCgVFo6ez9cI= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= @@ -127,15 +128,20 @@ github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe h1:QQ3GSy+MqSHxm/d8nCtnAiZdYFd45cYZPs8vOOIYKfk= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 h1:7To3pQ+pZo0i3dsWEbinPNFs5gPSBOsJtx3wTT94VBY= github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= @@ -178,6 +184,8 @@ github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM= github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= @@ -192,6 +200,7 @@ github.com/fsouza/fake-gcs-server v1.17.0 h1:OeH75kBZcZa3ZE+zz/mFdJ2btt9FgqfjI7g github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw= github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -233,10 +242,12 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -299,6 +310,7 @@ github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= @@ -393,6 +405,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/jonboulle/clockwork v0.3.0 h1:9BSCMi8C+0qdApAp4auwX0RkLGUjs956h0EkuQymUhg= +github.com/jonboulle/clockwork v0.3.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= @@ -513,6 +527,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rqlite/gorqlite v0.0.0-20230708021416-2acd02b70b79 h1:V7x0hCAgL8lNGezuex1RW1sh7VXXCqfw8nXZti66iFg= @@ -562,6 +577,10 @@ github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23n github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20231215113745-46f6d30f974a h1:9wx+kCrCQCdwmDe1AFW5yAHdzlo+RV7lcy6y7Zq661s= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20231215113745-46f6d30f974a/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= +github.com/ydb-platform/ydb-go-sdk/v3 v3.54.3 h1:pxVqQ1bLW7PeotN7Ko/AvqUC3CDp7IpOzGNUyXU0YOE= +github.com/ydb-platform/ydb-go-sdk/v3 v3.54.3/go.mod h1:PMwXjf6joVP1ID0u0FBiDRh+DeLD1wx+HyM5zv/PzlE= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -580,6 +599,7 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -667,6 +687,7 @@ golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -687,6 +708,7 @@ golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAG golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0= golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -850,6 +872,7 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= @@ -864,7 +887,10 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -878,6 +904,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -896,6 +924,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/cli/build_ydb.go b/internal/cli/build_ydb.go new file mode 100644 index 000000000..baf58a6de --- /dev/null +++ b/internal/cli/build_ydb.go @@ -0,0 +1,8 @@ +//go:build ydb +// +build ydb + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/ydb" +) From 1fb48c664c3d389c31737920ab593c2a4a9f5714 Mon Sep 17 00:00:00 2001 From: AlexisOMG Date: Fri, 12 Jan 2024 00:56:31 +0300 Subject: [PATCH 02/10] add README --- database/ydb/README.md | 12 ++++++++++++ database/ydb/examples/migrations/1_init.down.sql | 1 + database/ydb/examples/migrations/1_init.up.sql | 4 ++++ 3 files changed, 17 insertions(+) create mode 100644 database/ydb/README.md create mode 100644 database/ydb/examples/migrations/1_init.down.sql create mode 100644 database/ydb/examples/migrations/1_init.up.sql diff --git a/database/ydb/README.md b/database/ydb/README.md new file mode 100644 index 000000000..9e84a61e2 --- /dev/null +++ b/database/ydb/README.md @@ -0,0 +1,12 @@ +# YDB + +`grpc[s]://user:password@host:port/database?QUERY_PARAMS` + +| URL Query | Description | +|------------|-------------| +| `user` | The user to sign in as | +| `password` | The user's password | +| `host` | The host to connect to | +| `port` | The port to bind to | +| `database` | The name of the database to connect to | +| `x-migrations-table`| Name of the migrations table. Default: `schema_migrations` | diff --git a/database/ydb/examples/migrations/1_init.down.sql b/database/ydb/examples/migrations/1_init.down.sql new file mode 100644 index 000000000..0b014e965 --- /dev/null +++ b/database/ydb/examples/migrations/1_init.down.sql @@ -0,0 +1 @@ +DROP TABLE test; diff --git a/database/ydb/examples/migrations/1_init.up.sql b/database/ydb/examples/migrations/1_init.up.sql new file mode 100644 index 000000000..66c6e6fe4 --- /dev/null +++ b/database/ydb/examples/migrations/1_init.up.sql @@ -0,0 +1,4 @@ +CREATE TABLE test ( + id Int, + PRIMARY KEY(id) +); From f4ce1a7dcaefdefa7508be418414d911aa33b7e7 Mon Sep 17 00:00:00 2001 From: AlexisOMG Date: Fri, 12 Jan 2024 13:59:40 +0300 Subject: [PATCH 03/10] simplify ping db in test --- database/ydb/helpers.go | 1 - database/ydb/ydb_test.go | 9 +-------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/database/ydb/helpers.go b/database/ydb/helpers.go index 608be9d83..87ff97d72 100644 --- a/database/ydb/helpers.go +++ b/database/ydb/helpers.go @@ -51,7 +51,6 @@ func skipComments(statements string) (string, error) { stringDelimiter := byte(0) for i := 0; i < len(statements); i++ { - // Check for string literal start/end if (statements[i] == '\'' || statements[i] == '`') && (i == 0 || statements[i-1] != '\\') { if inString { if statements[i] == stringDelimiter { diff --git a/database/ydb/ydb_test.go b/database/ydb/ydb_test.go index 0416eb566..e88716d86 100644 --- a/database/ydb/ydb_test.go +++ b/database/ydb/ydb_test.go @@ -3,9 +3,7 @@ package ydb import ( "context" "database/sql" - sqldriver "database/sql/driver" "fmt" - "io" "log" "os" "testing" @@ -73,12 +71,7 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool { defer cancel() if err = db.PingContext(ctxWithTimeout); err != nil { - switch err { - case sqldriver.ErrBadConn, io.EOF: - return false - default: - log.Println(err) - } + log.Println(err) return false } From ff369d22dec95c417d0147b0a9db5dc3ce76c5d4 Mon Sep 17 00:00:00 2001 From: AlexisOMG Date: Wed, 17 Jan 2024 22:24:39 +0300 Subject: [PATCH 04/10] use scripting query mode --- README.md | 1 + database/ydb/helpers.go | 108 ----------------------------------- database/ydb/helpers_test.go | 42 -------------- database/ydb/ydb.go | 94 +++++++++++++++--------------- database/ydb/ydb_test.go | 12 ++-- 5 files changed, 50 insertions(+), 207 deletions(-) delete mode 100644 database/ydb/helpers.go delete mode 100644 database/ydb/helpers_test.go diff --git a/README.md b/README.md index ad1c73dc7..227587aaf 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ Database drivers run migrations. [Add a new database?](database/driver.go) * [Firebird](database/firebird) * [MS SQL Server](database/sqlserver) * [rqlite](database/rqlite) +* [YDB](database/ydb) ### Database URLs diff --git a/database/ydb/helpers.go b/database/ydb/helpers.go deleted file mode 100644 index 87ff97d72..000000000 --- a/database/ydb/helpers.go +++ /dev/null @@ -1,108 +0,0 @@ -package ydb - -import ( - "regexp" - "strings" -) - -const ( - createVersionTableQueryTemplate = ` - CREATE TABLE %s ( - version Int32, - dirty Bool, - applied_at Timestamp, - PRIMARY KEY (version) - ); - ` - - deleteVersionsQueryTemplate = ` - DELETE FROM %s;` - - setVersionQueryTemplate = ` - DECLARE $version AS Int32; - DECLARE $dirty AS Bool; - DECLARE $applied_at AS Timestamp; - UPSERT INTO %s (version, dirty, applied_at) - VALUES ($version, $dirty, $applied_at);` - - getVersionQueryTemplate = ` - SELECT version, dirty FROM %s - ORDER BY version DESC LIMIT 1;` - - dropTablesQueryTemplate = "DROP TABLE `%s`;" -) - -type queryMode int - -const ( - notSetMode queryMode = iota - ddlMode - dmlMode - unknownMode -) - -func skipComments(statements string) (string, error) { - type interval struct { - start int - end int - } - comments := make([]interval, 0) - inString := false - stringDelimiter := byte(0) - - for i := 0; i < len(statements); i++ { - if (statements[i] == '\'' || statements[i] == '`') && (i == 0 || statements[i-1] != '\\') { - if inString { - if statements[i] == stringDelimiter { - inString = false - } - } else { - inString = true - stringDelimiter = statements[i] - } - } - - if !inString { - if statements[i] == '-' && i+1 < len(statements) && statements[i+1] == '-' { - start := i - for ; i < len(statements) && statements[i] != '\n'; i++ { - } - comments = append(comments, interval{start: start, end: i}) - } else if statements[i] == '/' && i+1 < len(statements) && statements[i+1] == '*' { - start := i - for ; i < len(statements)-1 && !(statements[i] == '*' && statements[i+1] == '/'); i++ { - } - comments = append(comments, interval{start: start, end: i + 1}) - } - } - } - - res := strings.Builder{} - curPos := 0 - - for _, comment := range comments { - _, err := res.WriteString(statements[curPos:comment.start]) - if err != nil { - return "", err - } - curPos = comment.end + 1 - } - - if curPos < len(statements) { - _, err := res.WriteString(statements[curPos:]) - if err != nil { - return "", err - } - } - - return res.String(), nil -} - -func detectQueryMode(statement string) queryMode { - ddlReg := regexp.MustCompile(`^(?i)(CREATE|ALTER|DECLARE|GRANT|REVOKE|DROP).*`) - if ddlReg.MatchString(statement) { - return ddlMode - } - - return dmlMode -} diff --git a/database/ydb/helpers_test.go b/database/ydb/helpers_test.go deleted file mode 100644 index 128d5ef00..000000000 --- a/database/ydb/helpers_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package ydb - -import "testing" - -func TestSkipComments(t *testing.T) { - tests := []struct { - name string - input string - expected string - }{ - {"empty input", "", ""}, - {"no comments", "SELECT * FROM table;", "SELECT * FROM table;"}, - {"single line comments", "-- This is a comment\nSELECT * FROM table;", "SELECT * FROM table;"}, - {"multi-line comments", "/* This is a comment */SELECT * FROM table;", "SELECT * FROM table;"}, - {"mixed comments", "-- Single line comment\n/* Multi\nLine\nComment */SELECT * FROM table;\nDROP TABLE table;", "SELECT * FROM table;\nDROP TABLE table;"}, - {"comments at the start", "-- Comment\nSELECT * FROM table;", "SELECT * FROM table;"}, - {"comments at the end", "SELECT * FROM table;-- Comment", "SELECT * FROM table;"}, - {"comments with special characters", "/* Com!ment */SELECT * FROM table;", "SELECT * FROM table;"}, - {"single line comment at end of line", "SELECT * FROM table; -- Comment", "SELECT * FROM table; "}, - {"multi-line comment in middle of line", "SELECT /* Comment */ * FROM table;", "SELECT * FROM table;"}, - {"multiple single line comments consecutively", "-- Comment 1\n-- Comment 2\nSELECT * FROM table;", "SELECT * FROM table;"}, - {"multiple multi-line comments consecutively", "/* Comment 1 *//* Comment 2 */SELECT * FROM table;", "SELECT * FROM table;"}, - {"mixed comments in single line", "SELECT * -- Comment\nFROM /* Comment */table;", "SELECT * FROM table;"}, - {"comments with sql keywords", "-- SELECT * FROM table\nSELECT name FROM users;", "SELECT name FROM users;"}, - {"sql commands with comment-like syntax", "SELECT * FROM `--table--` WHERE name = '/* John */';", "SELECT * FROM `--table--` WHERE name = '/* John */';"}, - {"whitespace handling", " -- Comment\nSELECT * FROM table; ", " SELECT * FROM table; "}, - {"comments with escape characters", "-- Comment \n\t SELECT * FROM table;", "\t SELECT * FROM table;"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result, err := skipComments(tt.input) - if err != nil { - t.Errorf("skipComments() error = %v", err) - return - } - if result != tt.expected { - t.Errorf("skipComments() = %v, want %v", result, tt.expected) - } - }) - } -} diff --git a/database/ydb/ydb.go b/database/ydb/ydb.go index 7771d2832..c943367bc 100644 --- a/database/ydb/ydb.go +++ b/database/ydb/ydb.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "net/url" - "strings" "sync/atomic" "time" @@ -27,6 +26,35 @@ func init() { database.Register("ydb", &YDB{}) } +const ( + createVersionTableQueryTemplate = ` + CREATE TABLE %s ( + version Int32, + dirty Bool, + applied_at Timestamp, + PRIMARY KEY (version) + ); + ` + + deleteVersionsQueryTemplate = ` + DELETE FROM %s;` + + setVersionQueryTemplate = ` + DECLARE $version AS Int32; + DECLARE $dirty AS Bool; + DECLARE $applied_at AS Timestamp; + UPSERT INTO %s (version, dirty, applied_at) + VALUES ($version, $dirty, $applied_at);` + + getCurrentVersionQueryTemplate = ` + SELECT version, dirty FROM %s + ORDER BY version DESC LIMIT 1;` + + dropTablesQueryTemplate = "DROP TABLE `%s`;" + + getAllTablesQueryTemplate = "SELECT Path FROM `%s/.sys/partition_stats` WHERE Path NOT LIKE '%%/.sys%%'" +) + type YDB struct { db *sql.DB locked atomic.Bool @@ -105,8 +133,12 @@ func (y *YDB) createMigrationsTable(ctx context.Context) (err error) { ctx = ydb.WithQueryMode(ctx, ydb.SchemeQueryMode) - if _, err := y.db.ExecContext(ctx, fmt.Sprintf(createVersionTableQueryTemplate, y.tableWithPrefix(y.migrationsTable))); err != nil { - return err + createTableQuery := fmt.Sprintf(createVersionTableQueryTemplate, y.tableWithPrefix(y.migrationsTable)) + if _, err := y.db.ExecContext(ctx, createTableQuery); err != nil { + return database.Error{ + OrigErr: err, + Query: []byte(createTableQuery), + } } return nil @@ -118,7 +150,7 @@ func (y *YDB) Close() error { func (y *YDB) Drop() (err error) { tablesQuery := fmt.Sprintf( - "SELECT Path FROM `%s/.sys/partition_stats` WHERE Path NOT LIKE '%%/.sys%%'", + getAllTablesQueryTemplate, y.prefix, ) @@ -163,50 +195,12 @@ func (y *YDB) Run(migration io.Reader) error { return err } - statements, err := skipComments(string(data)) - if err != nil { - return database.Error{ - OrigErr: err, - Err: "failed to skip comments", - } - } - - currentMode := notSetMode - - for _, statement := range strings.Split(statements, ";") { - statement = strings.TrimSpace(statement) - - if statement == "" { - continue - } - - mode := detectQueryMode(statement) - - if currentMode == notSetMode { - currentMode = mode - } else if currentMode != mode { - return database.Error{ - Err: "mixed query modes in one migration", - Query: []byte(statements), - } - } - } - - ctx := context.Background() - - switch currentMode { - case ddlMode: - ctx = ydb.WithQueryMode(ctx, ydb.SchemeQueryMode) - case dmlMode: - ctx = ydb.WithQueryMode(ctx, ydb.DataQueryMode) - } - - _, err = y.db.ExecContext(ctx, statements) + _, err = y.db.ExecContext(ydb.WithQueryMode(context.Background(), ydb.ScriptingQueryMode), string(data)) if err != nil { return database.Error{ OrigErr: err, Err: "migration failed", - Query: []byte(statements), + Query: data, } } @@ -222,9 +216,10 @@ func (y *YDB) SetVersion(version int, dirty bool) error { return err } - deleteVersions := fmt.Sprintf(deleteVersionsQueryTemplate, y.tableWithPrefix(y.migrationsTable)) + ctx := ydb.WithQueryMode(context.Background(), ydb.DataQueryMode) - if _, err = tx.Exec(deleteVersions); err != nil { + deleteVersions := fmt.Sprintf(deleteVersionsQueryTemplate, y.tableWithPrefix(y.migrationsTable)) + if _, err = tx.ExecContext(ctx, deleteVersions); err != nil { if rollbackErr := tx.Rollback(); rollbackErr != nil { err = errors.Join(err, rollbackErr) } @@ -237,8 +232,8 @@ func (y *YDB) SetVersion(version int, dirty bool) error { } versionQuery := fmt.Sprintf(setVersionQueryTemplate, y.tableWithPrefix(y.migrationsTable)) - - _, err = tx.Exec( + _, err = tx.ExecContext( + ctx, versionQuery, sql.Named("version", version), sql.Named("dirty", dirty), @@ -252,6 +247,7 @@ func (y *YDB) SetVersion(version int, dirty bool) error { return database.Error{ OrigErr: err, Err: "failed to set version", + Query: []byte(versionQuery), } } @@ -259,7 +255,7 @@ func (y *YDB) SetVersion(version int, dirty bool) error { } func (y *YDB) Version() (version int, dirty bool, err error) { - versionQuery := fmt.Sprintf(getVersionQueryTemplate, y.tableWithPrefix(y.migrationsTable)) + versionQuery := fmt.Sprintf(getCurrentVersionQueryTemplate, y.tableWithPrefix(y.migrationsTable)) row, err := y.db.QueryContext(ydb.WithQueryMode(context.Background(), ydb.ScanQueryMode), versionQuery) if err != nil { diff --git a/database/ydb/ydb_test.go b/database/ydb/ydb_test.go index e88716d86..b6cad00c5 100644 --- a/database/ydb/ydb_test.go +++ b/database/ydb/ydb_test.go @@ -5,7 +5,6 @@ import ( "database/sql" "fmt" "log" - "os" "testing" "time" @@ -29,7 +28,8 @@ var ( ReadyTimeout: 15 * time.Second, Hostname: host, Env: map[string]string{ - "YDB_USE_IN_MEMORY_PDISKS": "true", + "YDB_USE_IN_MEMORY_PDISKS": "true", + "YDB_LOCAL_SURVIVE_RESTART": "true", }, PortBindings: nat.PortMap{ nat.Port(fmt.Sprintf("%s/tcp", port)): []nat.PortBinding{{ @@ -43,12 +43,8 @@ var ( image = "cr.yandex/yc/yandex-docker-local-ydb:latest" ) -func init() { - _ = os.Setenv("YDB_ANONYMOUS_CREDENTIALS", "1") -} - func isReady(ctx context.Context, c dktest.ContainerInfo) bool { - nativeDriver, err := ydb.Open(context.Background(), fmt.Sprintf("grpc://localhost:%s/%s", port, testDB)) + nativeDriver, err := ydb.Open(context.Background(), fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB)) if err != nil { log.Println(err) return false @@ -93,7 +89,7 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool { func Test(t *testing.T) { dktest.Run(t, image, opts, func(t *testing.T, c dktest.ContainerInfo) { - addr := fmt.Sprintf("grpc://localhost:%s/%s", port, testDB) + addr := fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB) p := &YDB{} d, err := p.Open(addr) if err != nil { From b2da2e0c36f15334a957d0aa374ffa360a8ad902 Mon Sep 17 00:00:00 2001 From: AlexisOMG Date: Sun, 21 Jan 2024 17:16:22 +0300 Subject: [PATCH 05/10] add open close test --- database/ydb/ydb_test.go | 49 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/database/ydb/ydb_test.go b/database/ydb/ydb_test.go index b6cad00c5..3ca186732 100644 --- a/database/ydb/ydb_test.go +++ b/database/ydb/ydb_test.go @@ -9,9 +9,12 @@ import ( "time" "github.com/docker/go-connections/nat" + "github.com/stretchr/testify/assert" ydb "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/ydb-platform/ydb-go-sdk/v3/balancers" "github.com/dhui/dktest" + "github.com/golang-migrate/migrate/v4/database" dt "github.com/golang-migrate/migrate/v4/database/testing" _ "github.com/golang-migrate/migrate/v4/source/file" ) @@ -44,7 +47,11 @@ var ( ) func isReady(ctx context.Context, c dktest.ContainerInfo) bool { - nativeDriver, err := ydb.Open(context.Background(), fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB)) + nativeDriver, err := ydb.Open( + context.Background(), + fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB), + ydb.WithBalancer(balancers.SingleConn()), + ) if err != nil { log.Println(err) return false @@ -71,7 +78,7 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool { return false } - ctxWithTimeout = ydb.WithQueryMode(ctxWithTimeout, ydb.SchemeQueryMode) + ctxWithTimeout = ydb.WithQueryMode(ctxWithTimeout, ydb.ScriptingQueryMode) _, err = db.ExecContext(ctxWithTimeout, ` CREATE TABLE test ( @@ -108,3 +115,41 @@ func Test(t *testing.T) { DROP TABLE test;`)) }) } + +func TestClose(t *testing.T) { + dktest.Run(t, image, opts, func(t *testing.T, c dktest.ContainerInfo) { + addr := fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB) + p := &YDB{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + if err := d.Close(); err != nil { + t.Error(err) + } + + _, _, err = d.Version() + assert.ErrorContains(t, err, "database is closed") + }) +} + +func TestOpen(t *testing.T) { + dktest.Run(t, image, opts, func(t *testing.T, c dktest.ContainerInfo) { + addr := fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB) + p := &YDB{} + d, err := p.Open(addr) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() + + version, dirty, err := d.Version() + assert.NoError(t, err) + assert.Equal(t, database.NilVersion, version) + assert.False(t, dirty) + }) +} From 263596edbf09db220e06a27f554c79b875e82d6a Mon Sep 17 00:00:00 2001 From: AlexisOMG Date: Sun, 21 Jan 2024 17:57:50 +0300 Subject: [PATCH 06/10] add warnings to README --- database/ydb/README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/database/ydb/README.md b/database/ydb/README.md index 9e84a61e2..973302f2e 100644 --- a/database/ydb/README.md +++ b/database/ydb/README.md @@ -1,6 +1,6 @@ -# YDB +# [YDB](https://ydb.tech/docs/en/) -`grpc[s]://user:password@host:port/database?QUERY_PARAMS` +`grpc[s]://[user:password@]host:port/database?QUERY_PARAMS` | URL Query | Description | |------------|-------------| @@ -10,3 +10,7 @@ | `port` | The port to bind to | | `database` | The name of the database to connect to | | `x-migrations-table`| Name of the migrations table. Default: `schema_migrations` | + +## Warning +- Beware of race conditions between migrations initiated from different processes (on the same machine or on different machines). +- Beware of partial migrations, because currently in YDB it is not possible to execute DDL SQL statements in a transaction. From 94c4a40c9dba17ed08147d373cc12510961347e0 Mon Sep 17 00:00:00 2001 From: AlexisOMG Date: Sun, 21 Jan 2024 18:46:07 +0300 Subject: [PATCH 07/10] add WithInstance --- database/ydb/ydb.go | 74 ++++++++++++++++++++++++++++++++-------- database/ydb/ydb_test.go | 70 +++++++++++++++++++++++-------------- 2 files changed, 105 insertions(+), 39 deletions(-) diff --git a/database/ydb/ydb.go b/database/ydb/ydb.go index c943367bc..736339966 100644 --- a/database/ydb/ydb.go +++ b/database/ydb/ydb.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/url" + "strings" "sync/atomic" "time" @@ -52,18 +53,55 @@ const ( dropTablesQueryTemplate = "DROP TABLE `%s`;" - getAllTablesQueryTemplate = "SELECT Path FROM `%s/.sys/partition_stats` WHERE Path NOT LIKE '%%/.sys%%'" + getAllTablesQueryTemplate = "SELECT Path FROM `%s.sys/partition_stats` WHERE Path NOT LIKE '%%.sys%%'" ) +type Config struct { + MigrationsTable string + Path string +} + +func WithInstance(db *sql.DB, config Config) (database.Driver, error) { + if err := db.Ping(); err != nil { + return nil, err + } + + if config.MigrationsTable == "" { + config.MigrationsTable = defaultMigrationsTable + } + + if config.Path != "" && !strings.HasSuffix(config.Path, "/") { + config.Path += "/" + } + + ydbDriver := &YDB{ + db: db, + config: config, + } + + err := ydbDriver.createMigrationsTable(context.Background()) + if err != nil { + return nil, database.Error{ + OrigErr: err, + Err: "failed to create migrations table", + } + } + + return ydbDriver, nil +} + type YDB struct { - db *sql.DB - locked atomic.Bool - migrationsTable string - prefix string + db *sql.DB + locked atomic.Bool + config Config } func (y *YDB) tableWithPrefix(table string) string { - return fmt.Sprintf("`%s/%s`", y.prefix, table) + if y.config.Path == "" { + return table + } + + return fmt.Sprintf("`%s%s`", y.config.Path, table) } func (y *YDB) Lock() error { @@ -103,10 +141,18 @@ func (y *YDB) Open(dsn string) (database.Driver, error) { migrationsTable = defaultMigrationsTable } + databaseName := nativeDriver.Name() + + if databaseName != "" { + databaseName += "/" + } + ydbDriver := &YDB{ - db: sql.OpenDB(connector), - migrationsTable: migrationsTable, - prefix: nativeDriver.Name(), + db: sql.OpenDB(connector), + config: Config{ + MigrationsTable: migrationsTable, + Path: databaseName, + }, } err = ydbDriver.createMigrationsTable(context.Background()) @@ -133,7 +179,7 @@ func (y *YDB) createMigrationsTable(ctx context.Context) (err error) { ctx = ydb.WithQueryMode(ctx, ydb.SchemeQueryMode) - createTableQuery := fmt.Sprintf(createVersionTableQueryTemplate, y.tableWithPrefix(y.migrationsTable)) + createTableQuery := fmt.Sprintf(createVersionTableQueryTemplate, y.tableWithPrefix(y.config.MigrationsTable)) if _, err := y.db.ExecContext(ctx, createTableQuery); err != nil { return database.Error{ OrigErr: err, @@ -151,7 +197,7 @@ func (y *YDB) Close() error { func (y *YDB) Drop() (err error) { tablesQuery := fmt.Sprintf( getAllTablesQueryTemplate, - y.prefix, + y.config.Path, ) rows, err := y.db.QueryContext(ydb.WithQueryMode(context.Background(), ydb.ScanQueryMode), tablesQuery) @@ -218,7 +264,7 @@ func (y *YDB) SetVersion(version int, dirty bool) error { ctx := ydb.WithQueryMode(context.Background(), ydb.DataQueryMode) - deleteVersions := fmt.Sprintf(deleteVersionsQueryTemplate, y.tableWithPrefix(y.migrationsTable)) + deleteVersions := fmt.Sprintf(deleteVersionsQueryTemplate, y.tableWithPrefix(y.config.MigrationsTable)) if _, err = tx.ExecContext(ctx, deleteVersions); err != nil { if rollbackErr := tx.Rollback(); rollbackErr != nil { err = errors.Join(err, rollbackErr) @@ -231,7 +277,7 @@ func (y *YDB) SetVersion(version int, dirty bool) error { } } - versionQuery := fmt.Sprintf(setVersionQueryTemplate, y.tableWithPrefix(y.migrationsTable)) + versionQuery := fmt.Sprintf(setVersionQueryTemplate, y.tableWithPrefix(y.config.MigrationsTable)) _, err = tx.ExecContext( ctx, versionQuery, @@ -255,7 +301,7 @@ func (y *YDB) SetVersion(version int, dirty bool) error { } func (y *YDB) Version() (version int, dirty bool, err error) { - versionQuery := fmt.Sprintf(getCurrentVersionQueryTemplate, y.tableWithPrefix(y.migrationsTable)) + versionQuery := fmt.Sprintf(getCurrentVersionQueryTemplate, y.tableWithPrefix(y.config.MigrationsTable)) row, err := y.db.QueryContext(ydb.WithQueryMode(context.Background(), ydb.ScanQueryMode), versionQuery) if err != nil { diff --git a/database/ydb/ydb_test.go b/database/ydb/ydb_test.go index 3ca186732..27b046d09 100644 --- a/database/ydb/ydb_test.go +++ b/database/ydb/ydb_test.go @@ -10,8 +10,8 @@ import ( "github.com/docker/go-connections/nat" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ydb "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/balancers" "github.com/dhui/dktest" "github.com/golang-migrate/migrate/v4/database" @@ -47,23 +47,11 @@ var ( ) func isReady(ctx context.Context, c dktest.ContainerInfo) bool { - nativeDriver, err := ydb.Open( - context.Background(), - fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB), - ydb.WithBalancer(balancers.SingleConn()), - ) + db, err := sql.Open("ydb", fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB)) if err != nil { log.Println(err) return false } - - connector, err := ydb.Connector(nativeDriver) - if err != nil { - log.Println("close error:", err) - return false - } - - db := sql.OpenDB(connector) defer func() { if err := db.Close(); err != nil { log.Println("close error:", err) @@ -94,7 +82,7 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool { return true } -func Test(t *testing.T) { +func TestOpen(t *testing.T) { dktest.Run(t, image, opts, func(t *testing.T, c dktest.ContainerInfo) { addr := fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB) p := &YDB{} @@ -107,12 +95,11 @@ func Test(t *testing.T) { t.Error(err) } }() - dt.Test(t, d, []byte(` - CREATE TABLE test ( - id Int, - PRIMARY KEY(id) - ); - DROP TABLE test;`)) + + version, dirty, err := d.Version() + assert.NoError(t, err) + assert.Equal(t, database.NilVersion, version) + assert.False(t, dirty) }) } @@ -133,7 +120,7 @@ func TestClose(t *testing.T) { }) } -func TestOpen(t *testing.T) { +func Test(t *testing.T) { dktest.Run(t, image, opts, func(t *testing.T, c dktest.ContainerInfo) { addr := fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB) p := &YDB{} @@ -146,10 +133,43 @@ func TestOpen(t *testing.T) { t.Error(err) } }() + dt.Test(t, d, []byte(` + CREATE TABLE test ( + id Int, + PRIMARY KEY(id) + ); + DROP TABLE test;`)) + }) +} + +func TestWithInstance(t *testing.T) { + dktest.Run(t, image, opts, func(t *testing.T, c dktest.ContainerInfo) { + addr := fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB) + db, err := sql.Open("ydb", addr) + if err != nil { + t.Fatal(err) + } + + d, err := WithInstance(db, Config{}) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + }() version, dirty, err := d.Version() - assert.NoError(t, err) - assert.Equal(t, database.NilVersion, version) - assert.False(t, dirty) + require.NoError(t, err) + require.Equal(t, database.NilVersion, version) + require.False(t, dirty) + + dt.Test(t, d, []byte(` + CREATE TABLE test ( + id Int, + PRIMARY KEY(id) + ); + DROP TABLE test;`)) }) } From cdb4786d0d5674eef6de39fb63cc8e316a65ee8b Mon Sep 17 00:00:00 2001 From: AlexisOMG Date: Mon, 22 Jan 2024 15:01:20 +0300 Subject: [PATCH 08/10] fix imports order --- database/ydb/ydb.go | 4 ++-- go.mod | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/database/ydb/ydb.go b/database/ydb/ydb.go index 736339966..2265621ec 100644 --- a/database/ydb/ydb.go +++ b/database/ydb/ydb.go @@ -11,9 +11,9 @@ import ( "sync/atomic" "time" - "github.com/golang-migrate/migrate/v4/database" - ydb "github.com/ydb-platform/ydb-go-sdk/v3" + + "github.com/golang-migrate/migrate/v4/database" ) const ( diff --git a/go.mod b/go.mod index 5706da935..e6578a95a 100644 --- a/go.mod +++ b/go.mod @@ -44,11 +44,6 @@ require ( modernc.org/sqlite v1.18.1 ) -require ( - github.com/jonboulle/clockwork v0.3.0 // indirect - github.com/ydb-platform/ydb-go-genproto v0.0.0-20231215113745-46f6d30f974a // indirect -) - require ( cloud.google.com/go v0.110.10 // indirect cloud.google.com/go/compute v1.23.3 // indirect @@ -126,6 +121,7 @@ require ( github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgtype v1.14.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jonboulle/clockwork v0.3.0 // indirect github.com/k0kubun/pp v2.3.0+incompatible // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect @@ -155,6 +151,7 @@ require ( github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.1 // indirect github.com/xdg-go/stringprep v1.0.3 // indirect + github.com/ydb-platform/ydb-go-genproto v0.0.0-20231215113745-46f6d30f974a // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/zeebo/xxh3 v1.0.2 // indirect gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b // indirect From 83e80061578606a80228d7355df55f2c43555356 Mon Sep 17 00:00:00 2001 From: AlexisOMG Date: Mon, 22 Jan 2024 23:40:32 +0300 Subject: [PATCH 09/10] fix connection string --- database/ydb/README.md | 3 ++- database/ydb/ydb.go | 20 +++++++++++++++----- database/ydb/ydb_test.go | 6 +++--- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/database/ydb/README.md b/database/ydb/README.md index 973302f2e..608e69cce 100644 --- a/database/ydb/README.md +++ b/database/ydb/README.md @@ -1,6 +1,6 @@ # [YDB](https://ydb.tech/docs/en/) -`grpc[s]://[user:password@]host:port/database?QUERY_PARAMS` +`ydb://[user:password@]host:port/database?QUERY_PARAMS` | URL Query | Description | |------------|-------------| @@ -10,6 +10,7 @@ | `port` | The port to bind to | | `database` | The name of the database to connect to | | `x-migrations-table`| Name of the migrations table. Default: `schema_migrations` | +| `x-use-grpcs` | Enable GRPCS protocol for connecting to YDB (default GRPC) | ## Warning - Beware of race conditions between migrations initiated from different processes (on the same machine or on different machines). diff --git a/database/ydb/ydb.go b/database/ydb/ydb.go index 2265621ec..346434641 100644 --- a/database/ydb/ydb.go +++ b/database/ydb/ydb.go @@ -13,12 +13,15 @@ import ( ydb "github.com/ydb-platform/ydb-go-sdk/v3" + "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database" ) const ( migrationsTableQueryParam = "x-migrations-table" - defaultMigrationsTable = "schema_migrations" + useGRPCSQueryParam = "x-use-grpcs" + + defaultMigrationsTable = "schema_migrations" ) var _ database.Driver = (*YDB)(nil) @@ -120,22 +123,29 @@ func (y *YDB) Unlock() error { } func (y *YDB) Open(dsn string) (database.Driver, error) { - nativeDriver, err := ydb.Open(context.Background(), dsn) + customUrl, err := url.Parse(dsn) if err != nil { return nil, err } - connector, err := ydb.Connector(nativeDriver) + connUrl := migrate.FilterCustomQuery(customUrl) + if customUrl.Query().Get(useGRPCSQueryParam) != "" { + connUrl.Scheme = "grpcs" + } else { + connUrl.Scheme = "grpc" + } + + nativeDriver, err := ydb.Open(context.Background(), connUrl.String()) if err != nil { return nil, err } - connUrl, err := url.Parse(dsn) + connector, err := ydb.Connector(nativeDriver) if err != nil { return nil, err } - migrationsTable := connUrl.Query().Get(migrationsTableQueryParam) + migrationsTable := customUrl.Query().Get(migrationsTableQueryParam) if migrationsTable == "" { migrationsTable = defaultMigrationsTable diff --git a/database/ydb/ydb_test.go b/database/ydb/ydb_test.go index 27b046d09..25fcce72d 100644 --- a/database/ydb/ydb_test.go +++ b/database/ydb/ydb_test.go @@ -84,7 +84,7 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool { func TestOpen(t *testing.T) { dktest.Run(t, image, opts, func(t *testing.T, c dktest.ContainerInfo) { - addr := fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB) + addr := fmt.Sprintf("ydb://%s:%s/%s", host, port, testDB) p := &YDB{} d, err := p.Open(addr) if err != nil { @@ -105,7 +105,7 @@ func TestOpen(t *testing.T) { func TestClose(t *testing.T) { dktest.Run(t, image, opts, func(t *testing.T, c dktest.ContainerInfo) { - addr := fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB) + addr := fmt.Sprintf("ydb://%s:%s/%s", host, port, testDB) p := &YDB{} d, err := p.Open(addr) if err != nil { @@ -122,7 +122,7 @@ func TestClose(t *testing.T) { func Test(t *testing.T) { dktest.Run(t, image, opts, func(t *testing.T, c dktest.ContainerInfo) { - addr := fmt.Sprintf("grpc://%s:%s/%s", host, port, testDB) + addr := fmt.Sprintf("ydb://%s:%s/%s", host, port, testDB) p := &YDB{} d, err := p.Open(addr) if err != nil { From 5f6ddb19ea7f24e81dc83c83fe4e412d63aa0ad4 Mon Sep 17 00:00:00 2001 From: AlexisOMG Date: Thu, 25 Jan 2024 22:06:38 +0300 Subject: [PATCH 10/10] supplement README --- database/ydb/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/database/ydb/README.md b/database/ydb/README.md index 608e69cce..ee546ed57 100644 --- a/database/ydb/README.md +++ b/database/ydb/README.md @@ -13,5 +13,6 @@ | `x-use-grpcs` | Enable GRPCS protocol for connecting to YDB (default GRPC) | ## Warning +- It is not possible to use DDL and DML queries simultaneously within a single migration. - Beware of race conditions between migrations initiated from different processes (on the same machine or on different machines). - Beware of partial migrations, because currently in YDB it is not possible to execute DDL SQL statements in a transaction.