From b1ef1fb36db0d777d8dd8914fd8b16551d5bd3e2 Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Fri, 6 Jul 2018 14:41:19 +0200 Subject: [PATCH 1/2] vendor: upgrade go-mysql-server Signed-off-by: Miguel Molina --- Gopkg.lock | 4 +- Gopkg.toml | 2 +- docs/using-gitbase/functions.md | 2 +- docs/using-gitbase/indexes.md | 2 +- docs/using-gitbase/supported-syntax.md | 2 +- .../src-d/go-mysql-server.v0/README.md | 11 ++ .../src-d/go-mysql-server.v0/engine_test.go | 21 +--- .../go-mysql-server.v0/sql/analyzer/rules.go | 7 +- .../sql/analyzer/rules_test.go | 53 ++++++++-- .../sql/expression/common.go | 10 ++ .../sql/expression/common_test.go | 12 +++ .../src-d/go-mysql-server.v0/sql/index.go | 15 ++- .../sql/index/pilosa/driver_test.go | 100 ++++-------------- .../go-mysql-server.v0/sql/index_test.go | 14 ++- .../go-mysql-server.v0/sql/plan/common.go | 10 ++ .../sql/plan/common_test.go | 16 +++ .../sql/plan/create_index.go | 24 +++-- .../sql/plan/create_index_test.go | 55 +++++++++- .../sql/plan/drop_index_test.go | 4 +- .../go-mysql-server.v0/sql/plan/pushdown.go | 3 +- 20 files changed, 227 insertions(+), 140 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 62b011415..eb5b894d8 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -485,7 +485,7 @@ "sql/parse", "sql/plan" ] - revision = "e03c0c8264d306f683a8c1e7933c5d2f4b200a5b" + revision = "9c3b77f244b0909fed15a669be2cdeb355462a7d" [[projects]] name = "gopkg.in/src-d/go-siva.v1" @@ -538,6 +538,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "a41d28886ed6617ccfd72902fad3d8acf95c7f3050e079db86024797a9e36897" + inputs-digest = "785fa54e48ad9295f1e211024fd82821e8996265c232a479ddfea19d4f2a91e0" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 4e1720b72..931f53d25 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,6 +1,6 @@ [[constraint]] name = "gopkg.in/src-d/go-mysql-server.v0" - revision = "e03c0c8264d306f683a8c1e7933c5d2f4b200a5b" + revision = "9c3b77f244b0909fed15a669be2cdeb355462a7d" [[constraint]] name = "github.com/jessevdk/go-flags" diff --git a/docs/using-gitbase/functions.md b/docs/using-gitbase/functions.md index 8e59884a0..b980c7a14 100644 --- a/docs/using-gitbase/functions.md +++ b/docs/using-gitbase/functions.md @@ -14,4 +14,4 @@ To make some common tasks easier for the user, there are some functions to inter ## Standard functions -You can check standard functions in [`go-mysql-server` documentation](https://github.com/src-d/go-mysql-server/tree/e03c0c8264d306f683a8c1e7933c5d2f4b200a5b#custom-functions). +You can check standard functions in [`go-mysql-server` documentation](https://github.com/src-d/go-mysql-server/tree/9c3b77f244b0909fed15a669be2cdeb355462a7d#custom-functions). diff --git a/docs/using-gitbase/indexes.md b/docs/using-gitbase/indexes.md index 16be800ad..dbc5ad050 100644 --- a/docs/using-gitbase/indexes.md +++ b/docs/using-gitbase/indexes.md @@ -10,4 +10,4 @@ Note that you can create an index either **on one or more columns** or **on a si You can find some more examples in the [examples](./examples.md#create-an-index-for-columns-on-a-table) section. -See [go-mysql-server](https://github.com/src-d/go-mysql-server/tree/e03c0c8264d306f683a8c1e7933c5d2f4b200a5b#indexes) documentation for more details +See [go-mysql-server](https://github.com/src-d/go-mysql-server/tree/9c3b77f244b0909fed15a669be2cdeb355462a7d#indexes) documentation for more details diff --git a/docs/using-gitbase/supported-syntax.md b/docs/using-gitbase/supported-syntax.md index a38ab7c53..7c80f8cc9 100644 --- a/docs/using-gitbase/supported-syntax.md +++ b/docs/using-gitbase/supported-syntax.md @@ -1,3 +1,3 @@ ## Supported syntax -To see the SQL subset currently supported take a look at [this list](https://github.com/src-d/go-mysql-server/blob/e03c0c8264d306f683a8c1e7933c5d2f4b200a5b/SUPPORTED.md) from [src-d/go-mysql-server](https://github.com/src-d/go-mysql-server). +To see the SQL subset currently supported take a look at [this list](https://github.com/src-d/go-mysql-server/blob/9c3b77f244b0909fed15a669be2cdeb355462a7d/SUPPORTED.md) from [src-d/go-mysql-server](https://github.com/src-d/go-mysql-server). diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/README.md b/vendor/gopkg.in/src-d/go-mysql-server.v0/README.md index d80607be3..a959640b2 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/README.md +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/README.md @@ -57,6 +57,7 @@ We are continuously adding more functionality to go-mysql-server. We support a s - `ARRAY_LENGTH(json)`: If the json representation is an array, this function returns its size. - `SPLIT(str,sep)`: Receives a string and a separator and returns the parts of the string split by the separator as a JSON array of strings. - `CONCAT(...)`: Concatenate any group of fields into a single string. + ## Example `go-mysql-server` contains a SQL engine and server implementation. So, if you want to start a server, first instantiate the engine and pass your `sql.Database` implementation. @@ -157,6 +158,7 @@ SELECT email FROM mytable WHERE name = 'Evil Bob' | evilbob@gmail.com | +-------------------+ ``` + ## Custom data source implementation To be able to create your own data source implementation you need to implement the following interfaces: @@ -174,6 +176,15 @@ To be able to create your own data source implementation you need to implement t You can see a really simple data source implementation on our `mem` package. +## Indexes + +`go-mysql-server` exposes a series of interfaces to allow you to implement your own indexes so you can speedup your queries. + +Taking a look at the main [index interface](https://github.com/src-d/go-mysql-server/blob/master/sql/index.go#L35), you must note a couple of constraints: + +- This abstraction lets you create an index for multiple columns (one or more) or for **only one** expression (e.g. function applied on multiple columns). +- If you want to index an expression that is not a column you will only be able to index **one and only one** expression at a time. + ## Powered by go-mysql-server * [gitbase](https://github.com/src-d/gitbase) diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/engine_test.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/engine_test.go index 207a3444a..0d04169ad 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/engine_test.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/engine_test.go @@ -13,7 +13,6 @@ import ( "gopkg.in/src-d/go-mysql-server.v0/mem" "gopkg.in/src-d/go-mysql-server.v0/sql" "gopkg.in/src-d/go-mysql-server.v0/sql/analyzer" - "gopkg.in/src-d/go-mysql-server.v0/sql/expression" "gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa" "gopkg.in/src-d/go-mysql-server.v0/sql/parse" "gopkg.in/src-d/go-mysql-server.v0/test" @@ -715,27 +714,11 @@ func TestIndexes(t *testing.T) { require.NoError(os.MkdirAll(tmpDir, 0644)) e.Catalog.RegisterIndexDriver(pilosa.NewIndexDriver(tmpDir)) - db, err := e.Catalog.Database("mydb") - require.NoError(err) - table := db.Tables()["mytable"].(sql.Indexable) - - driver := e.Catalog.IndexDriver(pilosa.DriverID) - conf := make(map[string]string) - expr := sql.NewExpressionHash(expression.NewGetFieldWithTable(0, sql.Int64, "mytable", "i", false)) - idx, err := driver.Create("mydb", "mytable", "myidx", []sql.ExpressionHash{expr}, conf) - require.NoError(err) - - created, err := e.Catalog.AddIndex(idx) + _, _, err = e.Query(sql.NewEmptyContext(), "CREATE INDEX myidx ON mytable (i) WITH (async = false)") require.NoError(err) - iter, err := table.IndexKeyValueIter(sql.NewEmptyContext(), []string{"i"}) - require.NoError(err) - - require.NoError(driver.Save(sql.NewEmptyContext(), idx, iter)) - created <- struct{}{} - defer func() { - done, err := e.Catalog.DeleteIndex("foo", "myidx", true) + done, err := e.Catalog.DeleteIndex("mydb", "myidx", true) require.NoError(err) <-done }() diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/analyzer/rules.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/analyzer/rules.go index 9b74f0f23..a67fa8a3d 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/analyzer/rules.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/analyzer/rules.go @@ -796,15 +796,12 @@ func resolveColumns(ctx *sql.Context, a *Analyzer, n sql.Node) (sql.Node, error) } var schema sql.Schema - switch n := n.(type) { // If expressioner and unary node we must take the // child's schema to correctly select the indexes // in the row is going to be evaluated in this node - case *plan.Project, *plan.Filter, *plan.GroupBy, *plan.Sort: + if plan.IsUnary(n) { schema = n.Children()[0].Schema() - case *plan.CreateIndex: - schema = n.Table.Schema() - default: + } else { schema = n.Schema() } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/analyzer/rules_test.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/analyzer/rules_test.go index 545480db5..c97cc8c93 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/analyzer/rules_test.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/analyzer/rules_test.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" "testing" - "time" "github.com/stretchr/testify/require" @@ -728,6 +727,43 @@ func TestResolveStar(t *testing.T) { table, ), }, + { // note that this behaviour deviates from MySQL + "star after some expressions", + plan.NewProject( + []sql.Expression{ + expression.NewUnresolvedColumn("foo"), + expression.NewStar(), + }, + table, + ), + plan.NewProject( + []sql.Expression{ + expression.NewUnresolvedColumn("foo"), + expression.NewGetFieldWithTable(0, sql.Int32, "mytable", "a", false), + expression.NewGetFieldWithTable(1, sql.Int32, "mytable", "b", false), + }, + table, + ), + }, + { // note that this behaviour deviates from MySQL + "unqualified star used multiple times", + plan.NewProject( + []sql.Expression{ + expression.NewStar(), + expression.NewStar(), + }, + table, + ), + plan.NewProject( + []sql.Expression{ + expression.NewGetFieldWithTable(0, sql.Int32, "mytable", "a", false), + expression.NewGetFieldWithTable(1, sql.Int32, "mytable", "b", false), + expression.NewGetFieldWithTable(0, sql.Int32, "mytable", "a", false), + expression.NewGetFieldWithTable(1, sql.Int32, "mytable", "b", false), + }, + table, + ), + }, } for _, tt := range testCases { @@ -1498,9 +1534,10 @@ func TestAssignIndexes(t *testing.T) { expression.NewGetFieldWithTable(0, sql.Int64, "t2", "bar", false), }, } - done, err := catalog.AddIndex(idx1) + done, ready, err := catalog.AddIndex(idx1) require.NoError(err) close(done) + <-ready idx2 := &dummyIndex{ "t1", @@ -1508,12 +1545,12 @@ func TestAssignIndexes(t *testing.T) { expression.NewGetFieldWithTable(0, sql.Int64, "t1", "foo", false), }, } - done, err = catalog.AddIndex(idx2) + done, ready, err = catalog.AddIndex(idx2) require.NoError(err) close(done) + <-ready - time.Sleep(50 * time.Millisecond) a := NewDefault(catalog) t1 := &indexableTable{ @@ -1958,12 +1995,12 @@ func TestGetIndexes(t *testing.T) { catalog := sql.NewCatalog() for _, idx := range indexes { - done, err := catalog.AddIndex(idx) + done, ready, err := catalog.AddIndex(idx) require.NoError(t, err) close(done) + <-ready } - time.Sleep(50 * time.Millisecond) a := NewDefault(catalog) for _, tt := range testCases { @@ -2015,12 +2052,12 @@ func TestGetMultiColumnIndexes(t *testing.T) { } for _, idx := range indexes { - done, err := catalog.AddIndex(idx) + done, ready, err := catalog.AddIndex(idx) require.NoError(err) close(done) + <-ready } - time.Sleep(50 * time.Millisecond) a := NewDefault(catalog) used := make(map[sql.Expression]struct{}) diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/common.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/common.go index 62ea4c714..8d1a5ef45 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/common.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/common.go @@ -4,6 +4,16 @@ import ( "gopkg.in/src-d/go-mysql-server.v0/sql" ) +// IsUnary returns whether the expression is unary or not. +func IsUnary(e sql.Expression) bool { + return len(e.Children()) == 1 +} + +// IsBinary returns whether the expression is binary or not. +func IsBinary(e sql.Expression) bool { + return len(e.Children()) == 2 +} + // UnaryExpression is an expression that has only one children. type UnaryExpression struct { Child sql.Expression diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/common_test.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/common_test.go index 71edd8de8..571008c69 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/common_test.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/expression/common_test.go @@ -13,3 +13,15 @@ func eval(t *testing.T, e sql.Expression, row sql.Row) interface{} { require.NoError(t, err) return v } + +func TestIsUnary(t *testing.T) { + require := require.New(t) + require.True(IsUnary(NewNot(nil))) + require.False(IsUnary(NewAnd(nil, nil))) +} + +func TestIsBinary(t *testing.T) { + require := require.New(t) + require.False(IsBinary(NewNot(nil))) + require.True(IsBinary(NewAnd(nil, nil))) +} diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index.go index 4336a35d5..3caa04e9d 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index.go @@ -467,9 +467,12 @@ func exprListsEqual(a, b []ExpressionHash) bool { // expression or id while the other is still being created. // When something is sent through the returned channel, it means the index has // finished it's creation and will be marked as ready. -func (r *IndexRegistry) AddIndex(idx Index) (chan<- struct{}, error) { +// Another channel is returned to notify the user when the index is ready. +func (r *IndexRegistry) AddIndex( + idx Index, +) (created chan<- struct{}, ready <-chan struct{}, err error) { if err := r.validateIndexToAdd(idx); err != nil { - return nil, err + return nil, nil, err } r.mut.Lock() @@ -479,15 +482,17 @@ func (r *IndexRegistry) AddIndex(idx Index) (chan<- struct{}, error) { r.indexOrder = append(r.indexOrder, key) r.mut.Unlock() - var created = make(chan struct{}) + var _created = make(chan struct{}) + var _ready = make(chan struct{}) go func() { - <-created + <-_created r.mut.Lock() defer r.mut.Unlock() r.setStatus(idx, IndexReady) + close(_ready) }() - return created, nil + return _created, _ready, nil } // DeleteIndex deletes an index from the registry by its id. First, it marks diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/driver_test.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/driver_test.go index 106b49eda..983ee0a23 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/driver_test.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index/pilosa/driver_test.go @@ -81,6 +81,11 @@ func withoutMapping(a sql.Index) sql.Index { return a } +type logLoc struct { + loc []byte + err error +} + func TestSaveAndLoad(t *testing.T) { if !dockerIsRunning { t.Skipf("Skip TestSaveAndLoad: %s", dockerCmdOutput) @@ -122,12 +127,27 @@ func TestSaveAndLoad(t *testing.T) { lit, err := lookup.Values() require.NoError(err) + var logs []logLoc for i := 0; ; i++ { loc, err := lit.Next() - t.Logf("[%d] values: %v location: %x loc: %x err: %v\n", i, r.values, r.location, loc, err) + + // make a copy of location to save in the log + loc2 := make([]byte, len(loc)) + copy(loc2, loc) + logs = append(logs, logLoc{loc2, err}) if err == io.EOF { - require.Truef(i > 0, "No data for r.values: %v\tr.location: %x", r.values, r.location) + if i == 0 { + for j, l := range logs { + t.Logf("[%d] values: %v location: %x loc: %x err: %v\n", + j, r.values, r.location, l.loc, l.err) + } + + t.Errorf("No data for r.values: %v\tr.location: %x", + r.values, r.location) + t.FailNow() + } + break } @@ -213,82 +233,6 @@ func TestLoadCorruptedIndex(t *testing.T) { require.True(os.IsNotExist(err)) } -func TestPilosaHiccup(t *testing.T) { - if !dockerIsRunning { - t.Skipf("Skip TestPilosaHiccup: %s", dockerCmdOutput) - } - require := require.New(t) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - db, table, id := "db_name", "table_name", "index_id" - expressions := makeExpressions("lang", "hash") - path, err := ioutil.TempDir(os.TempDir(), "indexes") - require.NoError(err) - defer os.RemoveAll(path) - - d := NewDriver(path, newClientWithTimeout(time.Second)) - sqlIdx, err := d.Create(db, table, id, expressions, nil) - require.NoError(err) - - it := &testIndexKeyValueIter{ - offset: 0, - total: 64, - expressions: expressions, - location: offsetLocation, - } - - // restart pilosa container every second - go pilosaHiccup(ctx, time.Second) - - // retry save index - if pilosa failed, reset iterator and start over - err = retry(ctx, func() error { - if e := d.Save(sql.NewContext(ctx), sqlIdx, it); e != nil { - t.Logf("Save err: %s", e) - // reset iterator! - it.Close() - return e - } - return nil - }) - require.NoError(err) - - // load indexes - it doesn't require pilosa, yet. - indexes, err := d.LoadAll(db, table) - require.NoError(err) - require.Equal(1, len(indexes)) - assertEqualIndexes(t, sqlIdx, indexes[0]) - - for i, r := range it.records { - var lookup sql.IndexLookup - // retry to get the next location - pilosa should recover - err = retry(ctx, func() error { - lookup, err = sqlIdx.Get(r.values...) - if err != nil { - t.Logf("Get err: %s", err) - } - return err - }) - require.NoError(err) - - lit, err := lookup.Values() - require.NoError(err) - - loc, err := lit.Next() - t.Logf("[%d] values: %v location: %x loc: %x err: %v\n", i, r.values, r.location, loc, err) - if err == io.EOF { - break - } - - require.NoError(err) - require.True(reflect.DeepEqual(r.location, loc), "Expected: %s\nGot: %v\n", hex.EncodeToString(r.location), hex.EncodeToString(loc)) - - err = lit.Close() - require.NoError(err) - } -} - func TestDelete(t *testing.T) { if !dockerIsRunning { t.Skipf("Skip TestDelete: %s", dockerCmdOutput) diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index_test.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index_test.go index 9dd22b5da..ec859f60a 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index_test.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/index_test.go @@ -4,7 +4,6 @@ import ( "crypto/sha1" "fmt" "testing" - "time" "github.com/stretchr/testify/require" ) @@ -53,7 +52,7 @@ func TestAddIndex(t *testing.T) { table: "foo", } - done, err := r.AddIndex(idx) + done, ready, err := r.AddIndex(idx) require.NoError(err) i := r.Index("foo", "foo") @@ -61,15 +60,15 @@ func TestAddIndex(t *testing.T) { done <- struct{}{} - <-time.After(25 * time.Millisecond) + <-ready i = r.Index("foo", "foo") require.True(r.CanUseIndex(i)) - _, err = r.AddIndex(idx) + _, _, err = r.AddIndex(idx) require.Error(err) require.True(ErrIndexIDAlreadyRegistered.Is(err)) - _, err = r.AddIndex(&dummyIdx{ + _, _, err = r.AddIndex(&dummyIdx{ id: "another", expr: []Expression{new(dummyExpr)}, database: "foo", @@ -163,13 +162,12 @@ func TestExpressionsWithIndexes(t *testing.T) { } for _, idx := range indexes { - done, err := r.AddIndex(idx) + done, ready, err := r.AddIndex(idx) require.NoError(err) close(done) + <-ready } - time.Sleep(50 * time.Millisecond) - exprs := r.ExpressionsWithIndexes( "foo", &dummyExpr{0, "foo"}, diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/common.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/common.go index d9045bb67..d089e2041 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/common.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/common.go @@ -2,6 +2,16 @@ package plan // import "gopkg.in/src-d/go-mysql-server.v0/sql/plan" import "gopkg.in/src-d/go-mysql-server.v0/sql" +// IsUnary returns whether the node is unary or not. +func IsUnary(node sql.Node) bool { + return len(node.Children()) == 1 +} + +// IsBinary returns whether the node is binary or not. +func IsBinary(node sql.Node) bool { + return len(node.Children()) == 2 +} + // UnaryNode is a node that has only one children. type UnaryNode struct { Child sql.Node diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/common_test.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/common_test.go index b7ad07066..0ea751431 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/common_test.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/common_test.go @@ -100,3 +100,19 @@ func collectRows(t *testing.T, node sql.Node) []sql.Row { rows = append(rows, row) } } + +func TestIsUnary(t *testing.T) { + require := require.New(t) + table := mem.NewTable("foo", nil) + + require.True(IsUnary(NewFilter(nil, table))) + require.False(IsUnary(NewCrossJoin(table, table))) +} + +func TestIsBinary(t *testing.T) { + require := require.New(t) + table := mem.NewTable("foo", nil) + + require.False(IsBinary(NewFilter(nil, table))) + require.True(IsBinary(NewCrossJoin(table, table))) +} diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/create_index.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/create_index.go index 6ad7c52c4..085ece04d 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/create_index.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/create_index.go @@ -33,6 +33,7 @@ type CreateIndex struct { Config map[string]string Catalog *sql.Catalog CurrentDatabase string + Async bool } // NewCreateIndex creates a new CreateIndex node. @@ -49,6 +50,7 @@ func NewCreateIndex( Exprs: exprs, Driver: driver, Config: config, + Async: config["async"] != "false", } } @@ -114,7 +116,7 @@ func (c *CreateIndex) RowIter(ctx *sql.Context) (sql.RowIter, error) { return nil, err } - done, err := c.Catalog.AddIndex(index) + created, ready, err := c.Catalog.AddIndex(index) if err != nil { return nil, err } @@ -124,22 +126,31 @@ func (c *CreateIndex) RowIter(ctx *sql.Context) (sql.RowIter, error) { "driver": index.Driver(), }) - go c.backgroundIndexCreate(ctx, log, driver, index, iter, done) + createIndex := func() { + c.createIndex(ctx, log, driver, index, iter, created, ready) + } + + log.WithField("async", c.Async).Info("starting to save the index") - log.Info("starting to save the index") + if c.Async { + go createIndex() + } else { + createIndex() + } return sql.RowsToRowIter(), nil } -func (c *CreateIndex) backgroundIndexCreate( +func (c *CreateIndex) createIndex( ctx *sql.Context, log *logrus.Entry, driver sql.IndexDriver, index sql.Index, iter sql.IndexKeyValueIter, done chan<- struct{}, + ready <-chan struct{}, ) { - span, ctx := ctx.Span("plan.backgroundIndexCreate", + span, ctx := ctx.Span("plan.createIndex", opentracing.Tags{ "index": index.ID(), "table": index.Table(), @@ -172,6 +183,7 @@ func (c *CreateIndex) backgroundIndexCreate( <-deleted } } else { + <-ready span.Finish() log.Info("index successfully created") } @@ -362,7 +374,7 @@ func newLoggingKeyValueIter( func (i *loggingKeyValueIter) Next() ([]interface{}, []byte, error) { if i.span == nil { - i.span, _ = i.ctx.Span("plan.backgroundIndexCreate.iterator", + i.span, _ = i.ctx.Span("plan.createIndex.iterator", opentracing.Tags{ "start": i.rows, }, diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/create_index_test.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/create_index_test.go index 2032aef54..c759ec62c 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/create_index_test.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/create_index_test.go @@ -57,7 +57,60 @@ func TestCreateIndex(t *testing.T) { found := false for _, span := range tracer.Spans { - if span == "plan.backgroundIndexCreate" { + if span == "plan.createIndex" { + found = true + break + } + } + + require.True(found) +} + +func TestCreateIndexSync(t *testing.T) { + require := require.New(t) + + table := &indexableTable{mem.NewTable("foo", sql.Schema{ + {Name: "a", Source: "foo"}, + {Name: "b", Source: "foo"}, + {Name: "c", Source: "foo"}, + })} + + driver := new(mockDriver) + catalog := sql.NewCatalog() + catalog.RegisterIndexDriver(driver) + db := mem.NewDatabase("foo") + db.AddTable("foo", table) + catalog.Databases = append(catalog.Databases, db) + + exprs := []sql.Expression{ + expression.NewGetFieldWithTable(2, sql.Int64, "foo", "c", true), + expression.NewGetFieldWithTable(0, sql.Int64, "foo", "a", true), + } + + ci := NewCreateIndex( + "idx", table, exprs, "mock", + map[string]string{"async": "false"}, + ) + ci.Catalog = catalog + ci.CurrentDatabase = "foo" + + tracer := new(test.MemTracer) + ctx := sql.NewContext(context.Background(), sql.WithTracer(tracer)) + _, err := ci.RowIter(ctx) + require.NoError(err) + + require.Len(driver.deleted, 0) + require.Equal([]string{"idx"}, driver.saved) + idx := catalog.IndexRegistry.Index("foo", "idx") + require.NotNil(idx) + require.Equal(&mockIndex{"foo", "foo", "idx", []sql.ExpressionHash{ + sql.NewExpressionHash(expression.NewGetFieldWithTable(0, sql.Int64, "foo", "c", true)), + sql.NewExpressionHash(expression.NewGetFieldWithTable(1, sql.Int64, "foo", "a", true)), + }}, idx) + + found := false + for _, span := range tracer.Spans { + if span == "plan.createIndex" { found = true break } diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/drop_index_test.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/drop_index_test.go index eb604cb9b..1b80d9ae8 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/drop_index_test.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/drop_index_test.go @@ -39,11 +39,11 @@ func TestDeleteIndex(t *testing.T) { hashes = append(hashes, exh) } - done, err := catalog.AddIndex(&mockIndex{id: "idx", db: "foo", table: "foo", exprs: hashes}) + done, ready, err := catalog.AddIndex(&mockIndex{id: "idx", db: "foo", table: "foo", exprs: hashes}) require.NoError(err) close(done) + <-ready - time.Sleep(50 * time.Millisecond) idx := catalog.Index("foo", "idx") require.NotNil(idx) catalog.ReleaseIndex(idx) diff --git a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/pushdown.go b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/pushdown.go index 26b6f6698..47945facd 100644 --- a/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/pushdown.go +++ b/vendor/gopkg.in/src-d/go-mysql-server.v0/sql/plan/pushdown.go @@ -196,8 +196,7 @@ type IndexableTable struct { Index sql.IndexLookup } -// NewPushdownProjectionAndFiltersTable creates a new -// PushdownProjectionAndFiltersTable node. +// NewIndexableTable creates a new IndexableTable node. func NewIndexableTable( columns []sql.Expression, filters []sql.Expression, From 77fce6c74dbc7ab7f0746374a6704d8f2ee2ce7c Mon Sep 17 00:00:00 2001 From: Miguel Molina Date: Fri, 6 Jul 2018 14:50:29 +0200 Subject: [PATCH 2/2] gitbase: create sync indexes instead of using low level calls Signed-off-by: Miguel Molina --- integration_test.go | 145 +++++++++++++------------------------------- 1 file changed, 43 insertions(+), 102 deletions(-) diff --git a/integration_test.go b/integration_test.go index 2ad14cdae..798da94cd 100644 --- a/integration_test.go +++ b/integration_test.go @@ -2,9 +2,11 @@ package gitbase_test import ( "context" + "fmt" "io/ioutil" "os" "path/filepath" + "strings" "testing" "github.com/src-d/gitbase/internal/rule" @@ -610,111 +612,67 @@ func col(t testing.TB, schema sql.Schema, name string) sql.Expression { } type indexData struct { - id string - expressions []sql.Expression - table sql.Table - columns []string + id string + table string + exprs []string } func createTestIndexes(t testing.TB, engine *sqle.Engine, ctx *sql.Context) func() { - db, err := engine.Catalog.Database("foo") - require.NoError(t, err) - tables := db.Tables() - var indexes = []indexData{ { - id: "refs_idx", - table: tables[gitbase.ReferencesTableName], - columns: []string{"ref_name"}, - expressions: []sql.Expression{ - col(t, gitbase.RefsSchema, "ref_name"), - }, + id: "refs_idx", + table: gitbase.ReferencesTableName, + exprs: []string{"ref_name"}, }, { - id: "remotes_idx", - table: tables[gitbase.RemotesTableName], - columns: []string{"remote_name"}, - expressions: []sql.Expression{ - col(t, gitbase.RemotesSchema, "remote_name"), - }, + id: "remotes_idx", + table: gitbase.RemotesTableName, + exprs: []string{"remote_name"}, }, { - id: "ref_commits_idx", - table: tables[gitbase.RefCommitsTableName], - columns: []string{"ref_name"}, - expressions: []sql.Expression{ - col(t, gitbase.RefCommitsSchema, "ref_name"), - }, + id: "ref_commits_idx", + table: gitbase.RefCommitsTableName, + exprs: []string{"ref_name"}, }, { - id: "commits_idx", - table: tables[gitbase.CommitsTableName], - columns: []string{"commit_hash"}, - expressions: []sql.Expression{ - col(t, gitbase.CommitsSchema, "commit_hash"), - }, + id: "commits_idx", + table: gitbase.CommitsTableName, + exprs: []string{"commit_hash"}, }, { - id: "commit_trees_idx", - table: tables[gitbase.CommitTreesTableName], - columns: []string{"commit_hash"}, - expressions: []sql.Expression{ - col(t, gitbase.CommitTreesSchema, "commit_hash"), - }, + id: "commit_trees_idx", + table: gitbase.CommitTreesTableName, + exprs: []string{"commit_hash"}, }, { - id: "commit_blobs_idx", - table: tables[gitbase.CommitBlobsTableName], - columns: []string{"commit_hash"}, - expressions: []sql.Expression{ - col(t, gitbase.CommitBlobsSchema, "commit_hash"), - }, + id: "commit_blobs_idx", + table: gitbase.CommitBlobsTableName, + exprs: []string{"commit_hash"}, }, { - id: "tree_entries_idx", - table: tables[gitbase.TreeEntriesTableName], - columns: []string{"tree_entry_name"}, - expressions: []sql.Expression{ - col(t, gitbase.TreeEntriesSchema, "tree_entry_name"), - }, + id: "tree_entries_idx", + table: gitbase.TreeEntriesTableName, + exprs: []string{"tree_entry_name"}, }, { - id: "blobs_idx", - table: tables[gitbase.BlobsTableName], - columns: []string{"blob_hash"}, - expressions: []sql.Expression{ - col(t, gitbase.BlobsSchema, "blob_hash"), - }, + id: "blobs_idx", + table: gitbase.BlobsTableName, + exprs: []string{"blob_hash"}, }, { - id: "commit_files_idx", - table: tables[gitbase.CommitFilesTableName], - columns: []string{"commit_hash"}, - expressions: []sql.Expression{ - col(t, gitbase.CommitFilesSchema, "commit_hash"), - }, + id: "commit_files_idx", + table: gitbase.CommitFilesTableName, + exprs: []string{"commit_hash"}, }, { - id: "files_idx", - table: tables[gitbase.FilesTableName], - columns: []string{"file_path"}, - expressions: []sql.Expression{ - col(t, gitbase.FilesSchema, "file_path"), - }, + id: "files_idx", + table: gitbase.FilesTableName, + exprs: []string{"file_path"}, }, { - id: "files_lang_idx", - table: tables[gitbase.FilesTableName], - columns: []string{"file_path"}, - expressions: []sql.Expression{ - func() sql.Expression { - f, _ := function.NewLanguage( - col(t, gitbase.FilesSchema, "file_path"), - col(t, gitbase.FilesSchema, "blob_content"), - ) - return f - }(), - }, + id: "files_lang_idx", + table: gitbase.FilesTableName, + exprs: []string{"language(file_path, blob_content)"}, }, } @@ -736,31 +694,14 @@ func createIndex( ctx *sql.Context, ) { t.Helper() - require := require.New(t) - driver := e.Catalog.IndexDriver(pilosa.DriverID) - require.NotNil(driver) - - var hashes []sql.ExpressionHash - for _, e := range data.expressions { - hashes = append(hashes, sql.NewExpressionHash(e)) - } - idx, err := driver.Create( - "foo", data.table.Name(), - data.id, hashes, - make(map[string]string), + query := fmt.Sprintf( + `CREATE INDEX %s ON %s (%s) WITH (async = false)`, + data.id, data.table, strings.Join(data.exprs, ", "), ) - require.NoError(err) - done, err := e.Catalog.AddIndex(idx) - require.NoError(err) - - iter, err := data.table.(sql.Indexable).IndexKeyValueIter(ctx, data.columns) - require.NoError(err) - - require.NoError(driver.Save(sql.NewEmptyContext(), idx, iter)) - - done <- struct{}{} + _, _, err := e.Query(ctx, query) + require.NoError(t, err) } func deleteIndex(