From ae149bd88c9ebc0c12e60fe711709fcc36b3c54d Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Tue, 23 Jun 2020 14:58:18 +0000 Subject: [PATCH 1/7] Use bulks in tests --- test/backup_test.go | 12 +++++++----- test/cursor_test.go | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/test/backup_test.go b/test/backup_test.go index 15f0db2e..e20036e2 100644 --- a/test/backup_test.go +++ b/test/backup_test.go @@ -869,17 +869,19 @@ func TestBackupRestoreWithViews(t *testing.T) { go func(i int) { defer wg.Done() + books := make([]BookWithAuthor, numDocs) + for j := 0; j < numDocs; j++ { - book := BookWithAuthor{ + books[j] = BookWithAuthor{ Title: fmt.Sprintf("Hello World - %d", j), Author: fmt.Sprintf("Author - %d", i), } + } - _, err := col.CreateDocument(ctx, book) - if err != nil { - t.Fatalf("Failed to create document %s", describe(err)) - } + _, _, err := col.CreateDocuments(ctx, books) + if err != nil { + t.Fatalf("Failed to create document %s", describe(err)) } }(k) } diff --git a/test/cursor_test.go b/test/cursor_test.go index ba80cda4..a46999f6 100644 --- a/test/cursor_test.go +++ b/test/cursor_test.go @@ -348,11 +348,13 @@ func TestCreateStreamCursor(t *testing.T) { } // This might take a few seconds - for i := 0; i < 10000; i++ { - user := UserDoc{Name: "John", Age: i} - if _, err := col.CreateDocument(ctx, user); err != nil { - t.Fatalf("Expected success, got %s", describe(err)) - } + docs := 10000 + users := make([]UserDoc, 0, docs) + for i := 0; i < docs; i++ { + users[i] = UserDoc{Name: "John", Age: i} + } + if _, _, err := col.CreateDocuments(ctx, users); err != nil { + t.Fatalf("Expected success, got %s", describe(err)) } t.Log("Completed inserting 10k docs") From c3bdef690ceb597aa8840b7f1536912268020911 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Wed, 24 Jun 2020 07:43:52 +0000 Subject: [PATCH 2/7] Fix list --- test/cursor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cursor_test.go b/test/cursor_test.go index a46999f6..40c72afc 100644 --- a/test/cursor_test.go +++ b/test/cursor_test.go @@ -349,7 +349,7 @@ func TestCreateStreamCursor(t *testing.T) { // This might take a few seconds docs := 10000 - users := make([]UserDoc, 0, docs) + users := make([]UserDoc, docs) for i := 0; i < docs; i++ { users[i] = UserDoc{Name: "John", Age: i} } From 682815d0fbe2193b7fce0adff4be06012be0840c Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Wed, 24 Jun 2020 08:01:43 +0000 Subject: [PATCH 3/7] Limit bulk to 1k --- test/backup_test.go | 14 +++----------- test/cursor_test.go | 10 +++------- test/util.go | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/test/backup_test.go b/test/backup_test.go index e20036e2..dc717835 100644 --- a/test/backup_test.go +++ b/test/backup_test.go @@ -869,20 +869,12 @@ func TestBackupRestoreWithViews(t *testing.T) { go func(i int) { defer wg.Done() - books := make([]BookWithAuthor, numDocs) - - for j := 0; j < numDocs; j++ { - - books[j] = BookWithAuthor{ + sendBulks(t, col, ctx, func(t *testing.T, j int) interface{} { + return BookWithAuthor{ Title: fmt.Sprintf("Hello World - %d", j), Author: fmt.Sprintf("Author - %d", i), } - } - - _, _, err := col.CreateDocuments(ctx, books) - if err != nil { - t.Fatalf("Failed to create document %s", describe(err)) - } + }, numDocs) }(k) } wg.Wait() diff --git a/test/cursor_test.go b/test/cursor_test.go index 40c72afc..de1e7358 100644 --- a/test/cursor_test.go +++ b/test/cursor_test.go @@ -349,13 +349,9 @@ func TestCreateStreamCursor(t *testing.T) { // This might take a few seconds docs := 10000 - users := make([]UserDoc, docs) - for i := 0; i < docs; i++ { - users[i] = UserDoc{Name: "John", Age: i} - } - if _, _, err := col.CreateDocuments(ctx, users); err != nil { - t.Fatalf("Expected success, got %s", describe(err)) - } + sendBulks(t, col, ctx, func(t *testing.T, i int) interface{} { + return UserDoc{Name: "John", Age: i} + }, docs) t.Log("Completed inserting 10k docs") const expectedResults int = 10 * 10000 diff --git a/test/util.go b/test/util.go index 2fa40a91..a539208a 100644 --- a/test/util.go +++ b/test/util.go @@ -23,6 +23,7 @@ package test import ( + "context" "encoding/hex" "encoding/json" "fmt" @@ -32,6 +33,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + driver "github.com/arangodb/go-driver" ) @@ -147,3 +150,38 @@ func retry(interval, timeout time.Duration, f func() error) error { } } } + +const bulkSize = 1000 + +func sendBulks(t *testing.T, col driver.Collection, ctx context.Context, creator func(t *testing.T, i int) interface{}, size int) { + current := 0 + + for { + stepSize := min(bulkSize, size-current) + if stepSize == 0 { + return + } + + objs := make([]interface{}, min(bulkSize, stepSize)) + for i := 0; i < stepSize; i++ { + objs[i] = creator(t, current+i) + } + + _, _, err := col.CreateDocuments(ctx, objs) + require.NoError(t, err) + + current += stepSize + } +} + +func min(ints ...int) int { + z := 0 + + for _, i := range ints { + if z > i { + z = i + } + } + + return z +} From d8779418b602c8c6dcbbc83f942896b45a882087 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Wed, 24 Jun 2020 14:03:31 +0000 Subject: [PATCH 4/7] Defer after error check --- test/clean.go | 4 ++++ test/cluster_test.go | 8 ++++---- test/server_mode_auth_test.go | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/test/clean.go b/test/clean.go index 9be9d862..59139e0b 100644 --- a/test/clean.go +++ b/test/clean.go @@ -34,5 +34,9 @@ type remove interface { } func clean(t *testing.T, ctx context.Context, col remove) { + if col == nil { + return + } + require.NoError(t, col.Remove(ctx)) } diff --git a/test/cluster_test.go b/test/cluster_test.go index 6a88a14f..38117833 100644 --- a/test/cluster_test.go +++ b/test/cluster_test.go @@ -264,10 +264,10 @@ func TestClusterMoveShard(t *testing.T) { col, err := db.CreateCollection(ctx, "test_move_shard", &driver.CreateCollectionOptions{ NumberOfShards: 12, }) - defer clean(t, ctx, col) if err != nil { t.Fatalf("CreateCollection failed: %s", describe(err)) } + defer clean(t, ctx, col) h, err := cl.Health(ctx) if err != nil { t.Fatalf("Health failed: %s", describe(err)) @@ -365,10 +365,10 @@ func TestClusterResignLeadership(t *testing.T) { NumberOfShards: 12, ReplicationFactor: 2, }) - defer clean(t, ctx, col) if err != nil { t.Fatalf("CreateCollection failed: %s", describe(err)) } + defer clean(t, ctx, col) inv, err := cl.DatabaseInventory(ctx, db) if err != nil { t.Fatalf("DatabaseInventory failed: %s", describe(err)) @@ -453,10 +453,10 @@ func TestClusterMoveShardWithViews(t *testing.T) { col, err := db.CreateCollection(ctx, "test_move_shard_with_view", &driver.CreateCollectionOptions{ NumberOfShards: 12, }) - clean(t, ctx, col) if err != nil { t.Fatalf("CreateCollection failed: %s", describe(err)) } + defer clean(t, ctx, col) opts := &driver.ArangoSearchViewProperties{ Links: driver.ArangoSearchLinks{ "test_move_shard_with_view": driver.ArangoSearchElementProperties{}, @@ -464,10 +464,10 @@ func TestClusterMoveShardWithViews(t *testing.T) { } viewName := "test_move_shard_view" view, err := db.CreateArangoSearchView(ctx, viewName, opts) - clean(t, ctx, view) if err != nil { t.Fatalf("Failed to create view '%s': %s", viewName, describe(err)) } + defer clean(t, ctx, view) h, err := cl.Health(ctx) if err != nil { t.Fatalf("Health failed: %s", describe(err)) diff --git a/test/server_mode_auth_test.go b/test/server_mode_auth_test.go index cd988060..b1d4eb8e 100644 --- a/test/server_mode_auth_test.go +++ b/test/server_mode_auth_test.go @@ -62,7 +62,7 @@ func TestServerModeAndGrants(t *testing.T) { db := ensureDatabase(ctx, c, "_system", nil, t) colName := "server_mode_and_grants_test1" col := ensureCollection(ctx, db, colName, nil, t) - clean(t, ctx, col) + defer clean(t, ctx, col) // Get database & collection access defaultDBAccess, err := u.GetDatabaseAccess(ctx, db) From 47e9ba06680beac61f8da5ae9eeb337790151248 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Wed, 24 Jun 2020 14:15:26 +0000 Subject: [PATCH 5/7] Debug bulk sending --- test/util.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/util.go b/test/util.go index a539208a..9d56f063 100644 --- a/test/util.go +++ b/test/util.go @@ -168,6 +168,7 @@ func sendBulks(t *testing.T, col driver.Collection, ctx context.Context, creator } _, _, err := col.CreateDocuments(ctx, objs) + t.Logf("Creating %d documents", len(objs)) require.NoError(t, err) current += stepSize From 8883e430129bf493daa55a684fa9db37724ebe61 Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Wed, 24 Jun 2020 14:25:48 +0000 Subject: [PATCH 6/7] Debug bulk sending --- test/util.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/util.go b/test/util.go index 9d56f063..29c91380 100644 --- a/test/util.go +++ b/test/util.go @@ -155,8 +155,10 @@ const bulkSize = 1000 func sendBulks(t *testing.T, col driver.Collection, ctx context.Context, creator func(t *testing.T, i int) interface{}, size int) { current := 0 + t.Logf("Creating %d documents", size) for { + t.Logf("Created %d/%d documents", current, size) stepSize := min(bulkSize, size-current) if stepSize == 0 { return From c4401fefd95176bf4ffc42d1e9dfee0c3a4949fb Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Wed, 24 Jun 2020 14:26:45 +0000 Subject: [PATCH 7/7] Debug bulk sending --- test/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/util.go b/test/util.go index 29c91380..892864fd 100644 --- a/test/util.go +++ b/test/util.go @@ -177,8 +177,8 @@ func sendBulks(t *testing.T, col driver.Collection, ctx context.Context, creator } } -func min(ints ...int) int { - z := 0 +func min(max int, ints ...int) int { + z := max for _, i := range ints { if z > i {