From eb57d89c1cebe83395e7973d65fe7737a542f6df Mon Sep 17 00:00:00 2001 From: Daniel Larkin-York Date: Wed, 11 Jul 2018 09:26:56 -0400 Subject: [PATCH 01/16] Initial empty test file. --- tests/load_balancer_test.go | 75 +++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/load_balancer_test.go diff --git a/tests/load_balancer_test.go b/tests/load_balancer_test.go new file mode 100644 index 000000000..af3a79bc8 --- /dev/null +++ b/tests/load_balancer_test.go @@ -0,0 +1,75 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package tests + +import ( + "context" + "testing" + + "github.com/dchest/uniuri" + "github.com/stretchr/testify/assert" + + driver "github.com/arangodb/go-driver" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/client" + "github.com/arangodb/kube-arangodb/pkg/util" +) + +// TestLoadBalancerCursorVST tests cursor forwarding with load-balanced VST +func TestLoadBalancerCursorVST(t *testing.T) { + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) + + // Prepare deployment config + depl := newDeployment("test-lb-" + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) + + // Create deployment + _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) + if err != nil { + t.Fatalf("Create deployment failed: %v", err) + } + // Prepare cleanup + defer removeDeployment(c, depl.GetName(), ns) + + // Wait for deployment to be ready + apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) + if err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } + + // Create a database client + ctx := context.Background() + client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) + + // Wait for cluster to be available + if err := waitUntilVersionUp(client, nil); err != nil { + t.Fatalf("Cluster not running returning version in time: %v", err) + } + + // TODO actual test code + + // Check server role + assert.NoError(t, testServerRole(ctx, client, driver.ServerRoleCoordinator)) +} From acb88baff629bc312ba1b48985267fa152864de2 Mon Sep 17 00:00:00 2001 From: Dan Larkin-York Date: Wed, 11 Jul 2018 19:22:40 -0400 Subject: [PATCH 02/16] First attempt to configure a VST connection. --- deps/github.com/arangodb/go-driver/context.go | 17 +- pkg/util/arangod/client.go | 60 ++++-- tests/load_balancer_test.go | 195 +++++++++++++++++- 3 files changed, 255 insertions(+), 17 deletions(-) diff --git a/deps/github.com/arangodb/go-driver/context.go b/deps/github.com/arangodb/go-driver/context.go index 170e2d5dd..9f39979f9 100644 --- a/deps/github.com/arangodb/go-driver/context.go +++ b/deps/github.com/arangodb/go-driver/context.go @@ -52,11 +52,12 @@ const ( keyIsSystem ContextKey = "arangodb-isSystem" keyIgnoreRevs ContextKey = "arangodb-ignoreRevs" keyEnforceReplicationFactor ContextKey = "arangodb-enforceReplicationFactor" - keyConfigured ContextKey = "arangodb-configured" + keyConfigured ContextKey = "arangodb-configured" keyFollowLeaderRedirect ContextKey = "arangodb-followLeaderRedirect" keyDBServerID ContextKey = "arangodb-dbserverID" keyBatchID ContextKey = "arangodb-batchID" keyJobIDResponse ContextKey = "arangodb-jobIDResponse" + keyUseVST ContextKey = "arangodb-useVST" ) // WithRevision is used to configure a context to make document @@ -221,6 +222,11 @@ func WithJobIDResponse(parent context.Context, jobID *string) context.Context { return context.WithValue(contextOrBackground(parent), keyJobIDResponse, jobID) } +// WithJobIDResponse is used to configure a client that will use VST for comm. +func WithUseVST(parent context.Context, value bool) context.Context { + return context.WithValue(contextOrBackground(parent), keyUseVST, value) +} + type contextSettings struct { Silent bool WaitForSync bool @@ -415,3 +421,12 @@ func withDocumentAt(ctx context.Context, index int) (context.Context, error) { return ctx, nil } + +// determines whether the context is configured to use VST for communications +func mustUseVST(ctx context.Context) bool { + if ctx == nil { return false } + if v := ctx.Value(keyUseVST); v != nil { + return reflect.ValueOf(v) + } + return false +} diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index 7774a63ce..38911c1b8 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -156,20 +156,38 @@ func CreateArangodImageIDClient(ctx context.Context, deployment k8sutil.APIObjec // CreateArangodClientForDNSName creates a go-driver client for a given DNS name. func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsName string) (driver.Client, error) { - connConfig, err := createArangodHTTPConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}) - if err != nil { - return nil, maskAny(err) - } - // TODO deal with TLS with proper CA checking - conn, err := http.NewConnection(connConfig) - if err != nil { - return nil, maskAny(err) + config := driver.ClientConfig{} + vst := driver.mustUseVST(context) + if vst { + connConfig, err := createArangodVSTConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}) + if err != nil { + return nil, maskAny(err) + } + // TODO deal with TLS with proper CA checking + conn, err := vst.NewConnection(connConfig) + if err != nil { + return nil, maskAny(err) + } + // Create client + config := driver.ClientConfig{ + Connection: conn, + } + } else { + connConfig, err := createArangodHTTPConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}) + if err != nil { + return nil, maskAny(err) + } + // TODO deal with TLS with proper CA checking + conn, err := http.NewConnection(connConfig) + if err != nil { + return nil, maskAny(err) + } + // Create client + config := driver.ClientConfig{ + Connection: conn, + } } - // Create client - config := driver.ClientConfig{ - Connection: conn, - } auth, err := createArangodClientAuthentication(ctx, cli, apiObject) if err != nil { return nil, maskAny(err) @@ -186,7 +204,7 @@ func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interfa func createArangodHTTPConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (http.ConnectionConfig, error) { scheme := "http" transport := sharedHTTPTransport - if apiObject != nil && apiObject.Spec.IsSecure() { + if apiObject != nil && apiObject.Spec.IsSecure() { scheme = "https" transport = sharedHTTPSTransport } @@ -200,6 +218,22 @@ func createArangodHTTPConfigForDNSNames(ctx context.Context, cli corev1.CoreV1In return connConfig, nil } +// createArangodVSTConfigForDNSNames creates a go-driver VST connection config for a given DNS names. +func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (http.ConnectionConfig, error) { + scheme := "vst" + if apiObject != nil && apiObject.Spec.IsSecure() { + // TODO set security for transport + } + transport := vst.NewTransport(/* TODO configure transport */) + connConfig := vst.ConnectionConfig{ + Transport: transport, + } + for _, dnsName := range dnsNames { + connConfig.Endpoints = append(connConfig.Endpoints, scheme+"://"+net.JoinHostPort(dnsName, strconv.Itoa(k8sutil.ArangoPort))) + } + return connConfig, nil +} + // createArangodClientAuthentication creates a go-driver authentication for the servers in the given deployment. func createArangodClientAuthentication(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment) (driver.Authentication, error) { if apiObject != nil && apiObject.Spec.IsAuthenticated() { diff --git a/tests/load_balancer_test.go b/tests/load_balancer_test.go index af3a79bc8..bbaa9db5e 100644 --- a/tests/load_balancer_test.go +++ b/tests/load_balancer_test.go @@ -24,7 +24,9 @@ package tests import ( "context" + "reflect" "testing" + "time" "github.com/dchest/uniuri" "github.com/stretchr/testify/assert" @@ -35,6 +37,20 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util" ) +type queryTest struct { + Query string + BindVars map[string]interface{} + ExpectSuccess bool + ExpectedDocuments []interface{} + DocumentType reflect.Type +} + +type queryTestContext struct { + Context context.Context + ExpectCount bool +} + + // TestLoadBalancerCursorVST tests cursor forwarding with load-balanced VST func TestLoadBalancerCursorVST(t *testing.T) { c := client.MustNewInCluster() @@ -68,8 +84,181 @@ func TestLoadBalancerCursorVST(t *testing.T) { t.Fatalf("Cluster not running returning version in time: %v", err) } - // TODO actual test code + // Create data set + collectionData := map[string][]interface{}{ + "books": []interface{}{ + Book{Title: "Book 01"}, + Book{Title: "Book 02"}, + Book{Title: "Book 03"}, + Book{Title: "Book 04"}, + Book{Title: "Book 05"}, + Book{Title: "Book 06"}, + Book{Title: "Book 07"}, + Book{Title: "Book 08"}, + Book{Title: "Book 09"}, + Book{Title: "Book 10"}, + Book{Title: "Book 11"}, + Book{Title: "Book 12"}, + Book{Title: "Book 13"}, + Book{Title: "Book 14"}, + Book{Title: "Book 15"}, + Book{Title: "Book 16"}, + Book{Title: "Book 17"}, + Book{Title: "Book 18"}, + Book{Title: "Book 19"}, + Book{Title: "Book 20"}, + }, + "users": []interface{}{ + UserDoc{Name: "John", Age: 13}, + UserDoc{Name: "Jake", Age: 25}, + UserDoc{Name: "Clair", Age: 12}, + UserDoc{Name: "Johnny", Age: 42}, + UserDoc{Name: "Blair", Age: 67}, + UserDoc{Name: "Zz", Age: 12}, + }, + } + for colName, colDocs := range collectionData { + col := ensureCollection(ctx, db, colName, nil, t) + if _, _, err := col.CreateDocuments(ctx, colDocs); err != nil { + t.Fatalf("Expected success, got %s", describe(err)) + } + } + + // Setup tests + tests := []queryTest{ + queryTest{ + Query: "FOR d IN books SORT d.Title RETURN d", + ExpectSuccess: true, + ExpectedDocuments: collectionData["books"], + DocumentType: reflect.TypeOf(Book{}), + }, + queryTest{ + Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", + BindVars: map[string]interface{}{"title": "Book 02"}, + ExpectSuccess: true, + ExpectedDocuments: []interface{}{collectionData["books"][1]}, + DocumentType: reflect.TypeOf(Book{}), + }, + queryTest{ + Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", + BindVars: map[string]interface{}{"somethingelse": "Book 02"}, + ExpectSuccess: false, // Unknown `@title` + }, + queryTest{ + Query: "FOR u IN users FILTER u.age>100 SORT u.name RETURN u", + ExpectSuccess: true, + ExpectedDocuments: []interface{}{}, + DocumentType: reflect.TypeOf(UserDoc{}), + }, + queryTest{ + Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", + BindVars: map[string]interface{}{"maxAge": 20}, + ExpectSuccess: true, + ExpectedDocuments: []interface{}{collectionData["users"][2], collectionData["users"][0], collectionData["users"][5]}, + DocumentType: reflect.TypeOf(UserDoc{}), + }, + queryTest{ + Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", + BindVars: map[string]interface{}{"maxage": 20}, + ExpectSuccess: false, // `@maxage` versus `@maxAge` + }, + queryTest{ + Query: "FOR u IN users SORT u.age RETURN u.age", + ExpectedDocuments: []interface{}{12, 12, 13, 25, 42, 67}, + DocumentType: reflect.TypeOf(12), + ExpectSuccess: true, + }, + queryTest{ + Query: "FOR p IN users COLLECT a = p.age WITH COUNT INTO c SORT a RETURN [a, c]", + ExpectedDocuments: []interface{}{[]int{12, 2}, []int{13, 1}, []int{25, 1}, []int{42, 1}, []int{67, 1}}, + DocumentType: reflect.TypeOf([]int{}), + ExpectSuccess: true, + }, + queryTest{ + Query: "FOR u IN users SORT u.name RETURN u.name", + ExpectedDocuments: []interface{}{"Blair", "Clair", "Jake", "John", "Johnny", "Zz"}, + DocumentType: reflect.TypeOf("foo"), + ExpectSuccess: true, + }, + } + + // Setup context alternatives + contexts := []queryTestContext{ + queryTestContext{driver.WithUseVST(nil, true), false}, + queryTestContext{driver.WithUseVST(context.Background(), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil), true), true}, + queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil, true), true), true}, + queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil, false), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryBatchSize(nil, 1), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil, true), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil, false), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryMemoryLimit(nil, 60000), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryTTL(nil, time.Minute), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryBatchSize(driver.WithQueryCount(nil), 1), true), true}, + queryTestContext{driver.WithUseVST(driver.WithQueryCache(driver.WithQueryCount(driver.WithQueryBatchSize(nil, 2))), true), true}, + } - // Check server role - assert.NoError(t, testServerRole(ctx, client, driver.ServerRoleCoordinator)) + // Run tests for every context alternative + for _, qctx := range contexts { + ctx := qctx.Context + for i, test := range tests { + cursor, err := db.Query(ctx, test.Query, test.BindVars) + if err == nil { + // Close upon exit of the function + defer cursor.Close() + } + if test.ExpectSuccess { + if err != nil { + t.Errorf("Expected success in query %d (%s), got '%s'", i, test.Query, describe(err)) + continue + } + count := cursor.Count() + if qctx.ExpectCount { + if count != int64(len(test.ExpectedDocuments)) { + t.Errorf("Expected count of %d, got %d in query %d (%s)", len(test.ExpectedDocuments), count, i, test.Query) + } + } else { + if count != 0 { + t.Errorf("Expected count of 0, got %d in query %d (%s)", count, i, test.Query) + } + } + var result []interface{} + for { + hasMore := cursor.HasMore() + doc := reflect.New(test.DocumentType) + if _, err := cursor.ReadDocument(ctx, doc.Interface()); driver.IsNoMoreDocuments(err) { + if hasMore { + t.Error("HasMore returned true, but ReadDocument returns a IsNoMoreDocuments error") + } + break + } else if err != nil { + t.Errorf("Failed to result document %d: %s", len(result), describe(err)) + } + if !hasMore { + t.Error("HasMore returned false, but ReadDocument returns a document") + } + result = append(result, doc.Elem().Interface()) + } + if len(result) != len(test.ExpectedDocuments) { + t.Errorf("Expected %d documents, got %d in query %d (%s)", len(test.ExpectedDocuments), len(result), i, test.Query) + } else { + for resultIdx, resultDoc := range result { + if !reflect.DeepEqual(resultDoc, test.ExpectedDocuments[resultIdx]) { + t.Errorf("Unexpected document in query %d (%s) at index %d: got %+v, expected %+v", i, test.Query, resultIdx, resultDoc, test.ExpectedDocuments[resultIdx]) + } + } + } + // Close anyway (this tests calling Close more than once) + if err := cursor.Close(); err != nil { + t.Errorf("Expected success in Close of cursor from query %d (%s), got '%s'", i, test.Query, describe(err)) + } + } else { + if err == nil { + t.Errorf("Expected error in query %d (%s), got '%s'", i, test.Query, describe(err)) + continue + } + } + } + } } From 013c87bb97ac9ff3420a2f70a8bacc35ea1232db Mon Sep 17 00:00:00 2001 From: Dan Larkin-York Date: Mon, 16 Jul 2018 08:20:54 -0400 Subject: [PATCH 03/16] Incorporate changes from review. --- deps/github.com/arangodb/go-driver/context.go | 17 +- pkg/util/arangod/client.go | 34 +- tests/load_balancer_test.go | 384 +++++++++--------- tests/test_util.go | 4 +- 4 files changed, 215 insertions(+), 224 deletions(-) diff --git a/deps/github.com/arangodb/go-driver/context.go b/deps/github.com/arangodb/go-driver/context.go index 9f39979f9..170e2d5dd 100644 --- a/deps/github.com/arangodb/go-driver/context.go +++ b/deps/github.com/arangodb/go-driver/context.go @@ -52,12 +52,11 @@ const ( keyIsSystem ContextKey = "arangodb-isSystem" keyIgnoreRevs ContextKey = "arangodb-ignoreRevs" keyEnforceReplicationFactor ContextKey = "arangodb-enforceReplicationFactor" - keyConfigured ContextKey = "arangodb-configured" + keyConfigured ContextKey = "arangodb-configured" keyFollowLeaderRedirect ContextKey = "arangodb-followLeaderRedirect" keyDBServerID ContextKey = "arangodb-dbserverID" keyBatchID ContextKey = "arangodb-batchID" keyJobIDResponse ContextKey = "arangodb-jobIDResponse" - keyUseVST ContextKey = "arangodb-useVST" ) // WithRevision is used to configure a context to make document @@ -222,11 +221,6 @@ func WithJobIDResponse(parent context.Context, jobID *string) context.Context { return context.WithValue(contextOrBackground(parent), keyJobIDResponse, jobID) } -// WithJobIDResponse is used to configure a client that will use VST for comm. -func WithUseVST(parent context.Context, value bool) context.Context { - return context.WithValue(contextOrBackground(parent), keyUseVST, value) -} - type contextSettings struct { Silent bool WaitForSync bool @@ -421,12 +415,3 @@ func withDocumentAt(ctx context.Context, index int) (context.Context, error) { return ctx, nil } - -// determines whether the context is configured to use VST for communications -func mustUseVST(ctx context.Context) bool { - if ctx == nil { return false } - if v := ctx.Value(keyUseVST); v != nil { - return reflect.ValueOf(v) - } - return false -} diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index 38911c1b8..1f66c2435 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -100,10 +100,10 @@ func CreateArangodClient(ctx context.Context, cli corev1.CoreV1Interface, apiObj } // CreateArangodDatabaseClient creates a go-driver client for accessing the entire cluster (or single server). -func CreateArangodDatabaseClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment) (driver.Client, error) { +func CreateArangodDatabaseClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, vst ...bool) (driver.Client, error) { // Create connection dnsName := k8sutil.CreateDatabaseClientServiceDNSName(apiObject) - c, err := createArangodClientForDNSName(ctx, cli, apiObject, dnsName) + c, err := createArangodClientForDNSName(ctx, cli, apiObject, dnsName, vst) if err != nil { return nil, maskAny(err) } @@ -155,37 +155,34 @@ func CreateArangodImageIDClient(ctx context.Context, deployment k8sutil.APIObjec } // CreateArangodClientForDNSName creates a go-driver client for a given DNS name. -func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsName string) (driver.Client, error) { +func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsName string, vst ...bool) (driver.Client, error) { config := driver.ClientConfig{} - vst := driver.mustUseVST(context) - if vst { + var conn driver.Connection + if len(vst) > 0 && vst[0] { connConfig, err := createArangodVSTConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}) if err != nil { return nil, maskAny(err) } // TODO deal with TLS with proper CA checking - conn, err := vst.NewConnection(connConfig) + conn, err = vst.NewConnection(connConfig) if err != nil { return nil, maskAny(err) } - // Create client - config := driver.ClientConfig{ - Connection: conn, - } } else { connConfig, err := createArangodHTTPConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}) if err != nil { return nil, maskAny(err) } // TODO deal with TLS with proper CA checking - conn, err := http.NewConnection(connConfig) + conn, err = http.NewConnection(connConfig) if err != nil { return nil, maskAny(err) } - // Create client - config := driver.ClientConfig{ - Connection: conn, - } + } + + // Create client + config := driver.ClientConfig{ + Connection: conn, } auth, err := createArangodClientAuthentication(ctx, cli, apiObject) @@ -220,11 +217,12 @@ func createArangodHTTPConfigForDNSNames(ctx context.Context, cli corev1.CoreV1In // createArangodVSTConfigForDNSNames creates a go-driver VST connection config for a given DNS names. func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (http.ConnectionConfig, error) { - scheme := "vst" + scheme := "http" + transport := vst.NewTransport(/* TODO configure transport */) if apiObject != nil && apiObject.Spec.IsSecure() { - // TODO set security for transport + scheme = "https" + transport = sharedHTTPSTransport } - transport := vst.NewTransport(/* TODO configure transport */) connConfig := vst.ConnectionConfig{ Transport: transport, } diff --git a/tests/load_balancer_test.go b/tests/load_balancer_test.go index bbaa9db5e..ffafec98a 100644 --- a/tests/load_balancer_test.go +++ b/tests/load_balancer_test.go @@ -51,214 +51,222 @@ type queryTestContext struct { } -// TestLoadBalancerCursorVST tests cursor forwarding with load-balanced VST -func TestLoadBalancerCursorVST(t *testing.T) { - c := client.MustNewInCluster() - kubecli := mustNewKubeClient(t) - ns := getNamespace(t) +// TestLoadBalancerCursorVST tests cursor forwarding with load-balanced conn. +func TestLoadBalancerCursor(t *testing.T) { + func runCursorLoadBalancingTest(t *testing.T, useVst bool) { + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) - // Prepare deployment config - depl := newDeployment("test-lb-" + uniuri.NewLen(4)) - depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) + // Prepare deployment config + depl := newDeployment("test-lb-" + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) - // Create deployment - _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) - if err != nil { - t.Fatalf("Create deployment failed: %v", err) - } - // Prepare cleanup - defer removeDeployment(c, depl.GetName(), ns) + // Create deployment + _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) + if err != nil { + t.Fatalf("Create deployment failed: %v", err) + } + // Prepare cleanup + defer removeDeployment(c, depl.GetName(), ns) - // Wait for deployment to be ready - apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) - if err != nil { - t.Fatalf("Deployment not running in time: %v", err) - } + // Wait for deployment to be ready + apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) + if err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } - // Create a database client - ctx := context.Background() - client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) + // Create a database client + ctx := context.Background() + client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t, useVst) - // Wait for cluster to be available - if err := waitUntilVersionUp(client, nil); err != nil { - t.Fatalf("Cluster not running returning version in time: %v", err) - } + // Wait for cluster to be available + if err := waitUntilVersionUp(client, nil); err != nil { + t.Fatalf("Cluster not running returning version in time: %v", err) + } - // Create data set - collectionData := map[string][]interface{}{ - "books": []interface{}{ - Book{Title: "Book 01"}, - Book{Title: "Book 02"}, - Book{Title: "Book 03"}, - Book{Title: "Book 04"}, - Book{Title: "Book 05"}, - Book{Title: "Book 06"}, - Book{Title: "Book 07"}, - Book{Title: "Book 08"}, - Book{Title: "Book 09"}, - Book{Title: "Book 10"}, - Book{Title: "Book 11"}, - Book{Title: "Book 12"}, - Book{Title: "Book 13"}, - Book{Title: "Book 14"}, - Book{Title: "Book 15"}, - Book{Title: "Book 16"}, - Book{Title: "Book 17"}, - Book{Title: "Book 18"}, - Book{Title: "Book 19"}, - Book{Title: "Book 20"}, - }, - "users": []interface{}{ - UserDoc{Name: "John", Age: 13}, - UserDoc{Name: "Jake", Age: 25}, - UserDoc{Name: "Clair", Age: 12}, - UserDoc{Name: "Johnny", Age: 42}, - UserDoc{Name: "Blair", Age: 67}, - UserDoc{Name: "Zz", Age: 12}, - }, - } - for colName, colDocs := range collectionData { - col := ensureCollection(ctx, db, colName, nil, t) - if _, _, err := col.CreateDocuments(ctx, colDocs); err != nil { - t.Fatalf("Expected success, got %s", describe(err)) + // Create data set + collectionData := map[string][]interface{}{ + "books": []interface{}{ + Book{Title: "Book 01"}, + Book{Title: "Book 02"}, + Book{Title: "Book 03"}, + Book{Title: "Book 04"}, + Book{Title: "Book 05"}, + Book{Title: "Book 06"}, + Book{Title: "Book 07"}, + Book{Title: "Book 08"}, + Book{Title: "Book 09"}, + Book{Title: "Book 10"}, + Book{Title: "Book 11"}, + Book{Title: "Book 12"}, + Book{Title: "Book 13"}, + Book{Title: "Book 14"}, + Book{Title: "Book 15"}, + Book{Title: "Book 16"}, + Book{Title: "Book 17"}, + Book{Title: "Book 18"}, + Book{Title: "Book 19"}, + Book{Title: "Book 20"}, + }, + "users": []interface{}{ + UserDoc{Name: "John", Age: 13}, + UserDoc{Name: "Jake", Age: 25}, + UserDoc{Name: "Clair", Age: 12}, + UserDoc{Name: "Johnny", Age: 42}, + UserDoc{Name: "Blair", Age: 67}, + UserDoc{Name: "Zz", Age: 12}, + }, + } + for colName, colDocs := range collectionData { + col := ensureCollection(ctx, db, colName, nil, t) + if _, _, err := col.CreateDocuments(ctx, colDocs); err != nil { + t.Fatalf("Expected success, got %s", describe(err)) + } } - } - // Setup tests - tests := []queryTest{ - queryTest{ - Query: "FOR d IN books SORT d.Title RETURN d", - ExpectSuccess: true, - ExpectedDocuments: collectionData["books"], - DocumentType: reflect.TypeOf(Book{}), - }, - queryTest{ - Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", - BindVars: map[string]interface{}{"title": "Book 02"}, - ExpectSuccess: true, - ExpectedDocuments: []interface{}{collectionData["books"][1]}, - DocumentType: reflect.TypeOf(Book{}), - }, - queryTest{ - Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", - BindVars: map[string]interface{}{"somethingelse": "Book 02"}, - ExpectSuccess: false, // Unknown `@title` - }, - queryTest{ - Query: "FOR u IN users FILTER u.age>100 SORT u.name RETURN u", - ExpectSuccess: true, - ExpectedDocuments: []interface{}{}, - DocumentType: reflect.TypeOf(UserDoc{}), - }, - queryTest{ - Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", - BindVars: map[string]interface{}{"maxAge": 20}, - ExpectSuccess: true, - ExpectedDocuments: []interface{}{collectionData["users"][2], collectionData["users"][0], collectionData["users"][5]}, - DocumentType: reflect.TypeOf(UserDoc{}), - }, - queryTest{ - Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", - BindVars: map[string]interface{}{"maxage": 20}, - ExpectSuccess: false, // `@maxage` versus `@maxAge` - }, - queryTest{ - Query: "FOR u IN users SORT u.age RETURN u.age", - ExpectedDocuments: []interface{}{12, 12, 13, 25, 42, 67}, - DocumentType: reflect.TypeOf(12), - ExpectSuccess: true, - }, - queryTest{ - Query: "FOR p IN users COLLECT a = p.age WITH COUNT INTO c SORT a RETURN [a, c]", - ExpectedDocuments: []interface{}{[]int{12, 2}, []int{13, 1}, []int{25, 1}, []int{42, 1}, []int{67, 1}}, - DocumentType: reflect.TypeOf([]int{}), - ExpectSuccess: true, - }, - queryTest{ - Query: "FOR u IN users SORT u.name RETURN u.name", - ExpectedDocuments: []interface{}{"Blair", "Clair", "Jake", "John", "Johnny", "Zz"}, - DocumentType: reflect.TypeOf("foo"), - ExpectSuccess: true, - }, - } + // Setup tests + tests := []queryTest{ + queryTest{ + Query: "FOR d IN books SORT d.Title RETURN d", + ExpectSuccess: true, + ExpectedDocuments: collectionData["books"], + DocumentType: reflect.TypeOf(Book{}), + }, + queryTest{ + Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", + BindVars: map[string]interface{}{"title": "Book 02"}, + ExpectSuccess: true, + ExpectedDocuments: []interface{}{collectionData["books"][1]}, + DocumentType: reflect.TypeOf(Book{}), + }, + queryTest{ + Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", + BindVars: map[string]interface{}{"somethingelse": "Book 02"}, + ExpectSuccess: false, // Unknown `@title` + }, + queryTest{ + Query: "FOR u IN users FILTER u.age>100 SORT u.name RETURN u", + ExpectSuccess: true, + ExpectedDocuments: []interface{}{}, + DocumentType: reflect.TypeOf(UserDoc{}), + }, + queryTest{ + Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", + BindVars: map[string]interface{}{"maxAge": 20}, + ExpectSuccess: true, + ExpectedDocuments: []interface{}{collectionData["users"][2], collectionData["users"][0], collectionData["users"][5]}, + DocumentType: reflect.TypeOf(UserDoc{}), + }, + queryTest{ + Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", + BindVars: map[string]interface{}{"maxage": 20}, + ExpectSuccess: false, // `@maxage` versus `@maxAge` + }, + queryTest{ + Query: "FOR u IN users SORT u.age RETURN u.age", + ExpectedDocuments: []interface{}{12, 12, 13, 25, 42, 67}, + DocumentType: reflect.TypeOf(12), + ExpectSuccess: true, + }, + queryTest{ + Query: "FOR p IN users COLLECT a = p.age WITH COUNT INTO c SORT a RETURN [a, c]", + ExpectedDocuments: []interface{}{[]int{12, 2}, []int{13, 1}, []int{25, 1}, []int{42, 1}, []int{67, 1}}, + DocumentType: reflect.TypeOf([]int{}), + ExpectSuccess: true, + }, + queryTest{ + Query: "FOR u IN users SORT u.name RETURN u.name", + ExpectedDocuments: []interface{}{"Blair", "Clair", "Jake", "John", "Johnny", "Zz"}, + DocumentType: reflect.TypeOf("foo"), + ExpectSuccess: true, + }, + } - // Setup context alternatives - contexts := []queryTestContext{ - queryTestContext{driver.WithUseVST(nil, true), false}, - queryTestContext{driver.WithUseVST(context.Background(), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil), true), true}, - queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil, true), true), true}, - queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil, false), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryBatchSize(nil, 1), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil, true), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil, false), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryMemoryLimit(nil, 60000), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryTTL(nil, time.Minute), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryBatchSize(driver.WithQueryCount(nil), 1), true), true}, - queryTestContext{driver.WithUseVST(driver.WithQueryCache(driver.WithQueryCount(driver.WithQueryBatchSize(nil, 2))), true), true}, - } + // Setup context alternatives + contexts := []queryTestContext{ + queryTestContext{driver.WithUseVST(nil, true), false}, + queryTestContext{driver.WithUseVST(context.Background(), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil), true), true}, + queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil, true), true), true}, + queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil, false), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryBatchSize(nil, 1), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil, true), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil, false), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryMemoryLimit(nil, 60000), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryTTL(nil, time.Minute), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryBatchSize(driver.WithQueryCount(nil), 1), true), true}, + queryTestContext{driver.WithUseVST(driver.WithQueryCache(driver.WithQueryCount(driver.WithQueryBatchSize(nil, 2))), true), true}, + } - // Run tests for every context alternative - for _, qctx := range contexts { - ctx := qctx.Context - for i, test := range tests { - cursor, err := db.Query(ctx, test.Query, test.BindVars) - if err == nil { - // Close upon exit of the function - defer cursor.Close() - } - if test.ExpectSuccess { - if err != nil { - t.Errorf("Expected success in query %d (%s), got '%s'", i, test.Query, describe(err)) - continue + // Run tests for every context alternative + for _, qctx := range contexts { + ctx := qctx.Context + for i, test := range tests { + cursor, err := db.Query(ctx, test.Query, test.BindVars) + if err == nil { + // Close upon exit of the function + defer cursor.Close() } - count := cursor.Count() - if qctx.ExpectCount { - if count != int64(len(test.ExpectedDocuments)) { - t.Errorf("Expected count of %d, got %d in query %d (%s)", len(test.ExpectedDocuments), count, i, test.Query) + if test.ExpectSuccess { + if err != nil { + t.Errorf("Expected success in query %d (%s), got '%s'", i, test.Query, describe(err)) + continue } - } else { - if count != 0 { - t.Errorf("Expected count of 0, got %d in query %d (%s)", count, i, test.Query) + count := cursor.Count() + if qctx.ExpectCount { + if count != int64(len(test.ExpectedDocuments)) { + t.Errorf("Expected count of %d, got %d in query %d (%s)", len(test.ExpectedDocuments), count, i, test.Query) + } + } else { + if count != 0 { + t.Errorf("Expected count of 0, got %d in query %d (%s)", count, i, test.Query) + } } - } - var result []interface{} - for { - hasMore := cursor.HasMore() - doc := reflect.New(test.DocumentType) - if _, err := cursor.ReadDocument(ctx, doc.Interface()); driver.IsNoMoreDocuments(err) { - if hasMore { - t.Error("HasMore returned true, but ReadDocument returns a IsNoMoreDocuments error") + var result []interface{} + for { + hasMore := cursor.HasMore() + doc := reflect.New(test.DocumentType) + if _, err := cursor.ReadDocument(ctx, doc.Interface()); driver.IsNoMoreDocuments(err) { + if hasMore { + t.Error("HasMore returned true, but ReadDocument returns a IsNoMoreDocuments error") + } + break + } else if err != nil { + t.Errorf("Failed to result document %d: %s", len(result), describe(err)) + } + if !hasMore { + t.Error("HasMore returned false, but ReadDocument returns a document") } - break - } else if err != nil { - t.Errorf("Failed to result document %d: %s", len(result), describe(err)) + result = append(result, doc.Elem().Interface()) } - if !hasMore { - t.Error("HasMore returned false, but ReadDocument returns a document") + if len(result) != len(test.ExpectedDocuments) { + t.Errorf("Expected %d documents, got %d in query %d (%s)", len(test.ExpectedDocuments), len(result), i, test.Query) + } else { + for resultIdx, resultDoc := range result { + if !reflect.DeepEqual(resultDoc, test.ExpectedDocuments[resultIdx]) { + t.Errorf("Unexpected document in query %d (%s) at index %d: got %+v, expected %+v", i, test.Query, resultIdx, resultDoc, test.ExpectedDocuments[resultIdx]) + } + } + } + // Close anyway (this tests calling Close more than once) + if err := cursor.Close(); err != nil { + t.Errorf("Expected success in Close of cursor from query %d (%s), got '%s'", i, test.Query, describe(err)) } - result = append(result, doc.Elem().Interface()) - } - if len(result) != len(test.ExpectedDocuments) { - t.Errorf("Expected %d documents, got %d in query %d (%s)", len(test.ExpectedDocuments), len(result), i, test.Query) } else { - for resultIdx, resultDoc := range result { - if !reflect.DeepEqual(resultDoc, test.ExpectedDocuments[resultIdx]) { - t.Errorf("Unexpected document in query %d (%s) at index %d: got %+v, expected %+v", i, test.Query, resultIdx, resultDoc, test.ExpectedDocuments[resultIdx]) - } + if err == nil { + t.Errorf("Expected error in query %d (%s), got '%s'", i, test.Query, describe(err)) + continue } } - // Close anyway (this tests calling Close more than once) - if err := cursor.Close(); err != nil { - t.Errorf("Expected success in Close of cursor from query %d (%s), got '%s'", i, test.Query, describe(err)) - } - } else { - if err == nil { - t.Errorf("Expected error in query %d (%s), got '%s'", i, test.Query, describe(err)) - continue - } } } } + + // run with VST + runCursorLoadBalancingTest(t, true) + + // run with HTTP + runCursorLoadBalancingTest(t, false) } diff --git a/tests/test_util.go b/tests/test_util.go index 3922b9a7c..d4cb46104 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -101,8 +101,8 @@ func mustNewKubeClient(t *testing.T) kubernetes.Interface { // mustNewArangodDatabaseClient creates a new database client, // failing the test on errors. -func mustNewArangodDatabaseClient(ctx context.Context, kubecli kubernetes.Interface, apiObject *api.ArangoDeployment, t *testing.T) driver.Client { - c, err := arangod.CreateArangodDatabaseClient(ctx, kubecli.CoreV1(), apiObject) +func mustNewArangodDatabaseClient(ctx context.Context, kubecli kubernetes.Interface, apiObject *api.ArangoDeployment, t *testing.T, vst ...bool) driver.Client { + c, err := arangod.CreateArangodDatabaseClient(ctx, kubecli.CoreV1(), apiObject, vst) if err != nil { t.Fatalf("Failed to create arango database client: %v", err) } From 14042eaa788994e05dadacf0cab9dfc14c9dc3af Mon Sep 17 00:00:00 2001 From: Dan Larkin-York Date: Mon, 16 Jul 2018 08:27:27 -0400 Subject: [PATCH 04/16] Formatting fixes. --- pkg/util/arangod/client.go | 4 +- tests/load_balancer_test.go | 391 ++++++++++++++++++------------------ 2 files changed, 198 insertions(+), 197 deletions(-) diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index 1f66c2435..147ce42eb 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -201,7 +201,7 @@ func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interfa func createArangodHTTPConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (http.ConnectionConfig, error) { scheme := "http" transport := sharedHTTPTransport - if apiObject != nil && apiObject.Spec.IsSecure() { + if apiObject != nil && apiObject.Spec.IsSecure() { scheme = "https" transport = sharedHTTPSTransport } @@ -218,7 +218,7 @@ func createArangodHTTPConfigForDNSNames(ctx context.Context, cli corev1.CoreV1In // createArangodVSTConfigForDNSNames creates a go-driver VST connection config for a given DNS names. func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (http.ConnectionConfig, error) { scheme := "http" - transport := vst.NewTransport(/* TODO configure transport */) + transport := vst.NewTransport( /* TODO configure transport */ ) if apiObject != nil && apiObject.Spec.IsSecure() { scheme = "https" transport = sharedHTTPSTransport diff --git a/tests/load_balancer_test.go b/tests/load_balancer_test.go index ffafec98a..38e111ba5 100644 --- a/tests/load_balancer_test.go +++ b/tests/load_balancer_test.go @@ -50,223 +50,224 @@ type queryTestContext struct { ExpectCount bool } +func TestLoadBalancingCursorVST(t *testing.T) { + // run with VST + TestLoadBalancingCursorSubtest(t, true) +} + +func TestLoadBalancingCursorHTTP(t *testing.T) { + // run with HTTP + TestCursorLoadBalancingTestSubtest(t, false) +} // TestLoadBalancerCursorVST tests cursor forwarding with load-balanced conn. -func TestLoadBalancerCursor(t *testing.T) { - func runCursorLoadBalancingTest(t *testing.T, useVst bool) { - c := client.MustNewInCluster() - kubecli := mustNewKubeClient(t) - ns := getNamespace(t) +func TestLoadBalancingCursorSubtest(t *testing.T, useVst bool) { + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) - // Prepare deployment config - depl := newDeployment("test-lb-" + uniuri.NewLen(4)) - depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) + // Prepare deployment config + depl := newDeployment("test-lb-" + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) - // Create deployment - _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) - if err != nil { - t.Fatalf("Create deployment failed: %v", err) - } - // Prepare cleanup - defer removeDeployment(c, depl.GetName(), ns) + // Create deployment + _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) + if err != nil { + t.Fatalf("Create deployment failed: %v", err) + } + // Prepare cleanup + defer removeDeployment(c, depl.GetName(), ns) - // Wait for deployment to be ready - apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) - if err != nil { - t.Fatalf("Deployment not running in time: %v", err) - } + // Wait for deployment to be ready + apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) + if err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } - // Create a database client - ctx := context.Background() - client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t, useVst) + // Create a database client + ctx := context.Background() + client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t, useVst) - // Wait for cluster to be available - if err := waitUntilVersionUp(client, nil); err != nil { - t.Fatalf("Cluster not running returning version in time: %v", err) - } + // Wait for cluster to be available + if err := waitUntilVersionUp(client, nil); err != nil { + t.Fatalf("Cluster not running returning version in time: %v", err) + } - // Create data set - collectionData := map[string][]interface{}{ - "books": []interface{}{ - Book{Title: "Book 01"}, - Book{Title: "Book 02"}, - Book{Title: "Book 03"}, - Book{Title: "Book 04"}, - Book{Title: "Book 05"}, - Book{Title: "Book 06"}, - Book{Title: "Book 07"}, - Book{Title: "Book 08"}, - Book{Title: "Book 09"}, - Book{Title: "Book 10"}, - Book{Title: "Book 11"}, - Book{Title: "Book 12"}, - Book{Title: "Book 13"}, - Book{Title: "Book 14"}, - Book{Title: "Book 15"}, - Book{Title: "Book 16"}, - Book{Title: "Book 17"}, - Book{Title: "Book 18"}, - Book{Title: "Book 19"}, - Book{Title: "Book 20"}, - }, - "users": []interface{}{ - UserDoc{Name: "John", Age: 13}, - UserDoc{Name: "Jake", Age: 25}, - UserDoc{Name: "Clair", Age: 12}, - UserDoc{Name: "Johnny", Age: 42}, - UserDoc{Name: "Blair", Age: 67}, - UserDoc{Name: "Zz", Age: 12}, - }, - } - for colName, colDocs := range collectionData { - col := ensureCollection(ctx, db, colName, nil, t) - if _, _, err := col.CreateDocuments(ctx, colDocs); err != nil { - t.Fatalf("Expected success, got %s", describe(err)) - } + // Create data set + collectionData := map[string][]interface{}{ + "books": []interface{}{ + Book{Title: "Book 01"}, + Book{Title: "Book 02"}, + Book{Title: "Book 03"}, + Book{Title: "Book 04"}, + Book{Title: "Book 05"}, + Book{Title: "Book 06"}, + Book{Title: "Book 07"}, + Book{Title: "Book 08"}, + Book{Title: "Book 09"}, + Book{Title: "Book 10"}, + Book{Title: "Book 11"}, + Book{Title: "Book 12"}, + Book{Title: "Book 13"}, + Book{Title: "Book 14"}, + Book{Title: "Book 15"}, + Book{Title: "Book 16"}, + Book{Title: "Book 17"}, + Book{Title: "Book 18"}, + Book{Title: "Book 19"}, + Book{Title: "Book 20"}, + }, + "users": []interface{}{ + UserDoc{Name: "John", Age: 13}, + UserDoc{Name: "Jake", Age: 25}, + UserDoc{Name: "Clair", Age: 12}, + UserDoc{Name: "Johnny", Age: 42}, + UserDoc{Name: "Blair", Age: 67}, + UserDoc{Name: "Zz", Age: 12}, + }, + } + for colName, colDocs := range collectionData { + col := ensureCollection(ctx, db, colName, nil, t) + if _, _, err := col.CreateDocuments(ctx, colDocs); err != nil { + t.Fatalf("Expected success, got %s", describe(err)) } + } - // Setup tests - tests := []queryTest{ - queryTest{ - Query: "FOR d IN books SORT d.Title RETURN d", - ExpectSuccess: true, - ExpectedDocuments: collectionData["books"], - DocumentType: reflect.TypeOf(Book{}), - }, - queryTest{ - Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", - BindVars: map[string]interface{}{"title": "Book 02"}, - ExpectSuccess: true, - ExpectedDocuments: []interface{}{collectionData["books"][1]}, - DocumentType: reflect.TypeOf(Book{}), - }, - queryTest{ - Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", - BindVars: map[string]interface{}{"somethingelse": "Book 02"}, - ExpectSuccess: false, // Unknown `@title` - }, - queryTest{ - Query: "FOR u IN users FILTER u.age>100 SORT u.name RETURN u", - ExpectSuccess: true, - ExpectedDocuments: []interface{}{}, - DocumentType: reflect.TypeOf(UserDoc{}), - }, - queryTest{ - Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", - BindVars: map[string]interface{}{"maxAge": 20}, - ExpectSuccess: true, - ExpectedDocuments: []interface{}{collectionData["users"][2], collectionData["users"][0], collectionData["users"][5]}, - DocumentType: reflect.TypeOf(UserDoc{}), - }, - queryTest{ - Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", - BindVars: map[string]interface{}{"maxage": 20}, - ExpectSuccess: false, // `@maxage` versus `@maxAge` - }, - queryTest{ - Query: "FOR u IN users SORT u.age RETURN u.age", - ExpectedDocuments: []interface{}{12, 12, 13, 25, 42, 67}, - DocumentType: reflect.TypeOf(12), - ExpectSuccess: true, - }, - queryTest{ - Query: "FOR p IN users COLLECT a = p.age WITH COUNT INTO c SORT a RETURN [a, c]", - ExpectedDocuments: []interface{}{[]int{12, 2}, []int{13, 1}, []int{25, 1}, []int{42, 1}, []int{67, 1}}, - DocumentType: reflect.TypeOf([]int{}), - ExpectSuccess: true, - }, - queryTest{ - Query: "FOR u IN users SORT u.name RETURN u.name", - ExpectedDocuments: []interface{}{"Blair", "Clair", "Jake", "John", "Johnny", "Zz"}, - DocumentType: reflect.TypeOf("foo"), - ExpectSuccess: true, - }, - } + // Setup tests + tests := []queryTest{ + queryTest{ + Query: "FOR d IN books SORT d.Title RETURN d", + ExpectSuccess: true, + ExpectedDocuments: collectionData["books"], + DocumentType: reflect.TypeOf(Book{}), + }, + queryTest{ + Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", + BindVars: map[string]interface{}{"title": "Book 02"}, + ExpectSuccess: true, + ExpectedDocuments: []interface{}{collectionData["books"][1]}, + DocumentType: reflect.TypeOf(Book{}), + }, + queryTest{ + Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", + BindVars: map[string]interface{}{"somethingelse": "Book 02"}, + ExpectSuccess: false, // Unknown `@title` + }, + queryTest{ + Query: "FOR u IN users FILTER u.age>100 SORT u.name RETURN u", + ExpectSuccess: true, + ExpectedDocuments: []interface{}{}, + DocumentType: reflect.TypeOf(UserDoc{}), + }, + queryTest{ + Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", + BindVars: map[string]interface{}{"maxAge": 20}, + ExpectSuccess: true, + ExpectedDocuments: []interface{}{collectionData["users"][2], collectionData["users"][0], collectionData["users"][5]}, + DocumentType: reflect.TypeOf(UserDoc{}), + }, + queryTest{ + Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", + BindVars: map[string]interface{}{"maxage": 20}, + ExpectSuccess: false, // `@maxage` versus `@maxAge` + }, + queryTest{ + Query: "FOR u IN users SORT u.age RETURN u.age", + ExpectedDocuments: []interface{}{12, 12, 13, 25, 42, 67}, + DocumentType: reflect.TypeOf(12), + ExpectSuccess: true, + }, + queryTest{ + Query: "FOR p IN users COLLECT a = p.age WITH COUNT INTO c SORT a RETURN [a, c]", + ExpectedDocuments: []interface{}{[]int{12, 2}, []int{13, 1}, []int{25, 1}, []int{42, 1}, []int{67, 1}}, + DocumentType: reflect.TypeOf([]int{}), + ExpectSuccess: true, + }, + queryTest{ + Query: "FOR u IN users SORT u.name RETURN u.name", + ExpectedDocuments: []interface{}{"Blair", "Clair", "Jake", "John", "Johnny", "Zz"}, + DocumentType: reflect.TypeOf("foo"), + ExpectSuccess: true, + }, + } - // Setup context alternatives - contexts := []queryTestContext{ - queryTestContext{driver.WithUseVST(nil, true), false}, - queryTestContext{driver.WithUseVST(context.Background(), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil), true), true}, - queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil, true), true), true}, - queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil, false), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryBatchSize(nil, 1), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil, true), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil, false), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryMemoryLimit(nil, 60000), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryTTL(nil, time.Minute), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryBatchSize(driver.WithQueryCount(nil), 1), true), true}, - queryTestContext{driver.WithUseVST(driver.WithQueryCache(driver.WithQueryCount(driver.WithQueryBatchSize(nil, 2))), true), true}, - } + // Setup context alternatives + contexts := []queryTestContext{ + queryTestContext{driver.WithUseVST(nil, true), false}, + queryTestContext{driver.WithUseVST(context.Background(), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil), true), true}, + queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil, true), true), true}, + queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil, false), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryBatchSize(nil, 1), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil, true), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil, false), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryMemoryLimit(nil, 60000), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryTTL(nil, time.Minute), true), false}, + queryTestContext{driver.WithUseVST(driver.WithQueryBatchSize(driver.WithQueryCount(nil), 1), true), true}, + queryTestContext{driver.WithUseVST(driver.WithQueryCache(driver.WithQueryCount(driver.WithQueryBatchSize(nil, 2))), true), true}, + } - // Run tests for every context alternative - for _, qctx := range contexts { - ctx := qctx.Context - for i, test := range tests { - cursor, err := db.Query(ctx, test.Query, test.BindVars) - if err == nil { - // Close upon exit of the function - defer cursor.Close() + // Run tests for every context alternative + for _, qctx := range contexts { + ctx := qctx.Context + for i, test := range tests { + cursor, err := db.Query(ctx, test.Query, test.BindVars) + if err == nil { + // Close upon exit of the function + defer cursor.Close() + } + if test.ExpectSuccess { + if err != nil { + t.Errorf("Expected success in query %d (%s), got '%s'", i, test.Query, describe(err)) + continue } - if test.ExpectSuccess { - if err != nil { - t.Errorf("Expected success in query %d (%s), got '%s'", i, test.Query, describe(err)) - continue - } - count := cursor.Count() - if qctx.ExpectCount { - if count != int64(len(test.ExpectedDocuments)) { - t.Errorf("Expected count of %d, got %d in query %d (%s)", len(test.ExpectedDocuments), count, i, test.Query) - } - } else { - if count != 0 { - t.Errorf("Expected count of 0, got %d in query %d (%s)", count, i, test.Query) - } + count := cursor.Count() + if qctx.ExpectCount { + if count != int64(len(test.ExpectedDocuments)) { + t.Errorf("Expected count of %d, got %d in query %d (%s)", len(test.ExpectedDocuments), count, i, test.Query) } - var result []interface{} - for { - hasMore := cursor.HasMore() - doc := reflect.New(test.DocumentType) - if _, err := cursor.ReadDocument(ctx, doc.Interface()); driver.IsNoMoreDocuments(err) { - if hasMore { - t.Error("HasMore returned true, but ReadDocument returns a IsNoMoreDocuments error") - } - break - } else if err != nil { - t.Errorf("Failed to result document %d: %s", len(result), describe(err)) - } - if !hasMore { - t.Error("HasMore returned false, but ReadDocument returns a document") - } - result = append(result, doc.Elem().Interface()) + } else { + if count != 0 { + t.Errorf("Expected count of 0, got %d in query %d (%s)", count, i, test.Query) } - if len(result) != len(test.ExpectedDocuments) { - t.Errorf("Expected %d documents, got %d in query %d (%s)", len(test.ExpectedDocuments), len(result), i, test.Query) - } else { - for resultIdx, resultDoc := range result { - if !reflect.DeepEqual(resultDoc, test.ExpectedDocuments[resultIdx]) { - t.Errorf("Unexpected document in query %d (%s) at index %d: got %+v, expected %+v", i, test.Query, resultIdx, resultDoc, test.ExpectedDocuments[resultIdx]) - } + } + var result []interface{} + for { + hasMore := cursor.HasMore() + doc := reflect.New(test.DocumentType) + if _, err := cursor.ReadDocument(ctx, doc.Interface()); driver.IsNoMoreDocuments(err) { + if hasMore { + t.Error("HasMore returned true, but ReadDocument returns a IsNoMoreDocuments error") } + break + } else if err != nil { + t.Errorf("Failed to result document %d: %s", len(result), describe(err)) } - // Close anyway (this tests calling Close more than once) - if err := cursor.Close(); err != nil { - t.Errorf("Expected success in Close of cursor from query %d (%s), got '%s'", i, test.Query, describe(err)) + if !hasMore { + t.Error("HasMore returned false, but ReadDocument returns a document") } + result = append(result, doc.Elem().Interface()) + } + if len(result) != len(test.ExpectedDocuments) { + t.Errorf("Expected %d documents, got %d in query %d (%s)", len(test.ExpectedDocuments), len(result), i, test.Query) } else { - if err == nil { - t.Errorf("Expected error in query %d (%s), got '%s'", i, test.Query, describe(err)) - continue + for resultIdx, resultDoc := range result { + if !reflect.DeepEqual(resultDoc, test.ExpectedDocuments[resultIdx]) { + t.Errorf("Unexpected document in query %d (%s) at index %d: got %+v, expected %+v", i, test.Query, resultIdx, resultDoc, test.ExpectedDocuments[resultIdx]) + } } } + // Close anyway (this tests calling Close more than once) + if err := cursor.Close(); err != nil { + t.Errorf("Expected success in Close of cursor from query %d (%s), got '%s'", i, test.Query, describe(err)) + } + } else { + if err == nil { + t.Errorf("Expected error in query %d (%s), got '%s'", i, test.Query, describe(err)) + continue + } } } } - - // run with VST - runCursorLoadBalancingTest(t, true) - - // run with HTTP - runCursorLoadBalancingTest(t, false) } From 3d47a4732cb702fa09fba74bafa4c9f3b1675e4f Mon Sep 17 00:00:00 2001 From: Dan Larkin-York Date: Mon, 16 Jul 2018 08:32:47 -0400 Subject: [PATCH 05/16] Remove bad fn call. --- tests/load_balancer_test.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/load_balancer_test.go b/tests/load_balancer_test.go index 38e111ba5..23fcdfa99 100644 --- a/tests/load_balancer_test.go +++ b/tests/load_balancer_test.go @@ -193,19 +193,19 @@ func TestLoadBalancingCursorSubtest(t *testing.T, useVst bool) { // Setup context alternatives contexts := []queryTestContext{ - queryTestContext{driver.WithUseVST(nil, true), false}, - queryTestContext{driver.WithUseVST(context.Background(), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil), true), true}, - queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil, true), true), true}, - queryTestContext{driver.WithUseVST(driver.WithQueryCount(nil, false), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryBatchSize(nil, 1), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil, true), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryCache(nil, false), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryMemoryLimit(nil, 60000), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryTTL(nil, time.Minute), true), false}, - queryTestContext{driver.WithUseVST(driver.WithQueryBatchSize(driver.WithQueryCount(nil), 1), true), true}, - queryTestContext{driver.WithUseVST(driver.WithQueryCache(driver.WithQueryCount(driver.WithQueryBatchSize(nil, 2))), true), true}, + queryTestContext{nil, true), false}, + queryTestContext{context.Background(), false}, + queryTestContext{driver.WithQueryCount(nil), true}, + queryTestContext{driver.WithQueryCount(nil, true), true}, + queryTestContext{driver.WithQueryCount(nil, false), false}, + queryTestContext{driver.WithQueryBatchSize(nil, 1), false}, + queryTestContext{driver.WithQueryCache(nil), false}, + queryTestContext{driver.WithQueryCache(nil, true), false}, + queryTestContext{driver.WithQueryCache(nil, false), false}, + queryTestContext{driver.WithQueryMemoryLimit(nil, 60000), false}, + queryTestContext{driver.WithQueryTTL(nil, time.Minute), false}, + queryTestContext{driver.WithQueryBatchSize(driver.WithQueryCount(nil), 1), true}, + queryTestContext{driver.WithQueryCache(driver.WithQueryCount(driver.WithQueryBatchSize(nil, 2))), true}, } // Run tests for every context alternative From 20d9bf6d7ce0f7a9265568147938b6e14b81045d Mon Sep 17 00:00:00 2001 From: Dan Larkin-York Date: Mon, 16 Jul 2018 10:31:15 -0400 Subject: [PATCH 06/16] Fix transport. --- pkg/util/arangod/client.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index 147ce42eb..25ad9e1a7 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -218,12 +218,16 @@ func createArangodHTTPConfigForDNSNames(ctx context.Context, cli corev1.CoreV1In // createArangodVSTConfigForDNSNames creates a go-driver VST connection config for a given DNS names. func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (http.ConnectionConfig, error) { scheme := "http" - transport := vst.NewTransport( /* TODO configure transport */ ) + tlsConfig := vst.config.TLSConfig if apiObject != nil && apiObject.Spec.IsSecure() { scheme = "https" - transport = sharedHTTPSTransport + tlsConfig = &tls.Config{} + } + transport := protocol.TransportConfig{ + Version: vst.protocol.Version1_1, } connConfig := vst.ConnectionConfig{ + TLSConfig: tlsConfig, Transport: transport, } for _, dnsName := range dnsNames { From 53738d6bedc2717026c58b4f6185760ea4b3e0f2 Mon Sep 17 00:00:00 2001 From: Dan Larkin-York Date: Mon, 16 Jul 2018 12:18:57 -0400 Subject: [PATCH 07/16] Fix some compilation issues. --- pkg/util/arangod/client.go | 20 +++++++++++--------- tests/load_balancer_test.go | 28 ++++++++++++++-------------- tests/test_util.go | 4 ++-- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index 25ad9e1a7..10b394c05 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -32,6 +32,8 @@ import ( "time" driver "github.com/arangodb/go-driver" + vst "github.com/arangodb/go-driver/vst" + vstProtocol "github.com/arangodb/go-driver/vst/protocol" "github.com/arangodb/go-driver/agency" "github.com/arangodb/go-driver/http" "github.com/arangodb/go-driver/jwt" @@ -100,10 +102,10 @@ func CreateArangodClient(ctx context.Context, cli corev1.CoreV1Interface, apiObj } // CreateArangodDatabaseClient creates a go-driver client for accessing the entire cluster (or single server). -func CreateArangodDatabaseClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, vst ...bool) (driver.Client, error) { +func CreateArangodDatabaseClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, useVst ...bool) (driver.Client, error) { // Create connection dnsName := k8sutil.CreateDatabaseClientServiceDNSName(apiObject) - c, err := createArangodClientForDNSName(ctx, cli, apiObject, dnsName, vst) + c, err := createArangodClientForDNSName(ctx, cli, apiObject, dnsName, useVst...) if err != nil { return nil, maskAny(err) } @@ -155,10 +157,10 @@ func CreateArangodImageIDClient(ctx context.Context, deployment k8sutil.APIObjec } // CreateArangodClientForDNSName creates a go-driver client for a given DNS name. -func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsName string, vst ...bool) (driver.Client, error) { +func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsName string, useVst ...bool) (driver.Client, error) { config := driver.ClientConfig{} var conn driver.Connection - if len(vst) > 0 && vst[0] { + if len(useVst) > 0 && useVst[0] { connConfig, err := createArangodVSTConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}) if err != nil { return nil, maskAny(err) @@ -181,7 +183,7 @@ func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interfa } // Create client - config := driver.ClientConfig{ + config = driver.ClientConfig{ Connection: conn, } @@ -216,15 +218,15 @@ func createArangodHTTPConfigForDNSNames(ctx context.Context, cli corev1.CoreV1In } // createArangodVSTConfigForDNSNames creates a go-driver VST connection config for a given DNS names. -func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (http.ConnectionConfig, error) { +func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (vst.ConnectionConfig, error) { scheme := "http" - tlsConfig := vst.config.TLSConfig + tlsConfig := &tls.Config{InsecureSkipVerify: true} if apiObject != nil && apiObject.Spec.IsSecure() { scheme = "https" tlsConfig = &tls.Config{} } - transport := protocol.TransportConfig{ - Version: vst.protocol.Version1_1, + transport := vstProtocol.TransportConfig{ + Version: vstProtocol.Version1_1, } connConfig := vst.ConnectionConfig{ TLSConfig: tlsConfig, diff --git a/tests/load_balancer_test.go b/tests/load_balancer_test.go index 23fcdfa99..b461476c5 100644 --- a/tests/load_balancer_test.go +++ b/tests/load_balancer_test.go @@ -29,15 +29,13 @@ import ( "time" "github.com/dchest/uniuri" - "github.com/stretchr/testify/assert" driver "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/client" - "github.com/arangodb/kube-arangodb/pkg/util" ) -type queryTest struct { +/*type queryTest struct { Query string BindVars map[string]interface{} ExpectSuccess bool @@ -45,23 +43,23 @@ type queryTest struct { DocumentType reflect.Type } -type queryTestContext struct { +type ueryTestContext struct { Context context.Context ExpectCount bool -} +}*/ func TestLoadBalancingCursorVST(t *testing.T) { // run with VST - TestLoadBalancingCursorSubtest(t, true) + LoadBalancingCursorSubtest(t, true) } func TestLoadBalancingCursorHTTP(t *testing.T) { // run with HTTP - TestCursorLoadBalancingTestSubtest(t, false) + LoadBalancingCursorSubtest(t, false) } // TestLoadBalancerCursorVST tests cursor forwarding with load-balanced conn. -func TestLoadBalancingCursorSubtest(t *testing.T, useVst bool) { +func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { c := client.MustNewInCluster() kubecli := mustNewKubeClient(t) ns := getNamespace(t) @@ -126,10 +124,12 @@ func TestLoadBalancingCursorSubtest(t *testing.T, useVst bool) { UserDoc{Name: "Zz", Age: 12}, }, } + + db := ensureDatabase(ctx, client, "lb_cursor_test", nil, t) for colName, colDocs := range collectionData { col := ensureCollection(ctx, db, colName, nil, t) if _, _, err := col.CreateDocuments(ctx, colDocs); err != nil { - t.Fatalf("Expected success, got %s", describe(err)) + t.Fatalf("Expected success, got %s", err) } } @@ -193,7 +193,7 @@ func TestLoadBalancingCursorSubtest(t *testing.T, useVst bool) { // Setup context alternatives contexts := []queryTestContext{ - queryTestContext{nil, true), false}, + queryTestContext{nil, false}, queryTestContext{context.Background(), false}, queryTestContext{driver.WithQueryCount(nil), true}, queryTestContext{driver.WithQueryCount(nil, true), true}, @@ -219,7 +219,7 @@ func TestLoadBalancingCursorSubtest(t *testing.T, useVst bool) { } if test.ExpectSuccess { if err != nil { - t.Errorf("Expected success in query %d (%s), got '%s'", i, test.Query, describe(err)) + t.Errorf("Expected success in query %d (%s), got '%s'", i, test.Query, err) continue } count := cursor.Count() @@ -242,7 +242,7 @@ func TestLoadBalancingCursorSubtest(t *testing.T, useVst bool) { } break } else if err != nil { - t.Errorf("Failed to result document %d: %s", len(result), describe(err)) + t.Errorf("Failed to result document %d: %s", len(result), err) } if !hasMore { t.Error("HasMore returned false, but ReadDocument returns a document") @@ -260,11 +260,11 @@ func TestLoadBalancingCursorSubtest(t *testing.T, useVst bool) { } // Close anyway (this tests calling Close more than once) if err := cursor.Close(); err != nil { - t.Errorf("Expected success in Close of cursor from query %d (%s), got '%s'", i, test.Query, describe(err)) + t.Errorf("Expected success in Close of cursor from query %d (%s), got '%s'", i, test.Query, err) } } else { if err == nil { - t.Errorf("Expected error in query %d (%s), got '%s'", i, test.Query, describe(err)) + t.Errorf("Expected error in query %d (%s), got '%s'", i, test.Query, err) continue } } diff --git a/tests/test_util.go b/tests/test_util.go index d4cb46104..faa0505aa 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -101,8 +101,8 @@ func mustNewKubeClient(t *testing.T) kubernetes.Interface { // mustNewArangodDatabaseClient creates a new database client, // failing the test on errors. -func mustNewArangodDatabaseClient(ctx context.Context, kubecli kubernetes.Interface, apiObject *api.ArangoDeployment, t *testing.T, vst ...bool) driver.Client { - c, err := arangod.CreateArangodDatabaseClient(ctx, kubecli.CoreV1(), apiObject, vst) +func mustNewArangodDatabaseClient(ctx context.Context, kubecli kubernetes.Interface, apiObject *api.ArangoDeployment, t *testing.T, useVst ...bool) driver.Client { + c, err := arangod.CreateArangodDatabaseClient(ctx, kubecli.CoreV1(), apiObject, useVst...) if err != nil { t.Fatalf("Failed to create arango database client: %v", err) } From 85ac0a08fb0536bcf8b3c560927acf1b3178c067 Mon Sep 17 00:00:00 2001 From: Dan Larkin-York Date: Tue, 17 Jul 2018 14:18:41 -0400 Subject: [PATCH 08/16] Moved client changes to test code. --- pkg/util/arangod/client.go | 62 ++++++------------------------- tests/load_balancer_test.go | 17 ++------- tests/test_util.go | 74 ++++++++++++++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 64 deletions(-) diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index 10b394c05..7774a63ce 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -32,8 +32,6 @@ import ( "time" driver "github.com/arangodb/go-driver" - vst "github.com/arangodb/go-driver/vst" - vstProtocol "github.com/arangodb/go-driver/vst/protocol" "github.com/arangodb/go-driver/agency" "github.com/arangodb/go-driver/http" "github.com/arangodb/go-driver/jwt" @@ -102,10 +100,10 @@ func CreateArangodClient(ctx context.Context, cli corev1.CoreV1Interface, apiObj } // CreateArangodDatabaseClient creates a go-driver client for accessing the entire cluster (or single server). -func CreateArangodDatabaseClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, useVst ...bool) (driver.Client, error) { +func CreateArangodDatabaseClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment) (driver.Client, error) { // Create connection dnsName := k8sutil.CreateDatabaseClientServiceDNSName(apiObject) - c, err := createArangodClientForDNSName(ctx, cli, apiObject, dnsName, useVst...) + c, err := createArangodClientForDNSName(ctx, cli, apiObject, dnsName) if err != nil { return nil, maskAny(err) } @@ -157,36 +155,21 @@ func CreateArangodImageIDClient(ctx context.Context, deployment k8sutil.APIObjec } // CreateArangodClientForDNSName creates a go-driver client for a given DNS name. -func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsName string, useVst ...bool) (driver.Client, error) { - config := driver.ClientConfig{} - var conn driver.Connection - if len(useVst) > 0 && useVst[0] { - connConfig, err := createArangodVSTConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}) - if err != nil { - return nil, maskAny(err) - } - // TODO deal with TLS with proper CA checking - conn, err = vst.NewConnection(connConfig) - if err != nil { - return nil, maskAny(err) - } - } else { - connConfig, err := createArangodHTTPConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}) - if err != nil { - return nil, maskAny(err) - } - // TODO deal with TLS with proper CA checking - conn, err = http.NewConnection(connConfig) - if err != nil { - return nil, maskAny(err) - } +func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsName string) (driver.Client, error) { + connConfig, err := createArangodHTTPConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}) + if err != nil { + return nil, maskAny(err) + } + // TODO deal with TLS with proper CA checking + conn, err := http.NewConnection(connConfig) + if err != nil { + return nil, maskAny(err) } // Create client - config = driver.ClientConfig{ + config := driver.ClientConfig{ Connection: conn, } - auth, err := createArangodClientAuthentication(ctx, cli, apiObject) if err != nil { return nil, maskAny(err) @@ -217,27 +200,6 @@ func createArangodHTTPConfigForDNSNames(ctx context.Context, cli corev1.CoreV1In return connConfig, nil } -// createArangodVSTConfigForDNSNames creates a go-driver VST connection config for a given DNS names. -func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (vst.ConnectionConfig, error) { - scheme := "http" - tlsConfig := &tls.Config{InsecureSkipVerify: true} - if apiObject != nil && apiObject.Spec.IsSecure() { - scheme = "https" - tlsConfig = &tls.Config{} - } - transport := vstProtocol.TransportConfig{ - Version: vstProtocol.Version1_1, - } - connConfig := vst.ConnectionConfig{ - TLSConfig: tlsConfig, - Transport: transport, - } - for _, dnsName := range dnsNames { - connConfig.Endpoints = append(connConfig.Endpoints, scheme+"://"+net.JoinHostPort(dnsName, strconv.Itoa(k8sutil.ArangoPort))) - } - return connConfig, nil -} - // createArangodClientAuthentication creates a go-driver authentication for the servers in the given deployment. func createArangodClientAuthentication(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment) (driver.Authentication, error) { if apiObject != nil && apiObject.Spec.IsAuthenticated() { diff --git a/tests/load_balancer_test.go b/tests/load_balancer_test.go index b461476c5..c717e9360 100644 --- a/tests/load_balancer_test.go +++ b/tests/load_balancer_test.go @@ -35,19 +35,6 @@ import ( "github.com/arangodb/kube-arangodb/pkg/client" ) -/*type queryTest struct { - Query string - BindVars map[string]interface{} - ExpectSuccess bool - ExpectedDocuments []interface{} - DocumentType reflect.Type -} - -type ueryTestContext struct { - Context context.Context - ExpectCount bool -}*/ - func TestLoadBalancingCursorVST(t *testing.T) { // run with VST LoadBalancingCursorSubtest(t, true) @@ -208,6 +195,10 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { queryTestContext{driver.WithQueryCache(driver.WithQueryCount(driver.WithQueryBatchSize(nil, 2))), true}, } + // keep track of whether at least one request was forwarded internally to the + // correct coordinator behind the load balancer + someRequestForwarded = false + // Run tests for every context alternative for _, qctx := range contexts { ctx := qctx.Context diff --git a/tests/test_util.go b/tests/test_util.go index faa0505aa..e9bec5375 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -24,6 +24,7 @@ package tests import ( "context" + "crypto/tls" "fmt" "net" "os" @@ -39,9 +40,12 @@ import ( "github.com/arangodb/arangosync/client" "github.com/arangodb/arangosync/tasks" driver "github.com/arangodb/go-driver" + vst "github.com/arangodb/go-driver/vst" + vstProtocol "github.com/arangodb/go-driver/vst/protocol" "github.com/pkg/errors" "github.com/rs/zerolog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" @@ -60,6 +64,68 @@ var ( showEnterpriseImageOnce sync.Once ) +// CreateArangodClientForDNSName creates a go-driver client for a given DNS name. +func createArangodVSTClientForDNSName(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsName string) (driver.Client, error) { + config := driver.ClientConfig{} + connConfig, err := createArangodVSTConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}) + if err != nil { + return nil, maskAny(err) + } + // TODO deal with TLS with proper CA checking + conn, err := vst.NewConnection(connConfig) + if err != nil { + return nil, maskAny(err) + } + + // Create client + config = driver.ClientConfig{ + Connection: conn, + } + + auth := driver.BasicAuthentication("root", "") + if err != nil { + return nil, maskAny(err) + } + config.Authentication = auth + c, err := driver.NewClient(config) + if err != nil { + return nil, maskAny(err) + } + return c, nil +} + +// createArangodVSTConfigForDNSNames creates a go-driver VST connection config for a given DNS names. +func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (vst.ConnectionConfig, error) { + scheme := "http" + tlsConfig := &tls.Config{InsecureSkipVerify: true} + if apiObject != nil && apiObject.Spec.IsSecure() { + scheme = "https" + tlsConfig = &tls.Config{} + } + transport := vstProtocol.TransportConfig{ + Version: vstProtocol.Version1_1, + } + connConfig := vst.ConnectionConfig{ + TLSConfig: tlsConfig, + Transport: transport, + } + for _, dnsName := range dnsNames { + connConfig.Endpoints = append(connConfig.Endpoints, scheme+"://"+net.JoinHostPort(dnsName, strconv.Itoa(k8sutil.ArangoPort))) + } + return connConfig, nil +} + +// CreateArangodDatabaseVSTClient creates a go-driver client for accessing the entire cluster (or single server) via VST +func CreateArangodDatabaseVSTClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment) (driver.Client, error) { + // Create connection + dnsName := k8sutil.CreateDatabaseClientServiceDNSName(apiObject) + c, err := createArangodVSTClientForDNSName(ctx, cli, apiObject, dnsName) + if err != nil { + return nil, maskAny(err) + } + return c, nil +} + // longOrSkip checks the short test flag. // If short is set, the current test is skipped. // If not, this function returns as normal. @@ -102,7 +168,13 @@ func mustNewKubeClient(t *testing.T) kubernetes.Interface { // mustNewArangodDatabaseClient creates a new database client, // failing the test on errors. func mustNewArangodDatabaseClient(ctx context.Context, kubecli kubernetes.Interface, apiObject *api.ArangoDeployment, t *testing.T, useVst ...bool) driver.Client { - c, err := arangod.CreateArangodDatabaseClient(ctx, kubecli.CoreV1(), apiObject, useVst...) + var c driver.Client + var err error + if len(useVst) > 0 && useVst[0] { + c, err = CreateArangodDatabaseVSTClient(ctx, kubecli.CoreV1(), apiObject) + } else { + c, err = arangod.CreateArangodDatabaseClient(ctx, kubecli.CoreV1(), apiObject) + } if err != nil { t.Fatalf("Failed to create arango database client: %v", err) } From 7c9b8fdecccf78ec76983baa9bbf3747aba6f1d0 Mon Sep 17 00:00:00 2001 From: Dan Larkin-York Date: Tue, 17 Jul 2018 14:52:55 -0400 Subject: [PATCH 09/16] Added placeholder checks for headers and fixed VST issue. --- tests/load_balancer_test.go | 8 +++++++- tests/test_util.go | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/load_balancer_test.go b/tests/load_balancer_test.go index c717e9360..c9e4a1972 100644 --- a/tests/load_balancer_test.go +++ b/tests/load_balancer_test.go @@ -197,7 +197,7 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { // keep track of whether at least one request was forwarded internally to the // correct coordinator behind the load balancer - someRequestForwarded = false + someRequestForwarded := false // Run tests for every context alternative for _, qctx := range contexts { @@ -239,6 +239,8 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { t.Error("HasMore returned false, but ReadDocument returns a document") } result = append(result, doc.Elem().Interface()) + // TODO extract `x-arango-request-served-by` header + // if header exists, set someRequestForwarded = true } if len(result) != len(test.ExpectedDocuments) { t.Errorf("Expected %d documents, got %d in query %d (%s)", len(test.ExpectedDocuments), len(result), i, test.Query) @@ -261,4 +263,8 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { } } } + + if !someRequestForwarded { + t.Error("Did not detect any request being forwarded behind load balancer!") + } } diff --git a/tests/test_util.go b/tests/test_util.go index e9bec5375..84aa2d6e8 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -97,10 +97,10 @@ func createArangodVSTClientForDNSName(ctx context.Context, cli corev1.CoreV1Inte // createArangodVSTConfigForDNSNames creates a go-driver VST connection config for a given DNS names. func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (vst.ConnectionConfig, error) { scheme := "http" - tlsConfig := &tls.Config{InsecureSkipVerify: true} + tlsConfig := &tls.Config{} if apiObject != nil && apiObject.Spec.IsSecure() { scheme = "https" - tlsConfig = &tls.Config{} + tlsConfig = &tls.Config{InsecureSkipVerify: true} } transport := vstProtocol.TransportConfig{ Version: vstProtocol.Version1_1, From 6601334fab14ed145dc0375be9881b426c392296 Mon Sep 17 00:00:00 2001 From: Dan Larkin-York Date: Wed, 18 Jul 2018 15:21:12 -0400 Subject: [PATCH 10/16] Further adjustments to use correct image and check header. --- tests/load_balancer_test.go | 47 ++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/tests/load_balancer_test.go b/tests/load_balancer_test.go index c9e4a1972..a55721a04 100644 --- a/tests/load_balancer_test.go +++ b/tests/load_balancer_test.go @@ -26,7 +26,7 @@ import ( "context" "reflect" "testing" - "time" + //"time" "github.com/dchest/uniuri" @@ -45,7 +45,12 @@ func TestLoadBalancingCursorHTTP(t *testing.T) { LoadBalancingCursorSubtest(t, false) } -// TestLoadBalancerCursorVST tests cursor forwarding with load-balanced conn. +func wasForwarded(r *driver.Response) bool { + h := (*r).Header("x-arango-request-served-by") + return h != "" +} + +// tests cursor forwarding with load-balanced conn. func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { c := client.MustNewInCluster() kubecli := mustNewKubeClient(t) @@ -54,6 +59,8 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { // Prepare deployment config depl := newDeployment("test-lb-" + uniuri.NewLen(4)) depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) + image := "dhly/arangodb:3.3.11-local" + depl.Spec.Image = &image // Create deployment _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) @@ -128,7 +135,7 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { ExpectedDocuments: collectionData["books"], DocumentType: reflect.TypeOf(Book{}), }, - queryTest{ +/* queryTest{ Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", BindVars: map[string]interface{}{"title": "Book 02"}, ExpectSuccess: true, @@ -175,24 +182,25 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { ExpectedDocuments: []interface{}{"Blair", "Clair", "Jake", "John", "Johnny", "Zz"}, DocumentType: reflect.TypeOf("foo"), ExpectSuccess: true, - }, + },*/ } + var r driver.Response // Setup context alternatives contexts := []queryTestContext{ - queryTestContext{nil, false}, - queryTestContext{context.Background(), false}, - queryTestContext{driver.WithQueryCount(nil), true}, - queryTestContext{driver.WithQueryCount(nil, true), true}, - queryTestContext{driver.WithQueryCount(nil, false), false}, - queryTestContext{driver.WithQueryBatchSize(nil, 1), false}, - queryTestContext{driver.WithQueryCache(nil), false}, - queryTestContext{driver.WithQueryCache(nil, true), false}, - queryTestContext{driver.WithQueryCache(nil, false), false}, - queryTestContext{driver.WithQueryMemoryLimit(nil, 60000), false}, - queryTestContext{driver.WithQueryTTL(nil, time.Minute), false}, - queryTestContext{driver.WithQueryBatchSize(driver.WithQueryCount(nil), 1), true}, - queryTestContext{driver.WithQueryCache(driver.WithQueryCount(driver.WithQueryBatchSize(nil, 2))), true}, + /*queryTestContext{driver.WithResponse(nil, &r), false}, + queryTestContext{driver.WithResponse(context.Background(), &r), false}, + queryTestContext{driver.WithResponse(driver.WithQueryCount(nil), &r), true}, + queryTestContext{driver.WithResponse(driver.WithQueryCount(nil, true), &r), true}, + queryTestContext{driver.WithResponse(driver.WithQueryCount(nil, false), &r), false},*/ + queryTestContext{driver.WithResponse(driver.WithQueryBatchSize(nil, 1), &r), false}, + /*queryTestContext{driver.WithResponse(driver.WithQueryCache(nil), &r), false}, + queryTestContext{driver.WithResponse(driver.WithQueryCache(nil, true), &r), false}, + queryTestContext{driver.WithResponse(driver.WithQueryCache(nil, false), &r), false}, + queryTestContext{driver.WithResponse(driver.WithQueryMemoryLimit(nil, 60000), &r), false}, + queryTestContext{driver.WithResponse(driver.WithQueryTTL(nil, time.Minute), &r), false}, + queryTestContext{driver.WithResponse(driver.WithQueryBatchSize(driver.WithQueryCount(nil), 1), &r), true}, + queryTestContext{driver.WithResponse(driver.WithQueryCache(driver.WithQueryCount(driver.WithQueryBatchSize(nil, 2))), &r), true},*/ } // keep track of whether at least one request was forwarded internally to the @@ -239,8 +247,9 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { t.Error("HasMore returned false, but ReadDocument returns a document") } result = append(result, doc.Elem().Interface()) - // TODO extract `x-arango-request-served-by` header - // if header exists, set someRequestForwarded = true + if (wasForwarded(&r)) { + someRequestForwarded = true + } } if len(result) != len(test.ExpectedDocuments) { t.Errorf("Expected %d documents, got %d in query %d (%s)", len(test.ExpectedDocuments), len(result), i, test.Query) From 9cbb4da6d6ae04125315eae6fb76ce342d4f78c2 Mon Sep 17 00:00:00 2001 From: Dan Larkin-York Date: Wed, 18 Jul 2018 16:58:07 -0400 Subject: [PATCH 11/16] Lowered keep-alive to trigger load-balancing. --- pkg/util/arangod/client.go | 8 ++++---- tests/load_balancer_test.go | 3 ++- tests/test_util.go | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index 7774a63ce..f0cfee783 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -65,11 +65,11 @@ var ( Proxy: nhttp.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, + KeepAlive: 100 * time.Millisecond, DualStack: true, }).DialContext, MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, + IdleConnTimeout: 100 * time.Millisecond, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, } @@ -77,11 +77,11 @@ var ( Proxy: nhttp.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, + KeepAlive: 100 * time.Millisecond, DualStack: true, }).DialContext, MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, + IdleConnTimeout: 100 * time.Millisecond, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, diff --git a/tests/load_balancer_test.go b/tests/load_balancer_test.go index a55721a04..ce1b58b87 100644 --- a/tests/load_balancer_test.go +++ b/tests/load_balancer_test.go @@ -26,7 +26,7 @@ import ( "context" "reflect" "testing" - //"time" + "time" "github.com/dchest/uniuri" @@ -250,6 +250,7 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { if (wasForwarded(&r)) { someRequestForwarded = true } + time.Sleep(200 * time.Millisecond) } if len(result) != len(test.ExpectedDocuments) { t.Errorf("Expected %d documents, got %d in query %d (%s)", len(test.ExpectedDocuments), len(result), i, test.Query) diff --git a/tests/test_util.go b/tests/test_util.go index 84aa2d6e8..da29c08f0 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -103,7 +103,8 @@ func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Int tlsConfig = &tls.Config{InsecureSkipVerify: true} } transport := vstProtocol.TransportConfig{ - Version: vstProtocol.Version1_1, + IdleConnTimeout: 100 * time.Millisecond, + Version: vstProtocol.Version1_1, } connConfig := vst.ConnectionConfig{ TLSConfig: tlsConfig, From d90fd0edf1cac41adae77e3976cefcca58c3b19e Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 19 Jul 2018 08:00:32 +0200 Subject: [PATCH 12/16] Formatted files --- tests/load_balancer_test.go | 98 ++++++++++++++++++------------------- tests/test_util.go | 4 +- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/tests/load_balancer_test.go b/tests/load_balancer_test.go index ce1b58b87..91b772877 100644 --- a/tests/load_balancer_test.go +++ b/tests/load_balancer_test.go @@ -135,54 +135,54 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { ExpectedDocuments: collectionData["books"], DocumentType: reflect.TypeOf(Book{}), }, -/* queryTest{ - Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", - BindVars: map[string]interface{}{"title": "Book 02"}, - ExpectSuccess: true, - ExpectedDocuments: []interface{}{collectionData["books"][1]}, - DocumentType: reflect.TypeOf(Book{}), - }, - queryTest{ - Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", - BindVars: map[string]interface{}{"somethingelse": "Book 02"}, - ExpectSuccess: false, // Unknown `@title` - }, - queryTest{ - Query: "FOR u IN users FILTER u.age>100 SORT u.name RETURN u", - ExpectSuccess: true, - ExpectedDocuments: []interface{}{}, - DocumentType: reflect.TypeOf(UserDoc{}), - }, - queryTest{ - Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", - BindVars: map[string]interface{}{"maxAge": 20}, - ExpectSuccess: true, - ExpectedDocuments: []interface{}{collectionData["users"][2], collectionData["users"][0], collectionData["users"][5]}, - DocumentType: reflect.TypeOf(UserDoc{}), - }, - queryTest{ - Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", - BindVars: map[string]interface{}{"maxage": 20}, - ExpectSuccess: false, // `@maxage` versus `@maxAge` - }, - queryTest{ - Query: "FOR u IN users SORT u.age RETURN u.age", - ExpectedDocuments: []interface{}{12, 12, 13, 25, 42, 67}, - DocumentType: reflect.TypeOf(12), - ExpectSuccess: true, - }, - queryTest{ - Query: "FOR p IN users COLLECT a = p.age WITH COUNT INTO c SORT a RETURN [a, c]", - ExpectedDocuments: []interface{}{[]int{12, 2}, []int{13, 1}, []int{25, 1}, []int{42, 1}, []int{67, 1}}, - DocumentType: reflect.TypeOf([]int{}), - ExpectSuccess: true, - }, - queryTest{ - Query: "FOR u IN users SORT u.name RETURN u.name", - ExpectedDocuments: []interface{}{"Blair", "Clair", "Jake", "John", "Johnny", "Zz"}, - DocumentType: reflect.TypeOf("foo"), - ExpectSuccess: true, - },*/ + /* queryTest{ + Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", + BindVars: map[string]interface{}{"title": "Book 02"}, + ExpectSuccess: true, + ExpectedDocuments: []interface{}{collectionData["books"][1]}, + DocumentType: reflect.TypeOf(Book{}), + }, + queryTest{ + Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", + BindVars: map[string]interface{}{"somethingelse": "Book 02"}, + ExpectSuccess: false, // Unknown `@title` + }, + queryTest{ + Query: "FOR u IN users FILTER u.age>100 SORT u.name RETURN u", + ExpectSuccess: true, + ExpectedDocuments: []interface{}{}, + DocumentType: reflect.TypeOf(UserDoc{}), + }, + queryTest{ + Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", + BindVars: map[string]interface{}{"maxAge": 20}, + ExpectSuccess: true, + ExpectedDocuments: []interface{}{collectionData["users"][2], collectionData["users"][0], collectionData["users"][5]}, + DocumentType: reflect.TypeOf(UserDoc{}), + }, + queryTest{ + Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", + BindVars: map[string]interface{}{"maxage": 20}, + ExpectSuccess: false, // `@maxage` versus `@maxAge` + }, + queryTest{ + Query: "FOR u IN users SORT u.age RETURN u.age", + ExpectedDocuments: []interface{}{12, 12, 13, 25, 42, 67}, + DocumentType: reflect.TypeOf(12), + ExpectSuccess: true, + }, + queryTest{ + Query: "FOR p IN users COLLECT a = p.age WITH COUNT INTO c SORT a RETURN [a, c]", + ExpectedDocuments: []interface{}{[]int{12, 2}, []int{13, 1}, []int{25, 1}, []int{42, 1}, []int{67, 1}}, + DocumentType: reflect.TypeOf([]int{}), + ExpectSuccess: true, + }, + queryTest{ + Query: "FOR u IN users SORT u.name RETURN u.name", + ExpectedDocuments: []interface{}{"Blair", "Clair", "Jake", "John", "Johnny", "Zz"}, + DocumentType: reflect.TypeOf("foo"), + ExpectSuccess: true, + },*/ } var r driver.Response @@ -247,7 +247,7 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { t.Error("HasMore returned false, but ReadDocument returns a document") } result = append(result, doc.Elem().Interface()) - if (wasForwarded(&r)) { + if wasForwarded(&r) { someRequestForwarded = true } time.Sleep(200 * time.Millisecond) diff --git a/tests/test_util.go b/tests/test_util.go index da29c08f0..6c14fee35 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -103,8 +103,8 @@ func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Int tlsConfig = &tls.Config{InsecureSkipVerify: true} } transport := vstProtocol.TransportConfig{ - IdleConnTimeout: 100 * time.Millisecond, - Version: vstProtocol.Version1_1, + IdleConnTimeout: 100 * time.Millisecond, + Version: vstProtocol.Version1_1, } connConfig := vst.ConnectionConfig{ TLSConfig: tlsConfig, From e7e3747a2dcf4cea86c70dbcd45ff68806830943 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 19 Jul 2018 09:16:59 +0200 Subject: [PATCH 13/16] Imported updated go-driver --- .../arangodb/go-driver/CHANGELOG.md | 131 ++++++++++++++++++ .../arangodb/go-driver/MAINTAINERS.md | 11 ++ deps/github.com/arangodb/go-driver/Makefile | 11 ++ .../go-driver/collection_document_impl.go | 67 ++++++++- .../go-driver/collection_documents.go | 6 + .../edge_collection_documents_impl.go | 70 +++++++++- .../go-driver/test/documents_create_test.go | 15 ++ .../go-driver/test/edges_create_test.go | 15 ++ .../go-driver/test/vertices_create_test.go | 15 ++ .../vertex_collection_documents_impl.go | 70 +++++++++- .../arangodb/go-driver/vst/authentication.go | 2 +- .../arangodb/go-driver/vst/connection.go | 2 +- .../go-driver/vst/protocol/connection.go | 4 +- .../go-driver/vst/protocol/transport.go | 2 +- .../arangodb/go-driver/vst/response.go | 5 +- 15 files changed, 399 insertions(+), 27 deletions(-) create mode 100644 deps/github.com/arangodb/go-driver/CHANGELOG.md create mode 100644 deps/github.com/arangodb/go-driver/MAINTAINERS.md diff --git a/deps/github.com/arangodb/go-driver/CHANGELOG.md b/deps/github.com/arangodb/go-driver/CHANGELOG.md new file mode 100644 index 000000000..0189636ab --- /dev/null +++ b/deps/github.com/arangodb/go-driver/CHANGELOG.md @@ -0,0 +1,131 @@ +# Change Log + +## [Master](https://github.com/arangodb/go-driver/tree/HEAD) + +**Closed issues:** + +- Structs with key specified don't read the key [\#138](https://github.com/arangodb/go-driver/issues/138) +- Edge document with user provided key is inserted as many times as the number of shards [\#137](https://github.com/arangodb/go-driver/issues/137) +- Query "return null" make the panic: runtime error [\#117](https://github.com/arangodb/go-driver/issues/117) +- driver does not seem to decode numeric timestamps to time.Time [\#102](https://github.com/arangodb/go-driver/issues/102) +- stable and full feature? [\#100](https://github.com/arangodb/go-driver/issues/100) +- collection with "Wait for sync:" setting, Create-/UpdateDocument return: ArangoError: Code 0, ErrorNum 0 [\#96](https://github.com/arangodb/go-driver/issues/96) +- Error when connecting to ArangoDB with SSL enabled [\#95](https://github.com/arangodb/go-driver/issues/95) +- Possible concurrency issues, using VST connection [\#86](https://github.com/arangodb/go-driver/issues/86) +- An example of a transactional request [\#84](https://github.com/arangodb/go-driver/issues/84) +- Multi-tenancy connection management [\#83](https://github.com/arangodb/go-driver/issues/83) +- Cursor returned from a query with empty Count [\#82](https://github.com/arangodb/go-driver/issues/82) +- CONTRIBUTING.md [\#79](https://github.com/arangodb/go-driver/issues/79) +- Querying documents + [\#76](https://github.com/arangodb/go-driver/issues/76) +- Is there an ArangoDB-Object on Struct? [\#75](https://github.com/arangodb/go-driver/issues/75) +- DB Session [\#73](https://github.com/arangodb/go-driver/issues/73) +- correct way to get multiple documents ? [\#70](https://github.com/arangodb/go-driver/issues/70) +- Revalidate whether workaround can be removed [\#68](https://github.com/arangodb/go-driver/issues/68) +- Makefile `SWTSECRET` vs `JWTSECRET` [\#66](https://github.com/arangodb/go-driver/issues/66) +- Implement transactions [\#56](https://github.com/arangodb/go-driver/issues/56) +- bulk import API [\#55](https://github.com/arangodb/go-driver/issues/55) +- Question: how to write auto-inc feild? [\#51](https://github.com/arangodb/go-driver/issues/51) +- add index attribute for new feature in arangodb3.2 [\#48](https://github.com/arangodb/go-driver/issues/48) +- Unable to connect to server in macos [\#41](https://github.com/arangodb/go-driver/issues/41) +- Handle 401 status code before checking content type. [\#38](https://github.com/arangodb/go-driver/issues/38) +- Support for raw string in query return [\#37](https://github.com/arangodb/go-driver/issues/37) +- \[ArangoDB Server 3.1.6\] Unsupported content type: client.DatabaseExists when Database already exists [\#36](https://github.com/arangodb/go-driver/issues/36) +- Can't connect because of 'Unsupported content type'? [\#35](https://github.com/arangodb/go-driver/issues/35) +- cursor implementation when running queries with non-Document return values [\#20](https://github.com/arangodb/go-driver/issues/20) +- Make failover with cursors more explicit [\#16](https://github.com/arangodb/go-driver/issues/16) +- Add non-context version of method calls [\#7](https://github.com/arangodb/go-driver/issues/7) + +**Merged pull requests:** + +- Added Collection.ReadDocuments [\#133](https://github.com/arangodb/go-driver/pull/133) +- Added support for fetching job ID in CleanoutServer [\#131](https://github.com/arangodb/go-driver/pull/131) +- Exclude high load test on VST+3.2 [\#129](https://github.com/arangodb/go-driver/pull/129) +- Test/prevent concurrent vst read write [\#128](https://github.com/arangodb/go-driver/pull/128) +- Added single+ssl tests. All tests now use starter [\#127](https://github.com/arangodb/go-driver/pull/127) +- Bugfix/read chunk loop [\#126](https://github.com/arangodb/go-driver/pull/126) +- Prevent possible endless read chunk look in in VST connection [\#125](https://github.com/arangodb/go-driver/pull/125) +- Fixing Query\(return nul\) resulting in panic [\#124](https://github.com/arangodb/go-driver/pull/124) +- Added WithAllowNoLeader [\#123](https://github.com/arangodb/go-driver/pull/123) +- Added Cluster.RemoveServer [\#122](https://github.com/arangodb/go-driver/pull/122) +- Added mutex guarding message chunks. [\#121](https://github.com/arangodb/go-driver/pull/121) +- Documentation/go refactor [\#120](https://github.com/arangodb/go-driver/pull/120) +- VST stream cursor test fixes & VST fail-quick fix [\#119](https://github.com/arangodb/go-driver/pull/119) +- Prevent a VST connection from being used before its configuration callback has finished [\#118](https://github.com/arangodb/go-driver/pull/118) +- Fixed AgencyConnection in the context of authentication [\#116](https://github.com/arangodb/go-driver/pull/116) +- Adding timeout for streaming cursor test [\#115](https://github.com/arangodb/go-driver/pull/115) +- Added package level docs [\#114](https://github.com/arangodb/go-driver/pull/114) +- Close the connection when the initial onCreatedCallback fails [\#113](https://github.com/arangodb/go-driver/pull/113) +- Adding extra concurrency-safety for VST [\#112](https://github.com/arangodb/go-driver/pull/112) +- Added upper limit to the number of concurrent requests to a single server. [\#111](https://github.com/arangodb/go-driver/pull/111) +- Added support for multiple VST connections per server [\#110](https://github.com/arangodb/go-driver/pull/110) +- Fixed expected status code for operations on collections that have waitForSync enabled [\#109](https://github.com/arangodb/go-driver/pull/109) +- Added helper to determine agency health [\#108](https://github.com/arangodb/go-driver/pull/108) +- Added ServerID function [\#107](https://github.com/arangodb/go-driver/pull/107) +- Added exclusive lock using agency [\#106](https://github.com/arangodb/go-driver/pull/106) +- Added helper for JWT secret based authentication [\#105](https://github.com/arangodb/go-driver/pull/105) +- Added Agency API [\#104](https://github.com/arangodb/go-driver/pull/104) +- Added option to not follow redirects and return the original response [\#103](https://github.com/arangodb/go-driver/pull/103) +- Add support for stream query cursor [\#101](https://github.com/arangodb/go-driver/pull/101) +- Active-Failover with Velocystream [\#99](https://github.com/arangodb/go-driver/pull/99) +- Added API functions for shutting down a server and cleaning it out [\#98](https://github.com/arangodb/go-driver/pull/98) +- Skip replication test on cluster [\#94](https://github.com/arangodb/go-driver/pull/94) +- Documented behavior for custom http.Transport wrt MaxIdleConnsPerHost field [\#93](https://github.com/arangodb/go-driver/pull/93) +- Fix ReturnOld/New for edge/vertex operations. [\#92](https://github.com/arangodb/go-driver/pull/92) +- Database.Info\(\) added [\#91](https://github.com/arangodb/go-driver/pull/91) +- Replication interface added. [\#90](https://github.com/arangodb/go-driver/pull/90) +- Grouping server specific info calls in ClientServerInfo, exposing ServerRole API [\#89](https://github.com/arangodb/go-driver/pull/89) +- Added `ReplicationFactor` to `SetCollectionPropertiesOptions` [\#88](https://github.com/arangodb/go-driver/pull/88) +- Allow for some time to reach intended status [\#87](https://github.com/arangodb/go-driver/pull/87) +- Fixing SWTSECRET -\> JWTSECRET [\#85](https://github.com/arangodb/go-driver/pull/85) +- Added CONTRIBUTING.md [\#80](https://github.com/arangodb/go-driver/pull/80) +- Resilientsingle support [\#77](https://github.com/arangodb/go-driver/pull/77) +- Adding cluster specific operations [\#72](https://github.com/arangodb/go-driver/pull/72) +- Added IsSmart, SmartGraphAttribute attributes to CreateCollectionOptions [\#71](https://github.com/arangodb/go-driver/pull/71) +- Adding Response.Header\(string\) & tests [\#69](https://github.com/arangodb/go-driver/pull/69) +- Server mode \(get/set\) [\#65](https://github.com/arangodb/go-driver/pull/65) +- Adding `WithConfigured` [\#64](https://github.com/arangodb/go-driver/pull/64) +- Added DistributeShardsLike field to CreateCollectionOptions [\#62](https://github.com/arangodb/go-driver/pull/62) +- Added WithEnforceReplicationFactor [\#61](https://github.com/arangodb/go-driver/pull/61) +- Added `WithIsSystem` [\#60](https://github.com/arangodb/go-driver/pull/60) +- Support synchronising endpoints on resilient single server mode [\#59](https://github.com/arangodb/go-driver/pull/59) +- Added `WithIgnoreRevisions` [\#58](https://github.com/arangodb/go-driver/pull/58) +- Basic transaction implementation [\#57](https://github.com/arangodb/go-driver/pull/57) +- Support `x-arango-dump` content type [\#54](https://github.com/arangodb/go-driver/pull/54) +- Added WithIsRestore \(not internal for normal client use!!!!\) [\#53](https://github.com/arangodb/go-driver/pull/53) +- Raw authentication [\#52](https://github.com/arangodb/go-driver/pull/52) +- Include travis tests for arangodb 3.1 [\#50](https://github.com/arangodb/go-driver/pull/50) +- Added NoDeduplicate field for hash & skiplist index options [\#49](https://github.com/arangodb/go-driver/pull/49) +- Supporting additional user access functions [\#47](https://github.com/arangodb/go-driver/pull/47) +- ftKnox - fix sprintf conversions [\#42](https://github.com/arangodb/go-driver/pull/42) +- Convert 401 text/plain response to proper ArangoError [\#39](https://github.com/arangodb/go-driver/pull/39) +- Adding Storage engine detection [\#34](https://github.com/arangodb/go-driver/pull/34) +- Starter update [\#33](https://github.com/arangodb/go-driver/pull/33) +- Fix reading the response body to ensure keep-alive [\#32](https://github.com/arangodb/go-driver/pull/32) +- Velocy stream support \(wip\) [\#31](https://github.com/arangodb/go-driver/pull/31) +- Supporting Velocypack content-type \(instead of JSON\) \(wip\) [\#29](https://github.com/arangodb/go-driver/pull/29) +- Fixed Cursor.ReadDocument for queries returning non-documents [\#27](https://github.com/arangodb/go-driver/pull/27) +- Added Collection.Statistics [\#25](https://github.com/arangodb/go-driver/pull/25) +- Adding Database.ValidateQuery [\#24](https://github.com/arangodb/go-driver/pull/24) +- Added Collection.DocumentExists [\#23](https://github.com/arangodb/go-driver/pull/23) +- Added `Database.Remove\(\)` [\#22](https://github.com/arangodb/go-driver/pull/22) +- Changing graph API to introduce VertexConstraints [\#21](https://github.com/arangodb/go-driver/pull/21) +- Endpoint reconfiguration [\#19](https://github.com/arangodb/go-driver/pull/19) +- Allow custom http.RoundTripper [\#18](https://github.com/arangodb/go-driver/pull/18) +- Added WithEndpoint, used to force a specific endpoint for a request. [\#17](https://github.com/arangodb/go-driver/pull/17) +- Adding failover tests [\#15](https://github.com/arangodb/go-driver/pull/15) +- Adding simple performance benchmarks [\#14](https://github.com/arangodb/go-driver/pull/14) +- Cluster tests [\#12](https://github.com/arangodb/go-driver/pull/12) +- ImportDocuments [\#11](https://github.com/arangodb/go-driver/pull/11) +- Adding Graph support \(wip\) [\#10](https://github.com/arangodb/go-driver/pull/10) +- Added collection status,count,rename,load,unload,truncate,properties [\#9](https://github.com/arangodb/go-driver/pull/9) +- Adding user API [\#8](https://github.com/arangodb/go-driver/pull/8) +- Adding index support [\#6](https://github.com/arangodb/go-driver/pull/6) +- Added Cursor support [\#5](https://github.com/arangodb/go-driver/pull/5) +- Adding multi document requests [\#4](https://github.com/arangodb/go-driver/pull/4) +- Creating interface. \(WIP\) [\#3](https://github.com/arangodb/go-driver/pull/3) +- Revert "Creating interface" [\#2](https://github.com/arangodb/go-driver/pull/2) +- Creating interface [\#1](https://github.com/arangodb/go-driver/pull/1) + + + +\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* \ No newline at end of file diff --git a/deps/github.com/arangodb/go-driver/MAINTAINERS.md b/deps/github.com/arangodb/go-driver/MAINTAINERS.md new file mode 100644 index 000000000..7e368ad50 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/MAINTAINERS.md @@ -0,0 +1,11 @@ +# Maintainer Instructions + +- Always preserve backward compatibility +- Build using `make clean && make` +- After merging PR, alway run `make changelog` and commit changes +- Set ArangoDB docker container (used for testing) using `export ARANGODB=` +- Run tests using: + - `make run-tests-single` + - `make run-tests-resilientsingle` + - `make run-tests-cluster`. +- Always create changes in a PR diff --git a/deps/github.com/arangodb/go-driver/Makefile b/deps/github.com/arangodb/go-driver/Makefile index b8f6f00d2..d49c309ef 100644 --- a/deps/github.com/arangodb/go-driver/Makefile +++ b/deps/github.com/arangodb/go-driver/Makefile @@ -113,6 +113,17 @@ $(GOBUILDDIR): GOPATH=$(GOBUILDDIR) go get github.com/arangodb/go-velocypack GOPATH=$(GOBUILDDIR) go get github.com/dgrijalva/jwt-go +.PHONY: changelog +changelog: + @docker run --rm \ + -e CHANGELOG_GITHUB_TOKEN=$(shell cat ~/.arangodb/github-token) \ + -v "$(ROOTDIR)":/usr/local/src/your-app \ + ferrarimarco/github-changelog-generator \ + --user arangodb \ + --project go-driver \ + --no-author \ + --unreleased-label "Master" + run-tests: run-tests-http run-tests-single run-tests-resilientsingle run-tests-cluster # Tests of HTTP package diff --git a/deps/github.com/arangodb/go-driver/collection_document_impl.go b/deps/github.com/arangodb/go-driver/collection_document_impl.go index bc09aa5f7..6ed995708 100644 --- a/deps/github.com/arangodb/go-driver/collection_document_impl.go +++ b/deps/github.com/arangodb/go-driver/collection_document_impl.go @@ -80,6 +80,55 @@ func (c *collection) ReadDocument(ctx context.Context, key string, result interf return meta, nil } +// ReadDocuments reads multiple documents with given keys from the collection. +// The documents data is stored into elements of the given results slice, +// the documents meta data is returned. +// If no document exists with a given key, a NotFoundError is returned at its errors index. +func (c *collection) ReadDocuments(ctx context.Context, keys []string, results interface{}) (DocumentMetaSlice, ErrorSlice, error) { + resultsVal := reflect.ValueOf(results) + switch resultsVal.Kind() { + case reflect.Array, reflect.Slice: + // OK + default: + return nil, nil, WithStack(InvalidArgumentError{Message: fmt.Sprintf("results data must be of kind Array, got %s", resultsVal.Kind())}) + } + if keys == nil { + return nil, nil, WithStack(InvalidArgumentError{Message: "keys nil"}) + } + resultCount := resultsVal.Len() + if len(keys) != resultCount { + return nil, nil, WithStack(InvalidArgumentError{Message: fmt.Sprintf("expected %d keys, got %d", resultCount, len(keys))}) + } + for _, key := range keys { + if err := validateKey(key); err != nil { + return nil, nil, WithStack(err) + } + } + req, err := c.conn.NewRequest("PUT", c.relPath("document")) + if err != nil { + return nil, nil, WithStack(err) + } + req = req.SetQuery("onlyget", "1") + cs := applyContextSettings(ctx, req) + if _, err := req.SetBodyArray(keys, nil); err != nil { + return nil, nil, WithStack(err) + } + resp, err := c.conn.Do(ctx, req) + if err != nil { + return nil, nil, WithStack(err) + } + if err := resp.CheckStatus(200); err != nil { + return nil, nil, WithStack(err) + } + // Parse response array + metas, errs, err := parseResponseArray(resp, resultCount, cs, results) + if err != nil { + return nil, nil, WithStack(err) + } + return metas, errs, nil + +} + // CreateDocument creates a single document in the collection. // The document data is loaded from the given document, the document meta data is returned. // If the document data already contains a `_key` field, this will be used as key of the new document, @@ -163,7 +212,7 @@ func (c *collection) CreateDocuments(ctx context.Context, documents interface{}) return nil, nil, nil } // Parse response array - metas, errs, err := parseResponseArray(resp, documentCount, cs) + metas, errs, err := parseResponseArray(resp, documentCount, cs, nil) if err != nil { return nil, nil, WithStack(err) } @@ -272,7 +321,7 @@ func (c *collection) UpdateDocuments(ctx context.Context, keys []string, updates return nil, nil, nil } // Parse response array - metas, errs, err := parseResponseArray(resp, updateCount, cs) + metas, errs, err := parseResponseArray(resp, updateCount, cs, nil) if err != nil { return nil, nil, WithStack(err) } @@ -381,7 +430,7 @@ func (c *collection) ReplaceDocuments(ctx context.Context, keys []string, docume return nil, nil, nil } // Parse response array - metas, errs, err := parseResponseArray(resp, documentCount, cs) + metas, errs, err := parseResponseArray(resp, documentCount, cs, nil) if err != nil { return nil, nil, WithStack(err) } @@ -464,7 +513,7 @@ func (c *collection) RemoveDocuments(ctx context.Context, keys []string) (Docume return nil, nil, nil } // Parse response array - metas, errs, err := parseResponseArray(resp, keyCount, cs) + metas, errs, err := parseResponseArray(resp, keyCount, cs, nil) if err != nil { return nil, nil, WithStack(err) } @@ -572,7 +621,7 @@ func createMergeArray(keys, revs []string) ([]map[string]interface{}, error) { } // parseResponseArray parses an array response in the given response -func parseResponseArray(resp Response, count int, cs contextSettings) (DocumentMetaSlice, ErrorSlice, error) { +func parseResponseArray(resp Response, count int, cs contextSettings, results interface{}) (DocumentMetaSlice, ErrorSlice, error) { resps, err := resp.ParseArrayBody() if err != nil { return nil, nil, WithStack(err) @@ -581,6 +630,7 @@ func parseResponseArray(resp Response, count int, cs contextSettings) (DocumentM errs := make(ErrorSlice, count) returnOldVal := reflect.ValueOf(cs.ReturnOld) returnNewVal := reflect.ValueOf(cs.ReturnNew) + resultsVal := reflect.ValueOf(results) for i := 0; i < count; i++ { resp := resps[i] var meta DocumentMeta @@ -606,6 +656,13 @@ func parseResponseArray(resp Response, count int, cs contextSettings) (DocumentM } } } + if results != nil { + // Parse compare result document + resultsEntryVal := resultsVal.Index(i).Addr() + if err := resp.ParseBody("", resultsEntryVal.Interface()); err != nil { + errs[i] = err + } + } } } return metas, errs, nil diff --git a/deps/github.com/arangodb/go-driver/collection_documents.go b/deps/github.com/arangodb/go-driver/collection_documents.go index aef8bc2ef..a600a053c 100644 --- a/deps/github.com/arangodb/go-driver/collection_documents.go +++ b/deps/github.com/arangodb/go-driver/collection_documents.go @@ -34,6 +34,12 @@ type CollectionDocuments interface { // If no document exists with given key, a NotFoundError is returned. ReadDocument(ctx context.Context, key string, result interface{}) (DocumentMeta, error) + // ReadDocuments reads multiple documents with given keys from the collection. + // The documents data is stored into elements of the given results slice, + // the documents meta data is returned. + // If no document exists with a given key, a NotFoundError is returned at its errors index. + ReadDocuments(ctx context.Context, keys []string, results interface{}) (DocumentMetaSlice, ErrorSlice, error) + // CreateDocument creates a single document in the collection. // The document data is loaded from the given document, the document meta data is returned. // If the document data already contains a `_key` field, this will be used as key of the new document, diff --git a/deps/github.com/arangodb/go-driver/edge_collection_documents_impl.go b/deps/github.com/arangodb/go-driver/edge_collection_documents_impl.go index 100526953..395cb32b5 100644 --- a/deps/github.com/arangodb/go-driver/edge_collection_documents_impl.go +++ b/deps/github.com/arangodb/go-driver/edge_collection_documents_impl.go @@ -43,33 +43,89 @@ func (c *edgeCollection) DocumentExists(ctx context.Context, key string) (bool, // The document data is stored into result, the document meta data is returned. // If no document exists with given key, a NotFoundError is returned. func (c *edgeCollection) ReadDocument(ctx context.Context, key string, result interface{}) (DocumentMeta, error) { - if err := validateKey(key); err != nil { + meta, _, err := c.readDocument(ctx, key, result) + if err != nil { return DocumentMeta{}, WithStack(err) } + return meta, nil +} + +func (c *edgeCollection) readDocument(ctx context.Context, key string, result interface{}) (DocumentMeta, contextSettings, error) { + if err := validateKey(key); err != nil { + return DocumentMeta{}, contextSettings{}, WithStack(err) + } escapedKey := pathEscape(key) req, err := c.conn.NewRequest("GET", path.Join(c.relPath(), escapedKey)) if err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } + cs := applyContextSettings(ctx, req) resp, err := c.conn.Do(ctx, req) if err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } if err := resp.CheckStatus(200); err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } // Parse metadata var meta DocumentMeta if err := resp.ParseBody("edge", &meta); err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } // Parse result if result != nil { if err := resp.ParseBody("edge", result); err != nil { - return meta, WithStack(err) + return meta, contextSettings{}, WithStack(err) } } - return meta, nil + return meta, cs, nil +} + +// ReadDocuments reads multiple documents with given keys from the collection. +// The documents data is stored into elements of the given results slice, +// the documents meta data is returned. +// If no document exists with a given key, a NotFoundError is returned at its errors index. +func (c *edgeCollection) ReadDocuments(ctx context.Context, keys []string, results interface{}) (DocumentMetaSlice, ErrorSlice, error) { + resultsVal := reflect.ValueOf(results) + switch resultsVal.Kind() { + case reflect.Array, reflect.Slice: + // OK + default: + return nil, nil, WithStack(InvalidArgumentError{Message: fmt.Sprintf("results data must be of kind Array, got %s", resultsVal.Kind())}) + } + if keys == nil { + return nil, nil, WithStack(InvalidArgumentError{Message: "keys nil"}) + } + resultCount := resultsVal.Len() + if len(keys) != resultCount { + return nil, nil, WithStack(InvalidArgumentError{Message: fmt.Sprintf("expected %d keys, got %d", resultCount, len(keys))}) + } + for _, key := range keys { + if err := validateKey(key); err != nil { + return nil, nil, WithStack(err) + } + } + metas := make(DocumentMetaSlice, resultCount) + errs := make(ErrorSlice, resultCount) + silent := false + for i := 0; i < resultCount; i++ { + result := resultsVal.Index(i).Addr() + ctx, err := withDocumentAt(ctx, i) + if err != nil { + return nil, nil, WithStack(err) + } + key := keys[i] + meta, cs, err := c.readDocument(ctx, key, result.Interface()) + if cs.Silent { + silent = true + } else { + metas[i], errs[i] = meta, err + } + } + if silent { + return nil, nil, nil + } + return metas, errs, nil } // CreateDocument creates a single document in the collection. diff --git a/deps/github.com/arangodb/go-driver/test/documents_create_test.go b/deps/github.com/arangodb/go-driver/test/documents_create_test.go index 30ae1e4af..1ee066acf 100644 --- a/deps/github.com/arangodb/go-driver/test/documents_create_test.go +++ b/deps/github.com/arangodb/go-driver/test/documents_create_test.go @@ -55,6 +55,21 @@ func TestCreateDocuments(t *testing.T) { } else if len(metas) != len(docs) { t.Errorf("Expected %d metas, got %d", len(docs), len(metas)) } else { + // Read back using ReadDocuments + keys := make([]string, len(docs)) + for i, m := range metas { + keys[i] = m.Key + } + readDocs := make([]UserDoc, len(docs)) + if _, _, err := col.ReadDocuments(nil, keys, readDocs); err != nil { + t.Fatalf("Failed to read documents: %s", describe(err)) + } + for i, d := range readDocs { + if !reflect.DeepEqual(docs[i], d) { + t.Errorf("Got wrong document. Expected %+v, got %+v", docs[i], d) + } + } + // Read back using individual ReadDocument requests for i := 0; i < len(docs); i++ { if err := errs[i]; err != nil { t.Errorf("Expected no error at index %d, got %s", i, describe(err)) diff --git a/deps/github.com/arangodb/go-driver/test/edges_create_test.go b/deps/github.com/arangodb/go-driver/test/edges_create_test.go index 2fab94d2a..8875b93c4 100644 --- a/deps/github.com/arangodb/go-driver/test/edges_create_test.go +++ b/deps/github.com/arangodb/go-driver/test/edges_create_test.go @@ -66,6 +66,21 @@ func TestCreateEdges(t *testing.T) { } else if len(metas) != len(docs) { t.Errorf("Expected %d metas, got %d", len(docs), len(metas)) } else { + // Read back using ReadDocuments + keys := make([]string, len(docs)) + for i, m := range metas { + keys[i] = m.Key + } + readDocs := make([]RouteEdge, len(docs)) + if _, _, err := ec.ReadDocuments(nil, keys, readDocs); err != nil { + t.Fatalf("Failed to read documents: %s", describe(err)) + } + for i, d := range readDocs { + if !reflect.DeepEqual(docs[i], d) { + t.Errorf("Got wrong document. Expected %+v, got %+v", docs[i], d) + } + } + // Read back using individual ReadDocument requests for i := 0; i < len(docs); i++ { if err := errs[i]; err != nil { t.Errorf("Expected no error at index %d, got %s", i, describe(err)) diff --git a/deps/github.com/arangodb/go-driver/test/vertices_create_test.go b/deps/github.com/arangodb/go-driver/test/vertices_create_test.go index db10b5745..bde766635 100644 --- a/deps/github.com/arangodb/go-driver/test/vertices_create_test.go +++ b/deps/github.com/arangodb/go-driver/test/vertices_create_test.go @@ -55,6 +55,21 @@ func TestCreateVertices(t *testing.T) { } else if len(metas) != len(docs) { t.Errorf("Expected %d metas, got %d", len(docs), len(metas)) } else { + // Read back using ReadDocuments + keys := make([]string, len(docs)) + for i, m := range metas { + keys[i] = m.Key + } + readDocs := make([]Book, len(docs)) + if _, _, err := vc.ReadDocuments(nil, keys, readDocs); err != nil { + t.Fatalf("Failed to read documents: %s", describe(err)) + } + for i, d := range readDocs { + if !reflect.DeepEqual(docs[i], d) { + t.Errorf("Got wrong document. Expected %+v, got %+v", docs[i], d) + } + } + // Read back using individual ReadDocument requests for i := 0; i < len(docs); i++ { if err := errs[i]; err != nil { t.Errorf("Expected no error at index %d, got %s", i, describe(err)) diff --git a/deps/github.com/arangodb/go-driver/vertex_collection_documents_impl.go b/deps/github.com/arangodb/go-driver/vertex_collection_documents_impl.go index 323e2c5b0..cf036afe6 100644 --- a/deps/github.com/arangodb/go-driver/vertex_collection_documents_impl.go +++ b/deps/github.com/arangodb/go-driver/vertex_collection_documents_impl.go @@ -42,33 +42,89 @@ func (c *vertexCollection) DocumentExists(ctx context.Context, key string) (bool // The document data is stored into result, the document meta data is returned. // If no document exists with given key, a NotFoundError is returned. func (c *vertexCollection) ReadDocument(ctx context.Context, key string, result interface{}) (DocumentMeta, error) { - if err := validateKey(key); err != nil { + meta, _, err := c.readDocument(ctx, key, result) + if err != nil { return DocumentMeta{}, WithStack(err) } + return meta, nil +} + +func (c *vertexCollection) readDocument(ctx context.Context, key string, result interface{}) (DocumentMeta, contextSettings, error) { + if err := validateKey(key); err != nil { + return DocumentMeta{}, contextSettings{}, WithStack(err) + } escapedKey := pathEscape(key) req, err := c.conn.NewRequest("GET", path.Join(c.relPath(), escapedKey)) if err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } + cs := applyContextSettings(ctx, req) resp, err := c.conn.Do(ctx, req) if err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } if err := resp.CheckStatus(200); err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } // Parse metadata var meta DocumentMeta if err := resp.ParseBody("vertex", &meta); err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } // Parse result if result != nil { if err := resp.ParseBody("vertex", result); err != nil { - return meta, WithStack(err) + return meta, contextSettings{}, WithStack(err) } } - return meta, nil + return meta, cs, nil +} + +// ReadDocuments reads multiple documents with given keys from the collection. +// The documents data is stored into elements of the given results slice, +// the documents meta data is returned. +// If no document exists with a given key, a NotFoundError is returned at its errors index. +func (c *vertexCollection) ReadDocuments(ctx context.Context, keys []string, results interface{}) (DocumentMetaSlice, ErrorSlice, error) { + resultsVal := reflect.ValueOf(results) + switch resultsVal.Kind() { + case reflect.Array, reflect.Slice: + // OK + default: + return nil, nil, WithStack(InvalidArgumentError{Message: fmt.Sprintf("results data must be of kind Array, got %s", resultsVal.Kind())}) + } + if keys == nil { + return nil, nil, WithStack(InvalidArgumentError{Message: "keys nil"}) + } + resultCount := resultsVal.Len() + if len(keys) != resultCount { + return nil, nil, WithStack(InvalidArgumentError{Message: fmt.Sprintf("expected %d keys, got %d", resultCount, len(keys))}) + } + for _, key := range keys { + if err := validateKey(key); err != nil { + return nil, nil, WithStack(err) + } + } + metas := make(DocumentMetaSlice, resultCount) + errs := make(ErrorSlice, resultCount) + silent := false + for i := 0; i < resultCount; i++ { + result := resultsVal.Index(i).Addr() + ctx, err := withDocumentAt(ctx, i) + if err != nil { + return nil, nil, WithStack(err) + } + key := keys[i] + meta, cs, err := c.readDocument(ctx, key, result.Interface()) + if cs.Silent { + silent = true + } else { + metas[i], errs[i] = meta, err + } + } + if silent { + return nil, nil, nil + } + return metas, errs, nil } // CreateDocument creates a single document in the collection. diff --git a/deps/github.com/arangodb/go-driver/vst/authentication.go b/deps/github.com/arangodb/go-driver/vst/authentication.go index 97a66dab6..4a9d4de49 100644 --- a/deps/github.com/arangodb/go-driver/vst/authentication.go +++ b/deps/github.com/arangodb/go-driver/vst/authentication.go @@ -141,7 +141,7 @@ func (a *vstAuthenticationImpl) PrepareFunc(vstConn *vstConnection) func(ctx con // Wait for response m := <-respChan - resp, err := newResponse(m, "", nil) + resp, err := newResponse(m.Data, "", nil) if err != nil { return driver.WithStack(err) } diff --git a/deps/github.com/arangodb/go-driver/vst/connection.go b/deps/github.com/arangodb/go-driver/vst/connection.go index cb8254d37..456ca6b1e 100644 --- a/deps/github.com/arangodb/go-driver/vst/connection.go +++ b/deps/github.com/arangodb/go-driver/vst/connection.go @@ -183,7 +183,7 @@ func (c *vstConnection) do(ctx context.Context, req driver.Request, transport me } } - vstResp, err := newResponse(msg, c.endpoint.String(), rawResponse) + vstResp, err := newResponse(msg.Data, c.endpoint.String(), rawResponse) if err != nil { fmt.Printf("Cannot decode msg %d: %#v\n", msg.ID, err) return nil, driver.WithStack(err) diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/connection.go b/deps/github.com/arangodb/go-driver/vst/protocol/connection.go index d1af33c4a..71b57853d 100644 --- a/deps/github.com/arangodb/go-driver/vst/protocol/connection.go +++ b/deps/github.com/arangodb/go-driver/vst/protocol/connection.go @@ -230,7 +230,7 @@ func (c *Connection) readChunkLoop() { if err != nil { if !c.IsClosed() { // Handle error - if err == io.EOF { + if driver.Cause(err) == io.EOF { // Connection closed c.Close() } else { @@ -286,5 +286,5 @@ func (c *Connection) updateLastActivity() { // IsIdle returns true when the last activity was more than the given timeout ago. func (c *Connection) IsIdle(idleTimeout time.Duration) bool { - return time.Since(c.lastActivity) > idleTimeout + return time.Since(c.lastActivity) > idleTimeout && c.msgStore.Size() == 0 } diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/transport.go b/deps/github.com/arangodb/go-driver/vst/protocol/transport.go index 36192c889..105428187 100644 --- a/deps/github.com/arangodb/go-driver/vst/protocol/transport.go +++ b/deps/github.com/arangodb/go-driver/vst/protocol/transport.go @@ -228,7 +228,7 @@ func (c *Transport) createConnection() (*Connection, error) { func (c *Transport) cleanup() { for { time.Sleep(c.IdleConnTimeout / 10) - remaining, _ := c.CloseIdleConnections() + _, remaining := c.CloseIdleConnections() if remaining == 0 { return } diff --git a/deps/github.com/arangodb/go-driver/vst/response.go b/deps/github.com/arangodb/go-driver/vst/response.go index e290e6c12..2beb225b9 100644 --- a/deps/github.com/arangodb/go-driver/vst/response.go +++ b/deps/github.com/arangodb/go-driver/vst/response.go @@ -28,7 +28,6 @@ import ( "sync" driver "github.com/arangodb/go-driver" - "github.com/arangodb/go-driver/vst/protocol" velocypack "github.com/arangodb/go-velocypack" ) @@ -46,9 +45,9 @@ type vstResponse struct { } // newResponse builds a vstResponse from given message. -func newResponse(msg protocol.Message, endpoint string, rawResponse *[]byte) (*vstResponse, error) { +func newResponse(msgData []byte, endpoint string, rawResponse *[]byte) (*vstResponse, error) { // Decode header - hdr := velocypack.Slice(msg.Data) + hdr := velocypack.Slice(msgData) if err := hdr.AssertType(velocypack.Array); err != nil { return nil, driver.WithStack(err) } From 7a86092da4bf6983a199804001bfbdde9ad9f116 Mon Sep 17 00:00:00 2001 From: Dan Larkin-York Date: Mon, 6 Aug 2018 08:54:59 -0400 Subject: [PATCH 14/16] Adjustments to timeouts and image. --- pkg/util/arangod/client.go | 47 +++++++++++++++++++---- tests/load_balancer_test.go | 75 ++++--------------------------------- tests/test_util.go | 23 +++++++----- 3 files changed, 60 insertions(+), 85 deletions(-) diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index f0cfee783..14550bc63 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -62,6 +62,31 @@ func WithRequireAuthentication(ctx context.Context) context.Context { var ( sharedHTTPTransport = &nhttp.Transport{ + Proxy: nhttp.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 90 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + sharedHTTPSTransport = &nhttp.Transport{ + Proxy: nhttp.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 90 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + sharedHTTPTransportShortTimeout = &nhttp.Transport{ Proxy: nhttp.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, @@ -73,7 +98,7 @@ var ( TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, } - sharedHTTPSTransport = &nhttp.Transport{ + sharedHTTPSTransportShortTimeout = &nhttp.Transport{ Proxy: nhttp.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, @@ -92,7 +117,7 @@ var ( func CreateArangodClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, group api.ServerGroup, id string) (driver.Client, error) { // Create connection dnsName := k8sutil.CreatePodDNSName(apiObject, group.AsRole(), id) - c, err := createArangodClientForDNSName(ctx, cli, apiObject, dnsName) + c, err := createArangodClientForDNSName(ctx, cli, apiObject, dnsName, false) if err != nil { return nil, maskAny(err) } @@ -100,10 +125,10 @@ func CreateArangodClient(ctx context.Context, cli corev1.CoreV1Interface, apiObj } // CreateArangodDatabaseClient creates a go-driver client for accessing the entire cluster (or single server). -func CreateArangodDatabaseClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment) (driver.Client, error) { +func CreateArangodDatabaseClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, shortTimeout bool) (driver.Client, error) { // Create connection dnsName := k8sutil.CreateDatabaseClientServiceDNSName(apiObject) - c, err := createArangodClientForDNSName(ctx, cli, apiObject, dnsName) + c, err := createArangodClientForDNSName(ctx, cli, apiObject, dnsName, shortTimeout) if err != nil { return nil, maskAny(err) } @@ -147,7 +172,7 @@ func CreateArangodAgencyClient(ctx context.Context, cli corev1.CoreV1Interface, func CreateArangodImageIDClient(ctx context.Context, deployment k8sutil.APIObject, role, id string) (driver.Client, error) { // Create connection dnsName := k8sutil.CreatePodDNSName(deployment, role, id) - c, err := createArangodClientForDNSName(ctx, nil, nil, dnsName) + c, err := createArangodClientForDNSName(ctx, nil, nil, dnsName, false) if err != nil { return nil, maskAny(err) } @@ -155,8 +180,8 @@ func CreateArangodImageIDClient(ctx context.Context, deployment k8sutil.APIObjec } // CreateArangodClientForDNSName creates a go-driver client for a given DNS name. -func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsName string) (driver.Client, error) { - connConfig, err := createArangodHTTPConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}) +func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsName string, shortTimeout bool) (driver.Client, error) { + connConfig, err := createArangodHTTPConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}, shortTimeout) if err != nil { return nil, maskAny(err) } @@ -183,12 +208,18 @@ func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interfa } // createArangodHTTPConfigForDNSNames creates a go-driver HTTP connection config for a given DNS names. -func createArangodHTTPConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (http.ConnectionConfig, error) { +func createArangodHTTPConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string, shortTimeout bool) (http.ConnectionConfig, error) { scheme := "http" transport := sharedHTTPTransport + if shortTimeout { + transport = sharedHTTPTransportShortTimeout + } if apiObject != nil && apiObject.Spec.IsSecure() { scheme = "https" transport = sharedHTTPSTransport + if shortTimeout { + transport = sharedHTTPSTransportShortTimeout + } } connConfig := http.ConnectionConfig{ Transport: transport, diff --git a/tests/load_balancer_test.go b/tests/load_balancer_test.go index 91b772877..47719bfe5 100644 --- a/tests/load_balancer_test.go +++ b/tests/load_balancer_test.go @@ -37,21 +37,21 @@ import ( func TestLoadBalancingCursorVST(t *testing.T) { // run with VST - LoadBalancingCursorSubtest(t, true) + loadBalancingCursorSubtest(t, true) } func TestLoadBalancingCursorHTTP(t *testing.T) { // run with HTTP - LoadBalancingCursorSubtest(t, false) + loadBalancingCursorSubtest(t, false) } -func wasForwarded(r *driver.Response) bool { - h := (*r).Header("x-arango-request-served-by") +func wasForwarded(r driver.Response) bool { + h := r.Header("x-arango-request-forwarded-to") return h != "" } // tests cursor forwarding with load-balanced conn. -func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { +func loadBalancingCursorSubtest(t *testing.T, useVst bool) { c := client.MustNewInCluster() kubecli := mustNewKubeClient(t) ns := getNamespace(t) @@ -59,8 +59,7 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { // Prepare deployment config depl := newDeployment("test-lb-" + uniuri.NewLen(4)) depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) - image := "dhly/arangodb:3.3.11-local" - depl.Spec.Image = &image + depl.Spec.Image = util.NewString("arangodb/arangodb:3.3.13") // Create deployment _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) @@ -135,72 +134,12 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { ExpectedDocuments: collectionData["books"], DocumentType: reflect.TypeOf(Book{}), }, - /* queryTest{ - Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", - BindVars: map[string]interface{}{"title": "Book 02"}, - ExpectSuccess: true, - ExpectedDocuments: []interface{}{collectionData["books"][1]}, - DocumentType: reflect.TypeOf(Book{}), - }, - queryTest{ - Query: "FOR d IN books FILTER d.Title==@title SORT d.Title RETURN d", - BindVars: map[string]interface{}{"somethingelse": "Book 02"}, - ExpectSuccess: false, // Unknown `@title` - }, - queryTest{ - Query: "FOR u IN users FILTER u.age>100 SORT u.name RETURN u", - ExpectSuccess: true, - ExpectedDocuments: []interface{}{}, - DocumentType: reflect.TypeOf(UserDoc{}), - }, - queryTest{ - Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", - BindVars: map[string]interface{}{"maxAge": 20}, - ExpectSuccess: true, - ExpectedDocuments: []interface{}{collectionData["users"][2], collectionData["users"][0], collectionData["users"][5]}, - DocumentType: reflect.TypeOf(UserDoc{}), - }, - queryTest{ - Query: "FOR u IN users FILTER u.age<@maxAge SORT u.name RETURN u", - BindVars: map[string]interface{}{"maxage": 20}, - ExpectSuccess: false, // `@maxage` versus `@maxAge` - }, - queryTest{ - Query: "FOR u IN users SORT u.age RETURN u.age", - ExpectedDocuments: []interface{}{12, 12, 13, 25, 42, 67}, - DocumentType: reflect.TypeOf(12), - ExpectSuccess: true, - }, - queryTest{ - Query: "FOR p IN users COLLECT a = p.age WITH COUNT INTO c SORT a RETURN [a, c]", - ExpectedDocuments: []interface{}{[]int{12, 2}, []int{13, 1}, []int{25, 1}, []int{42, 1}, []int{67, 1}}, - DocumentType: reflect.TypeOf([]int{}), - ExpectSuccess: true, - }, - queryTest{ - Query: "FOR u IN users SORT u.name RETURN u.name", - ExpectedDocuments: []interface{}{"Blair", "Clair", "Jake", "John", "Johnny", "Zz"}, - DocumentType: reflect.TypeOf("foo"), - ExpectSuccess: true, - },*/ } var r driver.Response // Setup context alternatives contexts := []queryTestContext{ - /*queryTestContext{driver.WithResponse(nil, &r), false}, - queryTestContext{driver.WithResponse(context.Background(), &r), false}, - queryTestContext{driver.WithResponse(driver.WithQueryCount(nil), &r), true}, - queryTestContext{driver.WithResponse(driver.WithQueryCount(nil, true), &r), true}, - queryTestContext{driver.WithResponse(driver.WithQueryCount(nil, false), &r), false},*/ queryTestContext{driver.WithResponse(driver.WithQueryBatchSize(nil, 1), &r), false}, - /*queryTestContext{driver.WithResponse(driver.WithQueryCache(nil), &r), false}, - queryTestContext{driver.WithResponse(driver.WithQueryCache(nil, true), &r), false}, - queryTestContext{driver.WithResponse(driver.WithQueryCache(nil, false), &r), false}, - queryTestContext{driver.WithResponse(driver.WithQueryMemoryLimit(nil, 60000), &r), false}, - queryTestContext{driver.WithResponse(driver.WithQueryTTL(nil, time.Minute), &r), false}, - queryTestContext{driver.WithResponse(driver.WithQueryBatchSize(driver.WithQueryCount(nil), 1), &r), true}, - queryTestContext{driver.WithResponse(driver.WithQueryCache(driver.WithQueryCount(driver.WithQueryBatchSize(nil, 2))), &r), true},*/ } // keep track of whether at least one request was forwarded internally to the @@ -247,7 +186,7 @@ func LoadBalancingCursorSubtest(t *testing.T, useVst bool) { t.Error("HasMore returned false, but ReadDocument returns a document") } result = append(result, doc.Elem().Interface()) - if wasForwarded(&r) { + if (wasForwarded(r)) { someRequestForwarded = true } time.Sleep(200 * time.Millisecond) diff --git a/tests/test_util.go b/tests/test_util.go index 6c14fee35..bae1bdb46 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -95,16 +95,20 @@ func createArangodVSTClientForDNSName(ctx context.Context, cli corev1.CoreV1Inte } // createArangodVSTConfigForDNSNames creates a go-driver VST connection config for a given DNS names. -func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (vst.ConnectionConfig, error) { +func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string, shortTimeout bool) (vst.ConnectionConfig, error) { scheme := "http" tlsConfig := &tls.Config{} + timeout := 90 * time.Second + if shortTimeout { + timeout = 100 * time.Millisecond + } if apiObject != nil && apiObject.Spec.IsSecure() { scheme = "https" tlsConfig = &tls.Config{InsecureSkipVerify: true} } transport := vstProtocol.TransportConfig{ - IdleConnTimeout: 100 * time.Millisecond, - Version: vstProtocol.Version1_1, + IdleConnTimeout: timeout, + Version: vstProtocol.Version1_1, } connConfig := vst.ConnectionConfig{ TLSConfig: tlsConfig, @@ -117,10 +121,10 @@ func createArangodVSTConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Int } // CreateArangodDatabaseVSTClient creates a go-driver client for accessing the entire cluster (or single server) via VST -func CreateArangodDatabaseVSTClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment) (driver.Client, error) { +func createArangodDatabaseVSTClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, shortTimeout bool) (driver.Client, error) { // Create connection dnsName := k8sutil.CreateDatabaseClientServiceDNSName(apiObject) - c, err := createArangodVSTClientForDNSName(ctx, cli, apiObject, dnsName) + c, err := createArangodVSTClientForDNSName(ctx, cli, apiObject, dnsName, shortTimeout) if err != nil { return nil, maskAny(err) } @@ -168,13 +172,14 @@ func mustNewKubeClient(t *testing.T) kubernetes.Interface { // mustNewArangodDatabaseClient creates a new database client, // failing the test on errors. -func mustNewArangodDatabaseClient(ctx context.Context, kubecli kubernetes.Interface, apiObject *api.ArangoDeployment, t *testing.T, useVst ...bool) driver.Client { +func mustNewArangodDatabaseClient(ctx context.Context, kubecli kubernetes.Interface, apiObject *api.ArangoDeployment, t *testing.T, flags ...bool) driver.Client { var c driver.Client var err error - if len(useVst) > 0 && useVst[0] { - c, err = CreateArangodDatabaseVSTClient(ctx, kubecli.CoreV1(), apiObject) + bool shortTimeout = len(flags) > 0 && flags[0] + if len(flags) > 1 && flags[1] { + c, err = createArangodDatabaseVSTClient(ctx, kubecli.CoreV1(), apiObject, shortTimeout) } else { - c, err = arangod.CreateArangodDatabaseClient(ctx, kubecli.CoreV1(), apiObject) + c, err = arangod.CreateArangodDatabaseClient(ctx, kubecli.CoreV1(), apiObject, shortTimeout) } if err != nil { t.Fatalf("Failed to create arango database client: %v", err) From 3c74e96164d8946a428c16d3f0751b71ebff74d9 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Mon, 6 Aug 2018 15:47:26 +0200 Subject: [PATCH 15/16] Revert changelog changes --- deps/github.com/arangodb/go-driver/CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deps/github.com/arangodb/go-driver/CHANGELOG.md b/deps/github.com/arangodb/go-driver/CHANGELOG.md index 2c7963cbf..239819a32 100644 --- a/deps/github.com/arangodb/go-driver/CHANGELOG.md +++ b/deps/github.com/arangodb/go-driver/CHANGELOG.md @@ -37,8 +37,11 @@ **Merged pull requests:** +- Properly closing idle VST connections [\#139](https://github.com/arangodb/go-driver/pull/139) +- Improving performance of reading the body of large responses [\#134](https://github.com/arangodb/go-driver/pull/134) - Added Collection.ReadDocuments [\#133](https://github.com/arangodb/go-driver/pull/133) - Added support for fetching job ID in CleanoutServer [\#131](https://github.com/arangodb/go-driver/pull/131) +- Exclude high load test on VST+3.2 [\#129](https://github.com/arangodb/go-driver/pull/129) - Test/prevent concurrent vst read write [\#128](https://github.com/arangodb/go-driver/pull/128) - Added single+ssl tests. All tests now use starter [\#127](https://github.com/arangodb/go-driver/pull/127) - Bugfix/read chunk loop [\#126](https://github.com/arangodb/go-driver/pull/126) From 289e776b80347f40a28051b50b4bf038f955d3fd Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Mon, 6 Aug 2018 16:03:05 +0200 Subject: [PATCH 16/16] Ensure BOTH forwarded as well as not-forwarded requests --- tests/load_balancer_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/load_balancer_test.go b/tests/load_balancer_test.go index f7dbd1712..c0542b093 100644 --- a/tests/load_balancer_test.go +++ b/tests/load_balancer_test.go @@ -155,7 +155,8 @@ func loadBalancingCursorSubtest(t *testing.T, useVst bool) { // keep track of whether at least one request was forwarded internally to the // correct coordinator behind the load balancer - someRequestForwarded := false + someRequestsForwarded := false + someRequestsNotForwarded := false // Run tests for every context alternative for i, test := range tests { @@ -189,7 +190,9 @@ func loadBalancingCursorSubtest(t *testing.T, useVst bool) { } result = append(result, doc.Elem().Interface()) if wasForwarded(r) { - someRequestForwarded = true + someRequestsForwarded = true + } else { + someRequestsNotForwarded = true } time.Sleep(200 * time.Millisecond) } @@ -214,7 +217,10 @@ func loadBalancingCursorSubtest(t *testing.T, useVst bool) { } } - if !someRequestForwarded { + if !someRequestsForwarded { t.Error("Did not detect any request being forwarded behind load balancer!") } + if !someRequestsNotForwarded { + t.Error("Did not detect any request NOT being forwarded behind load balancer!") + } }