From 6eeee601fa66c4369a2f3d104d2b3db220f66000 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Fri, 22 Jun 2018 14:42:53 +0200 Subject: [PATCH] Updated go-driver --- .../github.com/arangodb/go-driver/.travis.yml | 2 +- vendor/github.com/arangodb/go-driver/Makefile | 54 ++- .../github.com/arangodb/go-driver/README.md | 419 +----------------- .../arangodb/go-driver/agency/agency.go | 5 + .../go-driver/agency/agency_health.go | 20 +- .../arangodb/go-driver/agency/agency_impl.go | 5 + .../arangodb/go-driver/agency/doc.go | 31 +- .../github.com/arangodb/go-driver/cluster.go | 5 + .../arangodb/go-driver/cluster/doc.go | 26 ++ .../arangodb/go-driver/cluster_impl.go | 35 +- .../go-driver/collection_document_impl.go | 83 +++- .../go-driver/collection_documents.go | 6 + .../github.com/arangodb/go-driver/context.go | 25 +- .../arangodb/go-driver/cursor_impl.go | 25 +- .../Drivers/GO/ConnectionManagement/README.md | 84 ++++ .../docs/Drivers/GO/ExampleRequests/README.md | 182 ++++++++ .../docs/Drivers/GO/GettingStarted/README.md | 141 ++++++ .../go-driver/docs/Drivers/GO/README.md | 8 + .../edge_collection_documents_impl.go | 76 +++- .../arangodb/go-driver/encode-go_1_8.go | 5 +- .../github.com/arangodb/go-driver/encode.go | 2 +- .../arangodb/go-driver/encode_test.go | 38 ++ .../arangodb/go-driver/http/connection.go | 34 ++ .../github.com/arangodb/go-driver/http/doc.go | 69 +++ .../github.com/arangodb/go-driver/jwt/doc.go | 42 +- .../arangodb/go-driver/test/client_test.go | 16 +- .../go-driver/test/concurrency_test.go | 205 +++++++++ .../arangodb/go-driver/test/cursor_test.go | 67 ++- .../github.com/arangodb/go-driver/test/doc.go | 26 ++ .../go-driver/test/document_create_test.go | 32 ++ .../go-driver/test/document_remove_test.go | 33 ++ .../go-driver/test/document_replace_test.go | 35 ++ .../go-driver/test/document_update_test.go | 35 ++ .../go-driver/test/documents_create_test.go | 59 +++ .../go-driver/test/documents_import_test.go | 44 ++ .../go-driver/test/documents_remove_test.go | 33 ++ .../go-driver/test/documents_replace_test.go | 51 +++ .../go-driver/test/documents_update_test.go | 51 +++ .../go-driver/test/edges_create_test.go | 15 + .../arangodb/go-driver/test/user_auth_test.go | 66 ++- .../arangodb/go-driver/test/util.go | 16 + .../go-driver/test/vertices_create_test.go | 15 + .../github.com/arangodb/go-driver/util/doc.go | 26 ++ .../vertex_collection_documents_impl.go | 76 +++- .../arangodb/go-driver/vst/connection.go | 17 +- .../github.com/arangodb/go-driver/vst/doc.go | 68 +++ .../arangodb/go-driver/vst/protocol/chunk.go | 4 + .../go-driver/vst/protocol/chunk_1_0.go | 3 +- .../go-driver/vst/protocol/connection.go | 66 ++- .../arangodb/go-driver/vst/protocol/doc.go | 26 ++ .../go-driver/vst/protocol/message.go | 41 +- .../go-driver/vst/protocol/message_store.go | 12 +- .../go-driver/vst/protocol/transport.go | 48 +- 53 files changed, 2013 insertions(+), 595 deletions(-) create mode 100644 vendor/github.com/arangodb/go-driver/cluster/doc.go create mode 100644 vendor/github.com/arangodb/go-driver/docs/Drivers/GO/ConnectionManagement/README.md create mode 100644 vendor/github.com/arangodb/go-driver/docs/Drivers/GO/ExampleRequests/README.md create mode 100644 vendor/github.com/arangodb/go-driver/docs/Drivers/GO/GettingStarted/README.md create mode 100644 vendor/github.com/arangodb/go-driver/docs/Drivers/GO/README.md create mode 100644 vendor/github.com/arangodb/go-driver/encode_test.go create mode 100644 vendor/github.com/arangodb/go-driver/http/doc.go create mode 100644 vendor/github.com/arangodb/go-driver/test/concurrency_test.go create mode 100644 vendor/github.com/arangodb/go-driver/test/doc.go create mode 100644 vendor/github.com/arangodb/go-driver/util/doc.go create mode 100644 vendor/github.com/arangodb/go-driver/vst/doc.go create mode 100644 vendor/github.com/arangodb/go-driver/vst/protocol/doc.go diff --git a/vendor/github.com/arangodb/go-driver/.travis.yml b/vendor/github.com/arangodb/go-driver/.travis.yml index 5ab1337f..da58d729 100644 --- a/vendor/github.com/arangodb/go-driver/.travis.yml +++ b/vendor/github.com/arangodb/go-driver/.travis.yml @@ -7,7 +7,7 @@ language: go env: - TEST_SUITE=run-tests-http - - TEST_SUITE=run-tests-single ARANGODB=arangodb:3.1 + - TEST_SUITE=run-tests-single ARANGODB=arangodb:3.2 - TEST_SUITE=run-tests-single ARANGODB=arangodb/arangodb:latest - TEST_SUITE=run-tests-single ARANGODB=arangodb/arangodb-preview:latest diff --git a/vendor/github.com/arangodb/go-driver/Makefile b/vendor/github.com/arangodb/go-driver/Makefile index 85f2b321..b8f6f00d 100644 --- a/vendor/github.com/arangodb/go-driver/Makefile +++ b/vendor/github.com/arangodb/go-driver/Makefile @@ -3,7 +3,7 @@ SCRIPTDIR := $(shell pwd) ROOTDIR := $(shell cd $(SCRIPTDIR) && pwd) GOBUILDDIR := $(SCRIPTDIR)/.gobuild -GOVERSION := 1.9.2-alpine +GOVERSION := 1.10.1-alpine TMPDIR := $(GOBUILDDIR) ifndef ARANGODB @@ -58,13 +58,9 @@ else ifeq ("$(TEST_AUTH)", "jwt") ARANGOARGS := --server.jwt-secret=/jwtsecret endif -ifeq ("$(TEST_MODE)", "single") - TEST_NET := container:$(DBCONTAINER) - TEST_ENDPOINTS := http://localhost:8529 -else - TEST_NET := container:$(TESTCONTAINER)-ns - TEST_ENDPOINTS := http://localhost:7001 - TESTS := $(REPOPATH)/test +TEST_NET := container:$(TESTCONTAINER)-ns +TEST_ENDPOINTS := http://localhost:7001 +TESTS := $(REPOPATH)/test ifeq ("$(TEST_AUTH)", "rootpw") CLUSTERENV := JWTSECRET=testing TEST_AUTHENTICATION := basic:root: @@ -77,7 +73,6 @@ ifeq ("$(TEST_SSL)", "auto") CLUSTERENV := SSL=auto $(CLUSTERENV) TEST_ENDPOINTS = https://localhost:7001 endif -endif ifeq ("$(TEST_CONNECTION)", "vst") TESTS := $(REPOPATH)/test @@ -133,13 +128,13 @@ run-tests-http: $(GOBUILDDIR) # Single server tests run-tests-single: run-tests-single-json run-tests-single-vpack run-tests-single-vst-1.0 $(VST11_SINGLE_TESTS) -run-tests-single-json: run-tests-single-json-with-auth run-tests-single-json-no-auth +run-tests-single-json: run-tests-single-json-with-auth run-tests-single-json-no-auth run-tests-single-json-ssl -run-tests-single-vpack: run-tests-single-vpack-with-auth run-tests-single-vpack-no-auth +run-tests-single-vpack: run-tests-single-vpack-with-auth run-tests-single-vpack-no-auth run-tests-single-vpack-ssl -run-tests-single-vst-1.0: run-tests-single-vst-1.0-with-auth run-tests-single-vst-1.0-no-auth +run-tests-single-vst-1.0: run-tests-single-vst-1.0-with-auth run-tests-single-vst-1.0-no-auth run-tests-single-vst-1.0-ssl -run-tests-single-vst-1.1: run-tests-single-vst-1.1-with-auth run-tests-single-vst-1.1-jwt-auth run-tests-single-vst-1.1-no-auth +run-tests-single-vst-1.1: run-tests-single-vst-1.1-with-auth run-tests-single-vst-1.1-jwt-auth run-tests-single-vst-1.1-no-auth run-tests-single-vst-1.1-ssl run-tests-single-vst-1.1-jwt-ssl run-tests-single-json-no-auth: @echo "Single server, HTTP+JSON, no authentication" @@ -177,6 +172,26 @@ run-tests-single-vst-1.1-jwt-auth: @echo "Single server, Velocystream 1.1, JWT authentication" @${MAKE} TEST_MODE="single" TEST_AUTH="jwt" TEST_CONNECTION="vst" TEST_CVERSION="1.1" __run_tests +run-tests-single-json-ssl: + @echo "Single server, HTTP+JSON, with authentication, SSL" + @${MAKE} TEST_MODE="single" TEST_AUTH="rootpw" TEST_SSL="auto" TEST_CONTENT_TYPE="json" __run_tests + +run-tests-single-vpack-ssl: + @echo "Single server, HTTP+Velocypack, with authentication, SSL" + @${MAKE} TEST_MODE="single" TEST_AUTH="rootpw" TEST_SSL="auto" TEST_CONTENT_TYPE="vpack" __run_tests + +run-tests-single-vst-1.0-ssl: + @echo "Single server, Velocystream 1.0, with authentication, SSL" + @${MAKE} TEST_MODE="single" TEST_AUTH="rootpw" TEST_SSL="auto" TEST_CONNECTION="vst" TEST_CVERSION="1.0" __run_tests + +run-tests-single-vst-1.1-ssl: + @echo "Single server, Velocystream 1.1, with authentication, SSL" + @${MAKE} TEST_MODE="single" TEST_AUTH="rootpw" TEST_SSL="auto" TEST_CONNECTION="vst" TEST_CVERSION="1.1" __run_tests + +run-tests-single-vst-1.1-jwt-ssl: + @echo "Single server, Velocystream 1.1, JWT authentication, SSL" + @${MAKE} TEST_MODE="single" TEST_AUTH="jwt" TEST_SSL="auto" TEST_CONNECTION="vst" TEST_CVERSION="1.1" __run_tests + # ResilientSingle server tests run-tests-resilientsingle: run-tests-resilientsingle-json run-tests-resilientsingle-vpack run-tests-resilientsingle-vst-1.0 $(VST11_RESILIENTSINGLE_TESTS) @@ -292,11 +307,13 @@ __test_go_test: --net=$(TEST_NET) \ -v $(ROOTDIR):/usr/code \ -e GOPATH=/usr/code/.gobuild \ + -e GOCACHE=off \ -e TEST_ENDPOINTS=$(TEST_ENDPOINTS) \ -e TEST_AUTHENTICATION=$(TEST_AUTHENTICATION) \ -e TEST_CONNECTION=$(TEST_CONNECTION) \ -e TEST_CVERSION=$(TEST_CVERSION) \ -e TEST_CONTENT_TYPE=$(TEST_CONTENT_TYPE) \ + -e TEST_PPROF=$(TEST_PPROF) \ -w /usr/code/ \ golang:$(GOVERSION) \ go test $(TAGS) $(TESTOPTIONS) $(TESTVERBOSEOPTIONS) $(TESTS) @@ -308,25 +325,14 @@ else ifdef JWTSECRET echo "$JWTSECRET" > "${JWTSECRETFILE}" endif -ifeq ("$(TEST_MODE)", "single") - @-docker rm -f -v $(DBCONTAINER) $(TESTCONTAINER) &> /dev/null - docker run -d --name $(DBCONTAINER) \ - $(ARANGOENV) $(ARANGOVOL) \ - $(ARANGODB) --log.level requests=debug --log.use-microtime true $(ARANGOARGS) -else @-docker rm -f -v $(TESTCONTAINER) &> /dev/null @TESTCONTAINER=$(TESTCONTAINER) ARANGODB=$(ARANGODB) STARTER=$(STARTER) STARTERMODE=$(TEST_MODE) TMPDIR=${GOBUILDDIR} $(CLUSTERENV) $(ROOTDIR)/test/cluster.sh start endif -endif __test_cleanup: @docker rm -f -v $(TESTCONTAINER) &> /dev/null ifndef TEST_ENDPOINTS_OVERRIDE -ifeq ("$(TEST_MODE)", "single") - @docker rm -f -v $(DBCONTAINER) &> /dev/null -else @TESTCONTAINER=$(TESTCONTAINER) ARANGODB=$(ARANGODB) STARTER=$(STARTER) STARTERMODE=$(TEST_MODE) $(ROOTDIR)/test/cluster.sh cleanup -endif endif @sleep 3 diff --git a/vendor/github.com/arangodb/go-driver/README.md b/vendor/github.com/arangodb/go-driver/README.md index a25ddad9..88726a21 100644 --- a/vendor/github.com/arangodb/go-driver/README.md +++ b/vendor/github.com/arangodb/go-driver/README.md @@ -1,416 +1,13 @@ -# ArangoDB GO Driver. +![ArangoDB-Logo](https://docs.arangodb.com/assets/arangodb_logo_2016_inverted.png) +# ArangoDB GO Driver + +This project contains the official Go driver for the [ArangoDB database](https://arangodb.com). [![Build Status](https://travis-ci.org/arangodb/go-driver.svg?branch=master)](https://travis-ci.org/arangodb/go-driver) [![GoDoc](https://godoc.org/github.com/arangodb/g-driver?status.svg)](http://godoc.org/github.com/arangodb/go-driver) -API and implementation is considered stable, more protocols (Velocystream) are being added within the existing API. - -This project contains a Go driver for the [ArangoDB database](https://arangodb.com). - -## Supported versions - -- ArangoDB versions 3.1 and up. - - Single server & cluster setups - - With or without authentication -- Go 1.7 and up. - -## Go dependencies - -- None (Additional error libraries are supported). - -## Getting started - -To use the driver, first fetch the sources into your GOPATH. - -```sh -go get github.com/arangodb/go-driver -``` - -Using the driver, you always need to create a `Client`. -The following example shows how to create a `Client` for a single server -running on localhost. - -```go -import ( - "fmt" - - driver "github.com/arangodb/go-driver" - "github.com/arangodb/go-driver/http" -) - -... - -conn, err := http.NewConnection(http.ConnectionConfig{ - Endpoints: []string{"http://localhost:8529"}, -}) -if err != nil { - // Handle error -} -c, err := driver.NewClient(driver.ClientConfig{ - Connection: conn, -}) -if err != nil { - // Handle error -} -``` - -Once you have a `Client` you can access/create databases on the server, -access/create collections, graphs, documents and so on. - -The following example shows how to open an existing collection in an existing database -and create a new document in that collection. - -```go -// Open "examples_books" database -db, err := c.Database(nil, "examples_books") -if err != nil { - // Handle error -} - -// Open "books" collection -col, err := db.Collection(nil, "books") -if err != nil { - // Handle error -} - -// Create document -book := Book{ - Title: "ArangoDB Cookbook", - NoPages: 257, -} -meta, err := col.CreateDocument(nil, book) -if err != nil { - // Handle error -} -fmt.Printf("Created document in collection '%s' in database '%s'\n", col.Name(), db.Name()) -``` - -## API design - -### Concurrency - -All functions of the driver are stricly synchronous. They operate and only return a value (or error) -when they're done. - -If you want to run operations concurrently, use a go routine. All objects in the driver are designed -to be used from multiple concurrent go routines, except `Cursor`. - -All database objects (except `Cursor`) are considered static. After their creation they won't change. -E.g. after creating a `Collection` instance you can remove the collection, but the (Go) instance -will still be there. Calling functions on such a removed collection will of course fail. - -### Structured error handling & wrapping - -All functions of the driver that can fail return an `error` value. If that value is not `nil`, the -function call is considered to be failed. In that case all other return values are set to their `zero` -values. - -All errors are structured using error checking functions named `Is`. -E.g. `IsNotFound(error)` return true if the given error is of the category "not found". -There can be multiple internal error codes that all map onto the same category. - -All errors returned from any function of the driver (either internal or exposed) wrap errors -using the `WithStack` function. This can be used to provide detail stack trackes in case of an error. -All error checking functions use the `Cause` function to get the cause of an error instead of the error wrapper. - -Note that `WithStack` and `Cause` are actually variables to you can implement it using your own error -wrapper library. - -If you for example use https://github.com/pkg/errors, you want to initialize to go driver like this: -```go -import ( - driver "github.com/arangodb/go-driver" - "github.com/pkg/errors" -) - -func init() { - driver.WithStack = errors.WithStack - driver.Cause = errors.Cause -} -``` - -### Context aware - -All functions of the driver that involve some kind of long running operation or -support additional options not given as function arguments, have a `context.Context` argument. -This enables you cancel running requests, pass timeouts/deadlines and pass additional options. - -In all methods that take a `context.Context` argument you can pass `nil` as value. -This is equivalent to passing `context.Background()`. - -Many functions support 1 or more optional (and infrequently used) additional options. -These can be used with a `With` function. -E.g. to force a create document call to wait until the data is synchronized to disk, -use a prepared context like this: -```go -ctx := driver.WithWaitForSync(parentContext) -collection.CreateDocument(ctx, yourDocument) -``` - -### Failover - -The driver supports multiple endpoints to connect to. All request are in principle -send to the same endpoint until that endpoint fails to respond. -In that case a new endpoint is chosen and the operation is retried. - -The following example shows how to connect to a cluster of 3 servers. - -```go -conn, err := http.NewConnection(http.ConnectionConfig{ - Endpoints: []string{"http://server1:8529", "http://server2:8529", "http://server3:8529"}, -}) -if err != nil { - // Handle error -} -c, err := driver.NewClient(driver.ClientConfig{ - Connection: conn, -}) -if err != nil { - // Handle error -} -``` - -Note that a valid endpoint is an URL to either a standalone server, or a URL to a coordinator -in a cluster. - -### Failover: Exact behavior - -The driver monitors the request being send to a specific server (endpoint). -As soon as the request has been completely written, failover will no longer happen. -The reason for that is that several operations cannot be (safely) retried. -E.g. when a request to create a document has been send to a server and a timeout -occurs, the driver has no way of knowing if the server did or did not create -the document in the database. - -If the driver detects that a request has been completely written, but still gets -an error (other than an error response from Arango itself), it will wrap the -error in a `ResponseError`. The client can test for such an error using `IsResponseError`. - -If a client received a `ResponseError`, it can do one of the following: -- Retry the operation and be prepared for some kind of duplicate record / unique constraint violation. -- Perform a test operation to see if the "failed" operation did succeed after all. -- Simply consider the operation failed. This is risky, since it can still be the case that the operation did succeed. - -### Failover: Timeouts - -To control the timeout of any function in the driver, you must pass it a context -configured with `context.WithTimeout` (or `context.WithDeadline`). - -In the case of multiple endpoints, the actual timeout used for requests will be shorter than -the timeout given in the context. -The driver will divide the timeout by the number of endpoints with a maximum of 3. -This ensures that the driver can try up to 3 different endpoints (in case of failover) without -being canceled due to the timeout given by the client. -E.g. -- With 1 endpoint and a given timeout of 1 minute, the actual request timeout will be 1 minute. -- With 3 endpoints and a given timeout of 1 minute, the actual request timeout will be 20 seconds. -- With 8 endpoints and a given timeout of 1 minute, the actual request timeout will be 20 seconds. - -For most requests you want a actual request timeout of at least 30 seconds. - -### Secure connections (SSL) - -The driver supports endpoints that use SSL using the `https` URL scheme. - -The following example shows how to connect to a server that has a secure endpoint using -a self-signed certificate. - -```go -conn, err := http.NewConnection(http.ConnectionConfig{ - Endpoints: []string{"https://localhost:8529"}, - TLSConfig: &tls.Config{InsecureSkipVerify: true}, -}) -if err != nil { - // Handle error -} -c, err := driver.NewClient(driver.ClientConfig{ - Connection: conn, -}) -if err != nil { - // Handle error -} -``` - -# Sample requests - -## Connecting to ArangoDB - -```go -conn, err := http.NewConnection(http.ConnectionConfig{ - Endpoints: []string{"http://localhost:8529"}, - TLSConfig: &tls.Config{ /*...*/ }, -}) -if err != nil { - // Handle error -} -c, err := driver.NewClient(driver.ClientConfig{ - Connection: conn, - Authentication: driver.BasicAuthentication("user", "password"), -}) -if err != nil { - // Handle error -} -``` - -## Opening a database - -```go -ctx := context.Background() -db, err := client.Database(ctx, "myDB") -if err != nil { - // handle error -} -``` - -## Opening a collection - -```go -ctx := context.Background() -col, err := db.Collection(ctx, "myCollection") -if err != nil { - // handle error -} -``` - -## Checking if a collection exists - -```go -ctx := context.Background() -found, err := db.CollectionExists(ctx, "myCollection") -if err != nil { - // handle error -} -``` - -## Creating a collection - -```go -ctx := context.Background() -options := &driver.CreateCollectionOptions{ /* ... */ } -col, err := db.CreateCollection(ctx, "myCollection", options) -if err != nil { - // handle error -} -``` - -## Reading a document from a collection - -```go -var doc MyDocument -ctx := context.Background() -meta, err := col.ReadDocument(ctx, myDocumentKey, &doc) -if err != nil { - // handle error -} -``` - -## Reading a document from a collection with an explicit revision - -```go -var doc MyDocument -revCtx := driver.WithRevision(ctx, "mySpecificRevision") -meta, err := col.ReadDocument(revCtx, myDocumentKey, &doc) -if err != nil { - // handle error -} -``` - -## Creating a document - -```go -doc := MyDocument{ - Name: "jan", - Counter: 23, -} -ctx := context.Background() -meta, err := col.CreateDocument(ctx, doc) -if err != nil { - // handle error -} -fmt.Printf("Created document with key '%s', revision '%s'\n", meta.Key, meta.Rev) -``` - -## Removing a document - -```go -ctx := context.Background() -err := col.RemoveDocument(revCtx, myDocumentKey) -if err != nil { - // handle error -} -``` - -## Removing a document with an explicit revision - -```go -revCtx := driver.WithRevision(ctx, "mySpecificRevision") -err := col.RemoveDocument(revCtx, myDocumentKey) -if err != nil { - // handle error -} -``` - -## Updating a document - -```go -ctx := context.Background() -patch := map[string]interface{}{ - "Name": "Frank", -} -meta, err := col.UpdateDocument(ctx, myDocumentKey, patch) -if err != nil { - // handle error -} -``` - -## Querying documents, one document at a time - -```go -ctx := context.Background() -query := "FOR d IN myCollection LIMIT 10 RETURN d" -cursor, err := db.Query(ctx, query, nil) -if err != nil { - // handle error -} -defer cursor.Close() -for { - var doc MyDocument - meta, err := cursor.ReadDocument(ctx, &doc) - if driver.IsNoMoreDocuments(err) { - break - } else if err != nil { - // handle other errors - } - fmt.Printf("Got doc with key '%s' from query\n", meta.Key) -} -``` - -## Querying documents, fetching total count - -```go -ctx := driver.WithQueryCount(context.Background()) -query := "FOR d IN myCollection RETURN d" -cursor, err := db.Query(ctx, query, nil) -if err != nil { - // handle error -} -defer cursor.Close() -fmt.Printf("Query yields %d documents\n", cursor.Count()) -``` - -## Querying documents, with bind variables - -```go -ctx := context.Background() -query := "FOR d IN myCollection FILTER d.Name == @name RETURN d" -bindVars := map[string]interface{}{ - "name": "Some name", -} -cursor, err := db.Query(ctx, query, bindVars) -if err != nil { - // handle error -} -defer cursor.Close() -... -``` +- [Getting Started](docs/Drivers/GO/GettingStarted/README.md) +- [Example Requests](docs/Drivers/GO/ExampleRequests/README.md) +- [Connection Management](docs/Drivers/GO/ConnectionManagement/README.md) +- [Reference](https://godoc.org/github.com/arangodb/go-driver) diff --git a/vendor/github.com/arangodb/go-driver/agency/agency.go b/vendor/github.com/arangodb/go-driver/agency/agency.go index 77133cd1..cf666622 100644 --- a/vendor/github.com/arangodb/go-driver/agency/agency.go +++ b/vendor/github.com/arangodb/go-driver/agency/agency.go @@ -25,10 +25,15 @@ package agency import ( "context" "time" + + driver "github.com/arangodb/go-driver" ) // Agency provides API implemented by the ArangoDB agency. type Agency interface { + // Connection returns the connection used by this api. + Connection() driver.Connection + // ReadKey reads the value of a given key in the agency. ReadKey(ctx context.Context, key []string, value interface{}) error diff --git a/vendor/github.com/arangodb/go-driver/agency/agency_health.go b/vendor/github.com/arangodb/go-driver/agency/agency_health.go index b7dbd350..0dc826d7 100644 --- a/vendor/github.com/arangodb/go-driver/agency/agency_health.go +++ b/vendor/github.com/arangodb/go-driver/agency/agency_health.go @@ -34,7 +34,8 @@ import ( ) const ( - maxAgentResponseTime = time.Second * 10 + maxAgentResponseTime = time.Second * 10 + keyAllowNoLeader driver.ContextKey = "arangodb-agency-allow-no-leader" ) // agentStatus is a helper structure used in AreAgentsHealthy. @@ -44,6 +45,21 @@ type agentStatus struct { IsResponding bool } +// WithAllowNoLeader is used to configure a context to make AreAgentsHealthy +// accept the situation where it finds 0 leaders. +func WithAllowNoLeader(parent context.Context) context.Context { + if parent == nil { + parent = context.Background() + } + return context.WithValue(parent, keyAllowNoLeader, true) +} + +// hasAllowNoLeader returns true when the given context was +// prepared with WithAllowNoLeader. +func hasAllowNoLeader(ctx context.Context) bool { + return ctx != nil && ctx.Value(keyAllowNoLeader) != nil +} + // AreAgentsHealthy performs a health check on all given agents. // Of the given agents, 1 must respond as leader and all others must redirect to the leader. // The function returns nil when all agents are healthy or an error when something is wrong. @@ -101,7 +117,7 @@ func AreAgentsHealthy(ctx context.Context, clients []driver.Connection) error { } } } - if noLeaders != 1 { + if noLeaders != 1 && !hasAllowNoLeader(ctx) { return driver.WithStack(fmt.Errorf("Unexpected number of agency leaders: %d", noLeaders)) } return nil diff --git a/vendor/github.com/arangodb/go-driver/agency/agency_impl.go b/vendor/github.com/arangodb/go-driver/agency/agency_impl.go index 17799860..fb1f6a05 100644 --- a/vendor/github.com/arangodb/go-driver/agency/agency_impl.go +++ b/vendor/github.com/arangodb/go-driver/agency/agency_impl.go @@ -49,6 +49,11 @@ func NewAgency(conn driver.Connection) (Agency, error) { }, nil } +// Connection returns the connection used by this api. +func (c *agency) Connection() driver.Connection { + return c.conn +} + // ReadKey reads the value of a given key in the agency. func (c *agency) ReadKey(ctx context.Context, key []string, value interface{}) error { conn := c.conn diff --git a/vendor/github.com/arangodb/go-driver/agency/doc.go b/vendor/github.com/arangodb/go-driver/agency/doc.go index 23b0a268..76412a15 100644 --- a/vendor/github.com/arangodb/go-driver/agency/doc.go +++ b/vendor/github.com/arangodb/go-driver/agency/doc.go @@ -20,20 +20,19 @@ // Author Ewout Prangsma // -package agency +/* +Package agency provides an API to access the ArangoDB agency (it is unlikely that you need this package directly). -// -// The Agency is fault-tolerant and highly-available key-value store -// that is used to store critical, low-level information about -// an ArangoDB cluster. -// -// The API provided in this package gives access to the Agency. -// -// THIS API IS NOT USED FOR NORMAL DATABASE ACCESS. -// -// Reasons for using this API are: -// - You want to make use of an indepent Agency as your own HA key-value store. -// - You want access to low-level information of your database. USE WITH GREAT CARE! -// -// WARNING: Messing around in the Agency can quickly lead to a corrupt database! -// +The Agency is fault-tolerant and highly-available key-value store +that is used to store critical, low-level information about +an ArangoDB cluster. + +THIS API IS NOT USED FOR NORMAL DATABASE ACCESS. + +Reasons for using this API are: +- You want to make use of an indepent Agency as your own HA key-value store. +- You want access to low-level information of your database. USE WITH GREAT CARE! + +WARNING: Messing around in the Agency can quickly lead to a corrupt database! +*/ +package agency diff --git a/vendor/github.com/arangodb/go-driver/cluster.go b/vendor/github.com/arangodb/go-driver/cluster.go index d51a674f..fc414bc8 100644 --- a/vendor/github.com/arangodb/go-driver/cluster.go +++ b/vendor/github.com/arangodb/go-driver/cluster.go @@ -45,6 +45,11 @@ type Cluster interface { // IsCleanedOut checks if the dbserver with given ID has been cleaned out. IsCleanedOut(ctx context.Context, serverID string) (bool, error) + + // RemoveServer is a low-level option to remove a server from a cluster. + // This function is suitable for servers of type coordinator or dbserver. + // The use of `ClientServerAdmin.Shutdown` is highly recommended above this function. + RemoveServer(ctx context.Context, serverID ServerID) error } // ServerID identifies an arangod server in a cluster. diff --git a/vendor/github.com/arangodb/go-driver/cluster/doc.go b/vendor/github.com/arangodb/go-driver/cluster/doc.go new file mode 100644 index 00000000..53abf292 --- /dev/null +++ b/vendor/github.com/arangodb/go-driver/cluster/doc.go @@ -0,0 +1,26 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package cluster implements a driver.Connection that provides cluster failover support (it is not intended to be used directly). +*/ +package cluster diff --git a/vendor/github.com/arangodb/go-driver/cluster_impl.go b/vendor/github.com/arangodb/go-driver/cluster_impl.go index 3bf795f8..0ddd307f 100644 --- a/vendor/github.com/arangodb/go-driver/cluster_impl.go +++ b/vendor/github.com/arangodb/go-driver/cluster_impl.go @@ -123,6 +123,10 @@ type cleanOutServerRequest struct { Server string `json:"server"` } +type cleanOutServerResponse struct { + JobID string `json:"id"` +} + // CleanOutServer triggers activities to clean out a DBServers. func (c *cluster) CleanOutServer(ctx context.Context, serverID string) error { req, err := c.conn.NewRequest("POST", "_admin/cluster/cleanOutServer") @@ -135,7 +139,7 @@ func (c *cluster) CleanOutServer(ctx context.Context, serverID string) error { if _, err := req.SetBody(input); err != nil { return WithStack(err) } - applyContextSettings(ctx, req) + cs := applyContextSettings(ctx, req) resp, err := c.conn.Do(ctx, req) if err != nil { return WithStack(err) @@ -143,6 +147,13 @@ func (c *cluster) CleanOutServer(ctx context.Context, serverID string) error { if err := resp.CheckStatus(200, 202); err != nil { return WithStack(err) } + var result cleanOutServerResponse + if err := resp.ParseBody("", &result); err != nil { + return WithStack(err) + } + if cs.JobIDResponse != nil { + *cs.JobIDResponse = result.JobID + } return nil } @@ -188,3 +199,25 @@ func (c *cluster) NumberOfServers(ctx context.Context) (NumberOfServersResponse, } return result, nil } + +// RemoveServer is a low-level option to remove a server from a cluster. +// This function is suitable for servers of type coordinator or dbserver. +// The use of `ClientServerAdmin.Shutdown` is highly recommended above this function. +func (c *cluster) RemoveServer(ctx context.Context, serverID ServerID) error { + req, err := c.conn.NewRequest("POST", "_admin/cluster/removeServer") + if err != nil { + return WithStack(err) + } + if _, err := req.SetBody(serverID); err != nil { + return WithStack(err) + } + applyContextSettings(ctx, req) + resp, err := c.conn.Do(ctx, req) + if err != nil { + return WithStack(err) + } + if err := resp.CheckStatus(200, 202); err != nil { + return WithStack(err) + } + return nil +} diff --git a/vendor/github.com/arangodb/go-driver/collection_document_impl.go b/vendor/github.com/arangodb/go-driver/collection_document_impl.go index b9a83eb0..6ed99570 100644 --- a/vendor/github.com/arangodb/go-driver/collection_document_impl.go +++ b/vendor/github.com/arangodb/go-driver/collection_document_impl.go @@ -80,6 +80,55 @@ func (c *collection) ReadDocument(ctx context.Context, key string, result interf return meta, nil } +// ReadDocuments reads multiple documents with given keys from the collection. +// The documents data is stored into elements of the given results slice, +// the documents meta data is returned. +// If no document exists with a given key, a NotFoundError is returned at its errors index. +func (c *collection) ReadDocuments(ctx context.Context, keys []string, results interface{}) (DocumentMetaSlice, ErrorSlice, error) { + resultsVal := reflect.ValueOf(results) + switch resultsVal.Kind() { + case reflect.Array, reflect.Slice: + // OK + default: + return nil, nil, WithStack(InvalidArgumentError{Message: fmt.Sprintf("results data must be of kind Array, got %s", resultsVal.Kind())}) + } + if keys == nil { + return nil, nil, WithStack(InvalidArgumentError{Message: "keys nil"}) + } + resultCount := resultsVal.Len() + if len(keys) != resultCount { + return nil, nil, WithStack(InvalidArgumentError{Message: fmt.Sprintf("expected %d keys, got %d", resultCount, len(keys))}) + } + for _, key := range keys { + if err := validateKey(key); err != nil { + return nil, nil, WithStack(err) + } + } + req, err := c.conn.NewRequest("PUT", c.relPath("document")) + if err != nil { + return nil, nil, WithStack(err) + } + req = req.SetQuery("onlyget", "1") + cs := applyContextSettings(ctx, req) + if _, err := req.SetBodyArray(keys, nil); err != nil { + return nil, nil, WithStack(err) + } + resp, err := c.conn.Do(ctx, req) + if err != nil { + return nil, nil, WithStack(err) + } + if err := resp.CheckStatus(200); err != nil { + return nil, nil, WithStack(err) + } + // Parse response array + metas, errs, err := parseResponseArray(resp, resultCount, cs, results) + if err != nil { + return nil, nil, WithStack(err) + } + return metas, errs, nil + +} + // CreateDocument creates a single document in the collection. // The document data is loaded from the given document, the document meta data is returned. // If the document data already contains a `_key` field, this will be used as key of the new document, @@ -103,7 +152,7 @@ func (c *collection) CreateDocument(ctx context.Context, document interface{}) ( if err != nil { return DocumentMeta{}, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, WithStack(err) } if cs.Silent { @@ -155,7 +204,7 @@ func (c *collection) CreateDocuments(ctx context.Context, documents interface{}) if err != nil { return nil, nil, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return nil, nil, WithStack(err) } if cs.Silent { @@ -163,7 +212,7 @@ func (c *collection) CreateDocuments(ctx context.Context, documents interface{}) return nil, nil, nil } // Parse response array - metas, errs, err := parseResponseArray(resp, documentCount, cs) + metas, errs, err := parseResponseArray(resp, documentCount, cs, nil) if err != nil { return nil, nil, WithStack(err) } @@ -196,7 +245,7 @@ func (c *collection) UpdateDocument(ctx context.Context, key string, update inte if err != nil { return DocumentMeta{}, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, WithStack(err) } if cs.Silent { @@ -264,7 +313,7 @@ func (c *collection) UpdateDocuments(ctx context.Context, keys []string, updates if err != nil { return nil, nil, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return nil, nil, WithStack(err) } if cs.Silent { @@ -272,7 +321,7 @@ func (c *collection) UpdateDocuments(ctx context.Context, keys []string, updates return nil, nil, nil } // Parse response array - metas, errs, err := parseResponseArray(resp, updateCount, cs) + metas, errs, err := parseResponseArray(resp, updateCount, cs, nil) if err != nil { return nil, nil, WithStack(err) } @@ -305,7 +354,7 @@ func (c *collection) ReplaceDocument(ctx context.Context, key string, document i if err != nil { return DocumentMeta{}, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, WithStack(err) } if cs.Silent { @@ -373,7 +422,7 @@ func (c *collection) ReplaceDocuments(ctx context.Context, keys []string, docume if err != nil { return nil, nil, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return nil, nil, WithStack(err) } if cs.Silent { @@ -381,7 +430,7 @@ func (c *collection) ReplaceDocuments(ctx context.Context, keys []string, docume return nil, nil, nil } // Parse response array - metas, errs, err := parseResponseArray(resp, documentCount, cs) + metas, errs, err := parseResponseArray(resp, documentCount, cs, nil) if err != nil { return nil, nil, WithStack(err) } @@ -407,7 +456,7 @@ func (c *collection) RemoveDocument(ctx context.Context, key string) (DocumentMe if err != nil { return DocumentMeta{}, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(200, 202)); err != nil { + if err := resp.CheckStatus(200, 202); err != nil { return DocumentMeta{}, WithStack(err) } if cs.Silent { @@ -456,7 +505,7 @@ func (c *collection) RemoveDocuments(ctx context.Context, keys []string) (Docume if err != nil { return nil, nil, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(200, 202); err != nil { return nil, nil, WithStack(err) } if cs.Silent { @@ -464,7 +513,7 @@ func (c *collection) RemoveDocuments(ctx context.Context, keys []string) (Docume return nil, nil, nil } // Parse response array - metas, errs, err := parseResponseArray(resp, keyCount, cs) + metas, errs, err := parseResponseArray(resp, keyCount, cs, nil) if err != nil { return nil, nil, WithStack(err) } @@ -572,7 +621,7 @@ func createMergeArray(keys, revs []string) ([]map[string]interface{}, error) { } // parseResponseArray parses an array response in the given response -func parseResponseArray(resp Response, count int, cs contextSettings) (DocumentMetaSlice, ErrorSlice, error) { +func parseResponseArray(resp Response, count int, cs contextSettings, results interface{}) (DocumentMetaSlice, ErrorSlice, error) { resps, err := resp.ParseArrayBody() if err != nil { return nil, nil, WithStack(err) @@ -581,6 +630,7 @@ func parseResponseArray(resp Response, count int, cs contextSettings) (DocumentM errs := make(ErrorSlice, count) returnOldVal := reflect.ValueOf(cs.ReturnOld) returnNewVal := reflect.ValueOf(cs.ReturnNew) + resultsVal := reflect.ValueOf(results) for i := 0; i < count; i++ { resp := resps[i] var meta DocumentMeta @@ -606,6 +656,13 @@ func parseResponseArray(resp Response, count int, cs contextSettings) (DocumentM } } } + if results != nil { + // Parse compare result document + resultsEntryVal := resultsVal.Index(i).Addr() + if err := resp.ParseBody("", resultsEntryVal.Interface()); err != nil { + errs[i] = err + } + } } } return metas, errs, nil diff --git a/vendor/github.com/arangodb/go-driver/collection_documents.go b/vendor/github.com/arangodb/go-driver/collection_documents.go index aef8bc2e..a600a053 100644 --- a/vendor/github.com/arangodb/go-driver/collection_documents.go +++ b/vendor/github.com/arangodb/go-driver/collection_documents.go @@ -34,6 +34,12 @@ type CollectionDocuments interface { // If no document exists with given key, a NotFoundError is returned. ReadDocument(ctx context.Context, key string, result interface{}) (DocumentMeta, error) + // ReadDocuments reads multiple documents with given keys from the collection. + // The documents data is stored into elements of the given results slice, + // the documents meta data is returned. + // If no document exists with a given key, a NotFoundError is returned at its errors index. + ReadDocuments(ctx context.Context, keys []string, results interface{}) (DocumentMetaSlice, ErrorSlice, error) + // CreateDocument creates a single document in the collection. // The document data is loaded from the given document, the document meta data is returned. // If the document data already contains a `_key` field, this will be used as key of the new document, diff --git a/vendor/github.com/arangodb/go-driver/context.go b/vendor/github.com/arangodb/go-driver/context.go index 162991b6..170e2d5d 100644 --- a/vendor/github.com/arangodb/go-driver/context.go +++ b/vendor/github.com/arangodb/go-driver/context.go @@ -56,6 +56,7 @@ const ( keyFollowLeaderRedirect ContextKey = "arangodb-followLeaderRedirect" keyDBServerID ContextKey = "arangodb-dbserverID" keyBatchID ContextKey = "arangodb-batchID" + keyJobIDResponse ContextKey = "arangodb-jobIDResponse" ) // WithRevision is used to configure a context to make document @@ -213,6 +214,13 @@ func WithBatchID(parent context.Context, id string) context.Context { return context.WithValue(contextOrBackground(parent), keyBatchID, id) } +// WithJobIDResponse is used to configure a context that includes a reference to a JobID +// that is filled on a error-free response. +// This is used in cluster functions. +func WithJobIDResponse(parent context.Context, jobID *string) context.Context { + return context.WithValue(contextOrBackground(parent), keyJobIDResponse, jobID) +} + type contextSettings struct { Silent bool WaitForSync bool @@ -229,6 +237,7 @@ type contextSettings struct { FollowLeaderRedirect *bool DBServerID string BatchID string + JobIDResponse *string } // applyContextSettings returns the settings configured in the context in the given request. @@ -356,17 +365,13 @@ func applyContextSettings(ctx context.Context, req Request) contextSettings { result.BatchID = id } } - return result -} - -// okStatus returns one of the given status codes depending on the WaitForSync field value. -// If WaitForSync==true, statusWithWaitForSync is returned, otherwise statusWithoutWaitForSync is returned. -func (cs contextSettings) okStatus(statusWithWaitForSync, statusWithoutWaitForSync int) int { - if cs.WaitForSync { - return statusWithWaitForSync - } else { - return statusWithoutWaitForSync + // JobIDResponse + if v := ctx.Value(keyJobIDResponse); v != nil { + if idRef, ok := v.(*string); ok { + result.JobIDResponse = idRef + } } + return result } // contextOrBackground returns the given context if it is not nil. diff --git a/vendor/github.com/arangodb/go-driver/cursor_impl.go b/vendor/github.com/arangodb/go-driver/cursor_impl.go index d7ab022b..805d4baf 100644 --- a/vendor/github.com/arangodb/go-driver/cursor_impl.go +++ b/vendor/github.com/arangodb/go-driver/cursor_impl.go @@ -24,7 +24,9 @@ package driver import ( "context" + "encoding/json" "path" + "reflect" "sync" "sync/atomic" "time" @@ -165,12 +167,23 @@ func (c *cursor) ReadDocument(ctx context.Context, result interface{}) (Document } c.resultIndex++ var meta DocumentMeta - if err := c.conn.Unmarshal(*c.Result[index], &meta); err != nil { - // If a cursor returns something other than a document, this will fail. - // Just ignore it. - } - if err := c.conn.Unmarshal(*c.Result[index], result); err != nil { - return DocumentMeta{}, WithStack(err) + resultPtr := c.Result[index] + if resultPtr == nil { + // Got NULL result + rv := reflect.ValueOf(result) + if rv.Kind() != reflect.Ptr || rv.IsNil() { + return DocumentMeta{}, WithStack(&json.InvalidUnmarshalError{Type: reflect.TypeOf(result)}) + } + e := rv.Elem() + e.Set(reflect.Zero(e.Type())) + } else { + if err := c.conn.Unmarshal(*resultPtr, &meta); err != nil { + // If a cursor returns something other than a document, this will fail. + // Just ignore it. + } + if err := c.conn.Unmarshal(*resultPtr, result); err != nil { + return DocumentMeta{}, WithStack(err) + } } return meta, nil } diff --git a/vendor/github.com/arangodb/go-driver/docs/Drivers/GO/ConnectionManagement/README.md b/vendor/github.com/arangodb/go-driver/docs/Drivers/GO/ConnectionManagement/README.md new file mode 100644 index 00000000..e9ee5af2 --- /dev/null +++ b/vendor/github.com/arangodb/go-driver/docs/Drivers/GO/ConnectionManagement/README.md @@ -0,0 +1,84 @@ +# ArangoDB GO Driver - Connection Management +## Failover + +The driver supports multiple endpoints to connect to. All request are in principle +send to the same endpoint until that endpoint fails to respond. +In that case a new endpoint is chosen and the operation is retried. + +The following example shows how to connect to a cluster of 3 servers. + +```go +conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"http://server1:8529", "http://server2:8529", "http://server3:8529"}, +}) +if err != nil { + // Handle error +} +c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, +}) +if err != nil { + // Handle error +} +``` + +Note that a valid endpoint is an URL to either a standalone server, or a URL to a coordinator +in a cluster. + +## Failover: Exact behavior + +The driver monitors the request being send to a specific server (endpoint). +As soon as the request has been completely written, failover will no longer happen. +The reason for that is that several operations cannot be (safely) retried. +E.g. when a request to create a document has been send to a server and a timeout +occurs, the driver has no way of knowing if the server did or did not create +the document in the database. + +If the driver detects that a request has been completely written, but still gets +an error (other than an error response from Arango itself), it will wrap the +error in a `ResponseError`. The client can test for such an error using `IsResponseError`. + +If a client received a `ResponseError`, it can do one of the following: +- Retry the operation and be prepared for some kind of duplicate record / unique constraint violation. +- Perform a test operation to see if the "failed" operation did succeed after all. +- Simply consider the operation failed. This is risky, since it can still be the case that the operation did succeed. + +## Failover: Timeouts + +To control the timeout of any function in the driver, you must pass it a context +configured with `context.WithTimeout` (or `context.WithDeadline`). + +In the case of multiple endpoints, the actual timeout used for requests will be shorter than +the timeout given in the context. +The driver will divide the timeout by the number of endpoints with a maximum of 3. +This ensures that the driver can try up to 3 different endpoints (in case of failover) without +being canceled due to the timeout given by the client. +E.g. +- With 1 endpoint and a given timeout of 1 minute, the actual request timeout will be 1 minute. +- With 3 endpoints and a given timeout of 1 minute, the actual request timeout will be 20 seconds. +- With 8 endpoints and a given timeout of 1 minute, the actual request timeout will be 20 seconds. + +For most requests you want a actual request timeout of at least 30 seconds. + +## Secure connections (SSL) + +The driver supports endpoints that use SSL using the `https` URL scheme. + +The following example shows how to connect to a server that has a secure endpoint using +a self-signed certificate. + +```go +conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"https://localhost:8529"}, + TLSConfig: &tls.Config{InsecureSkipVerify: true}, +}) +if err != nil { + // Handle error +} +c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, +}) +if err != nil { + // Handle error +} +``` diff --git a/vendor/github.com/arangodb/go-driver/docs/Drivers/GO/ExampleRequests/README.md b/vendor/github.com/arangodb/go-driver/docs/Drivers/GO/ExampleRequests/README.md new file mode 100644 index 00000000..e7f3f893 --- /dev/null +++ b/vendor/github.com/arangodb/go-driver/docs/Drivers/GO/ExampleRequests/README.md @@ -0,0 +1,182 @@ +# ArangoDB GO Driver - Example requests + +## Connecting to ArangoDB + +```go +conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"http://localhost:8529"}, + TLSConfig: &tls.Config{ /*...*/ }, +}) +if err != nil { + // Handle error +} +c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, + Authentication: driver.BasicAuthentication("user", "password"), +}) +if err != nil { + // Handle error +} +``` + +## Opening a database + +```go +ctx := context.Background() +db, err := client.Database(ctx, "myDB") +if err != nil { + // handle error +} +``` + +## Opening a collection + +```go +ctx := context.Background() +col, err := db.Collection(ctx, "myCollection") +if err != nil { + // handle error +} +``` + +## Checking if a collection exists + +```go +ctx := context.Background() +found, err := db.CollectionExists(ctx, "myCollection") +if err != nil { + // handle error +} +``` + +## Creating a collection + +```go +ctx := context.Background() +options := &driver.CreateCollectionOptions{ /* ... */ } +col, err := db.CreateCollection(ctx, "myCollection", options) +if err != nil { + // handle error +} +``` + +## Reading a document from a collection + +```go +var doc MyDocument +ctx := context.Background() +meta, err := col.ReadDocument(ctx, myDocumentKey, &doc) +if err != nil { + // handle error +} +``` + +## Reading a document from a collection with an explicit revision + +```go +var doc MyDocument +revCtx := driver.WithRevision(ctx, "mySpecificRevision") +meta, err := col.ReadDocument(revCtx, myDocumentKey, &doc) +if err != nil { + // handle error +} +``` + +## Creating a document + +```go +doc := MyDocument{ + Name: "jan", + Counter: 23, +} +ctx := context.Background() +meta, err := col.CreateDocument(ctx, doc) +if err != nil { + // handle error +} +fmt.Printf("Created document with key '%s', revision '%s'\n", meta.Key, meta.Rev) +``` + +## Removing a document + +```go +ctx := context.Background() +err := col.RemoveDocument(revCtx, myDocumentKey) +if err != nil { + // handle error +} +``` + +## Removing a document with an explicit revision + +```go +revCtx := driver.WithRevision(ctx, "mySpecificRevision") +err := col.RemoveDocument(revCtx, myDocumentKey) +if err != nil { + // handle error +} +``` + +## Updating a document + +```go +ctx := context.Background() +patch := map[string]interface{}{ + "Name": "Frank", +} +meta, err := col.UpdateDocument(ctx, myDocumentKey, patch) +if err != nil { + // handle error +} +``` + +## Querying documents, one document at a time + +```go +ctx := context.Background() +query := "FOR d IN myCollection LIMIT 10 RETURN d" +cursor, err := db.Query(ctx, query, nil) +if err != nil { + // handle error +} +defer cursor.Close() +for { + var doc MyDocument + meta, err := cursor.ReadDocument(ctx, &doc) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + // handle other errors + } + fmt.Printf("Got doc with key '%s' from query\n", meta.Key) +} +``` + +## Querying documents, fetching total count + +```go +ctx := driver.WithQueryCount(context.Background()) +query := "FOR d IN myCollection RETURN d" +cursor, err := db.Query(ctx, query, nil) +if err != nil { + // handle error +} +defer cursor.Close() +fmt.Printf("Query yields %d documents\n", cursor.Count()) +``` + +## Querying documents, with bind variables + +```go +ctx := context.Background() +query := "FOR d IN myCollection FILTER d.Name == @name RETURN d" +bindVars := map[string]interface{}{ + "name": "Some name", +} +cursor, err := db.Query(ctx, query, bindVars) +if err != nil { + // handle error +} +defer cursor.Close() +... +``` diff --git a/vendor/github.com/arangodb/go-driver/docs/Drivers/GO/GettingStarted/README.md b/vendor/github.com/arangodb/go-driver/docs/Drivers/GO/GettingStarted/README.md new file mode 100644 index 00000000..59e1172d --- /dev/null +++ b/vendor/github.com/arangodb/go-driver/docs/Drivers/GO/GettingStarted/README.md @@ -0,0 +1,141 @@ +# ArangoDB GO Driver - Getting Started + +## Supported versions + +- ArangoDB versions 3.1 and up. + - Single server & cluster setups + - With or without authentication +- Go 1.7 and up. + +## Go dependencies + +- None (Additional error libraries are supported). + +## Configuration + +To use the driver, first fetch the sources into your GOPATH. + +```sh +go get github.com/arangodb/go-driver +``` + +Using the driver, you always need to create a `Client`. +The following example shows how to create a `Client` for a single server +running on localhost. + +```go +import ( + "fmt" + + driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/http" +) + +... + +conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"http://localhost:8529"}, +}) +if err != nil { + // Handle error +} +c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, +}) +if err != nil { + // Handle error +} +``` + +Once you have a `Client` you can access/create databases on the server, +access/create collections, graphs, documents and so on. + +The following example shows how to open an existing collection in an existing database +and create a new document in that collection. + +```go +// Open "examples_books" database +db, err := c.Database(nil, "examples_books") +if err != nil { + // Handle error +} + +// Open "books" collection +col, err := db.Collection(nil, "books") +if err != nil { + // Handle error +} + +// Create document +book := Book{ + Title: "ArangoDB Cookbook", + NoPages: 257, +} +meta, err := col.CreateDocument(nil, book) +if err != nil { + // Handle error +} +fmt.Printf("Created document in collection '%s' in database '%s'\n", col.Name(), db.Name()) +``` + +## API design + +### Concurrency + +All functions of the driver are stricly synchronous. They operate and only return a value (or error) +when they're done. + +If you want to run operations concurrently, use a go routine. All objects in the driver are designed +to be used from multiple concurrent go routines, except `Cursor`. + +All database objects (except `Cursor`) are considered static. After their creation they won't change. +E.g. after creating a `Collection` instance you can remove the collection, but the (Go) instance +will still be there. Calling functions on such a removed collection will of course fail. + +### Structured error handling & wrapping + +All functions of the driver that can fail return an `error` value. If that value is not `nil`, the +function call is considered to be failed. In that case all other return values are set to their `zero` +values. + +All errors are structured using error checking functions named `Is`. +E.g. `IsNotFound(error)` return true if the given error is of the category "not found". +There can be multiple internal error codes that all map onto the same category. + +All errors returned from any function of the driver (either internal or exposed) wrap errors +using the `WithStack` function. This can be used to provide detail stack trackes in case of an error. +All error checking functions use the `Cause` function to get the cause of an error instead of the error wrapper. + +Note that `WithStack` and `Cause` are actually variables to you can implement it using your own error +wrapper library. + +If you for example use https://github.com/pkg/errors, you want to initialize to go driver like this: +```go +import ( + driver "github.com/arangodb/go-driver" + "github.com/pkg/errors" +) + +func init() { + driver.WithStack = errors.WithStack + driver.Cause = errors.Cause +} +``` + +### Context aware + +All functions of the driver that involve some kind of long running operation or +support additional options not given as function arguments, have a `context.Context` argument. +This enables you cancel running requests, pass timeouts/deadlines and pass additional options. + +In all methods that take a `context.Context` argument you can pass `nil` as value. +This is equivalent to passing `context.Background()`. + +Many functions support 1 or more optional (and infrequently used) additional options. +These can be used with a `With` function. +E.g. to force a create document call to wait until the data is synchronized to disk, +use a prepared context like this: +```go +ctx := driver.WithWaitForSync(parentContext) +collection.CreateDocument(ctx, yourDocument) +``` diff --git a/vendor/github.com/arangodb/go-driver/docs/Drivers/GO/README.md b/vendor/github.com/arangodb/go-driver/docs/Drivers/GO/README.md new file mode 100644 index 00000000..b98c1e2e --- /dev/null +++ b/vendor/github.com/arangodb/go-driver/docs/Drivers/GO/README.md @@ -0,0 +1,8 @@ +# ArangoDB GO Driver + +The official [ArangoDB](https://arangodb.com) GO Driver + +- [Getting Started](GettingStarted/README.md) +- [Example Requests](ExampleRequests/README.md) +- [Connection Management](ConnectionManagement/README.md) +- [Reference](https://godoc.org/github.com/arangodb/go-driver) diff --git a/vendor/github.com/arangodb/go-driver/edge_collection_documents_impl.go b/vendor/github.com/arangodb/go-driver/edge_collection_documents_impl.go index c09d5d7b..395cb32b 100644 --- a/vendor/github.com/arangodb/go-driver/edge_collection_documents_impl.go +++ b/vendor/github.com/arangodb/go-driver/edge_collection_documents_impl.go @@ -43,33 +43,89 @@ func (c *edgeCollection) DocumentExists(ctx context.Context, key string) (bool, // The document data is stored into result, the document meta data is returned. // If no document exists with given key, a NotFoundError is returned. func (c *edgeCollection) ReadDocument(ctx context.Context, key string, result interface{}) (DocumentMeta, error) { - if err := validateKey(key); err != nil { + meta, _, err := c.readDocument(ctx, key, result) + if err != nil { return DocumentMeta{}, WithStack(err) } + return meta, nil +} + +func (c *edgeCollection) readDocument(ctx context.Context, key string, result interface{}) (DocumentMeta, contextSettings, error) { + if err := validateKey(key); err != nil { + return DocumentMeta{}, contextSettings{}, WithStack(err) + } escapedKey := pathEscape(key) req, err := c.conn.NewRequest("GET", path.Join(c.relPath(), escapedKey)) if err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } + cs := applyContextSettings(ctx, req) resp, err := c.conn.Do(ctx, req) if err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } if err := resp.CheckStatus(200); err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } // Parse metadata var meta DocumentMeta if err := resp.ParseBody("edge", &meta); err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } // Parse result if result != nil { if err := resp.ParseBody("edge", result); err != nil { - return meta, WithStack(err) + return meta, contextSettings{}, WithStack(err) } } - return meta, nil + return meta, cs, nil +} + +// ReadDocuments reads multiple documents with given keys from the collection. +// The documents data is stored into elements of the given results slice, +// the documents meta data is returned. +// If no document exists with a given key, a NotFoundError is returned at its errors index. +func (c *edgeCollection) ReadDocuments(ctx context.Context, keys []string, results interface{}) (DocumentMetaSlice, ErrorSlice, error) { + resultsVal := reflect.ValueOf(results) + switch resultsVal.Kind() { + case reflect.Array, reflect.Slice: + // OK + default: + return nil, nil, WithStack(InvalidArgumentError{Message: fmt.Sprintf("results data must be of kind Array, got %s", resultsVal.Kind())}) + } + if keys == nil { + return nil, nil, WithStack(InvalidArgumentError{Message: "keys nil"}) + } + resultCount := resultsVal.Len() + if len(keys) != resultCount { + return nil, nil, WithStack(InvalidArgumentError{Message: fmt.Sprintf("expected %d keys, got %d", resultCount, len(keys))}) + } + for _, key := range keys { + if err := validateKey(key); err != nil { + return nil, nil, WithStack(err) + } + } + metas := make(DocumentMetaSlice, resultCount) + errs := make(ErrorSlice, resultCount) + silent := false + for i := 0; i < resultCount; i++ { + result := resultsVal.Index(i).Addr() + ctx, err := withDocumentAt(ctx, i) + if err != nil { + return nil, nil, WithStack(err) + } + key := keys[i] + meta, cs, err := c.readDocument(ctx, key, result.Interface()) + if cs.Silent { + silent = true + } else { + metas[i], errs[i] = meta, err + } + } + if silent { + return nil, nil, nil + } + return metas, errs, nil } // CreateDocument creates a single document in the collection. @@ -103,7 +159,7 @@ func (c *edgeCollection) createDocument(ctx context.Context, document interface{ if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { @@ -318,7 +374,7 @@ func (c *edgeCollection) replaceDocument(ctx context.Context, key string, docume if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { @@ -433,7 +489,7 @@ func (c *edgeCollection) removeDocument(ctx context.Context, key string) (Docume if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(200, 202)); err != nil { + if err := resp.CheckStatus(200, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { diff --git a/vendor/github.com/arangodb/go-driver/encode-go_1_8.go b/vendor/github.com/arangodb/go-driver/encode-go_1_8.go index 1fc2d834..1f224717 100644 --- a/vendor/github.com/arangodb/go-driver/encode-go_1_8.go +++ b/vendor/github.com/arangodb/go-driver/encode-go_1_8.go @@ -20,7 +20,7 @@ // Author Ewout Prangsma // -// +build "go1.8" +// +build go1.8 package driver @@ -33,5 +33,6 @@ func pathEscape(s string) string { // pathUnescape unescapes the given value for use in a URL path. func pathUnescape(s string) string { - return url.PathUnescape(s) + r, _ := url.PathUnescape(s) + return r } diff --git a/vendor/github.com/arangodb/go-driver/encode.go b/vendor/github.com/arangodb/go-driver/encode.go index a490ab94..af8d4bb2 100644 --- a/vendor/github.com/arangodb/go-driver/encode.go +++ b/vendor/github.com/arangodb/go-driver/encode.go @@ -20,7 +20,7 @@ // Author Ewout Prangsma // -// +build !"go1.8" +// +build !go1.8 package driver diff --git a/vendor/github.com/arangodb/go-driver/encode_test.go b/vendor/github.com/arangodb/go-driver/encode_test.go new file mode 100644 index 00000000..25539ce0 --- /dev/null +++ b/vendor/github.com/arangodb/go-driver/encode_test.go @@ -0,0 +1,38 @@ +// +// DISCLAIMER +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package driver + +import "testing" + +func TestPathEscape(t *testing.T) { + tests := map[string]string{ // Input : Expected-Output + "abc": "abc", + "The Donald": "The%20Donald", + } + for input, expected := range tests { + result := pathEscape(input) + if result != expected { + t.Errorf("pathEscapse failed for '%s': Expected '%s', got '%s'", input, expected, result) + } + } +} diff --git a/vendor/github.com/arangodb/go-driver/http/connection.go b/vendor/github.com/arangodb/go-driver/http/connection.go index 92ee9f4b..e4c42889 100644 --- a/vendor/github.com/arangodb/go-driver/http/connection.go +++ b/vendor/github.com/arangodb/go-driver/http/connection.go @@ -43,6 +43,7 @@ import ( const ( DefaultMaxIdleConnsPerHost = 64 + DefaultConnLimit = 32 keyRawResponse driver.ContextKey = "arangodb-rawResponse" keyResponse driver.ContextKey = "arangodb-response" @@ -76,6 +77,10 @@ type ConnectionConfig struct { cluster.ConnectionConfig // ContentType specified type of content encoding to use. ContentType driver.ContentType + // ConnLimit is the upper limit to the number of connections to a single server. + // The default is 32 (DefaultConnLimit). + // Set this value to -1 if you do not want any upper limit. + ConnLimit int } // NewConnection creates a new HTTP connection based on the given configuration settings. @@ -95,6 +100,9 @@ func NewConnection(config ConnectionConfig) (driver.Connection, error) { // newHTTPConnection creates a new HTTP connection for a single endpoint and the remainder of the given configuration settings. func newHTTPConnection(endpoint string, config ConnectionConfig) (driver.Connection, error) { + if config.ConnLimit == 0 { + config.ConnLimit = DefaultConnLimit + } endpoint = util.FixupEndpointURLScheme(endpoint) u, err := url.Parse(endpoint) if err != nil { @@ -154,10 +162,19 @@ func newHTTPConnection(endpoint string, config ConnectionConfig) (driver.Connect } } } + var connPool chan int + if config.ConnLimit > 0 { + connPool = make(chan int, config.ConnLimit) + // Fill with available tokens + for i := 0; i < config.ConnLimit; i++ { + connPool <- i + } + } c := &httpConnection{ endpoint: *u, contentType: config.ContentType, client: httpClient, + connPool: connPool, } return c, nil } @@ -167,6 +184,7 @@ type httpConnection struct { endpoint url.URL contentType driver.ContentType client *http.Client + connPool chan int } // String returns the endpoint as string @@ -225,6 +243,22 @@ func (c *httpConnection) Do(ctx context.Context, req driver.Request) (driver.Res if err != nil { return nil, driver.WithStack(err) } + + // Block on too many concurrent connections + if c.connPool != nil { + select { + case t := <-c.connPool: + // Ok, we're allowed to continue + defer func() { + // Give back token + c.connPool <- t + }() + case <-rctx.Done(): + // Context cancelled or expired + return nil, driver.WithStack(rctx.Err()) + } + } + resp, err := c.client.Do(r) if err != nil { return nil, driver.WithStack(err) diff --git a/vendor/github.com/arangodb/go-driver/http/doc.go b/vendor/github.com/arangodb/go-driver/http/doc.go new file mode 100644 index 00000000..0cffc2b0 --- /dev/null +++ b/vendor/github.com/arangodb/go-driver/http/doc.go @@ -0,0 +1,69 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package http implements driver.Connection using an HTTP connection. + +This connection uses HTTP or HTTPS to connect to the ArangoDB database and +encodes its content as JSON or Velocypack, depending on the value +of the `ContentType` fields in the `http.ConnectionConfig`. + +Creating an Insecure Connection + +To create an HTTP connection, use code like this. + + // Create an HTTP connection to the database + conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"http://localhost:8529"}, + }) + if err != nil { + // Handle error + } + +The resulting connection is used to create a client which you will use +for normal database requests. + + // Create a client + c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, + }) + if err != nil { + // Handle error + } + +Creating a Secure Connection + +To create a secure HTTPS connection, use code like this. + + // Create an HTTPS connection to the database + conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"https://localhost:8529"}, + TLSConfig: &tls.Config{ + InsecureSkipVerify: trueWhenUsingNonPublicCertificates, + }, + }) + if err != nil { + // Handle error + } + +*/ +package http diff --git a/vendor/github.com/arangodb/go-driver/jwt/doc.go b/vendor/github.com/arangodb/go-driver/jwt/doc.go index 4a9b04ce..7d226ee3 100644 --- a/vendor/github.com/arangodb/go-driver/jwt/doc.go +++ b/vendor/github.com/arangodb/go-driver/jwt/doc.go @@ -20,12 +20,38 @@ // Author Ewout Prangsma // -package jwt +/* +Package jwt provides a helper function used to access ArangoDB +servers using a JWT secret. -// -// This package provides a help function used to access ArangoDB -// servers using a JWT secret. -// -// Authenticating with a JWT secret results in "super-user" access -// to the database. -// +Authenticating with a JWT secret results in "super-user" access +to the database. + +To use a JWT secret to access your database, use code like this: + + // Create an HTTP connection to the database + conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"http://localhost:8529"}, + }) + if err != nil { + // Handle error + } + + // Prepare authentication + hdr, err := CreateArangodJwtAuthorizationHeader("yourJWTSecret", "yourUniqueServerID") + if err != nil { + // Handle error + } + auth := driver.RawAuthentication(hdr) + + // Create a client + c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, + Authentication: auth, + }) + if err != nil { + // Handle error + } + +*/ +package jwt diff --git a/vendor/github.com/arangodb/go-driver/test/client_test.go b/vendor/github.com/arangodb/go-driver/test/client_test.go index 722b6608..986237e8 100644 --- a/vendor/github.com/arangodb/go-driver/test/client_test.go +++ b/vendor/github.com/arangodb/go-driver/test/client_test.go @@ -25,6 +25,7 @@ package test import ( "context" "crypto/tls" + "log" httplib "net/http" "os" "strconv" @@ -33,6 +34,8 @@ import ( "testing" "time" + _ "net/http/pprof" + driver "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/http" "github.com/arangodb/go-driver/vst" @@ -40,7 +43,8 @@ import ( ) var ( - logEndpointsOnce sync.Once + logEndpointsOnce sync.Once + runPProfServerOnce sync.Once ) // skipBelowVersion skips the test if the current server version is less than @@ -152,6 +156,16 @@ func createConnectionFromEnv(t testEnv) driver.Connection { // createClientFromEnv initializes a Client from information specified in environment variables. func createClientFromEnv(t testEnv, waitUntilReady bool, connection ...*driver.Connection) driver.Client { + runPProfServerOnce.Do(func() { + if os.Getenv("TEST_PPROF") != "" { + go func() { + // Start pprof server on port 6060 + // To use it in the test, run a command like: + // docker exec -it go-driver-test sh -c "apk add -U curl && curl http://localhost:6060/debug/pprof/goroutine?debug=1" + log.Println(httplib.ListenAndServe("localhost:6060", nil)) + }() + } + }) conn := createConnectionFromEnv(t) if len(connection) == 1 { *connection[0] = conn diff --git a/vendor/github.com/arangodb/go-driver/test/concurrency_test.go b/vendor/github.com/arangodb/go-driver/test/concurrency_test.go new file mode 100644 index 00000000..6af0d484 --- /dev/null +++ b/vendor/github.com/arangodb/go-driver/test/concurrency_test.go @@ -0,0 +1,205 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package test + +import ( + "context" + "crypto/rand" + "encoding/hex" + "os" + "strconv" + "sync" + "testing" + + driver "github.com/arangodb/go-driver" +) + +// TestConcurrentCreateSmallDocuments make a lot of concurrent CreateDocument calls. +// It then verifies that all documents "have arrived". +func TestConcurrentCreateSmallDocuments(t *testing.T) { + if testing.Short() { + t.Skip("Skip on short tests") + } + c := createClientFromEnv(t, true) + + version, err := c.Version(nil) + if err != nil { + t.Fatalf("Version failed: %s", describe(err)) + } + isv33p := version.Version.CompareTo("3.3") >= 0 + if !isv33p && os.Getenv("TEST_CONNECTION") == "vst" { + t.Skip("Skipping VST load test on 3.2") + } else { + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestConcurrentCreateSmallDocuments", nil, t) + + docChan := make(chan driver.DocumentMeta, 16*1024) + + creator := func(limit, interval int) { + for i := 0; i < limit; i++ { + ctx := context.Background() + doc := UserDoc{ + "Jan", + i * interval, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + docChan <- meta + } + } + + reader := func() { + for { + meta, ok := <-docChan + if !ok { + return + } + // Document must exists now + if found, err := col.DocumentExists(nil, meta.Key); err != nil { + t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err)) + } else if !found { + t.Errorf("DocumentExists returned false for '%s', expected true", meta.Key) + } + // Read document + var readDoc UserDoc + if _, err := col.ReadDocument(nil, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + } + } + + noCreators := getIntFromEnv("NOCREATORS", 25) + noReaders := getIntFromEnv("NOREADERS", 50) + noDocuments := getIntFromEnv("NODOCUMENTS", 1000) // per creator + + wgCreators := sync.WaitGroup{} + // Run N concurrent creators + for i := 0; i < noCreators; i++ { + wgCreators.Add(1) + go func() { + defer wgCreators.Done() + creator(noDocuments, noCreators) + }() + } + wgReaders := sync.WaitGroup{} + // Run M readers + for i := 0; i < noReaders; i++ { + wgReaders.Add(1) + go func() { + defer wgReaders.Done() + reader() + }() + } + wgCreators.Wait() + close(docChan) + wgReaders.Wait() + } +} + +// TestConcurrentCreateBigDocuments make a lot of concurrent CreateDocument calls. +// It then verifies that all documents "have arrived". +func TestConcurrentCreateBigDocuments(t *testing.T) { + if testing.Short() { + t.Skip("Skip on short tests") + } + c := createClientFromEnv(t, true) + + version, err := c.Version(nil) + if err != nil { + t.Fatalf("Version failed: %s", describe(err)) + } + isv33p := version.Version.CompareTo("3.3") >= 0 + if !isv33p && os.Getenv("TEST_CONNECTION") == "vst" { + t.Skip("Skipping VST load test on 3.2") + } else { + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestConcurrentCreateBigDocuments", nil, t) + + docChan := make(chan driver.DocumentMeta, 16*1024) + + creator := func(limit, interval int) { + data := make([]byte, 1024) + for i := 0; i < limit; i++ { + rand.Read(data) + ctx := context.Background() + doc := UserDoc{ + "Jan" + strconv.Itoa(i) + hex.EncodeToString(data), + i * interval, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + docChan <- meta + } + } + + reader := func() { + for { + meta, ok := <-docChan + if !ok { + return + } + // Document must exists now + if found, err := col.DocumentExists(nil, meta.Key); err != nil { + t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err)) + } else if !found { + t.Errorf("DocumentExists returned false for '%s', expected true", meta.Key) + } + // Read document + var readDoc UserDoc + if _, err := col.ReadDocument(nil, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + } + } + + noCreators := getIntFromEnv("NOCREATORS", 25) + noReaders := getIntFromEnv("NOREADERS", 50) + noDocuments := getIntFromEnv("NODOCUMENTS", 100) // per creator + + wgCreators := sync.WaitGroup{} + // Run N concurrent creators + for i := 0; i < noCreators; i++ { + wgCreators.Add(1) + go func() { + defer wgCreators.Done() + creator(noDocuments, noCreators) + }() + } + wgReaders := sync.WaitGroup{} + // Run M readers + for i := 0; i < noReaders; i++ { + wgReaders.Add(1) + go func() { + defer wgReaders.Done() + reader() + }() + } + wgCreators.Wait() + close(docChan) + wgReaders.Wait() + } +} diff --git a/vendor/github.com/arangodb/go-driver/test/cursor_test.go b/vendor/github.com/arangodb/go-driver/test/cursor_test.go index ec028b1d..2c806fae 100644 --- a/vendor/github.com/arangodb/go-driver/test/cursor_test.go +++ b/vendor/github.com/arangodb/go-driver/test/cursor_test.go @@ -229,10 +229,32 @@ func TestCreateCursor(t *testing.T) { } } +// TestCreateCursorReturnNull creates a cursor with a `RETURN NULL` query. +func TestCreateCursorReturnNull(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "cursor_test", nil, t) + + var result interface{} + query := "return null" + cursor, err := db.Query(ctx, query, nil) + if err != nil { + t.Fatalf("Query(return null) failed: %s", describe(err)) + } + defer cursor.Close() + if _, err := cursor.ReadDocument(ctx, &result); err != nil { + t.Fatalf("ReadDocument failed: %s", describe(err)) + } + if result != nil { + t.Errorf("Expected result to be nil, got %#v", result) + } +} + // Test stream query cursors. The goroutines are technically only // relevant for the MMFiles engine, but don't hurt on rocksdb either func TestCreateStreamCursor(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() c := createClientFromEnv(t, true) version, err := c.Version(nil) @@ -260,6 +282,7 @@ func TestCreateStreamCursor(t *testing.T) { t.Fatalf("Expected success, got %s", describe(err)) } } + t.Log("Completed inserting 10k docs") const expectedResults int = 10 * 10000 query := "FOR doc IN cursor_stream_test RETURN doc" @@ -269,14 +292,10 @@ func TestCreateStreamCursor(t *testing.T) { // create a bunch of read-only cursors for i := 0; i < 10; i++ { cursor, err := db.Query(ctx2, query, nil) - if err == nil { - // Close upon exit of the function - defer cursor.Close() - } if err != nil { - t.Errorf("Expected success in query %d (%s), got '%s'", i, query, describe(err)) - continue + t.Fatalf("Expected success in query %d (%s), got '%s'", i, query, describe(err)) } + defer cursor.Close() count := cursor.Count() if count != 0 { t.Errorf("Expected count of 0, got %d in query %d (%s)", count, i, query) @@ -293,30 +312,27 @@ func TestCreateStreamCursor(t *testing.T) { cursors = append(cursors, cursor) } - out := make(chan bool) - defer close(out) + t.Logf("Created %d cursors", len(cursors)) // start a write query on the same collection inbetween // contrary to normal cursors which are executed right // away this will block until all read cursors are resolved + testReady := make(chan bool) go func() { query = "FOR doc IN 1..5 LET y = SLEEP(0.01) INSERT {name:'Peter', age:0} INTO cursor_stream_test" cursor, err := db.Query(ctx2, query, nil) // should not return immediately - if err == nil { - // Close upon exit of the function - defer cursor.Close() - } if err != nil { - t.Errorf("Expected success in write-query %s, got '%s'", query, describe(err)) + t.Fatalf("Expected success in write-query %s, got '%s'", query, describe(err)) } + defer cursor.Close() for cursor.HasMore() { var data interface{} if _, err := cursor.ReadDocument(ctx2, &data); err != nil { - t.Errorf("Failed to read document, err: %s", describe(err)) + t.Fatalf("Failed to read document, err: %s", describe(err)) } } - out <- true // signal write done + testReady <- true // signal write done }() readCount := 0 @@ -326,22 +342,26 @@ func TestCreateStreamCursor(t *testing.T) { for cursor.HasMore() { var user UserDoc if _, err := cursor.ReadDocument(ctx2, &user); err != nil { - t.Errorf("Failed to result document %d: %s", i, describe(err)) + t.Fatalf("Failed to result document %d: %s", i, describe(err)) } readCount++ } } - out <- false // signal read done + testReady <- false // signal read done }() writeDone := false readDone := false for { - done := <-out - if done { - writeDone = true - } else { - readDone = true + select { + case <-ctx.Done(): + t.Fatal("Timeout") + case v := <-testReady: + if v { + writeDone = true + } else { + readDone = true + } } // On MMFiles the read-cursors have to finish first if writeDone && !readDone && info.Type == driver.EngineTypeMMFiles { @@ -349,6 +369,7 @@ func TestCreateStreamCursor(t *testing.T) { } if writeDone && readDone { + close(testReady) break } } diff --git a/vendor/github.com/arangodb/go-driver/test/doc.go b/vendor/github.com/arangodb/go-driver/test/doc.go new file mode 100644 index 00000000..45574733 --- /dev/null +++ b/vendor/github.com/arangodb/go-driver/test/doc.go @@ -0,0 +1,26 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package test implements add tests for the go-driver. +*/ +package test diff --git a/vendor/github.com/arangodb/go-driver/test/document_create_test.go b/vendor/github.com/arangodb/go-driver/test/document_create_test.go index 3d5b2e17..c9a2fadd 100644 --- a/vendor/github.com/arangodb/go-driver/test/document_create_test.go +++ b/vendor/github.com/arangodb/go-driver/test/document_create_test.go @@ -156,3 +156,35 @@ func TestCreateDocumentNil(t *testing.T) { t.Fatalf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestCreateDocumentInWaitForSyncCollection creates a document in a collection with waitForSync enabled, +// and then checks that it exists. +func TestCreateDocumentInWaitForSyncCollection(t *testing.T) { + c := createClientFromEnv(t, true) + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestCreateDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + doc := UserDoc{ + "Jan", + 40, + } + meta, err := col.CreateDocument(nil, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + // Document must exists now + if found, err := col.DocumentExists(nil, meta.Key); err != nil { + t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err)) + } else if !found { + t.Errorf("DocumentExists returned false for '%s', expected true", meta.Key) + } + // Read document + var readDoc UserDoc + if _, err := col.ReadDocument(nil, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + if !reflect.DeepEqual(doc, readDoc) { + t.Errorf("Got wrong document. Expected %+v, got %+v", doc, readDoc) + } +} diff --git a/vendor/github.com/arangodb/go-driver/test/document_remove_test.go b/vendor/github.com/arangodb/go-driver/test/document_remove_test.go index 8b3b227d..bd01d199 100644 --- a/vendor/github.com/arangodb/go-driver/test/document_remove_test.go +++ b/vendor/github.com/arangodb/go-driver/test/document_remove_test.go @@ -169,3 +169,36 @@ func TestRemoveDocumentKeyEmpty(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestReplaceDocumentInWaitForSyncCollection creates a document in a collection with waitForSync enabled, +// removes it and then checks the removal has succeeded. +func TestRemoveDocumentInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestRemoveDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + doc := UserDoc{ + "Piere", + 23, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + if _, err := col.RemoveDocument(ctx, meta.Key); err != nil { + t.Fatalf("Failed to remove document '%s': %s", meta.Key, describe(err)) + } + // Should not longer exist + var readDoc Account + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); !driver.IsNotFound(err) { + t.Fatalf("Expected NotFoundError, got %s", describe(err)) + } + // Document must exists now + if found, err := col.DocumentExists(ctx, meta.Key); err != nil { + t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err)) + } else if found { + t.Errorf("DocumentExists returned true for '%s', expected false", meta.Key) + } +} diff --git a/vendor/github.com/arangodb/go-driver/test/document_replace_test.go b/vendor/github.com/arangodb/go-driver/test/document_replace_test.go index f6d7bbeb..d033e219 100644 --- a/vendor/github.com/arangodb/go-driver/test/document_replace_test.go +++ b/vendor/github.com/arangodb/go-driver/test/document_replace_test.go @@ -214,3 +214,38 @@ func TestReplaceDocumentUpdateNil(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestReplaceDocumentInWaitForSyncCollection creates a document in a collection with waitForSync enabled, +// replaces it and then checks the replacement has succeeded. +func TestReplaceDocumentInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestReplaceDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + doc := UserDoc{ + "Piere", + 23, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + // Replacement doc + replacement := Account{ + ID: "foo", + User: &UserDoc{}, + } + if _, err := col.ReplaceDocument(ctx, meta.Key, replacement); err != nil { + t.Fatalf("Failed to replace document '%s': %s", meta.Key, describe(err)) + } + // Read replaces document + var readDoc Account + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + if !reflect.DeepEqual(replacement, readDoc) { + t.Errorf("Got wrong document. Expected %+v, got %+v", replacement, readDoc) + } +} diff --git a/vendor/github.com/arangodb/go-driver/test/document_update_test.go b/vendor/github.com/arangodb/go-driver/test/document_update_test.go index 6c3a7fe5..ab70f400 100644 --- a/vendor/github.com/arangodb/go-driver/test/document_update_test.go +++ b/vendor/github.com/arangodb/go-driver/test/document_update_test.go @@ -296,3 +296,38 @@ func TestUpdateDocumentUpdateNil(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestUpdateDocumentInWaitForSyncCollection creates a document in a collection with waitForSync enabled, +// updates it and then checks the update has succeeded. +func TestUpdateDocumentInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestUpdateDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + doc := UserDoc{ + "Piere", + 23, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + // Update document + update := map[string]interface{}{ + "name": "Updated", + } + if _, err := col.UpdateDocument(ctx, meta.Key, update); err != nil { + t.Fatalf("Failed to update document '%s': %s", meta.Key, describe(err)) + } + // Read updated document + var readDoc UserDoc + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + doc.Name = "Updated" + if !reflect.DeepEqual(doc, readDoc) { + t.Errorf("Got wrong document. Expected %+v, got %+v", doc, readDoc) + } +} diff --git a/vendor/github.com/arangodb/go-driver/test/documents_create_test.go b/vendor/github.com/arangodb/go-driver/test/documents_create_test.go index e10b3607..1ee066ac 100644 --- a/vendor/github.com/arangodb/go-driver/test/documents_create_test.go +++ b/vendor/github.com/arangodb/go-driver/test/documents_create_test.go @@ -55,6 +55,21 @@ func TestCreateDocuments(t *testing.T) { } else if len(metas) != len(docs) { t.Errorf("Expected %d metas, got %d", len(docs), len(metas)) } else { + // Read back using ReadDocuments + keys := make([]string, len(docs)) + for i, m := range metas { + keys[i] = m.Key + } + readDocs := make([]UserDoc, len(docs)) + if _, _, err := col.ReadDocuments(nil, keys, readDocs); err != nil { + t.Fatalf("Failed to read documents: %s", describe(err)) + } + for i, d := range readDocs { + if !reflect.DeepEqual(docs[i], d) { + t.Errorf("Got wrong document. Expected %+v, got %+v", docs[i], d) + } + } + // Read back using individual ReadDocument requests for i := 0; i < len(docs); i++ { if err := errs[i]; err != nil { t.Errorf("Expected no error at index %d, got %s", i, describe(err)) @@ -167,3 +182,47 @@ func TestCreateDocumentsNonSlice(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestCreateDocumentsInWaitForSyncCollection creates a few documents in a collection with waitForSync enabled and then checks that it exists. +func TestCreateDocumentsInWaitForSyncCollection(t *testing.T) { + c := createClientFromEnv(t, true) + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestCreateDocumentsInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDoc{ + UserDoc{ + "Jan", + 40, + }, + UserDoc{ + "Foo", + 41, + }, + UserDoc{ + "Frank", + 42, + }, + } + metas, errs, err := col.CreateDocuments(nil, docs) + if err != nil { + t.Fatalf("Failed to create new documents: %s", describe(err)) + } else if len(metas) != len(docs) { + t.Errorf("Expected %d metas, got %d", len(docs), len(metas)) + } else { + for i := 0; i < len(docs); i++ { + if err := errs[i]; err != nil { + t.Errorf("Expected no error at index %d, got %s", i, describe(err)) + } + + // Document must exists now + var readDoc UserDoc + if _, err := col.ReadDocument(nil, metas[i].Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", metas[i].Key, describe(err)) + } + if !reflect.DeepEqual(docs[i], readDoc) { + t.Errorf("Got wrong document. Expected %+v, got %+v", docs[i], readDoc) + } + } + } +} diff --git a/vendor/github.com/arangodb/go-driver/test/documents_import_test.go b/vendor/github.com/arangodb/go-driver/test/documents_import_test.go index 1d084041..414fb793 100644 --- a/vendor/github.com/arangodb/go-driver/test/documents_import_test.go +++ b/vendor/github.com/arangodb/go-driver/test/documents_import_test.go @@ -559,3 +559,47 @@ func TestImportDocumentsOverwriteNo(t *testing.T) { } } } + +// TestImportDocumentsWithKeysInWaitForSyncCollection imports documents into a collection with waitForSync enabled +// and then checks that it exists. +func TestImportDocumentsWithKeysInWaitForSyncCollection(t *testing.T) { + c := createClientFromEnv(t, true) + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestImportDocumentsWithKeysInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDocWithKey{ + UserDocWithKey{ + "jan", + "Jan", + 40, + }, + UserDocWithKey{ + "foo", + "Foo", + 41, + }, + UserDocWithKey{ + "frank", + "Frank", + 42, + }, + } + + var raw []byte + ctx := driver.WithRawResponse(nil, &raw) + stats, err := col.ImportDocuments(ctx, docs, nil) + if err != nil { + t.Fatalf("Failed to import documents: %s %#v", describe(err), err) + } else { + if stats.Created != int64(len(docs)) { + t.Errorf("Expected %d created documents, got %d (json %s)", len(docs), stats.Created, formatRawResponse(raw)) + } + if stats.Errors != 0 { + t.Errorf("Expected %d error documents, got %d (json %s)", 0, stats.Errors, formatRawResponse(raw)) + } + if stats.Empty != 0 { + t.Errorf("Expected %d empty documents, got %d (json %s)", 0, stats.Empty, formatRawResponse(raw)) + } + } +} diff --git a/vendor/github.com/arangodb/go-driver/test/documents_remove_test.go b/vendor/github.com/arangodb/go-driver/test/documents_remove_test.go index c3239181..a52f6936 100644 --- a/vendor/github.com/arangodb/go-driver/test/documents_remove_test.go +++ b/vendor/github.com/arangodb/go-driver/test/documents_remove_test.go @@ -227,3 +227,36 @@ func TestRemoveDocumentsKeyEmpty(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestRemoveDocumentsInWaitForSyncCollection creates documents in a collection with waitForSync enabled, +// removes them and then checks the removal has succeeded. +func TestRemoveDocumentsInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestRemoveDocumentsInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDoc{ + UserDoc{ + "Piere", + 23, + }, + } + metas, errs, err := col.CreateDocuments(ctx, docs) + if err != nil { + t.Fatalf("Failed to create new documents: %s", describe(err)) + } else if err := errs.FirstNonNil(); err != nil { + t.Fatalf("Expected no errors, got first: %s", describe(err)) + } + if _, _, err := col.RemoveDocuments(ctx, metas.Keys()); err != nil { + t.Fatalf("Failed to remove documents: %s", describe(err)) + } + // Should not longer exist + for i, meta := range metas { + var readDoc Account + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); !driver.IsNotFound(err) { + t.Fatalf("Expected NotFoundError at %d, got %s", i, describe(err)) + } + } +} diff --git a/vendor/github.com/arangodb/go-driver/test/documents_replace_test.go b/vendor/github.com/arangodb/go-driver/test/documents_replace_test.go index 2240a9eb..19748560 100644 --- a/vendor/github.com/arangodb/go-driver/test/documents_replace_test.go +++ b/vendor/github.com/arangodb/go-driver/test/documents_replace_test.go @@ -329,3 +329,54 @@ func TestReplaceDocumentsUpdateLenDiff(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestReplaceDocumentsInWaitForSyncCollection creates documents into a collection with waitForSync enabled, +// replaces them and then checks the replacements have succeeded. +func TestReplaceDocumentsInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestReplaceDocumentsInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDoc{ + UserDoc{ + "Piere", + 23, + }, + UserDoc{ + "Pioter", + 45, + }, + } + metas, errs, err := col.CreateDocuments(ctx, docs) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } else if err := errs.FirstNonNil(); err != nil { + t.Fatalf("Expected no errors, got first: %s", describe(err)) + } + // Replacement docs + replacements := []Account{ + Account{ + ID: "foo", + User: &UserDoc{}, + }, + Account{ + ID: "foo2", + User: &UserDoc{}, + }, + } + if _, _, err := col.ReplaceDocuments(ctx, metas.Keys(), replacements); err != nil { + t.Fatalf("Failed to replace documents: %s", describe(err)) + } + // Read replaced documents + for i, meta := range metas { + var readDoc Account + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + if !reflect.DeepEqual(replacements[i], readDoc) { + t.Errorf("Got wrong document %d. Expected %+v, got %+v", i, replacements[i], readDoc) + } + } +} diff --git a/vendor/github.com/arangodb/go-driver/test/documents_update_test.go b/vendor/github.com/arangodb/go-driver/test/documents_update_test.go index f601b375..a4ccad15 100644 --- a/vendor/github.com/arangodb/go-driver/test/documents_update_test.go +++ b/vendor/github.com/arangodb/go-driver/test/documents_update_test.go @@ -452,3 +452,54 @@ func TestUpdateDocumentsUpdateLenDiff(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestUpdateDocumentsInWaitForSyncCollection creates documents in a collection with waitForSync enabled, +// updates them and then checks the updates have succeeded. +func TestUpdateDocumentsInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestUpdateDocumentsInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDoc{ + UserDoc{ + "Piere", + 23, + }, + UserDoc{ + "Otto", + 43, + }, + } + metas, errs, err := col.CreateDocuments(ctx, docs) + if err != nil { + t.Fatalf("Failed to create new documents: %s", describe(err)) + } else if err := errs.FirstNonNil(); err != nil { + t.Fatalf("Expected no errors, got first: %s", describe(err)) + } + // Update documents + updates := []map[string]interface{}{ + map[string]interface{}{ + "name": "Updated1", + }, + map[string]interface{}{ + "name": "Updated2", + }, + } + if _, _, err := col.UpdateDocuments(ctx, metas.Keys(), updates); err != nil { + t.Fatalf("Failed to update documents: %s", describe(err)) + } + // Read updated documents + for i, meta := range metas { + var readDoc UserDoc + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + doc := docs[i] + doc.Name = fmt.Sprintf("Updated%d", i+1) + if !reflect.DeepEqual(doc, readDoc) { + t.Errorf("Got wrong document %d. Expected %+v, got %+v", i, doc, readDoc) + } + } +} diff --git a/vendor/github.com/arangodb/go-driver/test/edges_create_test.go b/vendor/github.com/arangodb/go-driver/test/edges_create_test.go index 2fab94d2..8875b93c 100644 --- a/vendor/github.com/arangodb/go-driver/test/edges_create_test.go +++ b/vendor/github.com/arangodb/go-driver/test/edges_create_test.go @@ -66,6 +66,21 @@ func TestCreateEdges(t *testing.T) { } else if len(metas) != len(docs) { t.Errorf("Expected %d metas, got %d", len(docs), len(metas)) } else { + // Read back using ReadDocuments + keys := make([]string, len(docs)) + for i, m := range metas { + keys[i] = m.Key + } + readDocs := make([]RouteEdge, len(docs)) + if _, _, err := ec.ReadDocuments(nil, keys, readDocs); err != nil { + t.Fatalf("Failed to read documents: %s", describe(err)) + } + for i, d := range readDocs { + if !reflect.DeepEqual(docs[i], d) { + t.Errorf("Got wrong document. Expected %+v, got %+v", docs[i], d) + } + } + // Read back using individual ReadDocument requests for i := 0; i < len(docs); i++ { if err := errs[i]; err != nil { t.Errorf("Expected no error at index %d, got %s", i, describe(err)) diff --git a/vendor/github.com/arangodb/go-driver/test/user_auth_test.go b/vendor/github.com/arangodb/go-driver/test/user_auth_test.go index 810da682..cc60a12b 100644 --- a/vendor/github.com/arangodb/go-driver/test/user_auth_test.go +++ b/vendor/github.com/arangodb/go-driver/test/user_auth_test.go @@ -257,35 +257,69 @@ func TestGrantUserDefaultDatabase(t *testing.T) { t.Fatalf("SetDatabaseAccess failed: %s", describe(err)) } - // wait for change to propagate (TODO add a check to the coordinators) - time.Sleep(time.Second * 5) - - // Try to create document in collection, should fail because there are no collection grants for this user and/or collection. - if _, err := authCol.CreateDocument(nil, Book{Title: "I cannot write"}); !driver.IsForbidden(err) { - t.Errorf("Expected failure, got %s", describe(err)) - } + // wait for change to propagate + { + deadline := time.Now().Add(time.Minute) + for { + // Try to create document in collection, should fail because there are no collection grants for this user and/or collection. + if _, err := authCol.CreateDocument(nil, Book{Title: "I cannot write"}); err == nil { + if time.Now().Before(deadline) { + t.Logf("Expected failure, got %s, trying again...", describe(err)) + time.Sleep(time.Second * 2) + continue + } + t.Errorf("Expected failure, got %s", describe(err)) + } - // Try to create collection, should fail - if _, err := authDb.CreateCollection(nil, "books_def_ro_db", nil); !driver.IsForbidden(err) { - t.Errorf("Expected failure, got %s", describe(err)) + // Try to create collection, should fail + if _, err := authDb.CreateCollection(nil, "books_def_ro_db", nil); err == nil { + t.Errorf("Expected failure, got %s", describe(err)) + } + break + } } // Grant no access to default database if err := u.SetDatabaseAccess(nil, nil, driver.GrantNone); err != nil { t.Fatalf("SetDatabaseAccess failed: %s", describe(err)) } - // Try to create collection, should fail - if _, err := authDb.CreateCollection(nil, "books_def_none_db", nil); !driver.IsUnauthorized(err) { - t.Errorf("Expected failure, got %s", describe(err)) + + // wait for change to propagate + { + deadline := time.Now().Add(time.Minute) + for { + // Try to create collection, should fail + if _, err := authDb.CreateCollection(nil, "books_def_none_db", nil); err == nil { + if time.Now().Before(deadline) { + t.Logf("Expected failure, got %s, trying again...", describe(err)) + time.Sleep(time.Second * 2) + continue + } + t.Errorf("Expected failure, got %s", describe(err)) + } + break + } } // Remove default database access, should fallback to "no-access" then if err := u.RemoveDatabaseAccess(nil, nil); err != nil { t.Fatalf("RemoveDatabaseAccess failed: %s", describe(err)) } - // Try to create collection, should fail - if _, err := authDb.CreateCollection(nil, "books_def_star_db", nil); !driver.IsUnauthorized(err) { - t.Errorf("Expected failure, got %s", describe(err)) + // wait for change to propagate + { + deadline := time.Now().Add(time.Minute) + for { + // Try to create collection, should fail + if _, err := authDb.CreateCollection(nil, "books_def_star_db", nil); err == nil { + if time.Now().Before(deadline) { + t.Logf("Expected failure, got %s, trying again...", describe(err)) + time.Sleep(time.Second * 2) + continue + } + t.Errorf("Expected failure, got %s", describe(err)) + } + break + } } } diff --git a/vendor/github.com/arangodb/go-driver/test/util.go b/vendor/github.com/arangodb/go-driver/test/util.go index 3d94ce88..40ea6d03 100644 --- a/vendor/github.com/arangodb/go-driver/test/util.go +++ b/vendor/github.com/arangodb/go-driver/test/util.go @@ -26,6 +26,9 @@ import ( "encoding/hex" "encoding/json" "fmt" + "os" + "strconv" + "strings" "testing" driver "github.com/arangodb/go-driver" @@ -81,3 +84,16 @@ func formatRawResponse(raw []byte) string { } return hex.EncodeToString(raw) } + +// getIntFromEnv looks for an environment variable with given key. +// If found, it parses the value to an int, if success that value is returned. +// In all other cases, the given default value is returned. +func getIntFromEnv(envKey string, defaultValue int) int { + v := strings.TrimSpace(os.Getenv(envKey)) + if v != "" { + if result, err := strconv.Atoi(v); err == nil { + return result + } + } + return defaultValue +} diff --git a/vendor/github.com/arangodb/go-driver/test/vertices_create_test.go b/vendor/github.com/arangodb/go-driver/test/vertices_create_test.go index db10b574..bde76663 100644 --- a/vendor/github.com/arangodb/go-driver/test/vertices_create_test.go +++ b/vendor/github.com/arangodb/go-driver/test/vertices_create_test.go @@ -55,6 +55,21 @@ func TestCreateVertices(t *testing.T) { } else if len(metas) != len(docs) { t.Errorf("Expected %d metas, got %d", len(docs), len(metas)) } else { + // Read back using ReadDocuments + keys := make([]string, len(docs)) + for i, m := range metas { + keys[i] = m.Key + } + readDocs := make([]Book, len(docs)) + if _, _, err := vc.ReadDocuments(nil, keys, readDocs); err != nil { + t.Fatalf("Failed to read documents: %s", describe(err)) + } + for i, d := range readDocs { + if !reflect.DeepEqual(docs[i], d) { + t.Errorf("Got wrong document. Expected %+v, got %+v", docs[i], d) + } + } + // Read back using individual ReadDocument requests for i := 0; i < len(docs); i++ { if err := errs[i]; err != nil { t.Errorf("Expected no error at index %d, got %s", i, describe(err)) diff --git a/vendor/github.com/arangodb/go-driver/util/doc.go b/vendor/github.com/arangodb/go-driver/util/doc.go new file mode 100644 index 00000000..b16feff9 --- /dev/null +++ b/vendor/github.com/arangodb/go-driver/util/doc.go @@ -0,0 +1,26 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package util provides some helper methods for the go-driver (it is unlikely that you need this package directly). +*/ +package util diff --git a/vendor/github.com/arangodb/go-driver/vertex_collection_documents_impl.go b/vendor/github.com/arangodb/go-driver/vertex_collection_documents_impl.go index 233231dc..cf036afe 100644 --- a/vendor/github.com/arangodb/go-driver/vertex_collection_documents_impl.go +++ b/vendor/github.com/arangodb/go-driver/vertex_collection_documents_impl.go @@ -42,33 +42,89 @@ func (c *vertexCollection) DocumentExists(ctx context.Context, key string) (bool // The document data is stored into result, the document meta data is returned. // If no document exists with given key, a NotFoundError is returned. func (c *vertexCollection) ReadDocument(ctx context.Context, key string, result interface{}) (DocumentMeta, error) { - if err := validateKey(key); err != nil { + meta, _, err := c.readDocument(ctx, key, result) + if err != nil { return DocumentMeta{}, WithStack(err) } + return meta, nil +} + +func (c *vertexCollection) readDocument(ctx context.Context, key string, result interface{}) (DocumentMeta, contextSettings, error) { + if err := validateKey(key); err != nil { + return DocumentMeta{}, contextSettings{}, WithStack(err) + } escapedKey := pathEscape(key) req, err := c.conn.NewRequest("GET", path.Join(c.relPath(), escapedKey)) if err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } + cs := applyContextSettings(ctx, req) resp, err := c.conn.Do(ctx, req) if err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } if err := resp.CheckStatus(200); err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } // Parse metadata var meta DocumentMeta if err := resp.ParseBody("vertex", &meta); err != nil { - return DocumentMeta{}, WithStack(err) + return DocumentMeta{}, contextSettings{}, WithStack(err) } // Parse result if result != nil { if err := resp.ParseBody("vertex", result); err != nil { - return meta, WithStack(err) + return meta, contextSettings{}, WithStack(err) } } - return meta, nil + return meta, cs, nil +} + +// ReadDocuments reads multiple documents with given keys from the collection. +// The documents data is stored into elements of the given results slice, +// the documents meta data is returned. +// If no document exists with a given key, a NotFoundError is returned at its errors index. +func (c *vertexCollection) ReadDocuments(ctx context.Context, keys []string, results interface{}) (DocumentMetaSlice, ErrorSlice, error) { + resultsVal := reflect.ValueOf(results) + switch resultsVal.Kind() { + case reflect.Array, reflect.Slice: + // OK + default: + return nil, nil, WithStack(InvalidArgumentError{Message: fmt.Sprintf("results data must be of kind Array, got %s", resultsVal.Kind())}) + } + if keys == nil { + return nil, nil, WithStack(InvalidArgumentError{Message: "keys nil"}) + } + resultCount := resultsVal.Len() + if len(keys) != resultCount { + return nil, nil, WithStack(InvalidArgumentError{Message: fmt.Sprintf("expected %d keys, got %d", resultCount, len(keys))}) + } + for _, key := range keys { + if err := validateKey(key); err != nil { + return nil, nil, WithStack(err) + } + } + metas := make(DocumentMetaSlice, resultCount) + errs := make(ErrorSlice, resultCount) + silent := false + for i := 0; i < resultCount; i++ { + result := resultsVal.Index(i).Addr() + ctx, err := withDocumentAt(ctx, i) + if err != nil { + return nil, nil, WithStack(err) + } + key := keys[i] + meta, cs, err := c.readDocument(ctx, key, result.Interface()) + if cs.Silent { + silent = true + } else { + metas[i], errs[i] = meta, err + } + } + if silent { + return nil, nil, nil + } + return metas, errs, nil } // CreateDocument creates a single document in the collection. @@ -102,7 +158,7 @@ func (c *vertexCollection) createDocument(ctx context.Context, document interfac if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { @@ -317,7 +373,7 @@ func (c *vertexCollection) replaceDocument(ctx context.Context, key string, docu if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { @@ -432,7 +488,7 @@ func (c *vertexCollection) removeDocument(ctx context.Context, key string) (Docu if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(200, 202)); err != nil { + if err := resp.CheckStatus(200, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { diff --git a/vendor/github.com/arangodb/go-driver/vst/connection.go b/vendor/github.com/arangodb/go-driver/vst/connection.go index 1250bd2d..cb8254d3 100644 --- a/vendor/github.com/arangodb/go-driver/vst/connection.go +++ b/vendor/github.com/arangodb/go-driver/vst/connection.go @@ -142,6 +142,9 @@ func (c *vstConnection) Do(ctx context.Context, req driver.Request) (driver.Resp // Do performs a given request, returning its response. func (c *vstConnection) do(ctx context.Context, req driver.Request, transport messageTransport) (driver.Response, error) { + if ctx == nil { + ctx = context.Background() + } vstReq, ok := req.(*vstRequest) if !ok { return nil, driver.WithStack(driver.InvalidArgumentError{Message: "request is not a *vstRequest"}) @@ -158,10 +161,16 @@ func (c *vstConnection) do(ctx context.Context, req driver.Request, transport me vstReq.WroteRequest() // Wait for response - msg, ok := <-resp - if !ok { - // Message was cancelled / timeout - return nil, driver.WithStack(context.DeadlineExceeded) + var msg protocol.Message + select { + case msg, ok = <-resp: + if !ok { + // Message was canceled / timeout + return nil, driver.WithStack(context.DeadlineExceeded) + } + case <-ctx.Done(): + // Context canceled while waiting here + return nil, driver.WithStack(ctx.Err()) } //fmt.Printf("Received msg: %d\n", msg.ID) diff --git a/vendor/github.com/arangodb/go-driver/vst/doc.go b/vendor/github.com/arangodb/go-driver/vst/doc.go new file mode 100644 index 00000000..2337aea2 --- /dev/null +++ b/vendor/github.com/arangodb/go-driver/vst/doc.go @@ -0,0 +1,68 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package vst implements driver.Connection using a VelocyStream connection. + +This connection uses VelocyStream (with optional TLS) to connect to the ArangoDB database. +It encodes its contents as Velocypack. + +Creating an Insecure Connection + +To create a VST connection, use code like this. + + // Create a VST connection to the database + conn, err := vst.NewConnection(vst.ConnectionConfig{ + Endpoints: []string{"http://localhost:8529"}, + }) + if err != nil { + // Handle error + } + +The resulting connection is used to create a client which you will use +for normal database requests. + + // Create a client + c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, + }) + if err != nil { + // Handle error + } + +Creating a Secure Connection + +To create a secure VST connection, use code like this. + + // Create a VST over TLS connection to the database + conn, err := vst.NewConnection(vst.ConnectionConfig{ + Endpoints: []string{"https://localhost:8529"}, + TLSConfig: &tls.Config{ + InsecureSkipVerify: trueWhenUsingNonPublicCertificates, + }, + }) + if err != nil { + // Handle error + } + +*/ +package vst diff --git a/vendor/github.com/arangodb/go-driver/vst/protocol/chunk.go b/vendor/github.com/arangodb/go-driver/vst/protocol/chunk.go index eb1e59bb..d16559ad 100644 --- a/vendor/github.com/arangodb/go-driver/vst/protocol/chunk.go +++ b/vendor/github.com/arangodb/go-driver/vst/protocol/chunk.go @@ -87,6 +87,10 @@ func buildChunks(messageID uint64, maxChunkSize uint32, messageParts ...[]byte) func readBytes(dst []byte, r io.Reader) error { offset := 0 remaining := len(dst) + if remaining == 0 { + // Nothing left to read + return nil + } for { n, err := r.Read(dst[offset:]) offset += n diff --git a/vendor/github.com/arangodb/go-driver/vst/protocol/chunk_1_0.go b/vendor/github.com/arangodb/go-driver/vst/protocol/chunk_1_0.go index 974b1dbe..6458c344 100644 --- a/vendor/github.com/arangodb/go-driver/vst/protocol/chunk_1_0.go +++ b/vendor/github.com/arangodb/go-driver/vst/protocol/chunk_1_0.go @@ -24,7 +24,6 @@ package protocol import ( "encoding/binary" - "fmt" "io" driver "github.com/arangodb/go-driver" @@ -45,7 +44,7 @@ func readChunkVST1_0(r io.Reader) (chunk, error) { if (1 == (chunkX & 0x1)) && ((chunkX >> 1) > 1) { // First chunk, numberOfChunks>1 -> read messageLength - fmt.Println("Reading maxHdr") + //fmt.Println("Reading maxHdr") if err := readBytes(hdr[minChunkHeaderSize:], r); err != nil { return chunk{}, driver.WithStack(err) } diff --git a/vendor/github.com/arangodb/go-driver/vst/protocol/connection.go b/vendor/github.com/arangodb/go-driver/vst/protocol/connection.go index 927f1aae..d1af33c4 100644 --- a/vendor/github.com/arangodb/go-driver/vst/protocol/connection.go +++ b/vendor/github.com/arangodb/go-driver/vst/protocol/connection.go @@ -43,12 +43,14 @@ type Connection struct { msgStore messageStore conn net.Conn writeMutex sync.Mutex - closing bool + closing int32 lastActivity time.Time + configured int32 // Set to 1 after the configuration callback has finished without errors. } const ( defaultMaxChunkSize = 30000 + maxRecentErrors = 64 ) var ( @@ -58,13 +60,8 @@ var ( // dial opens a new connection to the server on the given address. func dial(version Version, addr string, tlsConfig *tls.Config) (*Connection, error) { - var conn net.Conn - var err error - if tlsConfig != nil { - conn, err = tls.Dial("tcp", addr, tlsConfig) - } else { - conn, err = net.Dial("tcp", addr) - } + // Create TCP connection + conn, err := net.Dial("tcp", addr) if err != nil { return nil, driver.WithStack(err) } @@ -75,6 +72,12 @@ func dial(version Version, addr string, tlsConfig *tls.Config) (*Connection, err tcpConn.SetNoDelay(true) } + // Add TLS if needed + if tlsConfig != nil { + tlsConn := tls.Client(conn, tlsConfig) + conn = tlsConn + } + // Send protocol header switch version { case Version1_0: @@ -103,18 +106,20 @@ func dial(version Version, addr string, tlsConfig *tls.Config) (*Connection, err return c, nil } +// load returns an indication of the amount of work this connection has. +// 0 means no work at all, >0 means some work. +func (c *Connection) load() int { + return c.msgStore.Size() +} + // Close the connection to the server func (c *Connection) Close() error { - if !c.closing { - c.closing = true + if atomic.CompareAndSwapInt32(&c.closing, 0, 1) { if err := c.conn.Close(); err != nil { return driver.WithStack(err) } c.msgStore.ForEach(func(m *Message) { - if m.response != nil { - close(m.response) - m.response = nil - } + m.closeResponseChan() }) } return nil @@ -122,7 +127,12 @@ func (c *Connection) Close() error { // IsClosed returns true when the connection is closed, false otherwise. func (c *Connection) IsClosed() bool { - return c.closing + return atomic.LoadInt32(&c.closing) == 1 +} + +// IsConfigured returns true when the configuration callback has finished on this connection, without errors. +func (c *Connection) IsConfigured() bool { + return atomic.LoadInt32(&c.configured) == 1 } // Send sends a message (consisting of given parts) to the server and returns @@ -140,6 +150,7 @@ func (c *Connection) Send(ctx context.Context, messageParts ...[]byte) (<-chan M } // Prepare for receiving a response m := c.msgStore.Add(msgID) + responseChan := m.responseChan //panic(fmt.Sprintf("chunks: %d, messageParts: %d, first: %s", len(chunks), len(messageParts), hex.EncodeToString(messageParts[0]))) @@ -168,7 +179,7 @@ func (c *Connection) Send(ctx context.Context, messageParts ...[]byte) (<-chan M if err != nil { return nil, driver.WithStack(err) } - return m.response, nil + return responseChan, nil case <-ctx.Done(): return nil, ctx.Err() } @@ -198,8 +209,10 @@ func (c *Connection) sendChunk(deadline time.Time, chunk chunk) error { // readChunkLoop reads chunks from the connection until it is closed. func (c *Connection) readChunkLoop() { + recentErrors := 0 + goodChunks := 0 for { - if c.closing { + if c.IsClosed() { // Closing, we're done return } @@ -215,17 +228,27 @@ func (c *Connection) readChunkLoop() { } c.updateLastActivity() if err != nil { - if !c.closing { + if !c.IsClosed() { // Handle error if err == io.EOF { // Connection closed c.Close() } else { - fmt.Printf("readChunkLoop error: %#v\n", err) + recentErrors++ + fmt.Printf("readChunkLoop error: %#v (goodChunks=%d)\n", err, goodChunks) + if recentErrors > maxRecentErrors { + // When we get to many errors in a row, close this connection + c.Close() + } else { + // Backoff a bit, so we allow things to settle. + time.Sleep(time.Millisecond * time.Duration(recentErrors*5)) + } } } } else { // Process chunk + recentErrors = 0 + goodChunks++ go c.processChunk(chunk) } } @@ -252,10 +275,7 @@ func (c *Connection) processChunk(chunk chunk) { //fmt.Println("Chunk: " + hex.EncodeToString(chunk.Data) + "\nMessage: " + hex.EncodeToString(m.Data)) // Notify listener - if m.response != nil { - m.response <- *m - close(m.response) - } + m.notifyListener() } } diff --git a/vendor/github.com/arangodb/go-driver/vst/protocol/doc.go b/vendor/github.com/arangodb/go-driver/vst/protocol/doc.go new file mode 100644 index 00000000..84f503dc --- /dev/null +++ b/vendor/github.com/arangodb/go-driver/vst/protocol/doc.go @@ -0,0 +1,26 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package protocol implements the VelocyStream protocol (it is not intended to be used directly). +*/ +package protocol diff --git a/vendor/github.com/arangodb/go-driver/vst/protocol/message.go b/vendor/github.com/arangodb/go-driver/vst/protocol/message.go index efbaccc8..b7ba1a23 100644 --- a/vendor/github.com/arangodb/go-driver/vst/protocol/message.go +++ b/vendor/github.com/arangodb/go-driver/vst/protocol/message.go @@ -22,21 +22,51 @@ package protocol -import "sort" +import ( + "sort" + "sync" + "sync/atomic" +) // Message is what is send back to the client in response to a request. type Message struct { ID uint64 Data []byte - chunks []chunk - numberOfChunks uint32 - response chan Message + chunksMutex sync.Mutex + chunks []chunk + numberOfChunks uint32 + responseChanClosed int32 + responseChan chan Message +} + +// closes the response channel if needed. +func (m *Message) closeResponseChan() { + if atomic.CompareAndSwapInt32(&m.responseChanClosed, 0, 1) { + if ch := m.responseChan; ch != nil { + m.responseChan = nil + close(ch) + } + } +} + +// notifyListener pushes itself onto its response channel and closes the response channel afterwards. +func (m *Message) notifyListener() { + if atomic.CompareAndSwapInt32(&m.responseChanClosed, 0, 1) { + if ch := m.responseChan; ch != nil { + m.responseChan = nil + ch <- *m + close(ch) + } + } } // addChunk adds the given chunks to the list of chunks of the message. // If the given chunk is the first chunk, the expected number of chunks is recorded. func (m *Message) addChunk(c chunk) { + m.chunksMutex.Lock() + defer m.chunksMutex.Unlock() + m.chunks = append(m.chunks, c) if c.IsFirst() { m.numberOfChunks = c.NumberOfChunks() @@ -48,6 +78,9 @@ func (m *Message) addChunk(c chunk) { // is returned. // If all chunks are available, the Data field is build and set and true is returned. func (m *Message) assemble() bool { + m.chunksMutex.Lock() + defer m.chunksMutex.Unlock() + if m.Data != nil { // Already assembled return true diff --git a/vendor/github.com/arangodb/go-driver/vst/protocol/message_store.go b/vendor/github.com/arangodb/go-driver/vst/protocol/message_store.go index 5c60d4bf..e45da2cc 100644 --- a/vendor/github.com/arangodb/go-driver/vst/protocol/message_store.go +++ b/vendor/github.com/arangodb/go-driver/vst/protocol/message_store.go @@ -32,6 +32,14 @@ type messageStore struct { messages map[uint64]*Message } +// Size returns the number of messages in this store. +func (s *messageStore) Size() int { + s.mutex.RLock() + defer s.mutex.RUnlock() + + return len(s.messages) +} + // Get returns the message with given id, or nil if not found func (s *messageStore) Get(id uint64) *Message { s.mutex.RLock() @@ -58,8 +66,8 @@ func (s *messageStore) Add(id uint64) *Message { } m := &Message{ - ID: id, - response: make(chan Message), + ID: id, + responseChan: make(chan Message), } s.messages[id] = m return m diff --git a/vendor/github.com/arangodb/go-driver/vst/protocol/transport.go b/vendor/github.com/arangodb/go-driver/vst/protocol/transport.go index a943bc60..36192c88 100644 --- a/vendor/github.com/arangodb/go-driver/vst/protocol/transport.go +++ b/vendor/github.com/arangodb/go-driver/vst/protocol/transport.go @@ -26,6 +26,7 @@ import ( "context" "crypto/tls" "sync" + "sync/atomic" "time" driver "github.com/arangodb/go-driver" @@ -33,6 +34,7 @@ import ( const ( DefaultIdleConnTimeout = time.Minute + DefaultConnLimit = 3 ) // TransportConfig contains configuration options for Transport. @@ -43,6 +45,11 @@ type TransportConfig struct { // Zero means no limit. IdleConnTimeout time.Duration + // ConnLimit is the upper limit to the number of connections to a single server. + // Due to the nature of the VST protocol, this value does not have to be high. + // The default is 3 (DefaultConnLimit). + ConnLimit int + // Version specifies the version of the Velocystream protocol Version Version } @@ -63,6 +70,9 @@ func NewTransport(hostAddr string, tlsConfig *tls.Config, config TransportConfig if config.IdleConnTimeout == 0 { config.IdleConnTimeout = DefaultIdleConnTimeout } + if config.ConnLimit == 0 { + config.ConnLimit = DefaultConnLimit + } return &Transport{ TransportConfig: config, hostAddr: hostAddr, @@ -91,13 +101,16 @@ func (c *Transport) CloseIdleConnections() (closed, remaining int) { c.connMutex.Lock() defer c.connMutex.Unlock() - for i, conn := range c.connections { + for i := 0; i < len(c.connections); { + conn := c.connections[i] if conn.IsClosed() || conn.IsIdle(c.IdleConnTimeout) { // Remove connection from list c.connections = append(c.connections[:i], c.connections[i+1:]...) // Close connection go conn.Close() closed++ + } else { + i++ } } @@ -141,10 +154,14 @@ func (c *Transport) getConnection(ctx context.Context) (*Connection, error) { // Invoke callback if cb := c.onConnectionCreated; cb != nil { if err := cb(ctx, conn); err != nil { + conn.Close() return nil, driver.WithStack(err) } } + // Mark the connection as ready + atomic.StoreInt32(&conn.configured, 1) + return conn, nil } @@ -154,15 +171,36 @@ func (c *Transport) getAvailableConnection() *Connection { c.connMutex.Lock() defer c.connMutex.Unlock() + // Select the connection with the least amount of traffic + var bestConn *Connection + bestConnLoad := 0 + activeConnCount := 0 for _, conn := range c.connections { if !conn.IsClosed() { - conn.updateLastActivity() - return conn + activeConnCount++ + if conn.IsConfigured() { + connLoad := conn.load() + if bestConn == nil || connLoad < bestConnLoad { + bestConn = conn + bestConnLoad = connLoad + } + } } } - // No connections available - return nil + if bestConn == nil { + // No connections available + return nil + } + + // Is load is >0 AND the number of connections is below the limit, create a new one + if bestConnLoad > 0 && activeConnCount < c.ConnLimit { + return nil + } + + // Use the best connection found + bestConn.updateLastActivity() + return bestConn } // createConnection creates a new connection.