Merge pull request #903 from quartzmo/datastore-gax

Convert Datastore to GAPIC

google-cloud-datastore/acceptance/datastore_helper.rb

@@ -19,7 +19,7 @@
 require "google/cloud/datastore"
 
 # Create shared dataset object so we don't create new for each test
-$dataset = Google::Cloud.new.datastore retries: 10
+$dataset = Google::Cloud.new.datastore
 
 module Acceptance
   ##

google-cloud-datastore/lib/google-cloud-datastore.rb

@@ -38,9 +38,9 @@ module Cloud
     #   The default scope is:
     #
     #   * `https://www.googleapis.com/auth/datastore`
-    # @param [Integer] retries Number of times to retry requests on server
-    #   error. The default value is `3`. Optional.
     # @param [Integer] timeout Default timeout to use in requests. Optional.
+    # @param [Hash] client_config A hash of values to override the default
+    #   behavior of the API client. See Google::Gax::CallSettings. Optional.
     #
     # @return [Google::Cloud::Datastore::Dataset]
     #
@@ -66,10 +66,10 @@ module Cloud
     #   platform_scope = "https://www.googleapis.com/auth/cloud-platform"
     #   datastore = gcloud.datastore scope: platform_scope
     #
-    def datastore scope: nil, retries: nil, timeout: nil
+    def datastore scope: nil, timeout: nil, client_config: nil
       Google::Cloud.datastore @project, @keyfile,
-                              scope: scope, retries: (retries || @retries),
-                              timeout: (timeout || @timeout)
+                              scope: scope, timeout: (timeout || @timeout),
+                              client_config: client_config
     end
 
     ##
@@ -91,9 +91,9 @@ def datastore scope: nil, retries: nil, timeout: nil
     #   The default scope is:
     #
     #   * `https://www.googleapis.com/auth/datastore`
-    # @param [Integer] retries Number of times to retry requests on server
-    #   error. The default value is `3`. Optional.
     # @param [Integer] timeout Default timeout to use in requests. Optional.
+    # @param [Hash] client_config A hash of values to override the default
+    #   behavior of the API client. See Google::Gax::CallSettings. Optional.
     #
     # @return [Google::Cloud::Datastore::Dataset]
     #
@@ -108,16 +108,16 @@ def datastore scope: nil, retries: nil, timeout: nil
     #     t["done"] = false
     #     t["priority"] = 4
     #     t["description"] = "Learn Cloud Datastore"
-    #   end
+    #   end                                                                  ``
     #
     #   datastore.save task
     #
-    def self.datastore project = nil, keyfile = nil, scope: nil, retries: nil,
-                       timeout: nil
+    def self.datastore project = nil, keyfile = nil, scope: nil, timeout: nil,
+                       client_config: nil
       require "google/cloud/datastore"
       Google::Cloud::Datastore.new project: project, keyfile: keyfile,
-                                   scope: scope, retries: retries,
-                                   timeout: timeout
+                                   scope: scope, timeout: timeout,
+                                   client_config: client_config
     end
   end
 end

google-cloud-datastore/lib/google/cloud/datastore.rb

@@ -444,29 +444,16 @@ module Cloud
     # end
     # ```
     #
-    # ## Configuring retries and timeout
+    # ## Configuring timeout
     #
-    # You can configure how many times API requests may be automatically
-    # retried. When an API request fails, the response will be inspected to see
-    # if the request meets criteria indicating that it may succeed on retry,
-    # such as `500` and `503` status codes or a specific internal error code
-    # such as `rateLimitExceeded`. If it meets the criteria, the request will be
-    # retried after a delay. If another error occurs, the delay will be
-    # increased before a subsequent attempt, until the `retries` limit is
-    # reached.
-    #
-    # You can also set the request `timeout` value in seconds.
+    # You can configure the request `timeout` value in seconds.
     #
     # ```ruby
     # require "google/cloud/datastore"
     #
-    # datastore = Google::Cloud::Datastore.new retries: 10, timeout: 120
+    # datastore = Google::Cloud::Datastore.new timeout: 120
     # ```
     #
-    # See the [Datastore error
-    # codes](https://cloud.google.com/datastore/docs/concepts/errors#error_codes)
-    # for a list of error conditions.
-    #
     # ## The Cloud Datastore Emulator
     #
     # As of this release, the Cloud Datastore emulator that is part of the
@@ -531,9 +518,9 @@ module Datastore
       #   The default scope is:
       #
       #   * `https://www.googleapis.com/auth/datastore`
-      # @param [Integer] retries Number of times to retry requests on server
-      #   error. The default value is `3`. Optional.
       # @param [Integer] timeout Default timeout to use in requests. Optional.
+      # @param [Hash] client_config A hash of values to override the default
+      #   behavior of the API client. See Google::Gax::CallSettings. Optional.
       #
       # @return [Google::Cloud::Datastore::Dataset]
       #
@@ -554,8 +541,8 @@ module Datastore
       #
       #   datastore.save task
       #
-      def self.new project: nil, keyfile: nil, scope: nil, retries: nil,
-                   timeout: nil
+      def self.new project: nil, keyfile: nil, scope: nil, timeout: nil,
+                   client_config: nil
         project ||= Google::Cloud::Datastore::Dataset.default_project
         project = project.to_s # Always cast to a string
         fail ArgumentError, "project is missing" if project.empty?
@@ -564,20 +551,24 @@ def self.new project: nil, keyfile: nil, scope: nil, retries: nil,
           return Google::Cloud::Datastore::Dataset.new(
             Google::Cloud::Datastore::Service.new(
               project, :this_channel_is_insecure,
-              host: ENV["DATASTORE_EMULATOR_HOST"], retries: retries))
+              host: ENV["DATASTORE_EMULATOR_HOST"],
+              client_config: client_config))
         end
+        credentials = credentials_with_scope keyfile, scope
+        Google::Cloud::Datastore::Dataset.new(
+          Google::Cloud::Datastore::Service.new(
+            project, credentials, timeout: timeout,
+                                  client_config: client_config))
+      end
 
+      ##
+      # @private
+      def self.credentials_with_scope keyfile, scope
         if keyfile.nil?
-          credentials = Google::Cloud::Datastore::Credentials.default(
-            scope: scope)
+          Google::Cloud::Datastore::Credentials.default(scope: scope)
         else
-          credentials = Google::Cloud::Datastore::Credentials.new(
-            keyfile, scope: scope)
+          Google::Cloud::Datastore::Credentials.new(keyfile, scope: scope)
         end
-
-        Google::Cloud::Datastore::Dataset.new(
-          Google::Cloud::Datastore::Service.new(
-            project, credentials, retries: retries, timeout: timeout))
       end
     end
   end

google-cloud-datastore/lib/google/cloud/datastore/dataset.rb

@@ -54,7 +54,7 @@ module Datastore
       #
       class Dataset
         ##
-        # @private The gRPC Service object.
+        # @private The Service object.
         attr_accessor :service
 
         ##

google-cloud-datastore/lib/google/cloud/datastore/service.rb

@@ -13,46 +13,55 @@
 # limitations under the License.
 
 
+require "google/cloud/errors"
 require "google/cloud/datastore/credentials"
-require "google/datastore/v1/datastore_pb"
-require "google/cloud/core/grpc_backoff"
+require "google/cloud/datastore/version"
+require "google/cloud/datastore/v1"
+require "google/gax/errors"
 
 module Google
   module Cloud
     module Datastore
       ##
-      # @private Represents the gRPC Datastore service, including all the API
+      # @private Represents the GAX Datastore service, including all the API
       # methods.
       class Service
-        attr_accessor :project, :credentials, :host, :retries, :timeout
+        attr_accessor :project, :credentials, :host, :timeout, :client_config
 
         ##
         # Creates a new Service instance.
-        def initialize project, credentials, host: nil, retries: nil,
-                       timeout: nil
+        def initialize project, credentials, host: nil, timeout: nil,
+                       client_config: nil
           @project = project
           @credentials = credentials
-          @host = host || "datastore.googleapis.com"
-          @retries = retries
+          @host = host || V1::DatastoreApi::SERVICE_ADDRESS
           @timeout = timeout
+          @client_config = client_config || {}
         end
 
-        def creds
+        def channel
+          require "grpc"
+          GRPC::Core::Channel.new host, nil, chan_creds
+        end
+
+        def chan_creds
           return credentials if insecure?
+          require "grpc"
           GRPC::Core::ChannelCredentials.new.compose \
             GRPC::Core::CallCredentials.new credentials.client.updater_proc
         end
 
-        def datastore
-          return mocked_datastore if mocked_datastore
-          @datastore ||= begin
-            require "google/datastore/v1/datastore_services_pb"
-
-            Google::Datastore::V1::Datastore::Stub.new(
-              host, creds, timeout: timeout)
-          end
+        def service
+          return mocked_service if mocked_service
+          @service ||= V1::DatastoreApi.new(
+            service_path: host,
+            channel: channel,
+            timeout: timeout,
+            client_config: client_config,
+            app_name: "google-cloud-datastore",
+            app_version: Google::Cloud::Datastore::VERSION)
         end
-        attr_accessor :mocked_datastore
+        attr_accessor :mocked_service
 
         def insecure?
           credentials == :this_channel_is_insecure
@@ -62,99 +71,63 @@ def insecure?
         # Allocate IDs for incomplete keys.
         # (This is useful for referencing an entity before it is inserted.)
         def allocate_ids *incomplete_keys
-          allocate_req = Google::Datastore::V1::AllocateIdsRequest.new(
-            project_id: project,
-            keys: incomplete_keys
-          )
-
-          execute { datastore.allocate_ids allocate_req }
+          execute { service.allocate_ids project, incomplete_keys }
         end
 
         ##
         # Look up entities by keys.
         def lookup *keys, consistency: nil, transaction: nil
-          lookup_req = Google::Datastore::V1::LookupRequest.new(
-            project_id: project,
-            keys: keys
-          )
-          lookup_req.read_options = generate_read_options consistency,
-                                                          transaction
-
-          execute { datastore.lookup lookup_req }
+          read_options = generate_read_options consistency, transaction
+
+          execute { service.lookup project, read_options, keys }
         end
 
         # Query for entities.
         def run_query query, namespace = nil, consistency: nil, transaction: nil
-          run_req = Google::Datastore::V1::RunQueryRequest.new(
-            project_id: project)
-          if query.is_a? Google::Datastore::V1::Query
-            run_req["query"] = query
-          elsif query.is_a? Google::Datastore::V1::GqlQuery
-            run_req["gql_query"] = query
-          else
-            fail ArgumentError, "Unable to query with a #{query.class} object."
+          gql_query = nil
+          if query.is_a? Google::Datastore::V1::GqlQuery
+            gql_query = query
+            query = nil
           end
-          run_req.read_options = generate_read_options consistency, transaction
-
-          run_req.partition_id = Google::Datastore::V1::PartitionId.new(
+          read_options = generate_read_options consistency, transaction
+          partition_id = Google::Datastore::V1::PartitionId.new(
             namespace_id: namespace) if namespace
 
-          execute { datastore.run_query run_req }
+          execute do
+            service.run_query project,
+                              partition_id,
+                              read_options,
+                              query: query,
+                              gql_query: gql_query
+          end
         end
 
         ##
         # Begin a new transaction.
         def begin_transaction
-          tx_req = Google::Datastore::V1::BeginTransactionRequest.new(
-            project_id: project
-          )
-
-          execute { datastore.begin_transaction tx_req }
+          execute { service.begin_transaction project }
         end
 
         ##
         # Commit a transaction, optionally creating, deleting or modifying
         # some entities.
         def commit mutations, transaction: nil
-          commit_req = Google::Datastore::V1::CommitRequest.new(
-            project_id: project,
-            mode: :NON_TRANSACTIONAL,
-            mutations: mutations
-          )
-          if transaction
-            commit_req.mode = :TRANSACTIONAL
-            commit_req.transaction = transaction
+          mode =  transaction.nil? ? :NON_TRANSACTIONAL : :TRANSACTIONAL
+          execute do
+            service.commit project, mode, mutations, transaction: transaction
           end
-
-          execute { datastore.commit commit_req }
         end
 
         ##
         # Roll back a transaction.
         def rollback transaction
-          rb_req = Google::Datastore::V1::RollbackRequest.new(
-            project_id: project,
-            transaction: transaction
-          )
-
-          execute { datastore.rollback rb_req }
+          execute { service.rollback project, transaction }
         end
 
         def inspect
           "#{self.class}(#{@project})"
         end
 
-        ##
-        # Performs backoff and error handling
-        def execute
-          require "grpc" # Ensure GRPC is loaded before rescuing exception
-          Google::Cloud::Core::GrpcBackoff.new(retries: retries).execute do
-            yield
-          end
-        rescue GRPC::BadStatus => e
-          raise Google::Cloud::Error.from_error(e)
-        end
-
         protected
 
         def generate_read_options consistency, transaction
@@ -170,6 +143,13 @@ def generate_read_options consistency, transaction
           end
           nil
         end
+
+        def execute
+          yield
+        rescue Google::Gax::GaxError => e
+          # GaxError wraps BadStatus, but exposes it as #cause
+          raise Google::Cloud::Error.from_error(e.cause)
+        end
       end
     end
   end

google-cloud-datastore/lib/google/cloud/datastore/transaction.rb

@@ -59,7 +59,7 @@ class Transaction < Dataset
 
         ##
         # @private Creates a new Transaction instance.
-        # Takes a Connection and Service instead of project and Credentials.
+        # Takes a Service instead of project and Credentials.
         def initialize service
           @service = service
           reset!

google-cloud-datastore/lib/google/cloud/datastore/v1.rb

@@ -14,3 +14,5 @@
 
 
 require "google/cloud/datastore/v1/datastore_api"
+# Require the protobufs so we can create objects before GRPC is loaded.
+require "google/datastore/v1/datastore_pb"