Merge pull request #77 from 3scale/async-redis

Non-blocking Redis calls: preparation (import libs, write async client, adapt tests, etc.)

.circleci/config.yml

@@ -63,11 +63,6 @@ jobs:
           key: v1-dependencies-{{ checksum "Gemfile.lock" }}
 
       #run tests!
-      - run:
-          name: Run tests on Ruby 2.2
-          command: |
-            TEST_RUBY_VERSION=2.2 script/ci
-
       - run:
           name: Run tests on Ruby 2.3
           command: |

.ruby-version

@@ -1 +1 @@
-2.2.4
+2.3.6

3scale_backend.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   s.description = 'This gem provides a daemon that handles authorization and reporting of web services managed by 3scale.'
   s.license     = 'Apache-2.0'
 
-  s.required_ruby_version     = ">= 2.2.0"
+  s.required_ruby_version     = ">= 2.3.0"
   s.required_rubygems_version = ">= 1.3.7"
 
   s.files = Dir.glob('{lib,bin,app,config}/**/*')

Dockerfile.ci

@@ -272,7 +272,7 @@ RUN sudo chown -R "${USER_NAME}": /tmp/apisonator \
     && rbenv_update_env
 
 # specify all versions to be installed, partial versions also understood
-ARG RUBY_VERSIONS="2.2 2.3"
+ARG RUBY_VERSIONS="2.3"
 RUN cd /tmp/apisonator \
     && ruby_versions ${RUBY_VERSIONS}
 

Gemfile.base

@@ -27,6 +27,7 @@ group :test do
   gem 'rspec',         '~> 3.7.0', require: nil
   gem 'geminabox',     '~> 0.13.11', require: false
   gem 'codeclimate-test-reporter', '~> 0.6.0', require: nil
+  gem 'async-rspec'
 end
 
 group :development do
@@ -60,3 +61,4 @@ gem 'sinatra', '~> 2.0.3'
 gem 'sinatra-contrib', '~> 2.0.3'
 # Optional external error logging services
 gem 'bugsnag', '~> 6', require: nil
+gem 'async-redis', '~> 0.3.3', require: nil

Gemfile.lock

@@ -49,6 +49,16 @@ GEM
     airbrake (4.3.1)
       builder
       multi_json
+    async (1.15.2)
+      nio4r (~> 2.3)
+      timers (~> 4.1)
+    async-io (1.18.5)
+      async (~> 1.14)
+    async-redis (0.3.3)
+      async (~> 1.8)
+      async-io (~> 1.10)
+    async-rspec (1.12.0)
+      rspec (~> 3.0)
     aws-sdk (2.4.2)
       aws-sdk-resources (= 2.4.2)
     aws-sdk-core (2.4.2)
@@ -110,6 +120,7 @@ GEM
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
     net-ssh (4.2.0)
+    nio4r (2.3.1)
     nokogiri (1.9.1)
       mini_portile2 (~> 2.4.0)
     pg (0.20.0)
@@ -188,6 +199,7 @@ GEM
     thread_safe (0.3.6)
     tilt (2.0.8)
     timecop (0.9.1)
+    timers (4.3.0)
     tzinfo (1.2.4)
       thread_safe (~> 0.1)
     vegas (0.1.11)
@@ -205,6 +217,8 @@ PLATFORMS
 DEPENDENCIES
   3scale_backend!
   airbrake (= 4.3.1)
+  async-redis (~> 0.3.3)
+  async-rspec
   aws-sdk (= 2.4.2)
   benchmark-ips (~> 2.7.2)
   bugsnag (~> 6)

Gemfile.on_prem.lock

@@ -46,6 +46,16 @@ GEM
       i18n (~> 0.7)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
+    async (1.15.2)
+      nio4r (~> 2.3)
+      timers (~> 4.1)
+    async-io (1.18.5)
+      async (~> 1.14)
+    async-redis (0.3.3)
+      async (~> 1.8)
+      async-io (~> 1.10)
+    async-rspec (1.12.0)
+      rspec (~> 3.0)
     backports (3.11.3)
     benchmark-ips (2.7.2)
     bugsnag (6.6.4)
@@ -99,6 +109,7 @@ GEM
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
     net-ssh (4.2.0)
+    nio4r (2.3.1)
     nokogiri (1.9.1)
       mini_portile2 (~> 2.4.0)
     pkg-config (1.1.9)
@@ -174,6 +185,7 @@ GEM
     thread_safe (0.3.6)
     tilt (2.0.8)
     timecop (0.9.1)
+    timers (4.3.0)
     tzinfo (1.2.4)
       thread_safe (~> 0.1)
     vegas (0.1.11)
@@ -188,6 +200,8 @@ PLATFORMS
 
 DEPENDENCIES
   3scale_backend!
+  async-redis (~> 0.3.3)
+  async-rspec
   benchmark-ips (~> 2.7.2)
   bugsnag (~> 6)
   builder (= 3.2.3)

lib/3scale/backend/configuration.rb

@@ -43,7 +43,8 @@ def parse_int(value, default)
       config.add_section(:queues, :master_name, :sentinels, :role,
                          :connect_timeout, :read_timeout, :write_timeout)
       config.add_section(:redis, :url, :proxy, :sentinels, :role,
-                         :connect_timeout, :read_timeout, :write_timeout)
+                         :connect_timeout, :read_timeout, :write_timeout,
+                         :async)
       config.add_section(:analytics_redis, :server,
                          :connect_timeout, :read_timeout, :write_timeout)
       config.add_section(:hoptoad, :service, :api_key)

lib/3scale/backend/oauth/token_storage.rb

@@ -94,7 +94,7 @@ def all_by_service_and_app(service_id, app_id, user_id = nil)
                     Token.from_value token, service_id, value, ttl
                   end
                 end
-                .force.tap do
+                .tap do
                   # delete expired tokens (nil values) from token set
                   deltokens.each_slice(TOKEN_MAX_REDIS_SLICE_SIZE) do |delgrp|
                     storage.srem token_set, delgrp
@@ -185,10 +185,7 @@ def remove_whole_token_set(token_set, service_id)
             # TODO: provide a SSCAN interface with lazy enums because SMEMBERS
             # is prone to DoSing and timeouts
             def tokens_from(token_set)
-              # It is important that we make this a lazy enumerator. The
-              # laziness is maintained until some enumerator forces execution or
-              # the caller calls 'to_a' or 'force', whichever happens first.
-              storage.smembers(token_set).lazy
+              storage.smembers(token_set)
             end
 
             def tokens_n_keys(token_set, service_id)
@@ -198,15 +195,14 @@ def tokens_n_keys(token_set, service_id)
                   Key.for token, service_id
                 end
               end
-              # Note: this is returning two lazy enumerators
+
               [token_groups, key_groups]
             end
 
-            # Provides grouped data (as sourced from the lazy iterators) which
-            # matches respectively in each array position, ie. 1st group of data
-            # contains a group of tokens, keys and values with ttls, and
-            # position N of the tokens group has key in position N of the keys
-            # group, and so on.
+            # Provides grouped data which matches respectively in each array
+            # position, ie. 1st group of data contains a group of tokens, keys
+            # and values with ttls, and position N of the tokens group has key
+            # in position N of the keys group, and so on.
             #
             # [[[token group], [key group], [value_with_ttls_group]], ...]
             #

lib/3scale/backend/queue_storage.rb

@@ -9,7 +9,7 @@ def self.connection(env, cfg)
         options = Backend::Storage::Helpers.config_with(cfg.queues,
                                                         options: init_params)
 
-        Redis.new options
+        Storage.new(options)
       end
     end
   end

lib/3scale/backend/stats/storage.rb

@@ -92,7 +92,7 @@ def stats_storage
           def stats_storage_from_config
             options = Backend::Storage::Helpers.config_with(config.analytics_redis)
 
-            Redis.new(options)
+            Backend::Storage.new(options)
           end
 
           def config