Use LMDB key-value store for I18n backend

* Use i18n fork for CacheFile module
* Use LMDB for key-value store for loaded i18n translations
* add test_lmdb_key_value unit test for simple coverage
* Add I18n backend configuration to Cdo::I18nBackend
* Configure i18n backend with CDO.i18n_key_value setting
  Default false in production environment during controlled roll-out.

Gemfile

@@ -13,9 +13,11 @@ end
 gem 'rails', '~> 5.0.1'
 gem 'rails-controller-testing'
 
-# Fork to support numeric keys in the Simple backend.
+# Add CacheFile backend module.
+# Ref: https://github.com/svenfuchs/i18n/pull/423
+# Support numeric keys in Simple backend.
 # Ref: https://github.com/svenfuchs/i18n/pull/422
-gem 'i18n', github: 'wjordan/i18n', branch: 'simple_numeric_keys'
+gem 'i18n', github: 'wjordan/i18n', branch: 'cdo'
 
 # Compile Sprockets assets concurrently in `assets:precompile`.
 # Ref: https://github.com/rails/sprockets/pull/470
@@ -302,3 +304,4 @@ end
 
 gem 'activerecord-import'
 gem 'colorize'
+gem 'lmdb'

Gemfile.lock

@@ -73,8 +73,8 @@ GIT
 
 GIT
   remote: https://github.com/wjordan/i18n.git
-  revision: 4260c62cf628d60bd9fd45dd1315629c653cd8ab
-  branch: simple_numeric_keys
+  revision: f366d62ad73bd0fee939ed9aa38ab88c9f94a8df
+  branch: cdo
   specs:
     i18n (1.0.1)
       concurrent-ruby (~> 1.0)
@@ -450,6 +450,7 @@ GEM
       rb-inotify (~> 0.9, >= 0.9.7)
       ruby_dep (~> 1.2)
     little-plugger (1.1.4)
+    lmdb (0.4.8)
     logging (2.2.2)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
@@ -859,6 +860,7 @@ DEPENDENCIES
   kaminari
   launchy
   le (~> 2.2)
+  lmdb
   lograge
   loofah (~> 2.2.1)
   marked-rails

dashboard/config/application.rb

@@ -15,6 +15,7 @@
 
 require 'bootstrap-sass'
 require 'cdo/hash'
+require 'cdo/i18n_backend'
 
 # Require the gems listed in Gemfile, including any gems
 # you've limited to :test, :development, or :production.
@@ -80,6 +81,7 @@ class Application < Rails::Application
 
     # By default, config/locales/*.rb,yml are auto loaded.
     # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
+    config.i18n.backend = CDO.i18n_backend
     config.i18n.enforce_available_locales = false
     config.i18n.available_locales = ['en-US']
     config.i18n.fallbacks = {}

lib/cdo/i18n_backend.rb

@@ -0,0 +1,30 @@
+require 'i18n'
+require 'active_support/core_ext/numeric/bytes'
+require 'cdo/lmdb_key_value'
+
+module Cdo
+  # I18n backend instance used by the web application.
+  class I18nBackend < ::I18n::Backend::KeyValue
+    include ::I18n::Backend::CacheFile
+
+    CACHE_DIR = pegasus_dir('cache', 'i18n/cache')
+
+    # Maximum size of the i18n cache file.
+    # Used to set the fixed memory-map size.
+    MAX_CACHE_SIZE = 2.gigabytes
+
+    def initialize
+      store = LMDBKeyValue.new(CACHE_DIR, size: MAX_CACHE_SIZE)
+      super(store, false)
+      self.path_roots = [Gem.dir, deploy_dir]
+    end
+
+    alias init_translations load_translations
+    alias reload! load_translations
+  end
+end
+# Use custom i18n backend by enabling `CDO.i18n_key_value`.
+# Default false in production environment during controlled roll-out.
+CDO.i18n_backend = CDO.with_default(!rack_env?(:production)).i18n_key_value ?
+  Cdo::I18nBackend.new :
+  I18n::Backend::Simple.new

lib/cdo/lmdb_key_value.rb

@@ -0,0 +1,59 @@
+require 'lmdb'
+require 'fileutils'
+require 'digest'
+
+# Simple wrapper around LMDB to provide a key-value interface.
+# Use separate reader and writer objects to optimize the common read-only case.
+module Cdo
+  class LMDBKeyValue
+    def initialize(dir, size: 0)
+      FileUtils.mkpath(dir)
+      @dir = dir
+      @options = {
+        writemap: true,
+        mapasync: true,
+        nometasync: true,
+        mapsize: size
+      }
+    end
+
+    def reader
+      return @reader if @reader
+      @read_env = ::LMDB.new(@dir, @options.merge(rdonly: true))
+      @read_env.transaction(true) do
+        @reader = @read_env.database
+      end
+    rescue LMDB::Error => e
+      raise unless e.message == 'No such file or directory'
+      # Create database with writer first, then reload read-only environment.
+      writer
+      retry
+    end
+
+    def writer
+      return @writer if @writer
+      @write_env = ::LMDB.new(@dir, @options)
+      @writer = @write_env.database(nil, create: true)
+    end
+
+    def [](key)
+      reader.get(digest(key))
+    end
+
+    def []=(key, value)
+      writer.put(digest(key), value)
+    end
+
+    def close
+      @reader_env.close if @reader_env
+      @writer_env.close if @writer_env
+    end
+
+    protected
+
+    # Shorten key using hash digest to fit within LMDB's key-length limit.
+    def digest(key)
+      Digest::SHA2.hexdigest(key.to_s)
+    end
+  end
+end

pegasus/cache/.gitignore

@@ -11,3 +11,4 @@ milestone-cache_v2.json
 /.sass-cache
 /cloudfront_aliases*.json
 mysql-status-cache.json
+i18n/cache

pegasus/src/env.rb

@@ -2,6 +2,7 @@
 require 'cdo/pegasus'
 require 'i18n'
 require 'i18n/backend/fallbacks'
+require 'cdo/i18n_backend'
 require 'logger'
 require 'bcrypt'
 require 'chronic'
@@ -45,7 +46,8 @@ def self.create_logger
 def load_pegasus_settings
   $log = Pegasus.logger
 
-  I18n::Backend::Simple.send(:include, I18n::Backend::Fallbacks)
+  I18n.backend = CDO.i18n_backend
+  I18n.backend.class.send(:include, I18n::Backend::Fallbacks)
   if rack_env?(:development) && !CDO.load_locales
     I18n.load_path += Dir[cache_dir('i18n/en-US.yml')]
     I18n.load_path += Dir[cache_dir('i18n/es-ES.yml')]

shared/test/test_lmdb_key_value.rb

@@ -0,0 +1,16 @@
+require_relative 'test_helper'
+require 'cdo/lmdb_key_value'
+require 'active_support/core_ext/numeric/bytes'
+require 'tmpdir'
+
+class LMDBKeyValueTest < Minitest::Test
+  def test_lmdb_key_value
+    Dir.mktmpdir do |dir|
+      lmdb = Cdo::LMDBKeyValue.new(dir, size: 100.kilobytes)
+      assert_nil lmdb['key']
+      lmdb['key'] = 'value'
+      assert_equal 'value', lmdb['key']
+      lmdb.close
+    end
+  end
+end