[tokenizer] support for cyrillic symbols

0exp · Jun 28, 2019 · d469c69 · d469c69
1 parent b4e48b5
commit d469c69
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/lib/jaina/parser/tokenizer.rb b/lib/jaina/parser/tokenizer.rb
@@ -10,7 +10,7 @@ module Jaina::Parser::Tokenizer
   #
   # @api private
   # @since 0.1.0
-  TOKEN_SCAN_PATTERN = /\(|\)|[\<\=\-\>\:\"\'\.\,\w\[\]]+/.freeze
+  TOKEN_SCAN_PATTERN = /\(|\)|[\<\=\-\>\:\"\'\.\,\w\[\][а-яА-Я]]+/.freeze
 
   # @return [String]
   #

diff --git a/spec/smoke_test_spec.rb → spec/features_spec.rb b/spec/smoke_test_spec.rb → spec/features_spec.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-describe 'Smoke test' do
+describe 'Features' do
   specify do
     # NOTE: create new expressions
     a = Class.new(Jaina::TerminalExpr) { token 'A' }