Merge pull request #2074 from opal/elia/fix-parsing-unicode-v1.0

Fix parsing unicode on v1.0

lib/opal/parser/patch.rb

@@ -1,6 +1,21 @@
 # frozen_string_literal: true
 
 if RUBY_ENGINE == 'opal'
+  class Parser::Lexer
+    def source_buffer=(source_buffer)
+      @source_buffer = source_buffer
+
+      if @source_buffer
+        source = @source_buffer.source
+        # Force UTF8 unpacking even if JS works with UTF-16/UCS-2
+        # See: https://mathiasbynens.be/notes/javascript-encoding
+        @source_pts = source.unpack('U*')
+      else
+        @source_pts = nil
+      end
+    end
+  end
+
   class Parser::Lexer::Literal
     undef :extend_string
 

spec/opal/compiler/unicode_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+require 'opal-parser'
+
+describe Opal::Compiler do
+  describe "unicode support" do
+    it 'can compile code containing Unicode characters' do
+      -> { Opal::Compiler.new("'こんにちは'; p 1").compile }.should_not raise_error
+    end
+  end
+end