Use the multibyte scanner pervasively.

Closes #186

doc-src/SASS_CHANGELOG.md

@@ -3,6 +3,11 @@
 * Table of contents
 {:toc}
 
+## 3.1.13 (Unreleased)
+
+* Fix a smattering of subtle bugs that would crop up when using multibyte
+  character sets.
+
 ## 3.1.12
 
 * Compatibility with the `mathn` library

lib/sass/css.rb

@@ -1,7 +1,6 @@
 require File.dirname(__FILE__) + '/../sass'
 require 'sass/tree/node'
 require 'sass/scss/css_parser'
-require 'strscan'
 
 module Sass
   # This class converts CSS documents into Sass or SCSS templates.

lib/sass/engine.rb

@@ -1,4 +1,3 @@
-require 'strscan'
 require 'set'
 require 'digest/sha1'
 require 'sass/cache_stores'
@@ -565,7 +564,7 @@ def parse_line(parent, line, root)
     end
 
     def parse_property_or_rule(line)
-      scanner = StringScanner.new(line.text)
+      scanner = Sass::Util::MultibyteStringScanner.new(line.text)
       hack_char = scanner.scan(/[:\*\.]|\#(?!\{)/)
       parser = Sass::SCSS::SassParser.new(scanner, @options[:filename], @line)
 
@@ -749,7 +748,7 @@ def parse_import(line, value)
       raise SyntaxError.new("Illegal nesting: Nothing may be nested beneath import directives.",
         :line => @line + 1) unless line.children.empty?
 
-      scanner = StringScanner.new(value)
+      scanner = Sass::Util::MultibyteStringScanner.new(value)
       values = []
 
       loop do

lib/sass/script.rb

@@ -1,4 +1,3 @@
-require 'strscan'
 require 'sass/script/node'
 require 'sass/script/variable'
 require 'sass/script/funcall'

lib/sass/script/lexer.rb

@@ -1,7 +1,5 @@
 require 'sass/scss/rx'
 
-require 'strscan'
-
 module Sass
   module Script
     # The lexical analyzer for SassScript.
@@ -126,7 +124,7 @@ def string_re(open, close)
       # @param options [{Symbol => Object}] An options hash;
       #   see {file:SASS_REFERENCE.md#sass_options the Sass options documentation}
       def initialize(str, line, offset, options)
-        @scanner = str.is_a?(StringScanner) ? str : StringScanner.new(str)
+        @scanner = str.is_a?(StringScanner) ? str : Sass::Util::MultibyteStringScanner.new(str)
         @line = line
         @offset = offset
         @options = options

lib/sass/scss/parser.rb

@@ -1,4 +1,3 @@
-require 'strscan'
 require 'set'
 
 module Sass
@@ -50,7 +49,7 @@ def init_scanner!
           if @template.is_a?(StringScanner)
             @template
           else
-            StringScanner.new(@template.gsub("\r", ""))
+            Sass::Util::MultibyteStringScanner.new(@template.gsub("\r", ""))
           end
       end
 
@@ -906,7 +905,7 @@ def rethrow(err)
         if @throw_err
           throw :_sass_parser_error, err
         else
-          @scanner = StringScanner.new(@scanner.string)
+          @scanner = Sass::Util::MultibyteStringScanner.new(@scanner.string)
           @scanner.pos = err[:pos]
           @line = err[:line]
           @expected = err[:expected]

lib/sass/shared.rb

@@ -1,5 +1,3 @@
-require 'strscan'
-
 module Sass
   # This module contains functionality that's shared between Haml and Sass.
   module Shared
@@ -16,7 +14,7 @@ module Shared
     # @yieldparam scan [StringScanner] The scanner scanning through the string
     # @return [String] The text remaining in the scanner after all `#{`s have been processed
     def handle_interpolation(str)
-      scan = StringScanner.new(str)
+      scan = Sass::Util::MultibyteStringScanner.new(str)
       yield scan while scan.scan(/(.*?)(\\*)\#\{/m)
       scan.rest
     end
@@ -40,7 +38,7 @@ def handle_interpolation(str)
     #   `["Foo (Bar (Baz bang) bop)", " (Bang (bop bip))"]` in the example above.
     def balance(scanner, start, finish, count = 0)
       str = ''
-      scanner = StringScanner.new(scanner) unless scanner.is_a? StringScanner
+      scanner = Sass::Util::MultibyteStringScanner.new(scanner) unless scanner.is_a? StringScanner
       regexp = Regexp.new("(.*?)[\\#{start.chr}\\#{finish.chr}]", Regexp::MULTILINE)
       while scanner.scan(regexp)
         str << scanner.matched

lib/sass/util.rb

@@ -2,7 +2,6 @@
 require 'set'
 require 'enumerator'
 require 'stringio'
-require 'strscan'
 require 'rbconfig'
 
 require 'sass/root'

test/sass/engine_test.rb

@@ -2560,6 +2560,31 @@ def test_utf32be_bom
   a: b
 SASS
     end
+
+    # Encoding Regression Test
+
+    def test_multibyte_prop_name
+      assert_equal(<<CSS, render(<<SASS))
+@charset "UTF-8";
+#bar {
+  cölor: blue; }
+CSS
+#bar
+  cölor: blue
+SASS
+    end
+
+    def test_multibyte_and_interpolation
+      assert_equal(<<CSS, render(<<SCSS, :syntax => :scss))
+#bar {
+  background: a 0%; }
+CSS
+#bar {
+  // 
+  background: \#{a} 0%;
+}
+SCSS
+    end
   end
 
   def test_original_filename_set