Skip to content
This repository
Browse code

Merge branch 'master' into paint-integration

  • Loading branch information...
commit 916711c9983483c39f9a68c29e21a0ed40004bd2 2 parents 0c98047 + 2e4e83b
Kornelius Kalnbach authored June 11, 2013

Showing 32 changed files with 575 additions and 306 deletions. Show diff stats Hide diff stats

  1. 16  .gitignore
  2. 12  .travis.yml
  3. 10  Changes.textile
  4. 12  Gemfile
  5. 8  README.markdown
  6. 3  bin/coderay
  7. 3  lib/coderay/encoders/debug.rb
  8. 167  lib/coderay/encoders/html.rb
  9. 14  lib/coderay/encoders/html/css.rb
  10. 141  lib/coderay/encoders/terminal.rb
  11. 5  lib/coderay/helpers/file_type.rb
  12. 17  lib/coderay/helpers/plugin.rb
  13. 84  lib/coderay/scanner.rb
  14. 2  lib/coderay/scanners/c.rb
  15. 2  lib/coderay/scanners/cpp.rb
  16. 41  lib/coderay/scanners/css.rb
  17. 2  lib/coderay/scanners/diff.rb
  18. 2  lib/coderay/scanners/java.rb
  19. 36  lib/coderay/scanners/java_script.rb
  20. 2  lib/coderay/scanners/json.rb
  21. 2  lib/coderay/scanners/python.rb
  22. 2  lib/coderay/scanners/ruby.rb
  23. 227  lib/coderay/scanners/sass.rb
  24. 2  lib/coderay/scanners/sql.rb
  25. 6  lib/coderay/styles/alpha.rb
  26. 3  lib/coderay/token_kinds.rb
  27. 1  lib/coderay/tokens.rb
  28. 2  lib/coderay/version.rb
  29. 11  rake_tasks/documentation.rake
  30. 27  rake_tasks/generator.rake
  31. 14  test/functional/for_redcloth.rb
  32. 5  test/unit/plugin.rb
16  .gitignore
... ...
@@ -1,25 +1,15 @@
1 1
 .DS_Store
2  
-*.gem
3  
-*.rbc
4  
-.bundle
5  
-.config
  2
+.*~
6 3
 coverage
7  
-InstalledFiles
8  
-lib/bundler/man
9 4
 pkg
10  
-rdoc
11 5
 spec/reports
12  
-test/tmp
13  
-test/version_tmp
14  
-tmp
15 6
 doc
16 7
 Gemfile.lock
17 8
 .rvmrc
  9
+.ruby-gemset
  10
+.ruby-version
18 11
 test/executable/source.rb.html
19 12
 test/executable/source.rb.json
20 13
 test/scanners
21 14
 bench/test.div.html
22  
-diff.html
23  
-etc/CodeRay.tmproj
24  
-*.swp
25 15
 old-stuff
12  .travis.yml
... ...
@@ -1,15 +1,19 @@
1 1
 rvm:
2 2
   - 1.8.7
3  
-  - 1.9.2
  3
+  - ree
4 4
   - 1.9.3
  5
+  - 2.0.0
  6
+  - ruby-head
5 7
   - jruby-18mode
6 8
   - jruby-19mode
  9
+  - jruby-head
7 10
   - rbx-18mode
8 11
   - rbx-19mode
9  
-  - ruby-head  # test again later: RedCloth not compiling
10  
-  - jruby-head
11  
-  - ree
12 12
 branches:
13 13
   only:
14 14
     - master
  15
+matrix:
  16
+  allow_failures:
  17
+    - rvm: rbx-18mode
  18
+    - rvm: rbx-19mode
15 19
 script: "rake test" # test:scanners"
10  Changes.textile
Source Rendered
@@ -4,9 +4,19 @@ p=. _This files lists all changes in the CodeRay library since the 0.9.8 release
4 4
 
5 5
 h2. Changes in 1.1
6 6
 
  7
+* New scanner: Sass [#93]
7 8
 * Diff scanner: Highlight inline changes in multi-line changes [#99]
  9
+* JavaScript scanner: Highlight multi-line comments in diff correctly
8 10
 * Remove double-click toggle handler from HTML table output
  11
+* Fixes to CSS scanner (floats, pseudoclasses)
  12
+* Plugin does not warn about fallback when default is defined
9 13
 * Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom]
  14
+* Add .xaml file type [#121, thanks to Kozman Bálint]
  15
+* @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner]
  16
+* Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05]
  17
+* Accept keywords as Ruby 1.9 hash keys [#126]
  18
+* New token type @:id@ for CSS/Sass [#27]
  19
+* CSS scanner uses @:id@ and @:tag@ now [#27]
10 20
 
11 21
 h2. Changes in 1.0.9
12 22
 
12  Gemfile
... ...
@@ -1,4 +1,4 @@
1  
-source "http://rubygems.org"
  1
+source 'https://rubygems.org'
2 2
 
3 3
 # Specify your gem's dependencies in coderay.gemspec
4 4
 gemspec
@@ -9,10 +9,10 @@ gem 'paint', '~> 0.8.4'
9 9
 # Include everything needed to run rake, tests, features, etc.
10 10
 group :development do
11 11
   gem "bundler", ">= 1.0.0"
12  
-  gem "rake", "~> 0.9.2"
  12
+  gem "rake"
13 13
   gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3"
14  
-  gem "term-ansicolor"
15  
-  gem "shoulda-context", "~> 1.0.0" if RUBY_VERSION >= '1.8.7'
16  
-  gem "json" unless RUBY_VERSION >= '1.9.1'
17  
-  gem "rdoc" if RUBY_VERSION >= '1.8.7'
  14
+  gem "term-ansicolor", '~> 1.2.2'
  15
+  gem "shoulda-context", "~> 1.1.2"
  16
+  gem "json" if RUBY_VERSION < '1.9'
  17
+  gem "rdoc"
18 18
 end
8  README.markdown
Source Rendered
... ...
@@ -1,4 +1,8 @@
1  
-# CodeRay [![Build Status](https://travis-ci.org/rubychan/coderay.png)](https://travis-ci.org/rubychan/coderay)
  1
+# CodeRay
  2
+
  3
+[![Build Status](https://travis-ci.org/rubychan/coderay.png)](https://travis-ci.org/rubychan/coderay)
  4
+[![Gem Version](https://badge.fury.io/rb/coderay.png)](http://badge.fury.io/rb/coderay)
  5
+[![Dependency Status](https://gemnasium.com/rubychan/coderay.png)](https://gemnasium.com/rubychan/coderay)
2 6
 
3 7
 ## About
4 8
 
@@ -12,7 +16,7 @@ You put your code in, and you get it back colored; Keywords, strings, floats, co
12 16
 
13 17
 ### Dependencies
14 18
 
15  
-CodeRay needs Ruby 1.8.7+ or 1.9.2+. It also runs on Rubinius and JRuby.
  19
+CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0. It also runs on JRuby.
16 20
 
17 21
 ## Example Usage
18 22
 
3  bin/coderay
@@ -125,7 +125,7 @@ when 'highlight', nil
125 125
     end
126 126
     
127 127
     if output_file
128  
-      output_format ||= CodeRay::FileType[output_file]
  128
+      output_format ||= CodeRay::FileType[output_file] || :plain
129 129
     else
130 130
       output_format ||= :terminal_256
131 131
     end
@@ -143,7 +143,6 @@ when 'highlight', nil
143 143
         if output_file
144 144
           File.open output_file, 'w'
145 145
         else
146  
-          $stdout.sync = true
147 146
           $stdout
148 147
         end
149 148
       CodeRay.encode(input, input_lang, output_format, :out => file)
3  lib/coderay/encoders/debug.rb
@@ -24,11 +24,12 @@ def initialize options = {}
24 24
     end
25 25
     
26 26
     def text_token text, kind
  27
+      raise 'empty token' if $CODERAY_DEBUG && text.empty?
27 28
       if kind == :space
28 29
         @out << text
29 30
       else
30 31
         # TODO: Escape (
31  
-        text = text.gsub(/[)\\]/, '\\\\\0')  # escape ) and \
  32
+        text = text.gsub(/[)\\]/, '\\\\\0') if text.index(/[)\\]/)
32 33
         @out << kind.to_s << '(' << text << ')'
33 34
       end
34 35
     end
167  lib/coderay/encoders/html.rb
@@ -126,22 +126,21 @@ class HTML < Encoder
126 126
     
127 127
   protected
128 128
     
129  
-    HTML_ESCAPE = {  #:nodoc:
130  
-      '&' => '&amp;',
131  
-      '"' => '&quot;',
132  
-      '>' => '&gt;',
133  
-      '<' => '&lt;',
134  
-    }
  129
+    def self.make_html_escape_hash
  130
+      {
  131
+        '&' => '&amp;',
  132
+        '"' => '&quot;',
  133
+        '>' => '&gt;',
  134
+        '<' => '&lt;',
  135
+        # "\t" => will be set to ' ' * options[:tab_width] during setup
  136
+      }.tap do |hash|
  137
+        # Escape ASCII control codes except \x9 == \t and \xA == \n.
  138
+        (Array(0x00..0x8) + Array(0xB..0x1F)).each { |invalid| hash[invalid.chr] = ' ' }
  139
+      end
  140
+    end
135 141
     
136  
-    # This was to prevent illegal HTML.
137  
-    # Strange chars should still be avoided in codes.
138  
-    evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s]
139  
-    evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' }
140  
-    #ansi_chars = Array(0x7f..0xff)
141  
-    #ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
142  
-    # \x9 (\t) and \xA (\n) not included
143  
-    #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
144  
-    HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
  142
+    HTML_ESCAPE = make_html_escape_hash
  143
+    HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/
145 144
     
146 145
     TOKEN_KIND_TO_INFO = Hash.new do |h, kind|
147 146
       h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
@@ -172,59 +171,22 @@ def self.token_path_to_hint hint, kinds
172 171
     def setup options
173 172
       super
174 173
       
  174
+      check_options! options
  175
+      
175 176
       if options[:wrap] || options[:line_numbers]
176 177
         @real_out = @out
177 178
         @out = ''
178 179
       end
179 180
       
180  
-      options[:break_lines] = true if options[:line_numbers] == :inline
181  
-      
182 181
       @break_lines = (options[:break_lines] == true)
183 182
       
184  
-      @HTML_ESCAPE = HTML_ESCAPE.dup
185  
-      @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
  183
+      @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width])
186 184
       
187 185
       @opened = []
188 186
       @last_opened = nil
189 187
       @css = CSS.new options[:style]
190 188
       
191  
-      hint = options[:hint]
192  
-      if hint && ![:debug, :info, :info_long].include?(hint)
193  
-        raise ArgumentError, "Unknown value %p for :hint; \
194  
-          expected :info, :info_long, :debug, false, or nil." % hint
195  
-      end
196  
-      
197  
-      css_classes = TokenKinds
198  
-      case options[:css]
199  
-      when :class
200  
-        @span_for_kind = Hash.new do |h, k|
201  
-          if k.is_a? ::Symbol
202  
-            kind = k_dup = k
203  
-          else
204  
-            kind = k.first
205  
-            k_dup = k.dup
206  
-          end
207  
-          if kind != :space && (hint || css_class = css_classes[kind])
208  
-            title = HTML.token_path_to_hint hint, k if hint
209  
-            css_class ||= css_classes[kind]
210  
-            h[k_dup] = "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
211  
-          else
212  
-            h[k_dup] = nil
213  
-          end
214  
-        end
215  
-      when :style
216  
-        @span_for_kind = Hash.new do |h, k|
217  
-          kind = k.is_a?(Symbol) ? k : k.first
218  
-          h[k.is_a?(Symbol) ? k : k.dup] =
219  
-            if kind != :space && (hint || css_classes[kind])
220  
-              title = HTML.token_path_to_hint hint, k if hint
221  
-              style = @css.get_style Array(k).map { |c| css_classes[c] }
222  
-              "<span#{title}#{" style=\"#{style}\"" if style}>"
223  
-            end
224  
-        end
225  
-      else
226  
-        raise ArgumentError, "Unknown value %p for :css." % options[:css]
227  
-      end
  189
+      @span_for_kinds = make_span_for_kinds(options[:css], options[:hint])
228 190
       
229 191
       @set_last_opened = options[:hint] || options[:css] == :style
230 192
     end
@@ -255,20 +217,10 @@ def finish options
255 217
   public
256 218
     
257 219
     def text_token text, kind
258  
-      if text =~ /#{HTML_ESCAPE_PATTERN}/o
259  
-        text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
260  
-      end
  220
+      style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
261 221
       
262  
-      style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
263  
-      
264  
-      if @break_lines && (i = text.index("\n")) && (c = @opened.size + (style ? 1 : 0)) > 0
265  
-        close = '</span>' * c
266  
-        reopen = ''
267  
-        @opened.each_with_index do |k, index|
268  
-          reopen << (@span_for_kind[index > 0 ? [k, *@opened[0 ... index ]] : k] || '<span>')
269  
-        end
270  
-        text[i .. -1] = text[i .. -1].gsub("\n", "#{close}\n#{reopen}#{style}")
271  
-      end
  222
+      text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } if text =~ /#{HTML_ESCAPE_PATTERN}/o
  223
+      text = break_lines(text, style) if @break_lines && (style || @opened.size > 0) && text.index("\n")
272 224
       
273 225
       if style
274 226
         @out << style << text << '</span>'
@@ -279,25 +231,19 @@ def text_token text, kind
279 231
     
280 232
     # token groups, eg. strings
281 233
     def begin_group kind
282  
-      @out << (@span_for_kind[@last_opened ? [kind, *@opened] : kind] || '<span>')
  234
+      @out << (@span_for_kinds[@last_opened ? [kind, *@opened] : kind] || '<span>')
283 235
       @opened << kind
284 236
       @last_opened = kind if @set_last_opened
285 237
     end
286 238
     
287 239
     def end_group kind
288  
-      if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
289  
-        warn 'Malformed token stream: Trying to close a token (%p) ' \
290  
-          'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
291  
-      end
292  
-      if @opened.pop
293  
-        @out << '</span>'
294  
-        @last_opened = @opened.last if @last_opened
295  
-      end
  240
+      check_group_nesting 'token group', kind if $CODERAY_DEBUG
  241
+      close_span
296 242
     end
297 243
     
298 244
     # whole lines to be highlighted, eg. a deleted line in a diff
299 245
     def begin_line kind
300  
-      if style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
  246
+      if style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
301 247
         if style['class="']
302 248
           @out << style.sub('class="', 'class="line ')
303 249
         else
@@ -311,16 +257,71 @@ def begin_line kind
311 257
     end
312 258
     
313 259
     def end_line kind
314  
-      if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
315  
-        warn 'Malformed token stream: Trying to close a line (%p) ' \
316  
-          'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
  260
+      check_group_nesting 'line', kind if $CODERAY_DEBUG
  261
+      close_span
  262
+    end
  263
+    
  264
+  protected
  265
+    
  266
+    def check_options! options
  267
+      unless [false, nil, :debug, :info, :info_long].include? options[:hint]
  268
+        raise ArgumentError, "Unknown value %p for :hint; expected :info, :info_long, :debug, false, or nil." % [options[:hint]]
  269
+      end
  270
+      
  271
+      unless [:class, :style].include? options[:css]
  272
+        raise ArgumentError, 'Unknown value %p for :css.' % [options[:css]]
  273
+      end
  274
+      
  275
+      options[:break_lines] = true if options[:line_numbers] == :inline
  276
+    end
  277
+    
  278
+    def css_class_for_kinds kinds
  279
+      TokenKinds[kinds.is_a?(Symbol) ? kinds : kinds.first]
  280
+    end
  281
+    
  282
+    def style_for_kinds kinds
  283
+      css_classes = kinds.is_a?(Array) ? kinds.map { |c| TokenKinds[c] } : [TokenKinds[kinds]]
  284
+      @css.get_style_for_css_classes css_classes
  285
+    end
  286
+    
  287
+    def make_span_for_kinds method, hint
  288
+      Hash.new do |h, kinds|
  289
+        h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin
  290
+          css_class = css_class_for_kinds(kinds)
  291
+          title     = HTML.token_path_to_hint hint, kinds if hint
  292
+          
  293
+          if css_class || title
  294
+            if method == :style
  295
+              style = style_for_kinds(kinds)
  296
+              "<span#{title}#{" style=\"#{style}\"" if style}>"
  297
+            else
  298
+              "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
  299
+            end
  300
+          end
  301
+        end
  302
+      end
  303
+    end
  304
+    
  305
+    def check_group_nesting name, kind
  306
+      if @opened.empty? || @opened.last != kind
  307
+        warn "Malformed token stream: Trying to close a #{name} (%p) that is not open. Open are: %p." % [kind, @opened[1..-1]]
317 308
       end
  309
+    end
  310
+    
  311
+    def break_lines text, style
  312
+      reopen = ''
  313
+      @opened.each_with_index do |k, index|
  314
+        reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '<span>')
  315
+      end
  316
+      text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}")
  317
+    end
  318
+    
  319
+    def close_span
318 320
       if @opened.pop
319 321
         @out << '</span>'
320 322
         @last_opened = @opened.last if @last_opened
321 323
       end
322 324
     end
323  
-    
324 325
   end
325 326
   
326 327
 end
14  lib/coderay/encoders/html/css.rb
@@ -11,7 +11,7 @@ def CSS.load_stylesheet style = nil
11 11
       end
12 12
 
13 13
       def initialize style = :default
14  
-        @classes = Hash.new
  14
+        @styles = Hash.new
15 15
         style = CSS.load_stylesheet style
16 16
         @stylesheet = [
17 17
           style::CSS_MAIN_STYLES,
@@ -20,12 +20,12 @@ def initialize style = :default
20 20
         parse style::TOKEN_COLORS
21 21
       end
22 22
 
23  
-      def get_style styles
24  
-        cl = @classes[styles.first]
  23
+      def get_style_for_css_classes css_classes
  24
+        cl = @styles[css_classes.first]
25 25
         return '' unless cl
26 26
         style = ''
27  
-        1.upto styles.size do |offset|
28  
-          break if style = cl[styles[offset .. -1]]
  27
+        1.upto css_classes.size do |offset|
  28
+          break if style = cl[css_classes[offset .. -1]]
29 29
         end
30 30
         # warn 'Style not found: %p' % [styles] if style.empty?
31 31
         return style
@@ -52,8 +52,8 @@ def parse stylesheet
52 52
           for selector in selectors.split(',')
53 53
             classes = selector.scan(/[-\w]+/)
54 54
             cl = classes.pop
55  
-            @classes[cl] ||= Hash.new
56  
-            @classes[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
  55
+            @styles[cl] ||= Hash.new
  56
+            @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
57 57
           end
58 58
         end
59 59
       end
141  lib/coderay/encoders/terminal.rb
@@ -19,73 +19,73 @@ class Terminal < Encoder
19 19
       register_for :terminal
20 20
       
21 21
       TOKEN_COLORS = {
22  
-        :annotation => '35',
23  
-        :attribute_name => '33',
24  
-        :attribute_value => '31',
25  
-        :binary => '1;35',
  22
+        :annotation => "\e[35m",
  23
+        :attribute_name => "\e[33m",
  24
+        :attribute_value => "\e[31m",
  25
+        :binary => "\e[1;35m",
26 26
         :char => {
27  
-          :self => '36', :delimiter => '1;34'
  27
+          :self => "\e[36m", :delimiter => "\e[1;34m"
28 28
         },
29  
-        :class => '1;35',
30  
-        :class_variable => '36',
31  
-        :color => '32',
32  
-        :comment => '37',
33  
-        :complex => '1;34',
34  
-        :constant => ['1;34', '4'],
35  
-        :decoration => '35',
36  
-        :definition => '1;32',
37  
-        :directive => ['32', '4'],
38  
-        :doc => '46',
39  
-        :doctype => '1;30',
40  
-        :doc_string => ['31', '4'],
41  
-        :entity => '33',
42  
-        :error => ['1;33', '41'],
43  
-        :exception => '1;31',
44  
-        :float => '1;35',
45  
-        :function => '1;34',
46  
-        :global_variable => '42',
47  
-        :hex => '1;36',
48  
-        :include => '33',
49  
-        :integer => '1;34',
50  
-        :key => '35',
51  
-        :label => '1;15',
52  
-        :local_variable => '33',
53  
-        :octal => '1;35',
54  
-        :operator_name => '1;29',
55  
-        :predefined_constant => '1;36',
56  
-        :predefined_type => '1;30',
57  
-        :predefined => ['4', '1;34'],
58  
-        :preprocessor => '36',
59  
-        :pseudo_class => '1;34',
  29
+        :class => "\e[1;35m",
  30
+        :class_variable => "\e[36m",
  31
+        :color => "\e[32m",
  32
+        :comment => "\e[37m",
  33
+        :complex => "\e[1;34m",
  34
+        :constant => "\e[1;34m\e[4m",
  35
+        :decoration => "\e[35m",
  36
+        :definition => "\e[1;32m",
  37
+        :directive => "\e[32m\e[4m",
  38
+        :doc => "\e[46m",
  39
+        :doctype => "\e[1;30m",
  40
+        :doc_string => "\e[31m\e[4m",
  41
+        :entity => "\e[33m",
  42
+        :error => "\e[1;33m\e[41m",
  43
+        :exception => "\e[1;31m",
  44
+        :float => "\e[1;35m",
  45
+        :function => "\e[1;34m",
  46
+        :global_variable => "\e[42m",
  47
+        :hex => "\e[1;36m",
  48
+        :include => "\e[33m",
  49
+        :integer => "\e[1;34m",
  50
+        :key => "\e[35m",
  51
+        :label => "\e[1;15m",
  52
+        :local_variable => "\e[33m",
  53
+        :octal => "\e[1;35m",
  54
+        :operator_name => "\e[1;29m",
  55
+        :predefined_constant => "\e[1;36m",
  56
+        :predefined_type => "\e[1;30m",
  57
+        :predefined => "\e[4m\e[1;34m",
  58
+        :preprocessor => "\e[36m",
  59
+        :pseudo_class => "\e[1;34m",
60 60
         :regexp => {
61  
-          :self => '31',
62  
-          :content => '31',
63  
-          :delimiter => '1;29',
64  
-          :modifier => '35',
  61
+          :self => "\e[31m",
  62
+          :content => "\e[31m",
  63
+          :delimiter => "\e[1;29m",
  64
+          :modifier => "\e[35m",
65 65
         },
66  
-        :reserved => '1;31',
  66
+        :reserved => "\e[1;31m",
67 67
         :shell => {
68  
-          :self => '42',
69  
-          :content => '1;29',
70  
-          :delimiter => '37',
  68
+          :self => "\e[42m",
  69
+          :content => "\e[1;29m",
  70
+          :delimiter => "\e[37m",
71 71
         },
72 72
         :string => {
73  
-          :self => '32',
74  
-          :modifier => '1;32',
75  
-          :escape => '1;36',
76  
-          :delimiter => '1;32',
77  
-          :char => '1;36',
  73
+          :self => "\e[32m",
  74
+          :modifier => "\e[1;32m",
  75
+          :escape => "\e[1;36m",
  76
+          :delimiter => "\e[1;32m",
  77
+          :char => "\e[1;36m",
78 78
         },
79  
-        :symbol => '1;32',
80  
-        :tag => '1;34',
81  
-        :type => '1;34',
82  
-        :value => '36',
83  
-        :variable => '1;34',
  79
+        :symbol => "\e[1;32m",
  80
+        :tag => "\e[1;34m",
  81
+        :type => "\e[1;34m",
  82
+        :value => "\e[36m",
  83
+        :variable => "\e[1;34m",
84 84
         
85  
-        :insert => '42',
86  
-        :delete => '41',
87  
-        :change => '44',
88  
-        :head => '45'
  85
+        :insert => "\e[42m",
  86
+        :delete => "\e[41m",
  87
+        :change => "\e[44m",
  88
+        :head => "\e[45m"
89 89
       }
90 90
       TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved]
91 91
       TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
@@ -114,10 +114,10 @@ def text_token text, kind
114 114
             end
115 115
           end
116 116
           
117  
-          @out << ansi_colorize(color)
118  
-          @out << text.gsub("\n", ansi_clear + "\n" + ansi_colorize(color))
119  
-          @out << ansi_clear
120  
-          @out << ansi_colorize(@subcolors[:self]) if @subcolors && @subcolors[:self]
  117
+          @out << color
  118
+          @out << text.gsub("\n", "\e[0m\n" + color)
  119
+          @out << "\e[0m"
  120
+          @out << @subcolors[:self] if @subcolors
121 121
         else
122 122
           @out << text
123 123
         end
@@ -134,7 +134,7 @@ def end_group kind
134 134
           # nothing to close
135 135
         else
136 136
           @opened.pop
137  
-          @out << ansi_clear
  137
+          @out << "\e[0m"
138 138
           @out << open_token(@opened.last)
139 139
         end
140 140
       end
@@ -146,7 +146,7 @@ def end_line kind
146 146
           @opened.pop
147 147
           # whole lines to be highlighted,
148 148
           # eg. added/modified/deleted lines in a diff
149  
-          @out << "\t" * 100 + ansi_clear
  149
+          @out << (@line_filler ||= "\t" * 100 + "\e[0m")
150 150
           @out << open_token(@opened.last)
151 151
         end
152 152
       end
@@ -157,23 +157,16 @@ def open_token kind
157 157
         if color = TOKEN_COLORS[kind]
158 158
           if Hash === color
159 159
             @subcolors = color
160  
-            ansi_colorize(color[:self]) if color[:self]
  160
+            color[:self]
161 161
           else
162 162
             @subcolors = {}
163  
-            ansi_colorize(color)
  163
+            color
164 164
           end
165 165
         else
166 166
           @subcolors = nil
167 167
           ''
168 168
         end
169 169
       end
170  
-      
171  
-      def ansi_colorize(color)
172  
-        Array(color).map { |c| "\e[#{c}m" }.join
173  
-      end
174  
-      def ansi_clear
175  
-        ansi_colorize(0)
176  
-      end
177 170
     end
178 171
   end
179  
-end
  172
+end
5  lib/coderay/helpers/file_type.rb
@@ -99,6 +99,7 @@ def shebang filename
99 99
       'mab'      => :ruby,
100 100
       'pas'      => :delphi,
101 101
       'patch'    => :diff,
  102
+      'phtml'    => :php,
102 103
       'php'      => :php,
103 104
       'php3'     => :php,
104 105
       'php4'     => :php,
@@ -116,10 +117,10 @@ def shebang filename
116 117
       'rpdf'     => :ruby,
117 118
       'ru'       => :ruby,
118 119
       'rxml'     => :ruby,
119  
-      # 'sch'      => :scheme,
  120
+      'sass'     => :sass,
120 121
       'sql'      => :sql,
121  
-      # 'ss'       => :scheme,
122 122
       'tmproj'   => :xml,
  123
+      'xaml'     => :xml,
123 124
       'xhtml'    => :html,
124 125
       'xml'      => :xml,
125 126
       'yaml'     => :yaml,
17  lib/coderay/helpers/plugin.rb
@@ -131,7 +131,7 @@ def register plugin, id
131 131
     
132 132
     # A Hash of plugion_id => Plugin pairs.
133 133
     def plugin_hash
134  
-      @plugin_hash ||= make_plugin_hash
  134
+      @plugin_hash ||= (@plugin_hash = make_plugin_hash).tap { load_plugin_map }
135 135
     end
136 136
     
137 137
     # Returns an array of all .rb files in the plugin path.
@@ -158,7 +158,6 @@ def all_plugins
158 158
     # This is done automatically when plugin_path is called.
159 159
     def load_plugin_map
160 160
       mapfile = path_to '_map'
161  
-      @plugin_map_loaded = true
162 161
       if File.exist? mapfile
163 162
         require mapfile
164 163
         true
@@ -171,23 +170,16 @@ def load_plugin_map
171 170
     
172 171
     # Return a plugin hash that automatically loads plugins.
173 172
     def make_plugin_hash
174  
-      @plugin_map_loaded ||= false
175 173
       Hash.new do |h, plugin_id|
176 174
         id = validate_id(plugin_id)
177 175
         path = path_to id
178 176
         begin
179 177
           require path
180 178
         rescue LoadError => boom
181  
-          if @plugin_map_loaded
182  
-            if h.has_key?(:default)
183  
-              warn '%p could not load plugin %p; falling back to %p' % [self, id, h[:default]]
184  
-              h[:default]
185  
-            else
186  
-              raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom]
187  
-            end
  179
+          if h.has_key?(:default)
  180
+            h[:default]
188 181
           else
189  
-            load_plugin_map
190  
-            h[plugin_id]
  182
+            raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom]
191 183
           end
192 184
         else
193 185
           # Plugin should have registered by now
@@ -271,7 +263,6 @@ def plugin_host host = nil
271 263
     end
272 264
     
273 265
     def aliases
274  
-      plugin_host.load_plugin_map
275 266
       plugin_host.plugin_hash.inject [] do |aliases, (key, _)|
276 267
         aliases << key if plugin_host[key] == self
277 268
         aliases
84  lib/coderay/scanner.rb
@@ -182,16 +182,9 @@ def file_extension
182 182
       # Scan the code and returns all tokens in a Tokens object.
183 183
       def tokenize source = nil, options = {}
184 184
         options = @options.merge(options)
185  
-        @tokens = options[:tokens] || @tokens || Tokens.new
186  
-        @tokens.scanner = self if @tokens.respond_to? :scanner=
187  
-        case source
188  
-        when Array
189  
-          self.string = self.class.normalize(source.join)
190  
-        when nil
191  
-          reset
192  
-        else
193  
-          self.string = self.class.normalize(source)
194  
-        end
  185
+        
  186
+        set_tokens_from_options options
  187
+        set_string_from_source source
195 188
         
196 189
         begin
197 190
           scan_tokens @tokens, options
@@ -261,6 +254,22 @@ def binary_string
261 254
       def setup  # :doc:
262 255
       end
263 256
       
  257
+      def set_string_from_source source
  258
+        case source
  259
+        when Array
  260
+          self.string = self.class.normalize(source.join)
  261
+        when nil
  262
+          reset
  263
+        else
  264
+          self.string = self.class.normalize(source)
  265
+        end
  266
+      end
  267
+      
  268
+      def set_tokens_from_options options
  269
+        @tokens = options[:tokens] || @tokens || Tokens.new
  270
+        @tokens.scanner = self if @tokens.respond_to? :scanner=
  271
+      end
  272
+      
264 273
       # This is the central method, and commonly the only one a
265 274
       # subclass implements.
266 275
       #
@@ -277,19 +286,15 @@ def reset_instance
277 286
         @binary_string = nil if defined? @binary_string
278 287
       end
279 288
       
280  
-      # Scanner error with additional status information
281  
-      def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit = 30, backtrace = caller
282  
-        raise ScanError, <<-EOE % [
  289
+      SCAN_ERROR_MESSAGE = <<-MESSAGE
283 290
 
284 291
 
285  
-***ERROR in %s: %s (after %d tokens)
  292
+***ERROR in %s: %s (after %s tokens)
286 293
 
287 294
 tokens:
288 295
 %s
289 296
 
290  
-current line: %d  column: %d  pos: %d
291  
-matched: %p  state: %p
292  
-bol? = %p,  eos? = %p
  297
+%s
293 298
 
294 299
 surrounding code:
295 300
 %p  ~~  %p
@@ -297,16 +302,43 @@ def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit =
297 302
 
298 303
 ***ERROR***
299 304
 
300  
-        EOE
301  
-          File.basename(caller[0]),
302  
-          msg,
303  
-          tokens.respond_to?(:size) ? tokens.size : 0,
304  
-          tokens.respond_to?(:last) ? tokens.last(10).map { |t| t.inspect }.join("\n") : '',
305  
-          line, column, pos,
306  
-          matched, state, bol?, eos?,
  305
+      MESSAGE
  306
+      
  307
+      def raise_inspect_arguments message, tokens, state, ambit
  308
+        return File.basename(caller[0]),
  309
+          message,
  310
+          tokens_size(tokens),
  311
+          tokens_last(tokens, 10).map(&:inspect).join("\n"),
  312
+          scanner_state_info(state),
307 313
           binary_string[pos - ambit, ambit],
308  
-          binary_string[pos, ambit],
309  
-        ], backtrace
  314
+          binary_string[pos, ambit]
  315
+      end
  316
+      
  317
+      SCANNER_STATE_INFO = <<-INFO
  318
+current line: %d  column: %d  pos: %d
  319
+matched: %p  state: %p
  320
+bol?: %p,  eos?: %p
  321
+      INFO
  322
+      
  323
+      def scanner_state_info state
  324
+        SCANNER_STATE_INFO % [
  325
+          line, column, pos,
  326
+          matched, state || 'No state given!',
  327
+          bol?, eos?,
  328
+        ]
  329
+      end
  330
+      
  331
+      # Scanner error with additional status information
  332
+      def raise_inspect message, tokens, state = self.state, ambit = 30, backtrace = caller
  333
+        raise ScanError, SCAN_ERROR_MESSAGE % raise_inspect_arguments(message, tokens, state, ambit), backtrace
  334
+      end
  335
+      
  336
+      def tokens_size tokens
  337
+        tokens.size if tokens.respond_to?(:size)
  338
+      end
  339
+      
  340
+      def tokens_last tokens, n
  341
+        tokens.respond_to?(:last) ? tokens.last(n) : []
310 342
       end
311 343
       
312 344
       # Shorthand for scan_until(/\z/).
2  lib/coderay/scanners/c.rb
@@ -148,7 +148,7 @@ def scan_tokens encoder, options
148 148
             encoder.text_token match, :char
149 149
           elsif match = scan(/ \\ | $ /x)
150 150
             encoder.end_group :string
151  
-            encoder.text_token match, :error
  151
+            encoder.text_token match, :error unless match.empty?
152 152
             state = :initial
153 153
             label_expected = false
154 154
           else
2  lib/coderay/scanners/cpp.rb
@@ -160,7 +160,7 @@ def scan_tokens encoder, options
160 160
             encoder.text_token match, :char
161 161
           elsif match = scan(/ \\ | $ /x)
162 162
             encoder.end_group :string
163  
-            encoder.text_token match, :error
  163
+            encoder.text_token match, :error unless match.empty?
164 164
             state = :initial
165 165
             label_expected = false
166 166
           else
41  lib/coderay/scanners/css.rb
@@ -7,27 +7,25 @@ class CSS < Scanner
7 7
     
8 8
     KINDS_NOT_LOC = [
9 9
       :comment,
10  
-      :class, :pseudo_class, :type,
11  
-      :constant, :directive,
  10
+      :class, :pseudo_class, :tag,
  11
+      :id, :directive,
12 12
       :key, :value, :operator, :color, :float, :string,
13  
-      :error, :important,
  13
+      :error, :important, :type,
14 14
     ]  # :nodoc:
15 15
     
16 16
     module RE  # :nodoc:
17 17
       Hex = /[0-9a-fA-F]/
18  
-      Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
19  
-      Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
20  
-      NMChar = /[-_a-zA-Z0-9]|#{Escape}/
21  
-      NMStart = /[_a-zA-Z]|#{Escape}/
22  
-      NL = /\r\n|\r|\n|\f/
23  
-      String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/  # TODO: buggy regexp
24  
-      String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/  # TODO: buggy regexp
  18
+      Unicode = /\\#{Hex}{1,6}\b/ # differs from standard because it allows uppercase hex too
  19
+      Escape = /#{Unicode}|\\[^\n0-9a-fA-F]/
  20
+      NMChar = /[-_a-zA-Z0-9]/
  21
+      NMStart = /[_a-zA-Z]/
  22
+      String1 = /"(?:[^\n\\"]+|\\\n|#{Escape})*"?/  # TODO: buggy regexp
  23
+      String2 = /'(?:[^\n\\']+|\\\n|#{Escape})*'?/  # TODO: buggy regexp
25 24
       String = /#{String1}|#{String2}/
26 25
       
27 26
       HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
28  
-      Color = /#{HexColor}/
29 27
       
30  
-      Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
  28
+      Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)/
31 29
       Name = /#{NMChar}+/
32 30
       Ident = /-?#{NMStart}#{NMChar}*/
33 31
       AtKeyword = /@#{Ident}/
@@ -35,16 +33,15 @@ module RE  # :nodoc:
35 33
       
36 34
       reldimensions = %w[em ex px]
37 35
       absdimensions = %w[in cm mm pt pc]
38  
-      Unit = Regexp.union(*(reldimensions + absdimensions + %w[s]))
  36
+      Unit = Regexp.union(*(reldimensions + absdimensions + %w[s dpi dppx deg]))
39 37
       
40 38
       Dimension = /#{Num}#{Unit}/
41 39
       
42  
-      Comment = %r! /\* (?: .*? \*/ | .* ) !mx
43  
-      Function = /(?:url|alpha|attr|counters?)\((?:[^)\n\r\f]|\\\))*\)?/
  40
+      Function = /(?:url|alpha|attr|counters?)\((?:[^)\n]|\\\))*\)?/
44 41
       
45  
-      Id = /##{Name}/
  42
+      Id = /(?!#{HexColor}\b(?!-))##{Name}/
46 43
       Class = /\.#{Name}/
47  
-      PseudoClass = /:#{Name}/
  44
+      PseudoClass = /::?#{Ident}/
48 45
       AttributeSelector = /\[[^\]]*\]?/
49 46
     end
50 47
     
@@ -52,7 +49,7 @@ module RE  # :nodoc:
52 49
     
53 50
     def setup
54 51
       @state = :initial
55  
-      @value_expected = nil
  52
+      @value_expected = false
56 53
     end
57 54
     
58 55
     def scan_tokens encoder, options
@@ -67,13 +64,13 @@ def scan_tokens encoder, options
67 64
         elsif case states.last
68 65
           when :initial, :media
69 66
             if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox)
70  
-              encoder.text_token match, :type
  67
+              encoder.text_token match, :tag
71 68
               next
72 69
             elsif match = scan(RE::Class)
73 70
               encoder.text_token match, :class
74 71
               next
75 72
             elsif match = scan(RE::Id)
76  
-              encoder.text_token match, :constant
  73
+              encoder.text_token match, :id
77 74
               next
78 75
             elsif match = scan(RE::PseudoClass)
79 76
               encoder.text_token match, :pseudo_class
@@ -158,7 +155,7 @@ def scan_tokens encoder, options
158 155
         elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
159 156
           encoder.text_token match, :float
160 157
           
161  
-        elsif match = scan(/#{RE::Color}/o)
  158
+        elsif match = scan(/#{RE::HexColor}/o)
162 159
           encoder.text_token match, :color
163 160
           
164 161
         elsif match = scan(/! *important/)
@@ -170,7 +167,7 @@ def scan_tokens encoder, options
170 167
         elsif match = scan(RE::AtKeyword)
171 168
           encoder.text_token match, :directive
172 169
           
173  
-        elsif match = scan(/ [+>:;,.=()\/] /x)
  170
+        elsif match = scan(/ [+>~:;,.=()\/] /x)
174 171
           if match == ':'
175 172
             value_expected = true
176 173
           elsif match == ';'
2  lib/coderay/scanners/diff.rb
@@ -45,7 +45,7 @@ def scan_tokens encoder, options
45 45
           if match = scan(/--- |\+\+\+ |=+|_+/)
46 46
             encoder.begin_line line_kind = :head
47 47
             encoder.text_token match, :head
48  
-            if match = scan(/.*?(?=$|[\t\n\x00]|  \(revision)/)
  48
+            if match = scan(/[^\x00\n]+?(?=$|[\t\n]|  \(revision)/)
49 49
               encoder.text_token match, :filename
50 50
               if options[:highlight_code] && match != '/dev/null'
51 51
                 file_type = CodeRay::FileType.fetch(match, :text)
2  lib/coderay/scanners/java.rb
@@ -147,7 +147,7 @@ def scan_tokens encoder, options
147 147
           elsif match = scan(/ \\ | $ /x)
148 148
             encoder.end_group state
149 149
             state = :initial
150  
-            encoder.text_token match, :error
  150
+            encoder.text_token match, :error unless match.empty?
151 151
           else
152 152
             raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
153 153
           end
36  lib/coderay/scanners/java_script.rb
@@ -54,10 +54,17 @@ class JavaScript < Scanner
54 54
     
55 55
   protected
56 56
     
  57
+    def setup
  58
+      @state = :initial
  59
+    end
  60
+    
57 61
     def scan_tokens encoder, options
58 62
       
59  
-      state = :initial
60  
-      string_delimiter = nil
  63
+      state, string_delimiter = options[:state] || @state
  64
+      if string_delimiter
  65
+        encoder.begin_group state
  66
+      end
  67
+      
61 68
       value_expected = true
62 69
       key_expected = false
63 70
       function_expected = false
@@ -72,9 +79,10 @@ def scan_tokens encoder, options
72 79
             value_expected = true if !value_expected && match.index(?\n)
73 80
             encoder.text_token match, :space
74 81
             
75  
-          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
  82
+          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .*() ) !mx)
76 83
             value_expected = true
77 84
             encoder.text_token match, :comment
  85
+            state = :open_multi_line_comment if self[1]
78 86
             
79 87
           elsif check(/\.?\d/)
80 88
             key_expected = value_expected = false
@@ -175,20 +183,36 @@ def scan_tokens encoder, options
175 183
             encoder.text_token match, :content
176 184
           elsif match = scan(/ \\ | $ /x)
177 185
             encoder.end_group state
178  
-            encoder.text_token match, :error
  186
+            encoder.text_token match, :error unless match.empty?
  187
+            string_delimiter = nil
179 188
             key_expected = value_expected = false
180 189
             state = :initial
181 190
           else
182  
-            raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
  191
+            raise_inspect "else case #{string_delimiter} reached; %p not handled." % peek(1), encoder
183 192
           end
184 193
           
  194
+        when :open_multi_line_comment
  195
+          if match = scan(%r! .*? \*/ !mx)
  196
+            state = :initial
  197
+          else
  198
+            match = scan(%r! .+ !mx)
  199
+          end
  200
+          value_expected = true
  201
+          encoder.text_token match, :comment if match
  202
+          
185 203
         else
186  
-          raise_inspect 'Unknown state', encoder
  204
+          #:nocov:
  205
+          raise_inspect 'Unknown state: %p' % [state], encoder
  206
+          #:nocov:
187 207
           
188 208
         end
189 209
         
190 210
       end
191 211
       
  212
+      if options[:keep_state]
  213
+        @state = state, string_delimiter
  214
+      end
  215
+      
192 216
       if [:string, :regexp].include? state
193 217
         encoder.end_group state
194 218
       end
2  lib/coderay/scanners/json.rb
@@ -70,7 +70,7 @@ def scan_tokens encoder, options
70 70
             encoder.text_token match, :content
71 71
           elsif match = scan(/ \\ | $ /x)
72 72
             encoder.end_group state
73  
-            encoder.text_token match, :error
  73
+            encoder.text_token match, :error unless match.empty?
74 74
             state = :initial
75 75
           else
76 76
             raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
2  lib/coderay/scanners/python.rb
@@ -133,7 +133,7 @@ def scan_tokens encoder, options
133 133
           elsif match = scan(/ \\ | $ /x)
134 134
             encoder.end_group string_type
135 135
             string_type = nil
136  
-            encoder.text_token match, :error
  136
+            encoder.text_token match, :error unless match.empty?
137 137
             state = :initial
138 138
           else
139 139
             raise_inspect "else case \" reached; %p not handled." % peek(1), encoder, state
2  lib/coderay/scanners/ruby.rb
@@ -96,7 +96,7 @@ def scan_tokens encoder, options
96 96
                                       /#{patterns::METHOD_NAME}/o)
97 97
               
98 98
               kind = patterns::IDENT_KIND[match]
99  
-              if kind == :ident && value_expected != :colon_expected && scan(/:(?!:)/)
  99
+              if value_expected != :colon_expected && scan(/:(?!:)/)
100 100
                 value_expected = true
101 101
                 encoder.text_token match, :key
102 102
                 encoder.text_token ':',   :operator
227  lib/coderay/scanners/sass.rb
... ...
@@ -0,0 +1,227 @@
  1
+module CodeRay
  2
+module Scanners
  3
+  
  4
+  # A scanner for Sass.
  5
+  class Sass < CSS
  6
+    
  7
+    register_for :sass
  8
+    file_extension 'sass'
  9
+    
  10
+    STRING_CONTENT_PATTERN = {
  11
+      "'" => /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
  12
+      '"' => /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/,
  13
+    }
  14
+    
  15
+  protected
  16
+    
  17
+    def setup
  18
+      @state = :initial
  19
+    end
  20
+    
  21
+    def scan_tokens encoder, options
  22
+      states = Array(options[:state] || @state)
  23
+      string_delimiter = nil
  24
+      
  25
+      until eos?
  26
+        
  27
+        if bol? && (match = scan(/(?>( +)?(\/[\*\/])(.+)?)(?=\n)/))
  28
+          encoder.text_token self[1], :space if self[1]
  29
+          encoder.begin_group :comment
  30
+          encoder.text_token self[2], :delimiter
  31
+          encoder.text_token self[3], :content if self[3]
  32
+          if match = scan(/(?:\n+#{self[1]} .*)+/)
  33
+            encoder.text_token match, :content
  34
+          end
  35
+          encoder.end_group :comment
  36
+        elsif match = scan(/\n|[^\n\S]+\n?/)
  37
+          encoder.text_token match, :space
  38
+          if match.index(/\n/)
  39
+            value_expected = false
  40
+            states.pop if states.last == :include
  41
+          end
  42
+        
  43
+        elsif states.last == :sass_inline && (match = scan(/\}/))
  44
+          encoder.text_token match, :inline_delimiter
  45
+          encoder.end_group :inline
  46
+          states.pop
  47
+        
  48
+        elsif case states.last
  49
+          when :initial, :media, :sass_inline
  50
+            if match = scan(/(?>#{RE::Ident})(?!\()/ox)
  51
+              encoder.text_token match, value_expected ? :value : (check(/.*:/) ? :key : :tag)
  52
+              next
  53
+            elsif !value_expected && (match = scan(/\*/))
  54
+              encoder.text_token match, :tag
  55
+              next
  56