Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

more fixes

  • Loading branch information...
commit cf38ee2f3a39a4f10d6043a046815c24b18513ec 1 parent 581c52d
@Asherlc Asherlc authored
Showing with 55 additions and 32 deletions.
  1. +55 −32 lib/coderay/scanners/liquid.rb
View
87 lib/coderay/scanners/liquid.rb
@@ -3,6 +3,8 @@ module Scanners
class Liquid < Scanner
+ require 'csv'
+
register_for :liquid
DIRECTIVE_KEYWORDS = /endlist|list|endfor|for|endwrap|wrap|endif|if|endunless|unless|elsif|assignlist|assign|cycle|capture|end|capture|fill|endiflist|iflist|else/
@@ -11,13 +13,11 @@ class Liquid < Scanner
DIRECTIVE_PREPOSITIONS= /contains|in|#{MATH}/
- FILTER_KEYWORDS = /#{FILTER_WITH_VALUE_KEYWORDS}|textilize|capitalize|downcase|upcase|first|last|sort|map|size|escape_once|escape|strip_html|strip_newlines|newline_to_br/
-
FILTER_WITH_VALUE_KEYWORDS = /date|replace_first|replace|remove_first|remove_first|remove|minus|times|divided_by|modulo|mod|split|join|truncatewords|truncate|prepend|append/
- SELECTOR_KEYWORDS = /in|with|snippet|script|content_item|folder|widget|wrapper|category|asset_folder|asset/
+ FILTER_KEYWORDS = /#{FILTER_WITH_VALUE_KEYWORDS}|textilize|capitalize|downcase|upcase|first|last|sort|map|size|escape_once|escape|strip_html|strip_newlines|newline_to_br/
- DIRECTIVE_KEYS = /#{SELECTOR_KEYWORDS}|tabs|items_per_tab/
+ SELECTOR_KEYWORDS = /in|with|snippet|script|content_item|folder|widget|wrapper|category|asset_folder|asset/
LIQUID_DIRECTIVE_BLOCK = /
{{1,2}%
@@ -25,6 +25,8 @@ class Liquid < Scanner
%}{1,2}
/x
+ KEY_VALUE_REGEX = /(\w+)(:)(\w+|".*"|'.*?')/
+
def setup
@html_scanner = CodeRay.scanner(:html, tokens: @tokens, keep_tokens: true, keep_state: true)
end
@@ -35,35 +37,62 @@ def scan_spaces(encoder)
end
end
- def scan_key_value_pair(encoder, options, match)
- scan_spaces(encoder)
- if match =~ /#{SELECTOR_KEYWORDS}/
- encoder.text_token match, :directive
+ def scan_string(encoder, substring)
+ if substring and string_array = substring.match(/('|")(.+)('|")/)
+ delimiter = string_array.captures[0]
+ contents = string_array.captures[1]
+ delimiter_2 = string_array.captures[2]
+
+ encoder.begin_group :string
+ encoder.text_token delimiter, :delimiter
+ encoder.text_token contents, :contents
+ encoder.text_token delimiter, :delimiter
+ encoder.end_group :string
+
+ true
else
- encoder.text_token match, :key
+ false
end
- if delimiter = scan(/:/)
- encoder.text_token delimiter, :delimiter
- scan_spaces(encoder)
+ end
+
+ def scan_csv_list(encoder, list)
+ CSV.parse(list) do |row|
@korny
korny added a note

It's not a good idea to use the CSV scanner for scanning. CodeRay should never remove or change whitespace from the input.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ column_index = 0
+ row.each do |value|
+ column_index += 1
+ unless scan_string(encoder, value)
+ encoder.text_token value, :value
+ end
+ unless column_index >= row.length
+ encoder.text_token ',', :delimiter
+ end
+ end
end
- if value = scan(/\w+/)
- encoder.text_token value, :value
- elsif value = scan(/('\S+')|("\w+")/)
- encoder.text_token value, :string
+ end
+
+ def scan_key_value_pair(encoder, options, match)
+ scan_spaces(encoder)
+ if match = check(/#{KEY_VALUE_REGEX}/)
+ key = scan(/\w+/)
+ delimiter, values = scan(/(:)(\S+)|(".*?")|('.*?')/).match(/(:)(\S+)|(".*?")|('.*?')/).captures
+
+ if key =~ /#{SELECTOR_KEYWORDS}/
+ encoder.text_token key, :directive
+ else
+ encoder.text_token key, :key
+ end
+
+ encoder.text_token delimiter, :delimiter
+ scan_csv_list(encoder, values)
+ true
+ else
+ false
end
end
def scan_selector(encoder, options, match)
scan_spaces(encoder)
- Rails.logger.debug 'DEBUG: Looking for a selector'
- if match = scan(/#{DIRECTIVE_KEYS}/)
- if peek(1) == ':'
- Rails.logger.debug "DEBUG: Peek: #{peek(5)}"
- Rails.logger.debug 'DEBUG: Selector keyword found'
- scan_key_value_pair(encoder, options, match)
- else
- encoder.text_token match, :variable
- end
+ if scan_key_value_pair(encoder, options, match)
scan_selector(encoder, options, match)
else
false
@@ -71,7 +100,6 @@ def scan_selector(encoder, options, match)
end
def scan_directive(encoder, options, match)
- Rails.logger.debug 'DEBUG: Scanning directive'
encoder.text_token match, :tag
state = :liquid
scan_spaces(encoder)
@@ -106,9 +134,7 @@ def scan_directive(encoder, options, match)
def scan_output_filters(encoder, options, match)
encoder.text_token match, :operator
scan_spaces(encoder)
- if match = scan(/#{FILTER_WITH_VALUE_KEYWORDS}/)
- scan_key_value_pair(encoder, options, match)
- elsif directive = scan(/#{FILTER_KEYWORDS}/)
+ if !scan_key_value_pair(encoder, options, match) and directive = scan(/#{FILTER_KEYWORDS}/)
encoder.text_token directive, :directive
end
if next_filter = scan(/\s\|\s/)
@@ -117,7 +143,6 @@ def scan_output_filters(encoder, options, match)
end
def scan_output(encoder, options, match)
- Rails.logger.debug 'DEBUG: Scanning output'
encoder.text_token match, :tag
state = :liquid
scan_spaces(encoder)
@@ -135,12 +160,10 @@ def scan_output(encoder, options, match)
end
def scan_tokens(encoder, options)
- Rails.logger.debug "DEBUG: Scan started: #{self.string}"
state = :initial
until eos?
if (match = scan_until(/(?=({{2,3}|{{1,2}%))/) || scan_rest) and not match.empty? and state != :liquid
- Rails.logger.debug "DEBUG: HTML scanning: #{match}"
@html_scanner.tokenize(match, tokens: encoder)
state = :initial
scan_spaces(encoder)
Please sign in to comment.
Something went wrong with that request. Please try again.