Permalink
Browse files

Merge branch 'master' into bash-scanner

  • Loading branch information...
korny committed Feb 13, 2016
2 parents 3eca4cb + 0a1f500 commit 973fc94dc7fc134aec6b06ae3b9190a920418b1d
View
@@ -2,7 +2,10 @@ rvm:
- 1.8.7
- ree
- 1.9.3
- - 2.0.0
+ - 2.0
+ - 2.1
+ - 2.2
+ - 2.3.0
- ruby-head
- jruby-18mode
- jruby-19mode
@@ -19,3 +22,4 @@ matrix:
- rvm: rbx-18mode
- rvm: rbx-19mode
script: "rake test" # test:scanners"
+sudo: false
View
@@ -2,6 +2,17 @@ h1=. CodeRay Version History
p=. _This files lists all changes in the CodeRay library since the 0.9.8 release._
+h2. Changes in 1.1.1
+
+* SQL scanner: Allow @$@ signs in SQL identifiers [#164, thanks to jasir and Ben Basson]
+* SQL scanner: Fix open strings [#163, thanks to Adam]
+* Ruby scanner: Accept number literal suffixes @r@ and @i@ (Ruby 2.1)
+* Ruby scanner: Accept quoted hash keys like @{ "a": boss }@ (Ruby 2.2)
+* Ruby scanner: Accept save navigation operator @&.@ (Ruby 2.3)
+* Ruby scanner: Accept squiggly heredoc @<<~@ (Ruby 2.3)
+* Diff scanner: Prevent running out of regexp stack.
+* HTML encoder: You can keep tabs intact now by setting @tab_width: false@.
+
h2. Changes in 1.1
New scanners:
View
@@ -6,11 +6,13 @@ gemspec
# Add dependencies to develop your gem here.
# Include everything needed to run rake, tests, features, etc.
group :development do
- gem "bundler", ">= 1.0.0"
+ gem "bundler"
gem "rake"
gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3"
- gem "term-ansicolor", '~> 1.2.2'
- gem "shoulda-context", "~> 1.1.2"
+ gem "term-ansicolor"
+ gem 'tins', '~> 1.6.0'
+ gem "shoulda-context"
+ gem "test-unit"
gem "json" if RUBY_VERSION < '1.9'
gem "rdoc"
end
View
@@ -16,7 +16,7 @@ You put your code in, and you get it back colored; Keywords, strings, floats, co
### Dependencies
-CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0. It also runs on JRuby.
+CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0+. It also runs on JRuby.
## Example Usage
View
@@ -35,7 +35,7 @@ defaults:
common:
coderay file.rb # highlight file to terminal
- coderay file.rb > file.html # highlight file to HTML page
+ coderay file.rb -page > file.html # highlight file to HTML page
coderay file.rb -div > file.html # highlight file to HTML snippet
configure output:
@@ -25,7 +25,8 @@ module Encoders
# == Options
#
# === :tab_width
- # Convert \t characters to +n+ spaces (a number.)
+ # Convert \t characters to +n+ spaces (a number or false.)
+ # false will keep tab characters untouched.
#
# Default: 8
#
@@ -180,7 +181,7 @@ def setup options
@break_lines = (options[:break_lines] == true)
- @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width])
+ @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => options[:tab_width] ? ' ' * options[:tab_width] : "\t")
@opened = []
@last_opened = nil
@@ -100,7 +100,7 @@ def scan_tokens encoder, options
next
elsif match = scan(/-/)
deleted_lines_count += 1
- if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/)
+ if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && changed_lines_count <= 100_000 && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/)
deleted_lines = Array.new(changed_lines_count) { |i| skip(/\n\-/) if i > 0; scan(/.*/) }
inserted_lines = Array.new(changed_lines_count) { |i| skip(/\n\+/) ; scan(/.*/) }
@@ -164,15 +164,19 @@ def scan_tokens encoder, options
end
elsif match = scan(/ ' (?:(?>[^'\\]*) ')? | " (?:(?>[^"\\\#]*) ")? /mx)
- encoder.begin_group :string
if match.size == 1
+ kind = check(self.class::StringState.simple_key_pattern(match)) ? :key : :string
+ encoder.begin_group kind
encoder.text_token match, :delimiter
- state = self.class::StringState.new :string, match == '"', match # important for streaming
+ state = self.class::StringState.new kind, match == '"', match # important for streaming
else
+ kind = value_expected == true && scan(/:/) ? :key : :string
+ encoder.begin_group kind
encoder.text_token match[0,1], :delimiter
encoder.text_token match[1..-2], :content if match.size > 2
encoder.text_token match[-1,1], :delimiter
- encoder.end_group :string
+ encoder.end_group kind
+ encoder.text_token ':', :operator if kind == :key
value_expected = false
end
@@ -191,11 +195,14 @@ def scan_tokens encoder, options
encoder.text_token match, :error
method_call_expected = false
else
- encoder.text_token match, self[1] ? :float : :integer # TODO: send :hex/:octal/:binary
+ kind = self[1] ? :float : :integer # TODO: send :hex/:octal/:binary
+ match << 'r' if match !~ /e/i && scan(/r/)
+ match << 'i' if scan(/i/)
+ encoder.text_token match, kind
end
value_expected = false
- elsif match = scan(/ [-+!~^\/]=? | [:;] | [*|&]{1,2}=? | >>? /x)
+ elsif match = scan(/ [-+!~^\/]=? | [:;] | &\. | [*|&]{1,2}=? | >>? /x)
value_expected = true
encoder.text_token match, :operator
@@ -208,7 +215,7 @@ def scan_tokens encoder, options
encoder.end_group kind
heredocs ||= [] # create heredocs if empty
heredocs << self.class::StringState.new(kind, quote != "'", delim,
- self[1] == '-' ? :indented : :linestart)
+ self[1] ? :indented : :linestart)
value_expected = false
elsif value_expected && match = scan(/#{patterns::FANCY_STRING_START}/o)
@@ -114,7 +114,7 @@ module Ruby::Patterns # :nodoc: all
# NOTE: This is not completely correct, but
# nobody needs heredoc delimiters ending with \n.
HEREDOC_OPEN = /
- << (-)? # $1 = float
+ << ([-~])? # $1 = float
(?:
( [A-Za-z_0-9]+ ) # $2 = delim
|
@@ -37,6 +37,14 @@ class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
end
end
+ def self.simple_key_pattern delim
+ if delim == "'"
+ / (?> (?: [^\\']+ | \\. )* ) ' : /mx
+ else
+ / (?> (?: [^\\"\#]+ | \\. | \#\$[\\"] | \#\{[^\{\}]+\} | \#(?!\{) )* ) " : /mx
+ end
+ end
+
def initialize kind, interpreted, delim, heredoc = false
if heredoc
pattern = heredoc_pattern delim, interpreted, heredoc == :indented
@@ -57,6 +57,12 @@ class SQL < Scanner
STRING_PREFIXES = /[xnb]|_\w+/i
+ STRING_CONTENT_PATTERN = {
+ '"' => / (?: [^\\"] | "" )+ /x,
+ "'" => / (?: [^\\'] | '' )+ /x,
+ '`' => / (?: [^\\`] | `` )+ /x,
+ }
+
def scan_tokens encoder, options
state = :initial
@@ -90,7 +96,7 @@ def scan_tokens encoder, options
state = :string
encoder.text_token match, :delimiter
- elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x)
+ elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9\$]* /x)
encoder.text_token match, name_expected ? :ident : (match[0] == ?@ ? :variable : IDENT_KIND[match])
name_expected = false
@@ -115,40 +121,26 @@ def scan_tokens encoder, options
end
elsif state == :string
- if match = scan(/[^\\"'`]+/)
- string_content << match
- next
+ if match = scan(STRING_CONTENT_PATTERN[string_type])
+ encoder.text_token match, :content
elsif match = scan(/["'`]/)
if string_type == match
if peek(1) == string_type # doubling means escape
- string_content << string_type << getch
- next
- end
- unless string_content.empty?
- encoder.text_token string_content, :content
- string_content = ''
+ encoder.text_token match + getch, :content
+ else
+ encoder.text_token match, :delimiter
+ encoder.end_group :string
+ state = :initial
+ string_type = nil
end
- encoder.text_token match, :delimiter
- encoder.end_group :string
- state = :initial
- string_type = nil
else
- string_content << match
+ encoder.text_token match, :content
end
elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
- unless string_content.empty?
- encoder.text_token string_content, :content
- string_content = ''
- end
encoder.text_token match, :char
elsif match = scan(/ \\ . /mox)
- string_content << match
- next
+ encoder.text_token match, :content
elsif match = scan(/ \\ | $ /x)
- unless string_content.empty?
- encoder.text_token string_content, :content
- string_content = ''
- end
encoder.text_token match, :error unless match.empty?
encoder.end_group :string
state = :initial
View
@@ -1,3 +1,3 @@
module CodeRay
- VERSION = '1.1.0'
+ VERSION = '1.1.1'
end

0 comments on commit 973fc94

Please sign in to comment.