Permalink
Browse files

Merge branch 'master' into paint-integration

  • Loading branch information...
2 parents 916711c + 0a1f500 commit 0b8c69cfb7a65bec04c44e58e5776e323d2aa1af @korny korny committed Feb 13, 2016
Showing with 1,322 additions and 5,875 deletions.
  1. +0 −1 .gitignore
  2. +7 −1 .travis.yml
  3. +70 −14 Changes.textile
  4. +5 −3 Gemfile
  5. +2 −2 README.markdown
  6. +3 −1 Rakefile
  7. +31 −153 bench/bench.rb
  8. +4 −3 bin/coderay
  9. +2 −3 lib/coderay.rb
  10. +5 −18 lib/coderay/encoders/debug.rb
  11. +63 −0 lib/coderay/encoders/debug_lint.rb
  12. +17 −12 lib/coderay/encoders/html.rb
  13. +2 −2 lib/coderay/encoders/html/numbering.rb
  14. +59 −0 lib/coderay/encoders/lint.rb
  15. +0 −1 lib/coderay/encoders/statistic.rb
  16. +101 −78 lib/coderay/encoders/terminal.rb
  17. +57 −50 lib/coderay/helpers/file_type.rb
  18. +0 −41 lib/coderay/helpers/gzip.rb
  19. +11 −11 lib/coderay/helpers/plugin.rb
  20. +4 −4 lib/coderay/scanners/css.rb
  21. +18 −8 lib/coderay/scanners/debug.rb
  22. +2 −2 lib/coderay/scanners/diff.rb
  23. +208 −0 lib/coderay/scanners/go.rb
  24. +17 −4 lib/coderay/scanners/groovy.rb
  25. +38 −16 lib/coderay/scanners/html.rb
  26. +14 −11 lib/coderay/scanners/json.rb
  27. +280 −0 lib/coderay/scanners/lua.rb
  28. +22 −4 lib/coderay/scanners/php.rb
  29. +2 −2 lib/coderay/scanners/python.rb
  30. +27 −18 lib/coderay/scanners/raydebug.rb
  31. +14 −7 lib/coderay/scanners/ruby.rb
  32. +5 −2 lib/coderay/scanners/ruby/patterns.rb
  33. +15 −7 lib/coderay/scanners/ruby/string_state.rb
  34. +26 −21 lib/coderay/scanners/sass.rb
  35. +23 −28 lib/coderay/scanners/sql.rb
  36. +36 −0 lib/coderay/scanners/taskpaper.rb
  37. +2 −2 lib/coderay/scanners/yaml.rb
  38. +27 −23 lib/coderay/styles/alpha.rb
  39. +70 −74 lib/coderay/token_kinds.rb
  40. +22 −77 lib/coderay/tokens.rb
  41. +1 −1 lib/coderay/version.rb
  42. +0 −40 rake_helpers/ca.rb
  43. +0 −636 rake_helpers/coderay_rdoc_template.rb
  44. +0 −1,517 rake_helpers/html_coderay_generator.rb
  45. +1 −2 rake_tasks/benchmark.rake
  46. +0 −6 rake_tasks/bundler.rake
  47. 0 {rake_helpers → rake_tasks}/code_statistics.rb
  48. +0 −93 rake_tasks/diff.rake
  49. +0 −9 rake_tasks/documentation.rake
  50. +0 −10 rake_tasks/ruby-versions.rake
  51. +1 −1 rake_tasks/statistic.rake
  52. +0 −6 rake_tasks/test.rake
  53. +0 −1 sample/README
  54. +0 −2 sample/cache.expected
  55. +0 −12 sample/cache.rb
  56. +0 −1 sample/count.expected
  57. +0 −10 sample/count.rb
  58. +0 −130 sample/css.expected
  59. +0 −4 sample/css.rb
  60. +0 −17 sample/div.expected
  61. +0 −19 sample/div.rb
  62. +0 −21 sample/dump.expected
  63. +0 −15 sample/dump.rb
  64. +0 −65 sample/encoder.expected
  65. +0 −39 sample/encoder.rb
  66. +0 −3 sample/global_vars.expected
  67. +0 −13 sample/global_vars.rb
  68. +0 −10 sample/global_vars2.expected
  69. +0 −28 sample/global_vars2.rb
  70. +0 −175 sample/highlight.expected
  71. +0 −14 sample/highlight.rb
  72. +0 −919 sample/html.expected
  73. +0 −394 sample/html.rb
  74. +0 −185 sample/html2.expected
  75. +0 −11 sample/html2.rb
  76. +0 −160 sample/html_list.expected
  77. +0 −12 sample/html_list.rb
  78. +0 −8 sample/load_encoder.expected
  79. +0 −25 sample/load_encoder.rb
  80. +0 −8 sample/load_scanner.expected
  81. +0 −25 sample/load_scanner.rb
  82. +0 −2 sample/more.expected
  83. +0 −205 sample/more.rb
  84. +0 −16 sample/scanner.expected
  85. +0 −36 sample/scanner.rb
  86. +0 −110 sample/server.rb
  87. +0 −1 sample/simple.expected
  88. +0 −10 sample/simple.rb
  89. +0 −25 sample/stream.rb
  90. +0 −2 sample/stream2.expected
  91. +0 −8 sample/stream2.rb
  92. +0 −86 sample/suite.rb
  93. +0 −1 sample/tokens.expected
  94. +0 −3 sample/tokens.rb
  95. +1 −1 test/executable/suite.rb
  96. +1 −3 test/functional/basic.rb
  97. +1 −1 test/functional/examples.rb
  98. +5 −5 test/unit/debug.rb
  99. +0 −9 test/unit/tokens.rb
View
@@ -11,5 +11,4 @@ Gemfile.lock
test/executable/source.rb.html
test/executable/source.rb.json
test/scanners
-bench/test.div.html
old-stuff
View
@@ -2,7 +2,10 @@ rvm:
- 1.8.7
- ree
- 1.9.3
- - 2.0.0
+ - 2.0
+ - 2.1
+ - 2.2
+ - 2.3.0
- ruby-head
- jruby-18mode
- jruby-19mode
@@ -14,6 +17,9 @@ branches:
- master
matrix:
allow_failures:
+ - rvm: ruby-head
+ - rvm: jruby-head
- rvm: rbx-18mode
- rvm: rbx-19mode
script: "rake test" # test:scanners"
+sudo: false
View
@@ -2,21 +2,80 @@ h1=. CodeRay Version History
p=. _This files lists all changes in the CodeRay library since the 0.9.8 release._
+h2. Changes in 1.1.1
+
+* SQL scanner: Allow @$@ signs in SQL identifiers [#164, thanks to jasir and Ben Basson]
+* SQL scanner: Fix open strings [#163, thanks to Adam]
+* Ruby scanner: Accept number literal suffixes @r@ and @i@ (Ruby 2.1)
+* Ruby scanner: Accept quoted hash keys like @{ "a": boss }@ (Ruby 2.2)
+* Ruby scanner: Accept save navigation operator @&.@ (Ruby 2.3)
+* Ruby scanner: Accept squiggly heredoc @<<~@ (Ruby 2.3)
+* Diff scanner: Prevent running out of regexp stack.
+* HTML encoder: You can keep tabs intact now by setting @tab_width: false@.
+
h2. Changes in 1.1
-* New scanner: Sass [#93]
+New scanners:
+
+* Go [#28, thanks to Eric Guo and Nathan Youngman]
+* Lua [#21, #22, thanks to Quintus]
+* Sass [#93]
+* Taskpaper [#39, thanks to shimomura]
+
+More new stuff:
+
+* @.xaml@ file type [#121, thanks to Kozman Bálint]
+* recognize @Guardfile@, @Vagrantfile@, and @Appraisals@ as Ruby files [#121, thanks to Kozman Bálint]
+* new token kind @:id@ for CSS/Sass [#27]
+* new token kind @:done@ for Taskpaper [#39]
+* new token kind @:map@ for Lua, introducing a nice nested-shades trick [#22, thanks to Quintus and Nathan Youngman]
+* new token kind @:unknown@ for Debug scanner
+* new DebugLint encoder that checks for empty tokens and correct nesting
+
+Improvements:
+
+* CSS scanner uses @:id@ and @:tag@ now [#27]
* Diff scanner: Highlight inline changes in multi-line changes [#99]
* JavaScript scanner: Highlight multi-line comments in diff correctly
-* Remove double-click toggle handler from HTML table output
-* Fixes to CSS scanner (floats, pseudoclasses)
-* Plugin does not warn about fallback when default is defined
-* Display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom]
-* Add .xaml file type [#121, thanks to Kozman Bálint]
-* @CodeRay::TokenKinds@ should not be frozen [#130, thanks to Gavin Kistner]
-* Override Bootstrap's pre word-break setting for line numbers [#102, thanks to lightswitch05]
-* Accept keywords as Ruby 1.9 hash keys [#126]
-* New token type @:id@ for CSS/Sass [#27]
-* CSS scanner uses @:id@ and @:tag@ now [#27]
+* JSON scanner: simplify key/value heuristic, using look-ahead instead of a stack
+* HTML scanner displays style tags and attributes now [#145]
+* Ruby scanner: Accept @%i(…)@ and @%I(…)@ symbol lists (Ruby 2.0) [thanks to Nathan Youngman]
+* Ruby scanner: Accept keywords as Ruby hash keys [#126]
+* performance improvements to several scanners and encoders, especially Terminal and HTML
+* added @:keep_state@ functionality to more scanners so they work nicely with diff now [#116]
+* refactoring and cleanup to achieve better "Code Climate" ratings (but I don't really care)
+* updated and cleaned up the documentation,
+* documented list of TokenKinds
+* Alpha style: tweaked colors for @.binary@, @.local-variable@, and @.predefined-type@
+* @rake generate@ supports Git now instead of Subversion
+
+Removed:
+
+* @Tokens#dump@, @Tokens.load@, @Tokens::Undumping@, and @zlib@ dependency
+* double-click toggle handler from HTML table output
+* @rake_helpers@, @sample@ directories and several other ancient garbage
+
+Fixes:
+
+* fixes to CSS scanner (floats, pseudoclasses, nth-child) [#143]
+* fixed empty tokens and unclosed token groups in HTML, CSS, Diff, Goovy, PHP, Raydebug, Ruby, SQL, and YAML scanners [#144]
+* fixed @:docstring@ token type style
+* fixed several infinite Hash caches and dynamic Symbol creation that might have been exploited by an attacker [#148]
+* fixed HTML encoder when output is a StringIO (eg. when using @-HTML@ as a command line parameter)
+* TokenKinds should not be frozen [#130, thanks to Gavin Kistner]
+* display line numbers in HTML @:table@ mode even for single-line code (remove special case) [#41, thanks to Ariejan de Vroom]
+* override Bootstrap's @pre { word-break: break-all }@ styling for line numbers [#102, thanks to lightswitch05]
+* HTML encoder will not warn about unclosed token groups at the end of the stream
+* fixed problem with coderay/version.rb being loaded twice
+
+Internals:
+
+* The Debug scanner maps unknown token kinds to @:unknown@ (to avoid creating Symbols based on possibly unsafe input).
+* The Raydebug scanner highlights unknown token kinds as @:plain@.
+* The Debug encoder refactored; use DebugLint if you want strict checking now..
+* The Debug encoder will not warn about errors in the token stream.
+* Plugin does not warn about fallback when default is defined.
+* PluginHost now works with Strings instead of Symbols internally (to avoid using @#to_sym@).
h2. Changes in 1.0.9
@@ -426,6 +485,3 @@ The helper classes were cleaned up; see above for details.
* *CHANGED* @Plugin@ API was simplified and stripped of all unnecessary features.
* *CHANGED* Moved @GZip@ and @FileType@ libraries into @CodeRay@; cleaned them up.
-
-
-
View
@@ -8,11 +8,13 @@ gem 'paint', '~> 0.8.4'
# Add dependencies to develop your gem here.
# Include everything needed to run rake, tests, features, etc.
group :development do
- gem "bundler", ">= 1.0.0"
+ gem "bundler"
gem "rake"
gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3"
- gem "term-ansicolor", '~> 1.2.2'
- gem "shoulda-context", "~> 1.1.2"
+ gem "term-ansicolor"
+ gem 'tins', '~> 1.6.0'
+ gem "shoulda-context"
+ gem "test-unit"
gem "json" if RUBY_VERSION < '1.9'
gem "rdoc"
end
View
@@ -16,7 +16,7 @@ You put your code in, and you get it back colored; Keywords, strings, floats, co
### Dependencies
-CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0. It also runs on JRuby.
+CodeRay needs Ruby 1.8.7, 1.9.3 or 2.0+. It also runs on JRuby.
## Example Usage
@@ -28,4 +28,4 @@ html = CodeRay.scan("puts 'Hello, world!'", :ruby).div(:line_numbers => :table)
## Documentation
-See [http://coderay.rubychan.de/doc/](http://coderay.rubychan.de/doc/).
+See [rubydoc](http://rubydoc.info/gems/coderay).
View
@@ -1,3 +1,5 @@
+require 'bundler/gem_tasks'
+
$:.unshift File.dirname(__FILE__) unless $:.include? '.'
ROOT = '.'
@@ -32,4 +34,4 @@ else
rd.rdoc_dir = 'doc'
end
-end
+end
View
@@ -1,168 +1,46 @@
-# The most ugly test script I've ever written!
-# Shame on me!
-
-require 'pathname'
-require 'profile' if ARGV.include? '-p'
-
-MYDIR = File.dirname(__FILE__)
-LIBDIR = Pathname.new(MYDIR).join('..', 'lib').cleanpath.to_s
-$:.unshift MYDIR, LIBDIR
+require 'benchmark'
+$: << File.expand_path('../../lib', __FILE__)
require 'coderay'
-@size = ARGV.fetch(2, 100).to_i * 1000
-
-lang = ARGV.fetch(0) do
- puts <<-HELP
-Usage:
- ruby bench.rb (c|ruby|dump) (null|text|tokens|count|statistic|yaml|html) [size in kB] [stream]
-
- SIZE defaults to 100 kB (= 100,000 bytes).
- SIZE = 0 means the whole input.
- SIZE is ignored when dump is input.
-
--p generates a profile (slow! use with SIZE = 1)
--o shows the output
-stream enabled streaming mode
-
-Sorry for the strange interface. I will improve it in the next release.
- HELP
+if ARGV.include? '-h'
+ puts DATA.read
exit
end
-format = ARGV.fetch(1, 'html').downcase
-
-$stream = ARGV.include? 'stream'
-$optimize = ARGV.include? 'opt'
-$style = ARGV.include? 'style'
-
-require 'benchmark'
-require 'fileutils'
+lang = ARGV.fetch(0, 'ruby')
+data = nil
+File.open(File.expand_path("../example.#{lang}", __FILE__), 'rb') { |f| data = f.read }
+raise 'Example file is empty.' if data.empty?
-if format == 'comp'
- format = 'page'
- begin
- require 'syntax'
- require 'syntax/convertors/html.rb'
- rescue LoadError
- puts 'Syntax no found!! (Try % gem install syntax)'
- end
-end
-
-$dump_input = lang == 'dump'
-$dump_output = format == 'dump'
-require 'coderay/helpers/gzip_simple.rb' if $dump_input
+format = ARGV.fetch(1, 'html').downcase
+encoder = CodeRay.encoder(format)
-def here fn = nil
- return MYDIR unless fn
- File.join here, fn
+size = ARGV.fetch(2, 1000).to_i * 1000
+unless size.zero?
+ data += data until data.size >= size
+ data = data[0, size]
end
+size = data.size
+puts "encoding %d kB of #{lang} code to #{format}..." % [(size / 1000.0).round]
-n = ARGV.find { |a| a[/^N/] }
-N = if n then n[/\d+/].to_i else 1 end
-$filename = ARGV.include?('strange') ? 'strange' : 'example'
-
-Benchmark.bm(20) do |bm|
-N.times do
-
- data = nil
- File.open(here("#$filename." + lang), 'rb') { |f| data = f.read }
- if $dump_input
- @size = CodeRay::Tokens.load(data).text.size
- else
- raise 'Example file is empty.' if data.empty?
- unless @size.zero?
- data += data until data.size >= @size
- data = data[0, @size]
- end
- @size = data.size
- end
-
- options = {
- :tab_width => 2,
- # :line_numbers => :inline,
- :css => $style ? :style : :class,
- }
- $hl = CodeRay.encoder(format, options) unless $dump_output
- time = bm.report('CodeRay') do
- if $stream || true
- if $dump_input
- raise 'Can\'t stream dump.'
- elsif $dump_output
- raise 'Can\'t dump stream.'
- end
- $o = $hl.encode(data, lang, options)
- else
- if $dump_input
- tokens = CodeRay::Tokens.load data
- else
- tokens = CodeRay.scan(data, lang)
- end
- tokens.optimize! if $optimize
- if $dump_output
- $o = tokens.optimize.dump
- else
- $o = tokens.encode($hl)
- end
- end
- end
- $file_created = here('test.' +
- ($dump_output ? 'dump' : $hl.file_extension))
- File.open($file_created, 'wb') do |f|
- # f.write $o
- end
- Dir.chdir(here) do
- FileUtils.copy 'test.dump', 'example.dump' if $dump_output
- end
-
- time_real = time.real
-
- puts "\t%7.2f KB/s (%d.%d KB)" % [((@size / 1000.0) / time_real), @size / 1000, @size % 1000]
- puts $o if ARGV.include? '-o'
-
-end
+n = ARGV.fetch(3, 5).to_s[/\d+/].to_i
+require 'profile' if ARGV.include? '-p'
+n.times do |i|
+ time = Benchmark.realtime { encoder.encode(data, lang) }
+ puts "run %d: %5.2f s, %4.0f kB/s" % [i + 1, time, size / time / 1000.0]
end
-puts "Files created: #$file_created"
-STDIN.gets if ARGV.include? 'wait'
+STDIN.gets if ARGV.include? '-w'
__END__
-.ruby .normal {}
-.ruby .comment { color: #005; font-style: italic; }
-.ruby .keyword { color: #A00; font-weight: bold; }
-.ruby .method { color: #077; }
-.ruby .class { color: #074; }
-.ruby .module { color: #050; }
-.ruby .punct { color: #447; font-weight: bold; }
-.ruby .symbol { color: #099; }
-.ruby .string { color: #944; background: #FFE; }
-.ruby .char { color: #F07; }
-.ruby .ident { color: #004; }
-.ruby .constant { color: #07F; }
-.ruby .regex { color: #B66; background: #FEF; }
-.ruby .number { color: #F99; }
-.ruby .attribute { color: #7BB; }
-.ruby .global { color: #7FB; }
-.ruby .expr { color: #227; }
-.ruby .escape { color: #277; }
+Usage:
+ ruby bench.rb [lang] [format] [size in kB] [number of runs]
-.xml .normal {}
-.xml .namespace { color: #B66; font-weight: bold; }
-.xml .tag { color: #F88; }
-.xml .comment { color: #005; font-style: italic; }
-.xml .punct { color: #447; font-weight: bold; }
-.xml .string { color: #944; }
-.xml .number { color: #F99; }
-.xml .attribute { color: #BB7; }
+ - lang defaults to ruby.
+ - format defaults to html.
+ - size defaults to 1000 kB (= 1,000,000 bytes). 0 uses the whole example input.
+ - number of runs defaults to 5.
-.yaml .normal {}
-.yaml .document { font-weight: bold; color: #07F; }
-.yaml .type { font-weight: bold; color: #05C; }
-.yaml .key { color: #F88; }
-.yaml .comment { color: #005; font-style: italic; }
-.yaml .punct { color: #447; font-weight: bold; }
-.yaml .string { color: #944; }
-.yaml .number { color: #F99; }
-.yaml .time { color: #F99; }
-.yaml .date { color: #F99; }
-.yaml .ref { color: #944; }
-.yaml .anchor { color: #944; }
+-h prints this help
+-p generates a profile (slow, use with SIZE = 1)
+-w waits after the benchmark (for debugging memory usw)
View
@@ -35,7 +35,7 @@ defaults:
common:
coderay file.rb # highlight file to terminal
- coderay file.rb > file.html # highlight file to HTML page
+ coderay file.rb -page > file.html # highlight file to HTML page
coderay file.rb -div > file.html # highlight file to HTML snippet
configure output:
@@ -155,8 +155,9 @@ when 'highlight', nil
puts boom.message
end
# puts "I don't know this plugin: #{boom.message[/Could not load plugin (.*?): /, 1]}."
- rescue CodeRay::Scanners::Scanner::ScanError # FIXME: rescue Errno::EPIPE
- # this is sometimes raised by pagers; ignore [TODO: wtf?]
+ rescue CodeRay::Scanners::Scanner::ScanError
+ # this is sometimes raised by pagers; ignore
+ # FIXME: rescue Errno::EPIPE
ensure
file.close if output_file
end
Oops, something went wrong.

0 comments on commit 0b8c69c

Please sign in to comment.