Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Cleanups for C, C++, CSS, Delphi, Groovy, Java, PHP, Python, RHTML, X…

…ML, and YAML scanners.
  • Loading branch information...
commit 7fd577f8c2959902c4eb79d3c20b70a274e41618 1 parent cfa0257
@korny korny authored
View
27 lib/coderay/scanners/c.rb
@@ -1,45 +1,50 @@
module CodeRay
module Scanners
-
+
+ # Scanner for C.
+ #
+ # Alias: +h+
class C < Scanner
include Streamable
register_for :c
file_extension 'c'
-
+
RESERVED_WORDS = [
'asm', 'break', 'case', 'continue', 'default', 'do',
'else', 'enum', 'for', 'goto', 'if', 'return',
'sizeof', 'struct', 'switch', 'typedef', 'union', 'while',
'restrict', # added in C99
- ]
+ ] # :nodoc:
PREDEFINED_TYPES = [
'int', 'long', 'short', 'char',
'signed', 'unsigned', 'float', 'double',
'bool', 'complex', # added in C99
- ]
+ ] # :nodoc:
PREDEFINED_CONSTANTS = [
'EOF', 'NULL',
'true', 'false', # added in C99
- ]
+ ] # :nodoc:
DIRECTIVES = [
'auto', 'extern', 'register', 'static', 'void',
'const', 'volatile', # added in C89
'inline', # added in C99
- ]
+ ] # :nodoc:
IDENT_KIND = WordList.new(:ident).
add(RESERVED_WORDS, :reserved).
add(PREDEFINED_TYPES, :pre_type).
add(DIRECTIVES, :directive).
- add(PREDEFINED_CONSTANTS, :pre_constant)
-
- ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
- UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+ add(PREDEFINED_CONSTANTS, :pre_constant) # :nodoc:
+ ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc:
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc:
+
+ protected
+
def scan_tokens tokens, options
state = :initial
@@ -180,7 +185,7 @@ def scan_tokens tokens, options
end
match ||= matched
- if $DEBUG and not kind
+ if $CODERAY_DEBUG and not kind
raise_inspect 'Error token %p in line %d' %
[[match, kind], line], tokens
end
View
39 lib/coderay/scanners/cpp.rb
@@ -1,6 +1,9 @@
module CodeRay
module Scanners
+ # Scanner for C++.
+ #
+ # Aliases: +cplusplus+, c++
class CPlusPlus < Scanner
include Streamable
@@ -9,7 +12,7 @@ class CPlusPlus < Scanner
file_extension 'cpp'
title 'C++'
- # http://www.cppreference.com/wiki/keywords/start
+ #-- http://www.cppreference.com/wiki/keywords/start
RESERVED_WORDS = [
'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break',
'case', 'catch', 'class', 'compl', 'const_cast',
@@ -18,36 +21,38 @@ class CPlusPlus < Scanner
'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return',
'sizeof', 'static_cast', 'struct', 'switch', 'template',
'throw', 'try', 'typedef', 'typeid', 'typename', 'union',
- 'while', 'xor', 'xor_eq'
- ]
-
+ 'while', 'xor', 'xor_eq',
+ ] # :nodoc:
+
PREDEFINED_TYPES = [
'bool', 'char', 'double', 'float', 'int', 'long',
- 'short', 'signed', 'unsigned', 'wchar_t', 'string'
- ]
+ 'short', 'signed', 'unsigned', 'wchar_t', 'string',
+ ] # :nodoc:
PREDEFINED_CONSTANTS = [
'false', 'true',
'EOF', 'NULL',
- ]
+ ] # :nodoc:
PREDEFINED_VARIABLES = [
- 'this'
- ]
+ 'this',
+ ] # :nodoc:
DIRECTIVES = [
'auto', 'const', 'explicit', 'extern', 'friend', 'inline', 'mutable', 'operator',
'private', 'protected', 'public', 'register', 'static', 'using', 'virtual', 'void',
- 'volatile'
- ]
-
+ 'volatile',
+ ] # :nodoc:
+
IDENT_KIND = WordList.new(:ident).
add(RESERVED_WORDS, :reserved).
add(PREDEFINED_TYPES, :pre_type).
add(PREDEFINED_VARIABLES, :local_variable).
add(DIRECTIVES, :directive).
- add(PREDEFINED_CONSTANTS, :pre_constant)
-
- ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
- UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+ add(PREDEFINED_CONSTANTS, :pre_constant) # :nodoc:
+ ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc:
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc:
+
+ protected
+
def scan_tokens tokens, options
state = :initial
@@ -205,7 +210,7 @@ def scan_tokens tokens, options
end
match ||= matched
- if $DEBUG and not kind
+ if $CODERAY_DEBUG and not kind
raise_inspect 'Error token %p in line %d' %
[[match, kind], line], tokens
end
View
11 lib/coderay/scanners/css.rb
@@ -11,9 +11,9 @@ class CSS < Scanner
:constant, :directive,
:key, :value, :operator, :color, :float,
:error, :important,
- ]
+ ] # :nodoc:
- module RE
+ module RE # :nodoc:
NonASCII = /[\x80-\xFF]/
Hex = /[0-9a-fA-F]/
Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
@@ -47,9 +47,10 @@ module RE
Class = /\.#{Name}/
PseudoClass = /:#{Name}/
AttributeSelector = /\[[^\]]*\]?/
-
end
-
+
+ protected
+
def scan_tokens tokens, options
value_expected = nil
@@ -191,7 +192,7 @@ def scan_tokens tokens, options
end
match ||= matched
- if $DEBUG and not kind
+ if $CODERAY_DEBUG and not kind
raise_inspect 'Error token %p in line %d' %
[[match, kind], line], tokens
end
View
30 lib/coderay/scanners/delphi.rb
@@ -1,8 +1,11 @@
module CodeRay
module Scanners
+ # Scanner for the Delphi language (Object Pascal).
+ #
+ # Alias: +pascal+
class Delphi < Scanner
-
+
register_for :delphi
file_extension 'pas'
@@ -16,9 +19,9 @@ class Delphi < Scanner
'procedure', 'program', 'property', 'raise', 'record', 'repeat',
'resourcestring', 'set', 'shl', 'shr', 'string', 'then', 'threadvar',
'to', 'try', 'type', 'unit', 'until', 'uses', 'var', 'while', 'with',
- 'xor', 'on'
- ]
-
+ 'xor', 'on',
+ ] # :nodoc:
+
DIRECTIVES = [
'absolute', 'abstract', 'assembler', 'at', 'automated', 'cdecl',
'contains', 'deprecated', 'dispid', 'dynamic', 'export',
@@ -27,19 +30,20 @@ class Delphi < Scanner
'package', 'pascal', 'platform', 'private', 'protected', 'public',
'published', 'read', 'readonly', 'register', 'reintroduce',
'requires', 'resident', 'safecall', 'stdcall', 'stored', 'varargs',
- 'virtual', 'write', 'writeonly'
- ]
-
+ 'virtual', 'write', 'writeonly',
+ ] # :nodoc:
+
IDENT_KIND = CaseIgnoringWordList.new(:ident).
add(RESERVED_WORDS, :reserved).
- add(DIRECTIVES, :directive)
+ add(DIRECTIVES, :directive) # :nodoc:
NAME_FOLLOWS = CaseIgnoringWordList.new(false).
- add(%w(procedure function .))
-
- private
+ add(%w(procedure function .)) # :nodoc:
+
+ protected
+
def scan_tokens tokens, options
-
+
state = :initial
last_token = ''
@@ -130,7 +134,7 @@ def scan_tokens tokens, options
end
match ||= matched
- if $DEBUG and not kind
+ if $CODERAY_DEBUG and not kind
raise_inspect 'Error token %p in line %d' %
[[match, kind], line], tokens, state
end
View
30 lib/coderay/scanners/groovy.rb
@@ -2,28 +2,29 @@ module CodeRay
module Scanners
load :java
-
+
+ # Scanner for Groovy.
class Groovy < Java
include Streamable
register_for :groovy
- # TODO: Check this!
+ # TODO: check list of keywords
GROOVY_KEYWORDS = %w[
as assert def in
- ]
+ ] # :nodoc:
KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
case instanceof new return throw typeof while as assert in
- ]
- GROOVY_MAGIC_VARIABLES = %w[ it ]
+ ] # :nodoc:
+ GROOVY_MAGIC_VARIABLES = %w[ it ] # :nodoc:
IDENT_KIND = Java::IDENT_KIND.dup.
add(GROOVY_KEYWORDS, :keyword).
- add(GROOVY_MAGIC_VARIABLES, :local_variable)
+ add(GROOVY_MAGIC_VARIABLES, :local_variable) # :nodoc:
- ESCAPE = / [bfnrtv$\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
- UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # no 4-byte unicode chars? U[a-fA-F0-9]{8}
- REGEXP_ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x
+ ESCAPE = / [bfnrtv$\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc:
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # :nodoc: no 4-byte unicode chars? U[a-fA-F0-9]{8}
+ REGEXP_ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x # :nodoc:
# TODO: interpretation inside ', ", /
STRING_CONTENT_PATTERN = {
@@ -32,10 +33,12 @@ class Groovy < Java
"'''" => /(?>[^\\']+|'(?!''))+/,
'"""' => /(?>[^\\$"]+|"(?!""))+/,
'/' => /[^\\$\/\n]+/,
- }
+ } # :nodoc:
+
+ protected
def scan_tokens tokens, options
-
+
state = :initial
inline_block_stack = []
inline_block_paren_depth = nil
@@ -123,6 +126,7 @@ def scan_tokens tokens, options
next
end
end
+ kind = :operator
elsif check(/[\d.]/)
after_def = value_expected = false
@@ -143,7 +147,7 @@ def scan_tokens tokens, options
string_delimiter = match
kind = :delimiter
- # TODO: record.'name'
+ # TODO: record.'name' syntax
elsif match = scan(/["']/)
after_def = value_expected = false
state = match == '/' ? :regexp : :string
@@ -238,7 +242,7 @@ def scan_tokens tokens, options
end
match ||= matched
- if $DEBUG and not kind
+ if $CODERAY_DEBUG and not kind
raise_inspect 'Error token %p in line %d' %
[[match, kind], line], tokens
end
View
29 lib/coderay/scanners/java.rb
@@ -1,6 +1,7 @@
module CodeRay
module Scanners
-
+
+ # Scanner for Java.
class Java < Scanner
include Streamable
@@ -13,18 +14,18 @@ class Java < Scanner
finally for if instanceof import new package
return switch throw try typeof while
debugger export
- ]
- RESERVED = %w[ const goto ]
- CONSTANTS = %w[ false null true ]
- MAGIC_VARIABLES = %w[ this super ]
+ ] # :nodoc:
+ RESERVED = %w[ const goto ] # :nodoc:
+ CONSTANTS = %w[ false null true ] # :nodoc:
+ MAGIC_VARIABLES = %w[ this super ] # :nodoc:
TYPES = %w[
boolean byte char class double enum float int interface long
short void
- ] << '[]' # because int[] should be highlighted as a type
+ ] << '[]' # :nodoc: because int[] should be highlighted as a type
DIRECTIVES = %w[
abstract extends final implements native private protected public
static strictfp synchronized throws transient volatile
- ]
+ ] # :nodoc:
IDENT_KIND = WordList.new(:ident).
add(KEYWORDS, :keyword).
@@ -34,16 +35,18 @@ class Java < Scanner
add(TYPES, :type).
add(BuiltinTypes::List, :pre_type).
add(BuiltinTypes::List.select { |builtin| builtin[/(Error|Exception)$/] }, :exception).
- add(DIRECTIVES, :directive)
+ add(DIRECTIVES, :directive) # :nodoc:
- ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
- UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+ ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc:
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc:
STRING_CONTENT_PATTERN = {
"'" => /[^\\']+/,
'"' => /[^\\"]+/,
'/' => /[^\\\/]+/,
- }
- IDENT = /[a-zA-Z_][A-Za-z_0-9]*/
+ } # :nodoc:
+ IDENT = /[a-zA-Z_][A-Za-z_0-9]*/ # :nodoc:
+
+ protected
def scan_tokens tokens, options
@@ -151,7 +154,7 @@ def scan_tokens tokens, options
end
match ||= matched
- if $DEBUG and not kind
+ if $CODERAY_DEBUG and not kind
raise_inspect 'Error token %p in line %d' %
[[match, kind], line], tokens
end
View
12 lib/coderay/scanners/php.rb
@@ -3,6 +3,8 @@ module Scanners
load :html
+ # Scanner for PHP.
+ #
# Original by Stefan Walk.
class PHP < Scanner
@@ -11,6 +13,8 @@ class PHP < Scanner
KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
+ protected
+
def setup
@html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
end
@@ -20,7 +24,7 @@ def reset_instance
@html_scanner.reset
end
- module Words
+ module Words # :nodoc:
# according to http://www.php.net/manual/en/reserved.keywords.php
KEYWORDS = %w[
@@ -189,7 +193,7 @@ module Words
add(PREDEFINED, :predefined)
end
- module RE
+ module RE # :nodoc:
PHP_START = /
<script\s+[^>]*?language\s*=\s*"php"[^>]*?> |
@@ -223,6 +227,8 @@ module RE
end
+ protected
+
def scan_tokens tokens, options
if check(RE::PHP_START) || # starts with <?
@@ -507,7 +513,7 @@ def scan_tokens tokens, options
end
match ||= matched
- if $DEBUG and not kind
+ if $CODERAY_DEBUG and not kind
raise_inspect 'Error token %p in line %d' %
[[match, kind], line], tokens, states
end
View
46 lib/coderay/scanners/python.rb
@@ -1,7 +1,9 @@
module CodeRay
module Scanners
- # Bases on pygments' PythonLexer, see
+ # Scanner for Python. Supports Python 3.
+ #
+ # Based on pygments' PythonLexer, see
# http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.
class Python < Scanner
@@ -16,11 +18,11 @@ class Python < Scanner
'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not',
'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield',
'nonlocal', # new in Python 3
- ]
+ ] # :nodoc:
OLD_KEYWORDS = [
'exec', 'print', # gone in Python 3
- ]
+ ] # :nodoc:
PREDEFINED_METHODS_AND_TYPES = %w[
__import__ abs all any apply basestring bin bool buffer
@@ -32,7 +34,7 @@ class Python < Scanner
raw_input reduce reload repr reversed round set setattr slice
sorted staticmethod str sum super tuple type unichr unicode
vars xrange zip
- ]
+ ] # :nodoc:
PREDEFINED_EXCEPTIONS = %w[
ArithmeticError AssertionError AttributeError
@@ -47,23 +49,23 @@ class Python < Scanner
TypeError UnboundLocalError UnicodeDecodeError
UnicodeEncodeError UnicodeError UnicodeTranslateError
UnicodeWarning UserWarning ValueError Warning ZeroDivisionError
- ]
+ ] # :nodoc:
PREDEFINED_VARIABLES_AND_CONSTANTS = [
- 'False', 'True', 'None', # "keywords" since Python 3
+ 'False', 'True', 'None', # "keywords" since Python 3
'self', 'Ellipsis', 'NotImplemented',
- ]
+ ] # :nodoc:
IDENT_KIND = WordList.new(:ident).
add(KEYWORDS, :keyword).
add(OLD_KEYWORDS, :old_keyword).
add(PREDEFINED_METHODS_AND_TYPES, :predefined).
add(PREDEFINED_VARIABLES_AND_CONSTANTS, :pre_constant).
- add(PREDEFINED_EXCEPTIONS, :exception)
+ add(PREDEFINED_EXCEPTIONS, :exception) # :nodoc:
- NAME = / [^\W\d] \w* /x
- ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
- UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x
+ NAME = / [^\W\d] \w* /x # :nodoc:
+ ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc:
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x # :nodoc:
OPERATOR = /
\.\.\. | # ellipsis
@@ -73,26 +75,28 @@ class Python < Scanner
[-+*\/%&|^]=? | # ordinary math and binary logic
[~`] | # binary complement and inspection
<<=? | >>=? | [<>=]=? | != # comparison and assignment
- /x
+ /x # :nodoc:
- STRING_DELIMITER_REGEXP = Hash.new do |h, delimiter|
- h[delimiter] = Regexp.union delimiter
- end
+ STRING_DELIMITER_REGEXP = Hash.new { |h, delimiter|
+ h[delimiter] = Regexp.union delimiter # :nodoc:
+ }
- STRING_CONTENT_REGEXP = Hash.new do |h, delimiter|
- h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x
- end
+ STRING_CONTENT_REGEXP = Hash.new { |h, delimiter|
+ h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x # :nodoc:
+ }
DEF_NEW_STATE = WordList.new(:initial).
add(%w(def), :def_expected).
add(%w(import from), :include_expected).
- add(%w(class), :class_expected)
+ add(%w(class), :class_expected) # :nodoc:
DESCRIPTOR = /
#{NAME}
(?: \. #{NAME} )*
| \*
- /x
+ /x # :nodoc:
+
+ protected
def scan_tokens tokens, options
@@ -260,7 +264,7 @@ def scan_tokens tokens, options
end
match ||= matched
- if $DEBUG and not kind
+ if $CODERAY_DEBUG and not kind
raise_inspect 'Error token %p in line %d' %
[[match, kind], line], tokens, state
end
View
8 lib/coderay/scanners/rhtml.rb
@@ -4,7 +4,7 @@ module Scanners
load :html
load :ruby
- # RHTML Scanner
+ # Scanner for HTML ERB templates.
class RHTML < Scanner
include Streamable
@@ -23,13 +23,13 @@ class RHTML < Scanner
)*
)
(?: -?%> )?
- /x
+ /x # :nodoc:
START_OF_ERB = /
<%(?!%)
- /x
+ /x # :nodoc:
- private
+ protected
def setup
@ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
View
2  lib/coderay/scanners/xml.rb
@@ -3,7 +3,7 @@ module Scanners
load :html
- # XML Scanner
+ # Scanner for XML.
#
# Currently this is the same scanner as Scanners::HTML.
class XML < HTML
View
18 lib/coderay/scanners/yaml.rb
@@ -1,7 +1,7 @@
module CodeRay
module Scanners
- # YAML Scanner
+ # Scanner for YAML.
#
# Based on the YAML scanner from Syntax by Jamis Buck.
class YAML < Scanner
@@ -11,6 +11,8 @@ class YAML < Scanner
KINDS_NOT_LOC = :all
+ protected
+
def scan_tokens tokens, options
value_expected = nil
@@ -21,14 +23,7 @@ def scan_tokens tokens, options
kind = nil
match = nil
-
- if bol?
- key_indent = nil
- if $DEBUG
- indent = check(/ +/) ? matched.size : 0
- tokens << [indent.to_s, :debug]
- end
- end
+ key_indent = nil if bol?
if match = scan(/ +[\t ]*/)
kind = :space
@@ -128,7 +123,10 @@ def scan_tokens tokens, options
match ||= matched
- raise_inspect 'Error token %p in line %d' % [[match, kind], line], tokens if $DEBUG && !kind
+ if $CODERAY_DEBUG and not kind
+ raise_inspect 'Error token %p in line %d' %
+ [[match, kind], line], tokens
+ end
raise_inspect 'Empty token', tokens unless match
tokens << [match, kind]
Please sign in to comment.
Something went wrong with that request. Please try again.