Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

vendor syntax

  • Loading branch information...
commit c8ee394cc53ad6324c518fea7b920d590c8251a7 1 parent 128f72d
Adam Wiggins adamwiggins authored
2  lib/post.rb
... ... @@ -1,5 +1,5 @@
1 1 require File.dirname(__FILE__) + '/../vendor/maruku/maruku'
2   -require 'syntax/convertors/html'
  2 +require File.dirname(__FILE__) + '/../vendor/syntax/syntax/convertors/html'
3 3
4 4 class Post < Sequel::Model
5 5 set_primary_key [ :id ]
38 vendor/syntax/syntax.rb
... ... @@ -0,0 +1,38 @@
  1 +require 'syntax/common'
  2 +
  3 +module Syntax
  4 +
  5 + # A default tokenizer for handling syntaxes that are not explicitly handled
  6 + # elsewhere. It simply yields the given text as a single token.
  7 + class Default
  8 +
  9 + # Yield the given text as a single token.
  10 + def tokenize( text )
  11 + yield Token.new( text, :normal )
  12 + end
  13 +
  14 + end
  15 +
  16 + # A hash for registering syntax implementations.
  17 + SYNTAX = Hash.new( Default )
  18 +
  19 + # Load the implementation of the requested syntax. If the syntax cannot be
  20 + # found, or if it cannot be loaded for whatever reason, the Default syntax
  21 + # handler will be returned.
  22 + def load( syntax )
  23 + begin
  24 + require "syntax/lang/#{syntax}"
  25 + rescue LoadError
  26 + end
  27 + SYNTAX[ syntax ].new
  28 + end
  29 + module_function :load
  30 +
  31 + # Return an array of the names of supported syntaxes.
  32 + def all
  33 + lang_dir = File.join(File.dirname(__FILE__), "syntax", "lang")
  34 + Dir["#{lang_dir}/*.rb"].map { |path| File.basename(path, ".rb") }
  35 + end
  36 + module_function :all
  37 +
  38 +end
163 vendor/syntax/syntax/common.rb
... ... @@ -0,0 +1,163 @@
  1 +require 'strscan'
  2 +
  3 +module Syntax
  4 +
  5 + # A single token extracted by a tokenizer. It is simply the lexeme
  6 + # itself, decorated with a 'group' attribute to identify the type of the
  7 + # lexeme.
  8 + class Token < String
  9 +
  10 + # the type of the lexeme that was extracted.
  11 + attr_reader :group
  12 +
  13 + # the instruction associated with this token (:none, :region_open, or
  14 + # :region_close)
  15 + attr_reader :instruction
  16 +
  17 + # Create a new Token representing the given text, and belonging to the
  18 + # given group.
  19 + def initialize( text, group, instruction = :none )
  20 + super text
  21 + @group = group
  22 + @instruction = instruction
  23 + end
  24 +
  25 + end
  26 +
  27 + # The base class of all tokenizers. It sets up the scanner and manages the
  28 + # looping until all tokens have been extracted. It also provides convenience
  29 + # methods to make sure adjacent tokens of identical groups are returned as
  30 + # a single token.
  31 + class Tokenizer
  32 +
  33 + # The current group being processed by the tokenizer
  34 + attr_reader :group
  35 +
  36 + # The current chunk of text being accumulated
  37 + attr_reader :chunk
  38 +
  39 + # Start tokenizing. This sets up the state in preparation for tokenization,
  40 + # such as creating a new scanner for the text and saving the callback block.
  41 + # The block will be invoked for each token extracted.
  42 + def start( text, &block )
  43 + @chunk = ""
  44 + @group = :normal
  45 + @callback = block
  46 + @text = StringScanner.new( text )
  47 + setup
  48 + end
  49 +
  50 + # Subclasses may override this method to provide implementation-specific
  51 + # setup logic.
  52 + def setup
  53 + end
  54 +
  55 + # Finish tokenizing. This flushes the buffer, yielding any remaining text
  56 + # to the client.
  57 + def finish
  58 + start_group nil
  59 + teardown
  60 + end
  61 +
  62 + # Subclasses may override this method to provide implementation-specific
  63 + # teardown logic.
  64 + def teardown
  65 + end
  66 +
  67 + # Subclasses must implement this method, which is called for each iteration
  68 + # of the tokenization process. This method may extract multiple tokens.
  69 + def step
  70 + raise NotImplementedError, "subclasses must implement #step"
  71 + end
  72 +
  73 + # Begins tokenizing the given text, calling #step until the text has been
  74 + # exhausted.
  75 + def tokenize( text, &block )
  76 + start text, &block
  77 + step until @text.eos?
  78 + finish
  79 + end
  80 +
  81 + # Specify a set of tokenizer-specific options. Each tokenizer may (or may
  82 + # not) publish any options, but if a tokenizer does those options may be
  83 + # used to specify optional behavior.
  84 + def set( opts={} )
  85 + ( @options ||= Hash.new ).update opts
  86 + end
  87 +
  88 + # Get the value of the specified option.
  89 + def option(opt)
  90 + @options ? @options[opt] : nil
  91 + end
  92 +
  93 + private
  94 +
  95 + EOL = /(?=\r\n?|\n|$)/
  96 +
  97 + # A convenience for delegating method calls to the scanner.
  98 + def self.delegate( sym )
  99 + define_method( sym ) { |*a| @text.__send__( sym, *a ) }
  100 + end
  101 +
  102 + delegate :bol?
  103 + delegate :eos?
  104 + delegate :scan
  105 + delegate :scan_until
  106 + delegate :check
  107 + delegate :check_until
  108 + delegate :getch
  109 + delegate :matched
  110 + delegate :pre_match
  111 + delegate :peek
  112 + delegate :pos
  113 +
  114 + # Access the n-th subgroup from the most recent match.
  115 + def subgroup(n)
  116 + @text[n]
  117 + end
  118 +
  119 + # Append the given data to the currently active chunk.
  120 + def append( data )
  121 + @chunk << data
  122 + end
  123 +
  124 + # Request that a new group be started. If the current group is the same
  125 + # as the group being requested, a new group will not be created. If a new
  126 + # group is created and the current chunk is not empty, the chunk's
  127 + # contents will be yielded to the client as a token, and then cleared.
  128 + #
  129 + # After the new group is started, if +data+ is non-nil it will be appended
  130 + # to the chunk.
  131 + def start_group( gr, data=nil )
  132 + flush_chunk if gr != @group
  133 + @group = gr
  134 + @chunk << data if data
  135 + end
  136 +
  137 + def start_region( gr, data=nil )
  138 + flush_chunk
  139 + @group = gr
  140 + @callback.call( Token.new( data||"", @group, :region_open ) )
  141 + end
  142 +
  143 + def end_region( gr, data=nil )
  144 + flush_chunk
  145 + @group = gr
  146 + @callback.call( Token.new( data||"", @group, :region_close ) )
  147 + end
  148 +
  149 + def flush_chunk
  150 + @callback.call( Token.new( @chunk, @group ) ) unless @chunk.empty?
  151 + @chunk = ""
  152 + end
  153 +
  154 + def subtokenize( syntax, text )
  155 + tokenizer = Syntax.load( syntax )
  156 + tokenizer.set @options if @options
  157 + flush_chunk
  158 + tokenizer.tokenize( text, &@callback )
  159 + end
  160 +
  161 + end
  162 +
  163 +end
27 vendor/syntax/syntax/convertors/abstract.rb
... ... @@ -0,0 +1,27 @@
  1 +require 'syntax'
  2 +
  3 +module Syntax
  4 + module Convertors
  5 +
  6 + # The abstract ancestor class for all convertors. It implements a few
  7 + # convenience methods to provide a common interface for all convertors.
  8 + class Abstract
  9 +
  10 + # A reference to the tokenizer used by this convertor.
  11 + attr_reader :tokenizer
  12 +
  13 + # A convenience method for instantiating a new convertor for a
  14 + # specific syntax.
  15 + def self.for_syntax( syntax )
  16 + new( Syntax.load( syntax ) )
  17 + end
  18 +
  19 + # Creates a new convertor that uses the given tokenizer.
  20 + def initialize( tokenizer )
  21 + @tokenizer = tokenizer
  22 + end
  23 +
  24 + end
  25 +
  26 + end
  27 +end
51 vendor/syntax/syntax/convertors/html.rb
... ... @@ -0,0 +1,51 @@
  1 +require 'syntax/convertors/abstract'
  2 +
  3 +module Syntax
  4 + module Convertors
  5 +
  6 + # A simple class for converting a text into HTML.
  7 + class HTML < Abstract
  8 +
  9 + # Converts the given text to HTML, using spans to represent token groups
  10 + # of any type but <tt>:normal</tt> (which is always unhighlighted). If
  11 + # +pre+ is +true+, the html is automatically wrapped in pre tags.
  12 + def convert( text, pre=true )
  13 + html = ""
  14 + html << "<pre>" if pre
  15 + regions = []
  16 + @tokenizer.tokenize( text ) do |tok|
  17 + value = html_escape(tok)
  18 + case tok.instruction
  19 + when :region_close then
  20 + regions.pop
  21 + html << "</span>"
  22 + when :region_open then
  23 + regions.push tok.group
  24 + html << "<span class=\"#{tok.group}\">#{value}"
  25 + else
  26 + if tok.group == ( regions.last || :normal )
  27 + html << value
  28 + else
  29 + html << "<span class=\"#{tok.group}\">#{value}</span>"
  30 + end
  31 + end
  32 + end
  33 + html << "</span>" while regions.pop
  34 + html << "</pre>" if pre
  35 + html
  36 + end
  37 +
  38 + private
  39 +
  40 + # Replaces some characters with their corresponding HTML entities.
  41 + def html_escape( string )
  42 + string.gsub( /&/, "&amp;" ).
  43 + gsub( /</, "&lt;" ).
  44 + gsub( />/, "&gt;" ).
  45 + gsub( /"/, "&quot;" )
  46 + end
  47 +
  48 + end
  49 +
  50 + end
  51 +end
317 vendor/syntax/syntax/lang/ruby.rb
... ... @@ -0,0 +1,317 @@
  1 +require 'syntax'
  2 +
  3 +module Syntax
  4 +
  5 + # A tokenizer for the Ruby language. It recognizes all common syntax
  6 + # (and some less common syntax) but because it is not a true lexer, it
  7 + # will make mistakes on some ambiguous cases.
  8 + class Ruby < Tokenizer
  9 +
  10 + # The list of all identifiers recognized as keywords.
  11 + KEYWORDS =
  12 + %w{if then elsif else end begin do rescue ensure while for
  13 + class module def yield raise until unless and or not when
  14 + case super undef break next redo retry in return alias
  15 + defined?}
  16 +
  17 + # Perform ruby-specific setup
  18 + def setup
  19 + @selector = false
  20 + @allow_operator = false
  21 + @heredocs = []
  22 + end
  23 +
  24 + # Step through a single iteration of the tokenization process.
  25 + def step
  26 + case
  27 + when bol? && check( /=begin/ )
  28 + start_group( :comment, scan_until( /^=end#{EOL}/ ) )
  29 + when bol? && check( /__END__#{EOL}/ )
  30 + start_group( :comment, scan_until( /\Z/ ) )
  31 + else
  32 + case
  33 + when check( /def\s+/ )
  34 + start_group :keyword, scan( /def\s+/ )
  35 + start_group :method, scan_until( /(?=[;(\s]|#{EOL})/ )
  36 + when check( /class\s+/ )
  37 + start_group :keyword, scan( /class\s+/ )
  38 + start_group :class, scan_until( /(?=[;\s<]|#{EOL})/ )
  39 + when check( /module\s+/ )
  40 + start_group :keyword, scan( /module\s+/ )
  41 + start_group :module, scan_until( /(?=[;\s]|#{EOL})/ )
  42 + when check( /::/ )
  43 + start_group :punct, scan(/::/)
  44 + when check( /:"/ )
  45 + start_group :symbol, scan(/:/)
  46 + scan_delimited_region :symbol, :symbol, "", true
  47 + @allow_operator = true
  48 + when check( /:'/ )
  49 + start_group :symbol, scan(/:/)
  50 + scan_delimited_region :symbol, :symbol, "", false
  51 + @allow_operator = true
  52 + when scan( /:[_a-zA-Z@$][$@\w]*[=!?]?/ )
  53 + start_group :symbol, matched
  54 + @allow_operator = true
  55 + when scan( /\?(\\[^\n\r]|[^\\\n\r\s])/ )
  56 + start_group :char, matched
  57 + @allow_operator = true
  58 + when check( /(__FILE__|__LINE__|true|false|nil|self)[?!]?/ )
  59 + if @selector || matched[-1] == ?? || matched[-1] == ?!
  60 + start_group :ident,
  61 + scan(/(__FILE__|__LINE__|true|false|nil|self)[?!]?/)
  62 + else
  63 + start_group :constant,
  64 + scan(/(__FILE__|__LINE__|true|false|nil|self)/)
  65 + end
  66 + @selector = false
  67 + @allow_operator = true
  68 + when scan(/0([bB][01]+|[oO][0-7]+|[dD][0-9]+|[xX][0-9a-fA-F]+)/)
  69 + start_group :number, matched
  70 + @allow_operator = true
  71 + else
  72 + case peek(2)
  73 + when "%r"
  74 + scan_delimited_region :punct, :regex, scan( /../ ), true
  75 + @allow_operator = true
  76 + when "%w", "%q"
  77 + scan_delimited_region :punct, :string, scan( /../ ), false
  78 + @allow_operator = true
  79 + when "%s"
  80 + scan_delimited_region :punct, :symbol, scan( /../ ), false
  81 + @allow_operator = true
  82 + when "%W", "%Q", "%x"
  83 + scan_delimited_region :punct, :string, scan( /../ ), true
  84 + @allow_operator = true
  85 + when /%[^\sa-zA-Z0-9]/
  86 + scan_delimited_region :punct, :string, scan( /./ ), true
  87 + @allow_operator = true
  88 + when "<<"
  89 + saw_word = ( chunk[-1,1] =~ /[\w!?]/ )
  90 + start_group :punct, scan( /<</ )
  91 + if saw_word
  92 + @allow_operator = false
  93 + return
  94 + end
  95 +
  96 + float_right = scan( /-/ )
  97 + append "-" if float_right
  98 + if ( type = scan( /['"]/ ) )
  99 + append type
  100 + delim = scan_until( /(?=#{type})/ )
  101 + if delim.nil?
  102 + append scan_until( /\Z/ )
  103 + return
  104 + end
  105 + else
  106 + delim = scan( /\w+/ ) or return
  107 + end
  108 + start_group :constant, delim
  109 + start_group :punct, scan( /#{type}/ ) if type
  110 + @heredocs << [ float_right, type, delim ]
  111 + @allow_operator = true
  112 + else
  113 + case peek(1)
  114 + when /[\n\r]/
  115 + unless @heredocs.empty?
  116 + scan_heredoc(*@heredocs.shift)
  117 + else
  118 + start_group :normal, scan( /\s+/ )
  119 + end
  120 + @allow_operator = false
  121 + when /\s/
  122 + start_group :normal, scan( /\s+/ )
  123 + when "#"
  124 + start_group :comment, scan( /#[^\n\r]*/ )
  125 + when /[A-Z]/
  126 + start_group @selector ? :ident : :constant, scan( /\w+/ )
  127 + @allow_operator = true
  128 + when /[a-z_]/
  129 + word = scan( /\w+[?!]?/ )
  130 + if !@selector && KEYWORDS.include?( word )
  131 + start_group :keyword, word
  132 + @allow_operator = false
  133 + elsif
  134 + start_group :ident, word
  135 + @allow_operator = true
  136 + end
  137 + @selector = false
  138 + when /\d/
  139 + start_group :number,
  140 + scan( /[\d_]+(\.[\d_]+)?([eE][\d_]+)?/ )
  141 + @allow_operator = true
  142 + when '"'
  143 + scan_delimited_region :punct, :string, "", true
  144 + @allow_operator = true
  145 + when '/'
  146 + if @allow_operator
  147 + start_group :punct, scan(%r{/})
  148 + @allow_operator = false
  149 + else
  150 + scan_delimited_region :punct, :regex, "", true
  151 + @allow_operator = true
  152 + end
  153 + when "'"
  154 + scan_delimited_region :punct, :string, "", false
  155 + @allow_operator = true
  156 + when "."
  157 + dots = scan( /\.{1,3}/ )
  158 + start_group :punct, dots
  159 + @selector = ( dots.length == 1 )
  160 + when /[@]/
  161 + start_group :attribute, scan( /@{1,2}\w*/ )
  162 + @allow_operator = true
  163 + when /[$]/
  164 + start_group :global, scan(/\$/)
  165 + start_group :global, scan( /\w+|./ ) if check(/./)
  166 + @allow_operator = true
  167 + when /[-!?*\/+=<>(\[\{}:;,&|%]/
  168 + start_group :punct, scan(/./)
  169 + @allow_operator = false
  170 + when /[)\]]/
  171 + start_group :punct, scan(/./)
  172 + @allow_operator = true
  173 + else
  174 + # all else just falls through this, to prevent
  175 + # infinite loops...
  176 + append getch
  177 + end
  178 + end
  179 + end
  180 + end
  181 + end
  182 +
  183 + private
  184 +
  185 + # Scan a delimited region of text. This handles the simple cases (strings
  186 + # delimited with quotes) as well as the more complex cases of %-strings
  187 + # and here-documents.
  188 + #
  189 + # * +delim_group+ is the group to use to classify the delimiters of the
  190 + # region
  191 + # * +inner_group+ is the group to use to classify the contents of the
  192 + # region
  193 + # * +starter+ is the text to use as the starting delimiter
  194 + # * +exprs+ is a boolean flag indicating whether the region is an
  195 + # interpolated string or not
  196 + # * +delim+ is the text to use as the delimiter of the region. If +nil+,
  197 + # the next character will be treated as the delimiter.
  198 + # * +heredoc+ is either +false+, meaning the region is not a heredoc, or
  199 + # <tt>:flush</tt> (meaning the delimiter must be flushed left), or
  200 + # <tt>:float</tt> (meaning the delimiter doens't have to be flush left).
  201 + def scan_delimited_region( delim_group, inner_group, starter, exprs,
  202 + delim=nil, heredoc=false )
  203 + # begin
  204 + if !delim
  205 + start_group delim_group, starter
  206 + delim = scan( /./ )
  207 + append delim
  208 +
  209 + delim = case delim
  210 + when '{' then '}'
  211 + when '(' then ')'
  212 + when '[' then ']'
  213 + when '<' then '>'
  214 + else delim
  215 + end
  216 + end
  217 +
  218 + start_region inner_group
  219 +
  220 + items = "\\\\|"
  221 + if heredoc
  222 + items << "(^"
  223 + items << '\s*' if heredoc == :float
  224 + items << "#{Regexp.escape(delim)}\s*?)#{EOL}"
  225 + else
  226 + items << "#{Regexp.escape(delim)}"
  227 + end
  228 + items << "|#(\\$|@@?|\\{)" if exprs
  229 + items = Regexp.new( items )
  230 +
  231 + loop do
  232 + p = pos
  233 + match = scan_until( items )
  234 + if match.nil?
  235 + start_group inner_group, scan_until( /\Z/ )
  236 + break
  237 + else
  238 + text = pre_match[p..-1]
  239 + start_group inner_group, text if text.length > 0
  240 + case matched.strip
  241 + when "\\"
  242 + unless exprs
  243 + case peek(1)
  244 + when "'"
  245 + scan(/./)
  246 + start_group :escape, "\\'"
  247 + when "\\"
  248 + scan(/./)
  249 + start_group :escape, "\\\\"
  250 + else
  251 + start_group inner_group, "\\"
  252 + end
  253 + else
  254 + start_group :escape, "\\"
  255 + c = getch
  256 + append c
  257 + case c
  258 + when 'x'
  259 + append scan( /[a-fA-F0-9]{1,2}/ )
  260 + when /[0-7]/
  261 + append scan( /[0-7]{0,2}/ )
  262 + end
  263 + end
  264 + when delim
  265 + end_region inner_group
  266 + start_group delim_group, matched
  267 + break
  268 + when /^#/
  269 + do_highlight = (option(:expressions) == :highlight)
  270 + start_region :expr if do_highlight
  271 + start_group :expr, matched
  272 + case matched[1]
  273 + when ?{
  274 + depth = 1
  275 + content = ""
  276 + while depth > 0
  277 + p = pos
  278 + c = scan_until( /[\{}]/ )
  279 + if c.nil?
  280 + content << scan_until( /\Z/ )
  281 + break
  282 + else
  283 + depth += ( matched == "{" ? 1 : -1 )
  284 + content << pre_match[p..-1]
  285 + content << matched if depth > 0
  286 + end
  287 + end
  288 + if do_highlight
  289 + subtokenize "ruby", content
  290 + start_group :expr, "}"
  291 + else
  292 + append content + "}"
  293 + end
  294 + when ?$, ?@
  295 + append scan( /\w+/ )
  296 + end
  297 + end_region :expr if do_highlight
  298 + else raise "unexpected match on #{matched}"
  299 + end
  300 + end
  301 + end
  302 + end
  303 +
  304 + # Scan a heredoc beginning at the current position.
  305 + #
  306 + # * +float+ indicates whether the delimiter may be floated to the right
  307 + # * +type+ is +nil+, a single quote, or a double quote
  308 + # * +delim+ is the delimiter to look for
  309 + def scan_heredoc(float, type, delim)
  310 + scan_delimited_region( :constant, :string, "", type != "'",
  311 + delim, float ? :float : :flush )
  312 + end
  313 + end
  314 +
  315 + SYNTAX["ruby"] = Ruby
  316 +
  317 +end
108 vendor/syntax/syntax/lang/xml.rb
... ... @@ -0,0 +1,108 @@
  1 +require 'syntax'
  2 +
  3 +module Syntax
  4 +
  5 + # A simple implementation of an XML lexer. It handles most cases. It is
  6 + # not a validating lexer, meaning it will happily process invalid XML without
  7 + # complaining.
  8 + class XML < Tokenizer
  9 +
  10 + # Initialize the lexer.
  11 + def setup
  12 + @in_tag = false
  13 + end
  14 +
  15 + # Step through a single iteration of the tokenization process. This will
  16 + # yield (potentially) many tokens, and possibly zero tokens.
  17 + def step
  18 + start_group :normal, matched if scan( /\s+/ )
  19 + if @in_tag
  20 + case
  21 + when scan( /([-\w]+):([-\w]+)/ )
  22 + start_group :namespace, subgroup(1)
  23 + start_group :punct, ":"
  24 + start_group :attribute, subgroup(2)
  25 + when scan( /\d+/ )
  26 + start_group :number, matched
  27 + when scan( /[-\w]+/ )
  28 + start_group :attribute, matched
  29 + when scan( %r{[/?]?>} )
  30 + @in_tag = false
  31 + start_group :punct, matched
  32 + when scan( /=/ )
  33 + start_group :punct, matched
  34 + when scan( /["']/ )
  35 + scan_string matched
  36 + else
  37 + append getch
  38 + end
  39 + elsif ( text = scan_until( /(?=[<&])/ ) )
  40 + start_group :normal, text unless text.empty?
  41 + if scan(/<!--.*?(-->|\Z)/m)
  42 + start_group :comment, matched
  43 + else
  44 + case peek(1)
  45 + when "<"
  46 + start_group :punct, getch
  47 + case peek(1)
  48 + when "?"
  49 + append getch
  50 + when "/"
  51 + append getch
  52 + when "!"
  53 + append getch
  54 + end
  55 + start_group :normal, matched if scan( /\s+/ )
  56 + if scan( /([-\w]+):([-\w]+)/ )
  57 + start_group :namespace, subgroup(1)
  58 + start_group :punct, ":"
  59 + start_group :tag, subgroup(2)
  60 + elsif scan( /[-\w]+/ )
  61 + start_group :tag, matched
  62 + end
  63 + @in_tag = true
  64 + when "&"
  65 + if scan( /&\S{1,10};/ )
  66 + start_group :entity, matched
  67 + else
  68 + start_group :normal, scan( /&/ )
  69 + end
  70 + end
  71 + end
  72 + else
  73 + append scan_until( /\Z/ )
  74 + end
  75 + end
  76 +
  77 + private
  78 +
  79 + # Scan the string starting at the current position, with the given
  80 + # delimiter character.
  81 + def scan_string( delim )
  82 + start_group :punct, delim
  83 + match = /(?=[&\\]|#{delim})/
  84 + loop do
  85 + break unless ( text = scan_until( match ) )
  86 + start_group :string, text unless text.empty?
  87 + case peek(1)
  88 + when "&"
  89 + if scan( /&\S{1,10};/ )
  90 + start_group :entity, matched
  91 + else
  92 + start_group :string, getch
  93 + end
  94 + when "\\"
  95 + start_group :string, getch
  96 + append getch || ""
  97 + when delim
  98 + start_group :punct, getch
  99 + break
  100 + end
  101 + end
  102 + end
  103 +
  104 + end
  105 +
  106 + SYNTAX["xml"] = XML
  107 +
  108 +end
105 vendor/syntax/syntax/lang/yaml.rb
... ... @@ -0,0 +1,105 @@
  1 +require 'syntax'
  2 +
  3 +module Syntax
  4 +
  5 + # A simple implementation of an YAML lexer. It handles most cases. It is
  6 + # not a validating lexer.
  7 + class YAML < Tokenizer
  8 +
  9 + # Step through a single iteration of the tokenization process. This will
  10 + # yield (potentially) many tokens, and possibly zero tokens.
  11 + def step
  12 + if bol?
  13 + case
  14 + when scan(/---(\s*.+)?$/)
  15 + start_group :document, matched
  16 + when scan(/(\s*)([a-zA-Z][-\w]*)(\s*):/)
  17 + start_group :normal, subgroup(1)
  18 + start_group :key, subgroup(2)
  19 + start_group :normal, subgroup(3)
  20 + start_group :punct, ":"
  21 + when scan(/(\s*)-/)
  22 + start_group :normal, subgroup(1)
  23 + start_group :punct, "-"
  24 + when scan(/\s*$/)
  25 + start_group :normal, matched
  26 + when scan(/#.*$/)
  27 + start_group :comment, matched
  28 + else
  29 + append getch
  30 + end
  31 + else
  32 + case
  33 + when scan(/[\n\r]+/)
  34 + start_group :normal, matched
  35 + when scan(/[ \t]+/)
  36 + start_group :normal, matched
  37 + when scan(/!+(.*?^)?\S+/)
  38 + start_group :type, matched
  39 + when scan(/&\S+/)
  40 + start_group :anchor, matched
  41 + when scan(/\*\S+/)
  42 + start_group :ref, matched
  43 + when scan(/\d\d:\d\d:\d\d/)
  44 + start_group :time, matched
  45 + when scan(/\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d(\.\d+)? [-+]\d\d:\d\d/)
  46 + start_group :date, matched
  47 + when scan(/['"]/)
  48 + start_group :punct, matched
  49 + scan_string matched
  50 + when scan(/:\w+/)
  51 + start_group :symbol, matched
  52 + when scan(/[:]/)
  53 + start_group :punct, matched
  54 + when scan(/#.*$/)
  55 + start_group :comment, matched
  56 + when scan(/>-?/)
  57 + start_group :punct, matched
  58 + start_group :normal, scan(/.*$/)
  59 + append getch until eos? || bol?
  60 + return if eos?
  61 + indent = check(/ */)
  62 + start_group :string
  63 + loop do
  64 + line = check_until(/[\n\r]|\Z/)
  65 + break if line.nil?
  66 + if line.chomp.length > 0
  67 + this_indent = line.chomp.match( /^\s*/ )[0]
  68 + break if this_indent.length < indent.length
  69 + end
  70 + append scan_until(/[\n\r]|\Z/)
  71 + end
  72 + else
  73 + start_group :normal, scan_until(/(?=$|#)/)
  74 + end
  75 + end
  76 + end
  77 +
  78 + private
  79 +
  80 + def scan_string( delim )
  81 + regex = /(?=[#{delim=="'" ? "" : "\\\\"}#{delim}])/
  82 + loop do
  83 + text = scan_until( regex )
  84 + if text.nil?
  85 + start_group :string, scan_until( /\Z/ )
  86 + break
  87 + else
  88 + start_group :string, text unless text.empty?
  89 + end
  90 +
  91 + case peek(1)
  92 + when "\\"
  93 + start_group :expr, scan(/../)
  94 + else
  95 + start_group :punct, getch
  96 + break
  97 + end
  98 + end
  99 + end
  100 +
  101 + end
  102 +
  103 + SYNTAX["yaml"] = YAML
  104 +
  105 +end
9 vendor/syntax/syntax/version.rb
... ... @@ -0,0 +1,9 @@
  1 +module Syntax
  2 + module Version
  3 + MAJOR=1
  4 + MINOR=0
  5 + TINY=0
  6 +
  7 + STRING=[MAJOR,MINOR,TINY].join('.')
  8 + end
  9 +end

0 comments on commit c8ee394

Please sign in to comment.
Something went wrong with that request. Please try again.