Permalink
Browse files

A bit of tidying.

  • Loading branch information...
1 parent 4dd65b7 commit c50f2fb4a82f292ff86d94512011e9d989e3193c @gigamonkey committed Apr 17, 2012
Showing with 24 additions and 15 deletions.
  1. +23 −14 markup.rb
  2. +1 −1 tests.rb
View
@@ -94,7 +94,11 @@ def to_a
# TextCleaner is responsible for converting tabs to spaces and
# removing trailing whitespace from lines. The TextCleaner#clean
# method iterates over tokens representing the characters in the text
-# with some of them removed or replaced by other characters.
+# with some of them removed or replaced by other characters. The
+# tokens are marked with the actual line and column they came from in
+# the file. (This does mean that, for instance, when a tab character
+# is converted to spaces, all the space tokens have the same line and
+# column values.)
#
class TextCleaner
@@ -156,6 +160,14 @@ def clean(text)
end
end
+#
+# A Tokenizer takes the tokens coming out of a TextCleaner and
+# converts them to slightly higher-level semantic tokens. A single
+# newline at the end of a line becomes a :newline token; multiple
+# newlines are converted to :blank tokens. And we track changes of
+# indentation here, emitting :open_blockquote, :close_blockquote,
+# :open_verbatim, and :close_verbatim tokens as appropriate.
+#
class Tokenizer
def initialize
@@ -287,7 +299,7 @@ def initialize(markup, subdoc=false, section=false)
def grok(token)
case token.value
when :blank, :newline
- #raise "Parse error #{token}"
+ # ignore
when "*"
@markup.push_parser(HeaderParser.new(@markup))
when '-'
@@ -389,7 +401,7 @@ def grok(token)
@tokens << token
elsif @tokens.length == 3 and token.value == :blank
@markup.pop_parser # this one
- @markup.pop_parser # document parser
+ @markup.pop_parser # section's document parser
@markup.close_element(@section)
else
raise "Parser error: #{token}"
@@ -519,6 +531,10 @@ def grok(token)
end
+#
+# Parse something that could either be a link at the start of a
+# paragraph or a linkdef.
+#
class AmbiguousLinkParser < Parser
def initialize(markup)
@@ -533,7 +549,7 @@ def grok(token)
elsif @tokens.length == 1 and token.value == '<'
@element.tag = :link_def
@markup.pop_parser # ourself
- @markup.push_parser(LinkdefParser.new(@markup, false))
+ @markup.push_parser(LinkdefParser.new(@markup))
@markup.current_parser.grok(token)
else
@element.tag = :p
@@ -545,17 +561,11 @@ def grok(token)
end
end
-
-
class LinkdefParser < Parser
- def initialize(markup, open_element=true)
+ def initialize(markup)
super(markup)
- if open_element
- @linkdef = @markup.open_element(:link_def)
- else
- @linkdef = @markup.current_element
- end
+ @linkdef = @markup.current_element
end
def grok(token)
@@ -654,8 +664,7 @@ def grok(token)
[:blockquote, IndentedElementParser]
end
- element = @markup.open_element(tag)
- parser = parserClass.new(@markup, element)
+ parser = parserClass.new(@markup, @markup.open_element(tag))
@markup.pop_parser
@markup.push_parser(parser)
parser.grok(token)
View
@@ -207,7 +207,7 @@ def test_files
base = File.basename(json, ".json")
expect = json_to_array(json)
subdocs = Set.new([:note, :comment])
- puts "Testing #{dir}/#{base}.txt"
+ #puts "Testing #{dir}/#{base}.txt"
begin
got = Markup.new(subdocs).parse_file("#{dir}/#{base}.txt").to_a
assert_equal expect, got, "Error in test file #{base}"

0 comments on commit c50f2fb

Please sign in to comment.