Skip to content

Commit

Permalink
馃拕 two blank lines between top-level commits
Browse files Browse the repository at this point in the history
  • Loading branch information
miyuchina committed Aug 18, 2017
1 parent 4b426e2 commit 4453dba
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 0 deletions.
18 changes: 18 additions & 0 deletions mistletoe/block_token.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@
import mistletoe.block_tokenizer as tokenizer
import mistletoe.span_token as span_token


"""
The items and ordering of this __all__ matters to mistletoe (see
tokenize). Don't mess with it unless you know what you are doing!
"""
__all__ = ['Heading', 'Quote', 'BlockCode', 'Separator', 'List', 'Table',
'FootnoteBlock']


def tokenize(lines, root=None):
"""
A wrapper around block_tokenizer.tokenize. Pass in all block-level
Expand All @@ -28,14 +30,17 @@ def tokenize(lines, root=None):
fallback_token = Paragraph
return tokenizer.tokenize(lines, token_types, fallback_token, root)


def add_token(token_cls):
globals()[token_cls.__name__] = token_cls
__all__.insert(1, token_cls.__name__)


def remove_token(token_cls):
del globals()[token_cls.__name__]
__all__.remove(token_cls.__name__)


class BlockToken(object):
"""
Base class for span-level tokens. Recursively parse inner tokens.
Expand All @@ -57,6 +62,7 @@ class BlockToken(object):
def __init__(self, lines, tokenize_func):
self.children = tokenize_func(lines)


class Document(BlockToken):
"""
Document token.
Expand All @@ -67,6 +73,7 @@ def __init__(self, lines):
# Useful for footnotes, etc.
self.children = list(tokenize(lines, root=self))


class Heading(BlockToken):
"""
Heading token. (["### some heading ###\n"])
Expand Down Expand Up @@ -100,6 +107,7 @@ def match(lines):
return len(lines) > 1 and (lines[-1].startswith('---')
or lines[-1].startswith('==='))


class Quote(BlockToken):
"""
Quote token. (["> # heading\n", "> paragraph\n"])
Expand All @@ -117,6 +125,7 @@ def __init__(self, lines):
def match(lines):
return lines[0].startswith('> ')


class Paragraph(BlockToken):
"""
Paragraph token. (["some\n", "continuous\n", "lines\n"])
Expand All @@ -133,6 +142,7 @@ def match(lines):
return False
return True


class BlockCode(BlockToken):
"""
Block code. (["```sh\n", "rm -rf /", ..., "```"])
Expand Down Expand Up @@ -160,6 +170,7 @@ def match(lines):
return False
return True


class List(BlockToken):
"""
List tokens. (["- item\n", " - nested item\n", "- item\n" ])
Expand Down Expand Up @@ -246,6 +257,7 @@ def match(lines):
return False
return True


class ListItem(BlockToken):
"""
List item token. (["- item 1\n", "continued\n"])
Expand All @@ -258,6 +270,7 @@ def __init__(self, lines):
content = line.split(' ', 1)[1].strip()
super().__init__(content, span_token.tokenize_inner)


class Table(BlockToken):
"""
Table token.
Expand Down Expand Up @@ -312,6 +325,7 @@ def match(lines):
return False
return True


class TableRow(BlockToken):
"""
Table row token.
Expand All @@ -324,6 +338,7 @@ def __init__(self, line, row_align=[None]):
self.children = (TableCell(cell.strip(), align)
for cell, align in zip_longest(cells, row_align))


class TableCell(BlockToken):
"""
Table cell token.
Expand All @@ -339,6 +354,7 @@ def __init__(self, content, align=None):
self.align = align
super().__init__(content, span_token.tokenize_inner)


class FootnoteBlock(BlockToken):
"""
Footnote tokens.
Expand All @@ -358,6 +374,7 @@ def match(lines):
return False
return True


class FootnoteEntry(BlockToken):
"""
Footnote entry tokens.
Expand All @@ -374,6 +391,7 @@ def __init__(self, line):
self.key = key[1:]
self.value = value.strip()


class Separator(BlockToken):
"""
Separator token (a.k.a. horizontal rule.)
Expand Down
3 changes: 3 additions & 0 deletions mistletoe/html_token.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@
import mistletoe.span_token as span_token
import mistletoe.block_token as block_token


__all__ = ['HTMLBlock', 'HTMLSpan']


class HTMLBlock(block_token.BlockToken):
"""
Block-level HTML tokens.
Expand All @@ -35,6 +37,7 @@ def match(lines):
return False
return True


class HTMLSpan(span_token.SpanToken):
"""
Span-level HTML tokens.
Expand Down
18 changes: 18 additions & 0 deletions mistletoe/span_token.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import re
import mistletoe.span_tokenizer as tokenizer


"""
The items and ordering of this __all__ matters to mistletoe (see
tokenize_inner). Don't mess with it unless you know what you are doing!
Expand All @@ -13,6 +14,7 @@
'Strikethrough', 'Image', 'FootnoteImage', 'Link',
'FootnoteLink', 'AutoLink']


def tokenize_inner(content):
"""
A wrapper around span_tokenizer.tokenize. Pass in all span-level token
Expand All @@ -28,17 +30,21 @@ def tokenize_inner(content):
fallback_token = RawText
yield from tokenizer.tokenize(content, token_types, fallback_token)


def add_token(token_cls):
globals()[token_cls.__name__] = token_cls
__all__.insert(1, token_cls.__name__)


def remove_token(token_cls):
del globals()[token_cls.__name__]
__all__.remove(token_cls.__name__)


def _first_not_none_group(match_obj):
return next(group for group in match_obj.groups() if group is not None)


class SpanToken(object):
"""
Base class for span-level tokens. Recursively parse inner tokens.
Expand All @@ -62,6 +68,7 @@ class SpanToken(object):
def __init__(self, match_obj):
self.children = tokenize_inner(match_obj.group(1))


class Strong(SpanToken):
"""
Strong tokens. ("**some text**")
Expand All @@ -72,6 +79,7 @@ class Strong(SpanToken):
def __init__(self, match_obj):
self.children = tokenize_inner(_first_not_none_group(match_obj))


class Emphasis(SpanToken):
"""
Emphasis tokens. ("*some text*")
Expand All @@ -82,6 +90,7 @@ class Emphasis(SpanToken):
def __init__(self, match_obj):
self.children = tokenize_inner(_first_not_none_group(match_obj))


class InlineCode(SpanToken):
"""
Inline code tokens. ("`some code`")
Expand All @@ -92,6 +101,7 @@ class InlineCode(SpanToken):
def __init__(self, match_obj):
self.children = iter([RawText(match_obj.group(1))])


class Strikethrough(SpanToken):
"""
Strikethrough tokens. ("~~some text~~")
Expand All @@ -100,6 +110,7 @@ class Strikethrough(SpanToken):
"""
pattern = re.compile(r"~~(.+)~~")


class Image(SpanToken):
"""
Image tokens. ("![alt](src "title")")
Expand All @@ -116,6 +127,7 @@ def __init__(self, match_obj):
self.src = match_obj.group(2)
self.title = match_obj.group(3)


class FootnoteImage(SpanToken):
"""
Footnote image tokens. ("![alt] [some key]")
Expand All @@ -129,6 +141,7 @@ def __init__(self, match_obj):
self.children = iter([RawText(match_obj.group(1))])
self.src = FootnoteAnchor(match_obj.group(2))


class Link(SpanToken):
"""
Link tokens. ("[name](target)")
Expand All @@ -142,6 +155,7 @@ def __init__(self, match_obj):
super().__init__(match_obj)
self.target = match_obj.group(2)


class FootnoteLink(SpanToken):
"""
Footnote-style links. ("[name] [some target]")
Expand All @@ -155,6 +169,7 @@ def __init__(self, match_obj):
super().__init__(match_obj)
self.target = FootnoteAnchor(match_obj.group(2))


class AutoLink(SpanToken):
"""
Autolink tokens. ("<http://www.google.com>")
Expand All @@ -169,6 +184,7 @@ def __init__(self, match_obj):
self.children = iter([RawText(match_obj.group(1))])
self.target = match_obj.group(1)


class EscapeSequence(SpanToken):
"""
Escape sequences. ("\*")
Expand All @@ -177,13 +193,15 @@ class EscapeSequence(SpanToken):
def __init__(self, match_obj):
self.children = iter([RawText(match_obj.group(1))])


class RawText(SpanToken):
"""
Raw text. A leaf node.
"""
def __init__(self, raw):
self.content = raw


class FootnoteAnchor(SpanToken):
"""
Footnote anchor.
Expand Down

0 comments on commit 4453dba

Please sign in to comment.