Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
173 lines (137 sloc) 3.89 KB
"""
Support spellchecking various file formats by converting them to plain text
"""
import json
import sys
import xml.etree.ElementTree
supported_extensions = ["txt", "html", "md", "markdown", "rst", "ipynb", "json", "xliff"]
def convert(source, texttype):
"""
Convert files of various types to plaintext
Args:
texttype (str):
file extension of the input file
source (str):
content of the input file
Returns:
str:
plaintext output
"""
if texttype == "html":
return html2text(source)
if texttype in ["md", "markdown"]:
return html2text(markdown2html(source))
if texttype in ["rst"]:
return html2text(rst2html(source))
if texttype == "ipynb":
return html2text(markdown2html(ipynb2markdown(source)))
if texttype == "json":
return transifexjson2txt(source)
if texttype == "xliff":
return xliff2txt(source)
if texttype != "txt":
print("filetype not detected, assuming plaintext")
return source
def html2text(html):
"""
convert HTML to plaintext by parsing it with BeautifulSoup and removing code
Args:
html (str):
HTML string
Returns:
str: plaintext
"""
try:
from bs4 import BeautifulSoup
except ImportError:
notinstalled("beautifulsoup4", "HTML", "text")
sys.exit(4)
soup = BeautifulSoup(html, "html.parser")
# remove scripts from html
for script in soup(["script", "style", "code", "pre"]) + soup("span", {"class": "literal"}):
script.extract()
return soup.get_text()
def markdown2html(markdown):
"""
convert Markdown to HTML via ``markdown2``
Args:
markdown (str):
Markdown text
Returns:
str: HTML
"""
try:
import markdown2
except ImportError:
notinstalled("markdown2", "markdown", "HTML")
sys.exit(4)
return markdown2.markdown(markdown)
def ipynb2markdown(ipynb):
"""
Extract Markdown cells from iPython Notebook
Args:
ipynb (str):
iPython notebook JSON file
Returns:
str: Markdown
"""
j = json.loads(ipynb)
markdown = ""
for cell in j["cells"]:
if cell["cell_type"] == "markdown":
markdown += "".join(cell["source"]) + "\n"
return markdown
def rst2html(rst):
"""
convert reStructuredText to HTML with ``docutils``
Args:
rst (str):
reStructuredText
Returns:
str: HTML
"""
try:
from docutils.core import publish_string
except ImportError:
notinstalled("docutils", "ReStructuredText", "HTML")
sys.exit(4)
return publish_string(rst, writer_name="html5")
def transifexjson2txt(jsondata):
"""
extract translations from Transifex JSON file
Args:
jsondata (str):
Transifex export file
Returns:
str: Plaintext translations
"""
data = json.loads(jsondata)
text = ""
for category, content in data.items():
for key, value in content.items():
text += value + "\n"
return text
def xliff2txt(source):
"""
extract translations from ``XLIFF`` file
Args:
source (str):
XLIFF XML string
Returns:
str: Plaintext translations
"""
root = xml.etree.ElementTree.fromstring(source)
text = ""
for file in root:
for body in file:
for trans_unit in body:
value = trans_unit.find("{urn:oasis:names:tc:xliff:document:1.1}target").text
if value:
text += value + "\n\n"
return text
def notinstalled(package, convert_from, convert_to):
print(
"""{package} is needed to convert {source} to {target}
you can install it with pip:
pip install {package}""".format(package=package, source=convert_from, target=convert_to)
)
You can’t perform that action at this time.