Skip to content
Browse files

Fixed

  • Loading branch information...
1 parent ba8d922 commit 2034777af3f6945e5fa38ae9ab9bd17dba57a4b3 @il-vladislav committed Nov 16, 2012
View
21 bs4/__init__.py
@@ -26,11 +26,6 @@
import re
import warnings
-str = str
-unicode = str
-bytes = bytes
-basestring = (str,bytes)
-
from .builder import builder_registry
from .dammit import UnicodeDammit
from .element import (
@@ -49,7 +44,7 @@
# The very first thing we do is give a useful error if someone is
# running this code under Python 3 without converting it.
-syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
+syntax_error = 'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
class BeautifulSoup(Tag):
"""
@@ -73,7 +68,7 @@ class BeautifulSoup(Tag):
like HTML's <br> tag), call handle_starttag and then
handle_endtag.
"""
- ROOT_TAG_NAME = u'[document]'
+ ROOT_TAG_NAME = '[document]'
# If the end-user gives no indication which tree builder they
# want, look for one with these features.
@@ -143,12 +138,12 @@ def deprecated_argument(old_name, new_name):
"fromEncoding", "from_encoding")
if len(kwargs) > 0:
- arg = kwargs.keys().pop()
+ arg = list(kwargs.keys()).pop()
raise TypeError(
"__init__() got an unexpected keyword argument '%s'" % arg)
if builder is None:
- if isinstance(features, basestring):
+ if isinstance(features, str):
features = [features]
if features is None or len(features) == 0:
features = self.DEFAULT_BUILDER_FEATURES
@@ -234,7 +229,7 @@ def pushTag(self, tag):
def endData(self, containerClass=NavigableString):
if self.currentData:
- currentData = u''.join(self.currentData)
+ currentData = ''.join(self.currentData)
if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and
not set([tag.name for tag in self.tagStack]).intersection(
self.builder.preserve_whitespace_tags)):
@@ -328,9 +323,9 @@ def decode(self, pretty_print=False,
encoding_part = ''
if eventual_encoding != None:
encoding_part = ' encoding="%s"' % eventual_encoding
- prefix = u'<?xml version="1.0"%s?>\n' % encoding_part
+ prefix = '<?xml version="1.0"%s?>\n' % encoding_part
else:
- prefix = u''
+ prefix = ''
if not pretty_print:
indent_level = None
else:
@@ -361,4 +356,4 @@ class FeatureNotFound(ValueError):
if __name__ == '__main__':
import sys
soup = BeautifulSoup(sys.stdin)
- print (soup.prettify())
+ print(soup.prettify())
View
7 bs4/builder/__init__.py
@@ -22,11 +22,6 @@
HTML = 'html'
HTML_5 = 'html5'
-str = str
-unicode = str
-bytes = bytes
-basestring = (str,bytes)
-
class TreeBuilderRegistry(object):
@@ -162,7 +157,7 @@ def _replace_cdata_list_attribute_values(self, tag_name, attrs):
# value is a whitespace-separated list of CSS
# classes. Split it into a list.
value = attrs[cdata_list_attr]
- if isinstance(value, (str,bytes)):
+ if isinstance(value, str):
values = whitespace_re.split(value)
else:
# html5lib sometimes calls setAttributes twice
View
6 bs4/builder/_html5lib.py
@@ -37,7 +37,7 @@ def feed(self, markup):
doc = parser.parse(markup, encoding=self.user_specified_encoding)
# Set the character encoding detected by the tokenizer.
- if isinstance(markup, unicode):
+ if isinstance(markup, str):
# We need to special-case this because html5lib sets
# charEncoding to UTF-8 if it gets Unicode input.
doc.original_encoding = None
@@ -51,7 +51,7 @@ def create_treebuilder(self, namespaceHTMLElements):
def test_fragment_to_document(self, fragment):
"""See `TreeBuilder`."""
- return u'<html><head></head><body>%s</body></html>' % fragment
+ return '<html><head></head><body>%s</body></html>' % fragment
class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
@@ -150,7 +150,7 @@ def setAttributes(self, attributes):
self.soup.builder._replace_cdata_list_attribute_values(
self.name, attributes)
- for name, value in attributes.items():
+ for name, value in list(attributes.items()):
self.element[name] = value
# The attributes may contain variables that need substitution.
View
13 bs4/builder/_htmlparser.py
@@ -11,11 +11,6 @@
import sys
import warnings
-
-str = str
-unicode = str
-bytes = bytes
-basestring = (str,bytes)
# Starting in Python 3.2, the HTMLParser constructor takes a 'strict'
# argument, which we'd like to set to False. Unfortunately,
# http://bugs.python.org/issue13273 makes strict=True a better bet
@@ -68,8 +63,8 @@ def handle_charref(self, name):
try:
data = chr(real_name)
- except (ValueError, OverflowError):
- data = u"\N{REPLACEMENT CHARACTER}"
+ except (ValueError, OverflowError) as e:
+ data = "\N{REPLACEMENT CHARACTER}"
self.handle_data(data)
@@ -134,7 +129,7 @@ def prepare_markup(self, markup, user_specified_encoding=None,
declared within markup, whether any characters had to be
replaced with REPLACEMENT CHARACTER).
"""
- if isinstance(markup, unicode):
+ if isinstance(markup, str):
return markup, None, None, False
try_encodings = [user_specified_encoding, document_declared_encoding]
@@ -149,7 +144,7 @@ def feed(self, markup):
parser.soup = self.soup
try:
parser.feed(markup)
- except HTMLParseError:
+ except HTMLParseError as e:
warnings.warn(RuntimeWarning(
"Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
raise e
View
16 bs4/builder/_lxml.py
@@ -3,7 +3,7 @@
'LXMLTreeBuilder',
]
-from StringIO import StringIO
+from io import StringIO
import collections
from lxml import etree
from bs4.element import Comment, Doctype, NamespacedAttribute
@@ -61,7 +61,7 @@ def prepare_markup(self, markup, user_specified_encoding=None,
:return: A 3-tuple (markup, original encoding, encoding
declared within markup).
"""
- if isinstance(markup, unicode):
+ if isinstance(markup, str):
return markup, None, None, False
try_encodings = [user_specified_encoding, document_declared_encoding]
@@ -71,7 +71,7 @@ def prepare_markup(self, markup, user_specified_encoding=None,
dammit.contains_replacement_characters)
def feed(self, markup):
- if isinstance(markup, basestring):
+ if isinstance(markup, str):
markup = StringIO(markup)
# Call feed() at least once, even if the markup is empty,
# or the parser won't be initialized.
@@ -102,12 +102,12 @@ def start(self, name, attrs, nsmap={}):
# A new namespace mapping has come into play.
if self.nsmaps is None:
self.nsmaps = []
- inverted_nsmap = dict((value, key) for key, value in nsmap.items())
+ inverted_nsmap = dict((value, key) for key, value in list(nsmap.items()))
self.nsmaps.append(inverted_nsmap)
# Also treat the namespace mapping as a set of attributes on the
# tag, so we can recreate it later.
attrs = attrs.copy()
- for prefix, namespace in nsmap.items():
+ for prefix, namespace in list(nsmap.items()):
attribute = NamespacedAttribute(
"xmlns", prefix, "http://www.w3.org/2000/xmlns/")
attrs[attribute] = namespace
@@ -117,7 +117,7 @@ def start(self, name, attrs, nsmap={}):
# from lxml with namespaces attached to their names, and
# turn then into NamespacedAttribute objects.
new_attrs = {}
- for attr, value in attrs.items():
+ for attr, value in list(attrs.items()):
namespace, attr = self._getNsTag(attr)
if namespace is None:
new_attrs[attr] = value
@@ -178,7 +178,7 @@ def comment(self, content):
def test_fragment_to_document(self, fragment):
"""See `TreeBuilder`."""
- return u'<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
+ return '<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
@@ -196,4 +196,4 @@ def feed(self, markup):
def test_fragment_to_document(self, fragment):
"""See `TreeBuilder`."""
- return u'<html><body>%s</body></html>' % fragment
+ return '<html><body>%s</body></html>' % fragment
View
1,606 bs4/dammit.py
@@ -1,803 +1,803 @@
-# -*- coding: utf-8 -*-
-"""Beautiful Soup bonus library: Unicode, Dammit
-
-This class forces XML data into a standard format (usually to UTF-8 or
-Unicode). It is heavily based on code from Mark Pilgrim's Universal
-Feed Parser. It does not rewrite the XML or HTML to reflect a new
-encoding; that's the tree builder's job.
-"""
-
-import codecs
-from html.entities import codepoint2name
-import re
-import logging
-
-# Import a library to autodetect character encodings.
-chardet_type = None
-try:
- # First try the fast C implementation.
- # PyPI package: cchardet
- import cchardet
- def chardet_dammit(s):
- return cchardet.detect(s)['encoding']
-except ImportError:
- try:
- # Fall back to the pure Python implementation
- # Debian package: python-chardet
- # PyPI package: chardet
- import chardet
- def chardet_dammit(s):
- return chardet.detect(s)['encoding']
- #import chardet.constants
- #chardet.constants._debug = 1
- except ImportError:
- # No chardet available.
- def chardet_dammit(s):
- return None
-
-# Available from http://cjkpython.i18n.org/.
-try:
- import iconv_codec
-except ImportError:
- pass
-
-xml_encoding_re = re.compile(
- '^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I)
-html_meta_re = re.compile(
- '<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I)
-
-class EntitySubstitution(object):
-
- """Substitute XML or HTML entities for the corresponding characters."""
-
- def _populate_class_variables():
- lookup = {}
- reverse_lookup = {}
- characters_for_re = []
- for codepoint, name in list(codepoint2name.items()):
- character = chr(codepoint)
- if codepoint != 34:
- # There's no point in turning the quotation mark into
- # &quot;, unless it happens within an attribute value, which
- # is handled elsewhere.
- characters_for_re.append(character)
- lookup[character] = name
- # But we do want to turn &quot; into the quotation mark.
- reverse_lookup[name] = character
- re_definition = "[%s]" % "".join(characters_for_re)
- return lookup, reverse_lookup, re.compile(re_definition)
- (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER,
- CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables()
-
- CHARACTER_TO_XML_ENTITY = {
- "'": "apos",
- '"': "quot",
- "&": "amp",
- "<": "lt",
- ">": "gt",
- }
-
- BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
- "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
- ")")
-
- @classmethod
- def _substitute_html_entity(cls, matchobj):
- entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
- return "&%s;" % entity
-
- @classmethod
- def _substitute_xml_entity(cls, matchobj):
- """Used with a regular expression to substitute the
- appropriate XML entity for an XML special character."""
- entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
- return "&%s;" % entity
-
- @classmethod
- def quoted_attribute_value(self, value):
- """Make a value into a quoted XML attribute, possibly escaping it.
-
- Most strings will be quoted using double quotes.
-
- Bob's Bar -> "Bob's Bar"
-
- If a string contains double quotes, it will be quoted using
- single quotes.
-
- Welcome to "my bar" -> 'Welcome to "my bar"'
-
- If a string contains both single and double quotes, the
- double quotes will be escaped, and the string will be quoted
- using double quotes.
-
- Welcome to "Bob's Bar" -> "Welcome to &quot;Bob's bar&quot;
- """
- quote_with = '"'
- if '"' in value:
- if "'" in value:
- # The string contains both single and double
- # quotes. Turn the double quotes into
- # entities. We quote the double quotes rather than
- # the single quotes because the entity name is
- # "&quot;" whether this is HTML or XML. If we
- # quoted the single quotes, we'd have to decide
- # between &apos; and &squot;.
- replace_with = "&quot;"
- value = value.replace('"', replace_with)
- else:
- # There are double quotes but no single quotes.
- # We can use single quotes to quote the attribute.
- quote_with = "'"
- return quote_with + value + quote_with
-
- @classmethod
- def substitute_xml(cls, value, make_quoted_attribute=False):
- """Substitute XML entities for special XML characters.
-
- :param value: A string to be substituted. The less-than sign will
- become &lt;, the greater-than sign will become &gt;, and any
- ampersands that are not part of an entity defition will
- become &amp;.
-
- :param make_quoted_attribute: If True, then the string will be
- quoted, as befits an attribute value.
- """
- # Escape angle brackets, and ampersands that aren't part of
- # entities.
- value = cls.BARE_AMPERSAND_OR_BRACKET.sub(
- cls._substitute_xml_entity, value)
-
- if make_quoted_attribute:
- value = cls.quoted_attribute_value(value)
- return value
-
- @classmethod
- def substitute_html(cls, s):
- """Replace certain Unicode characters with named HTML entities.
-
- This differs from data.encode(encoding, 'xmlcharrefreplace')
- in that the goal is to make the result more readable (to those
- with ASCII displays) rather than to recover from
- errors. There's absolutely nothing wrong with a UTF-8 string
- containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that
- character with "&eacute;" will make it more readable to some
- people.
- """
- return cls.CHARACTER_TO_HTML_ENTITY_RE.sub(
- cls._substitute_html_entity, s)
-
-
-class UnicodeDammit:
- """A class for detecting the encoding of a *ML document and
- converting it to a Unicode string. If the source encoding is
- windows-1252, can replace MS smart quotes with their HTML or XML
- equivalents."""
-
- # This dictionary maps commonly seen values for "charset" in HTML
- # meta tags to the corresponding Python codec names. It only covers
- # values that aren't in Python's aliases and can't be determined
- # by the heuristics in find_codec.
- CHARSET_ALIASES = {"macintosh": "mac-roman",
- "x-sjis": "shift-jis"}
-
- ENCODINGS_WITH_SMART_QUOTES = [
- "windows-1252",
- "iso-8859-1",
- "iso-8859-2",
- ]
-
- def __init__(self, markup, override_encodings=[],
- smart_quotes_to=None, is_html=False):
- self.declared_html_encoding = None
- self.smart_quotes_to = smart_quotes_to
- self.tried_encodings = []
- self.contains_replacement_characters = False
-
- if markup == '' or isinstance(markup, unicode):
- self.markup = markup
- self.unicode_markup = unicode(markup)
- self.original_encoding = None
- return
-
- new_markup, document_encoding, sniffed_encoding = \
- self._detectEncoding(markup, is_html)
- self.markup = new_markup
-
- u = None
- if new_markup != markup:
- # _detectEncoding modified the markup, then converted it to
- # Unicode and then to UTF-8. So convert it from UTF-8.
- u = self._convert_from("utf8")
- self.original_encoding = sniffed_encoding
-
- if not u:
- for proposed_encoding in (
- override_encodings + [document_encoding, sniffed_encoding]):
- if proposed_encoding is not None:
- u = self._convert_from(proposed_encoding)
- if u:
- break
-
- # If no luck and we have auto-detection library, try that:
- if not u and not isinstance(self.markup, unicode):
- u = self._convert_from(chardet_dammit(self.markup))
-
- # As a last resort, try utf-8 and windows-1252:
- if not u:
- for proposed_encoding in ("utf-8", "windows-1252"):
- u = self._convert_from(proposed_encoding)
- if u:
- break
-
- # As an absolute last resort, try the encodings again with
- # character replacement.
- if not u:
- for proposed_encoding in (
- override_encodings + [
- document_encoding, sniffed_encoding, "utf-8", "windows-1252"]):
- if proposed_encoding != "ascii":
- u = self._convert_from(proposed_encoding, "replace")
- if u is not None:
- logging.warning(
- "Some characters could not be decoded, and were "
- "replaced with REPLACEMENT CHARACTER.")
- self.contains_replacement_characters = True
- break
-
- # We could at this point force it to ASCII, but that would
- # destroy so much data that I think giving up is better
- self.unicode_markup = u
- if not u:
- self.original_encoding = None
-
- def _sub_ms_char(self, match):
- """Changes a MS smart quote character to an XML or HTML
- entity, or an ASCII character."""
- orig = match.group(1)
- if self.smart_quotes_to == 'ascii':
- sub = self.MS_CHARS_TO_ASCII.get(orig).encode()
- else:
- sub = self.MS_CHARS.get(orig)
- if type(sub) == tuple:
- if self.smart_quotes_to == 'xml':
- sub = '&#x'.encode() + sub[1].encode() + ';'.encode()
- else:
- sub = '&'.encode() + sub[0].encode() + ';'.encode()
- else:
- sub = sub.encode()
- return sub
-
- def _convert_from(self, proposed, errors="strict"):
- proposed = self.find_codec(proposed)
- if not proposed or (proposed, errors) in self.tried_encodings:
- return None
- self.tried_encodings.append((proposed, errors))
- markup = self.markup
-
- # Convert smart quotes to HTML if coming from an encoding
- # that might have them.
- if (self.smart_quotes_to is not None
- and proposed.lower() in self.ENCODINGS_WITH_SMART_QUOTES):
- smart_quotes_re = b"([\x80-\x9f])"
- smart_quotes_compiled = re.compile(smart_quotes_re)
- markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)
-
- try:
- #print "Trying to convert document to %s (errors=%s)" % (
- # proposed, errors)
- u = self._to_unicode(markup, proposed, errors)
- self.markup = u
- self.original_encoding = proposed
- except Exception as e:
- #print "That didn't work!"
- #print e
- return None
- #print "Correct encoding: %s" % proposed
- return self.markup
-
- def _to_unicode(self, data, encoding, errors="strict"):
- '''Given a string and its encoding, decodes the string into Unicode.
- %encoding is a string recognized by encodings.aliases'''
-
- # strip Byte Order Mark (if present)
- if (len(data) >= 4) and (data[:2] == '\xfe\xff') \
- and (data[2:4] != '\x00\x00'):
- encoding = 'utf-16be'
- data = data[2:]
- elif (len(data) >= 4) and (data[:2] == '\xff\xfe') \
- and (data[2:4] != '\x00\x00'):
- encoding = 'utf-16le'
- data = data[2:]
- elif data[:3] == '\xef\xbb\xbf':
- encoding = 'utf-8'
- data = data[3:]
- elif data[:4] == '\x00\x00\xfe\xff':
- encoding = 'utf-32be'
- data = data[4:]
- elif data[:4] == '\xff\xfe\x00\x00':
- encoding = 'utf-32le'
- data = data[4:]
- newdata = unicode(data, encoding, errors)
- return newdata
-
- def _detectEncoding(self, xml_data, is_html=False):
- """Given a document, tries to detect its XML encoding."""
- xml_encoding = sniffed_xml_encoding = None
- try:
- if xml_data[:4] == b'\x4c\x6f\xa7\x94':
- # EBCDIC
- xml_data = self._ebcdic_to_ascii(xml_data)
- elif xml_data[:4] == b'\x00\x3c\x00\x3f':
- # UTF-16BE
- sniffed_xml_encoding = 'utf-16be'
- xml_data = unicode(xml_data, 'utf-16be').encode('utf-8')
- elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xfe\xff') \
- and (xml_data[2:4] != b'\x00\x00'):
- # UTF-16BE with BOM
- sniffed_xml_encoding = 'utf-16be'
- xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8')
- elif xml_data[:4] == b'\x3c\x00\x3f\x00':
- # UTF-16LE
- sniffed_xml_encoding = 'utf-16le'
- xml_data = unicode(xml_data, 'utf-16le').encode('utf-8')
- elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xff\xfe') and \
- (xml_data[2:4] != b'\x00\x00'):
- # UTF-16LE with BOM
- sniffed_xml_encoding = 'utf-16le'
- xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8')
- elif xml_data[:4] == b'\x00\x00\x00\x3c':
- # UTF-32BE
- sniffed_xml_encoding = 'utf-32be'
- xml_data = unicode(xml_data, 'utf-32be').encode('utf-8')
- elif xml_data[:4] == b'\x3c\x00\x00\x00':
- # UTF-32LE
- sniffed_xml_encoding = 'utf-32le'
- xml_data = unicode(xml_data, 'utf-32le').encode('utf-8')
- elif xml_data[:4] == b'\x00\x00\xfe\xff':
- # UTF-32BE with BOM
- sniffed_xml_encoding = 'utf-32be'
- xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8')
- elif xml_data[:4] == b'\xff\xfe\x00\x00':
- # UTF-32LE with BOM
- sniffed_xml_encoding = 'utf-32le'
- xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8')
- elif xml_data[:3] == b'\xef\xbb\xbf':
- # UTF-8 with BOM
- sniffed_xml_encoding = 'utf-8'
- xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8')
- else:
- sniffed_xml_encoding = 'ascii'
- pass
- except:
- xml_encoding_match = None
- xml_encoding_match = xml_encoding_re.match(xml_data)
- if not xml_encoding_match and is_html:
- xml_encoding_match = html_meta_re.search(xml_data)
- if xml_encoding_match is not None:
- xml_encoding = xml_encoding_match.groups()[0].decode(
- 'ascii').lower()
- if is_html:
- self.declared_html_encoding = xml_encoding
- if sniffed_xml_encoding and \
- (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode',
- 'iso-10646-ucs-4', 'ucs-4', 'csucs4',
- 'utf-16', 'utf-32', 'utf_16', 'utf_32',
- 'utf16', 'u16')):
- xml_encoding = sniffed_xml_encoding
- return xml_data, xml_encoding, sniffed_xml_encoding
-
- def find_codec(self, charset):
- return self._codec(self.CHARSET_ALIASES.get(charset, charset)) \
- or (charset and self._codec(charset.replace("-", ""))) \
- or (charset and self._codec(charset.replace("-", "_"))) \
- or charset
-
- def _codec(self, charset):
- if not charset:
- return charset
- codec = None
- try:
- codecs.lookup(charset)
- codec = charset
- except (LookupError, ValueError):
- pass
- return codec
-
- EBCDIC_TO_ASCII_MAP = None
-
- def _ebcdic_to_ascii(self, s):
- c = self.__class__
- if not c.EBCDIC_TO_ASCII_MAP:
- emap = (0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15,
- 16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31,
- 128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7,
- 144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26,
- 32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33,
- 38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94,
- 45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63,
- 186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34,
- 195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,
- 201,202,106,107,108,109,110,111,112,113,114,203,204,205,
- 206,207,208,209,126,115,116,117,118,119,120,121,122,210,
- 211,212,213,214,215,216,217,218,219,220,221,222,223,224,
- 225,226,227,228,229,230,231,123,65,66,67,68,69,70,71,72,
- 73,232,233,234,235,236,237,125,74,75,76,77,78,79,80,81,
- 82,238,239,240,241,242,243,92,159,83,84,85,86,87,88,89,
- 90,244,245,246,247,248,249,48,49,50,51,52,53,54,55,56,57,
- 250,251,252,253,254,255)
- import string
- c.EBCDIC_TO_ASCII_MAP = string.maketrans(
- ''.join(map(chr, list(range(256)))), ''.join(map(chr, emap)))
- return s.translate(c.EBCDIC_TO_ASCII_MAP)
-
- # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.
- MS_CHARS = {b'\x80': ('euro', '20AC'),
- b'\x81': ' ',
- b'\x82': ('sbquo', '201A'),
- b'\x83': ('fnof', '192'),
- b'\x84': ('bdquo', '201E'),
- b'\x85': ('hellip', '2026'),
- b'\x86': ('dagger', '2020'),
- b'\x87': ('Dagger', '2021'),
- b'\x88': ('circ', '2C6'),
- b'\x89': ('permil', '2030'),
- b'\x8A': ('Scaron', '160'),
- b'\x8B': ('lsaquo', '2039'),
- b'\x8C': ('OElig', '152'),
- b'\x8D': '?',
- b'\x8E': ('#x17D', '17D'),
- b'\x8F': '?',
- b'\x90': '?',
- b'\x91': ('lsquo', '2018'),
- b'\x92': ('rsquo', '2019'),
- b'\x93': ('ldquo', '201C'),
- b'\x94': ('rdquo', '201D'),
- b'\x95': ('bull', '2022'),
- b'\x96': ('ndash', '2013'),
- b'\x97': ('mdash', '2014'),
- b'\x98': ('tilde', '2DC'),
- b'\x99': ('trade', '2122'),
- b'\x9a': ('scaron', '161'),
- b'\x9b': ('rsaquo', '203A'),
- b'\x9c': ('oelig', '153'),
- b'\x9d': '?',
- b'\x9e': ('#x17E', '17E'),
- b'\x9f': ('Yuml', ''),}
-
- # A parochial partial mapping of ISO-Latin-1 to ASCII. Contains
- # horrors like stripping diacritical marks to turn á into a, but also
- # contains non-horrors like turning “ into ".
- MS_CHARS_TO_ASCII = {
- b'\x80' : 'EUR',
- b'\x81' : ' ',
- b'\x82' : ',',
- b'\x83' : 'f',
- b'\x84' : ',,',
- b'\x85' : '...',
- b'\x86' : '+',
- b'\x87' : '++',
- b'\x88' : '^',
- b'\x89' : '%',
- b'\x8a' : 'S',
- b'\x8b' : '<',
- b'\x8c' : 'OE',
- b'\x8d' : '?',
- b'\x8e' : 'Z',
- b'\x8f' : '?',
- b'\x90' : '?',
- b'\x91' : "'",
- b'\x92' : "'",
- b'\x93' : '"',
- b'\x94' : '"',
- b'\x95' : '*',
- b'\x96' : '-',
- b'\x97' : '--',
- b'\x98' : '~',
- b'\x99' : '(TM)',
- b'\x9a' : 's',
- b'\x9b' : '>',
- b'\x9c' : 'oe',
- b'\x9d' : '?',
- b'\x9e' : 'z',
- b'\x9f' : 'Y',
- b'\xa0' : ' ',
- b'\xa1' : '!',
- b'\xa2' : 'c',
- b'\xa3' : 'GBP',
- b'\xa4' : '$', #This approximation is especially parochial--this is the
- #generic currency symbol.
- b'\xa5' : 'YEN',
- b'\xa6' : '|',
- b'\xa7' : 'S',
- b'\xa8' : '..',
- b'\xa9' : '',
- b'\xaa' : '(th)',
- b'\xab' : '<<',
- b'\xac' : '!',
- b'\xad' : ' ',
- b'\xae' : '(R)',
- b'\xaf' : '-',
- b'\xb0' : 'o',
- b'\xb1' : '+-',
- b'\xb2' : '2',
- b'\xb3' : '3',
- b'\xb4' : ("'", 'acute'),
- b'\xb5' : 'u',
- b'\xb6' : 'P',
- b'\xb7' : '*',
- b'\xb8' : ',',
- b'\xb9' : '1',
- b'\xba' : '(th)',
- b'\xbb' : '>>',
- b'\xbc' : '1/4',
- b'\xbd' : '1/2',
- b'\xbe' : '3/4',
- b'\xbf' : '?',
- b'\xc0' : 'A',
- b'\xc1' : 'A',
- b'\xc2' : 'A',
- b'\xc3' : 'A',
- b'\xc4' : 'A',
- b'\xc5' : 'A',
- b'\xc6' : 'AE',
- b'\xc7' : 'C',
- b'\xc8' : 'E',
- b'\xc9' : 'E',
- b'\xca' : 'E',
- b'\xcb' : 'E',
- b'\xcc' : 'I',
- b'\xcd' : 'I',
- b'\xce' : 'I',
- b'\xcf' : 'I',
- b'\xd0' : 'D',
- b'\xd1' : 'N',
- b'\xd2' : 'O',
- b'\xd3' : 'O',
- b'\xd4' : 'O',
- b'\xd5' : 'O',
- b'\xd6' : 'O',
- b'\xd7' : '*',
- b'\xd8' : 'O',
- b'\xd9' : 'U',
- b'\xda' : 'U',
- b'\xdb' : 'U',
- b'\xdc' : 'U',
- b'\xdd' : 'Y',
- b'\xde' : 'b',
- b'\xdf' : 'B',
- b'\xe0' : 'a',
- b'\xe1' : 'a',
- b'\xe2' : 'a',
- b'\xe3' : 'a',
- b'\xe4' : 'a',
- b'\xe5' : 'a',
- b'\xe6' : 'ae',
- b'\xe7' : 'c',
- b'\xe8' : 'e',
- b'\xe9' : 'e',
- b'\xea' : 'e',
- b'\xeb' : 'e',
- b'\xec' : 'i',
- b'\xed' : 'i',
- b'\xee' : 'i',
- b'\xef' : 'i',
- b'\xf0' : 'o',
- b'\xf1' : 'n',
- b'\xf2' : 'o',
- b'\xf3' : 'o',
- b'\xf4' : 'o',
- b'\xf5' : 'o',
- b'\xf6' : 'o',
- b'\xf7' : '/',
- b'\xf8' : 'o',
- b'\xf9' : 'u',
- b'\xfa' : 'u',
- b'\xfb' : 'u',
- b'\xfc' : 'u',
- b'\xfd' : 'y',
- b'\xfe' : 'b',
- b'\xff' : 'y',
- }
-
- # A map used when removing rogue Windows-1252/ISO-8859-1
- # characters in otherwise UTF-8 documents.
- #
- # Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in
- # Windows-1252.
- WINDOWS_1252_TO_UTF8 = {
- 0x80 : b'\xe2\x82\xac', # €
- 0x82 : b'\xe2\x80\x9a', # ‚
- 0x83 : b'\xc6\x92', # ƒ
- 0x84 : b'\xe2\x80\x9e', # „
- 0x85 : b'\xe2\x80\xa6', # …
- 0x86 : b'\xe2\x80\xa0', # †
- 0x87 : b'\xe2\x80\xa1', # ‡
- 0x88 : b'\xcb\x86', # ˆ
- 0x89 : b'\xe2\x80\xb0', # ‰
- 0x8a : b'\xc5\xa0', # Š
- 0x8b : b'\xe2\x80\xb9', # ‹
- 0x8c : b'\xc5\x92', # Œ
- 0x8e : b'\xc5\xbd', # Ž
- 0x91 : b'\xe2\x80\x98', # ‘
- 0x92 : b'\xe2\x80\x99', # ’
- 0x93 : b'\xe2\x80\x9c', # “
- 0x94 : b'\xe2\x80\x9d', # ”
- 0x95 : b'\xe2\x80\xa2', # •
- 0x96 : b'\xe2\x80\x93', # –
- 0x97 : b'\xe2\x80\x94', # —
- 0x98 : b'\xcb\x9c', # ˜
- 0x99 : b'\xe2\x84\xa2', # ™
- 0x9a : b'\xc5\xa1', # š
- 0x9b : b'\xe2\x80\xba', # ›
- 0x9c : b'\xc5\x93', # œ
- 0x9e : b'\xc5\xbe', # ž
- 0x9f : b'\xc5\xb8', # Ÿ
- 0xa0 : b'\xc2\xa0', #  
- 0xa1 : b'\xc2\xa1', # ¡
- 0xa2 : b'\xc2\xa2', # ¢
- 0xa3 : b'\xc2\xa3', # £
- 0xa4 : b'\xc2\xa4', # ¤
- 0xa5 : b'\xc2\xa5', # ¥
- 0xa6 : b'\xc2\xa6', # ¦
- 0xa7 : b'\xc2\xa7', # §
- 0xa8 : b'\xc2\xa8', # ¨
- 0xa9 : b'\xc2\xa9', # ©
- 0xaa : b'\xc2\xaa', # ª
- 0xab : b'\xc2\xab', # «
- 0xac : b'\xc2\xac', # ¬
- 0xad : b'\xc2\xad', # ­
- 0xae : b'\xc2\xae', # ®
- 0xaf : b'\xc2\xaf', # ¯
- 0xb0 : b'\xc2\xb0', # °
- 0xb1 : b'\xc2\xb1', # ±
- 0xb2 : b'\xc2\xb2', # ²
- 0xb3 : b'\xc2\xb3', # ³
- 0xb4 : b'\xc2\xb4', # ´
- 0xb5 : b'\xc2\xb5', # µ
- 0xb6 : b'\xc2\xb6', # ¶
- 0xb7 : b'\xc2\xb7', # ·
- 0xb8 : b'\xc2\xb8', # ¸
- 0xb9 : b'\xc2\xb9', # ¹
- 0xba : b'\xc2\xba', # º
- 0xbb : b'\xc2\xbb', # »
- 0xbc : b'\xc2\xbc', # ¼
- 0xbd : b'\xc2\xbd', # ½
- 0xbe : b'\xc2\xbe', # ¾
- 0xbf : b'\xc2\xbf', # ¿
- 0xc0 : b'\xc3\x80', # À
- 0xc1 : b'\xc3\x81', # Á
- 0xc2 : b'\xc3\x82', # Â
- 0xc3 : b'\xc3\x83', # Ã
- 0xc4 : b'\xc3\x84', # Ä
- 0xc5 : b'\xc3\x85', # Å
- 0xc6 : b'\xc3\x86', # Æ
- 0xc7 : b'\xc3\x87', # Ç
- 0xc8 : b'\xc3\x88', # È
- 0xc9 : b'\xc3\x89', # É
- 0xca : b'\xc3\x8a', # Ê
- 0xcb : b'\xc3\x8b', # Ë
- 0xcc : b'\xc3\x8c', # Ì
- 0xcd : b'\xc3\x8d', # Í
- 0xce : b'\xc3\x8e', # Î
- 0xcf : b'\xc3\x8f', # Ï
- 0xd0 : b'\xc3\x90', # Ð
- 0xd1 : b'\xc3\x91', # Ñ
- 0xd2 : b'\xc3\x92', # Ò
- 0xd3 : b'\xc3\x93', # Ó
- 0xd4 : b'\xc3\x94', # Ô
- 0xd5 : b'\xc3\x95', # Õ
- 0xd6 : b'\xc3\x96', # Ö
- 0xd7 : b'\xc3\x97', # ×
- 0xd8 : b'\xc3\x98', # Ø
- 0xd9 : b'\xc3\x99', # Ù
- 0xda : b'\xc3\x9a', # Ú
- 0xdb : b'\xc3\x9b', # Û
- 0xdc : b'\xc3\x9c', # Ü
- 0xdd : b'\xc3\x9d', # Ý
- 0xde : b'\xc3\x9e', # Þ
- 0xdf : b'\xc3\x9f', # ß
- 0xe0 : b'\xc3\xa0', # à
- 0xe1 : b'\xa1', # á
- 0xe2 : b'\xc3\xa2', # â
- 0xe3 : b'\xc3\xa3', # ã
- 0xe4 : b'\xc3\xa4', # ä
- 0xe5 : b'\xc3\xa5', # å
- 0xe6 : b'\xc3\xa6', # æ
- 0xe7 : b'\xc3\xa7', # ç
- 0xe8 : b'\xc3\xa8', # è
- 0xe9 : b'\xc3\xa9', # é
- 0xea : b'\xc3\xaa', # ê
- 0xeb : b'\xc3\xab', # ë
- 0xec : b'\xc3\xac', # ì
- 0xed : b'\xc3\xad', # í
- 0xee : b'\xc3\xae', # î
- 0xef : b'\xc3\xaf', # ï
- 0xf0 : b'\xc3\xb0', # ð
- 0xf1 : b'\xc3\xb1', # ñ
- 0xf2 : b'\xc3\xb2', # ò
- 0xf3 : b'\xc3\xb3', # ó
- 0xf4 : b'\xc3\xb4', # ô
- 0xf5 : b'\xc3\xb5', # õ
- 0xf6 : b'\xc3\xb6', # ö
- 0xf7 : b'\xc3\xb7', # ÷
- 0xf8 : b'\xc3\xb8', # ø
- 0xf9 : b'\xc3\xb9', # ù
- 0xfa : b'\xc3\xba', # ú
- 0xfb : b'\xc3\xbb', # û
- 0xfc : b'\xc3\xbc', # ü
- 0xfd : b'\xc3\xbd', # ý
- 0xfe : b'\xc3\xbe', # þ
- }
-
- MULTIBYTE_MARKERS_AND_SIZES = [
- (0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF
- (0xe0, 0xef, 3), # 3-byte characters start with E0-EF
- (0xf0, 0xf4, 4), # 4-byte characters start with F0-F4
- ]
-
- FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0]
- LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1]
-
- @classmethod
- def detwingle(cls, in_bytes, main_encoding="utf8",
- embedded_encoding="windows-1252"):
- """Fix characters from one encoding embedded in some other encoding.
-
- Currently the only situation supported is Windows-1252 (or its
- subset ISO-8859-1), embedded in UTF-8.
-
- The input must be a bytestring. If you've already converted
- the document to Unicode, you're too late.
-
- The output is a bytestring in which `embedded_encoding`
- characters have been converted to their `main_encoding`
- equivalents.
- """
- if embedded_encoding.replace('_', '-').lower() not in (
- 'windows-1252', 'windows_1252'):
- raise NotImplementedError(
- "Windows-1252 and ISO-8859-1 are the only currently supported "
- "embedded encodings.")
-
- if main_encoding.lower() not in ('utf8', 'utf-8'):
- raise NotImplementedError(
- "UTF-8 is the only currently supported main encoding.")
-
- byte_chunks = []
-
- chunk_start = 0
- pos = 0
- while pos < len(in_bytes):
- byte = in_bytes[pos]
- if not isinstance(byte, int):
- # Python 2.x
- byte = ord(byte)
- if (byte >= cls.FIRST_MULTIBYTE_MARKER
- and byte <= cls.LAST_MULTIBYTE_MARKER):
- # This is the start of a UTF-8 multibyte character. Skip
- # to the end.
- for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
- if byte >= start and byte <= end:
- pos += size
- break
- elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
- # We found a Windows-1252 character!
- # Save the string up to this point as a chunk.
- byte_chunks.append(in_bytes[chunk_start:pos])
-
- # Now translate the Windows-1252 character into UTF-8
- # and add it as another, one-byte chunk.
- byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
- pos += 1
- chunk_start = pos
- else:
- # Go on to the next character.
- pos += 1
- if chunk_start == 0:
- # The string is unchanged.
- return in_bytes
- else:
- # Store the final chunk.
- byte_chunks.append(in_bytes[chunk_start:])
- return b''.join(byte_chunks)
-
+# -*- coding: utf-8 -*-
+"""Beautiful Soup bonus library: Unicode, Dammit
+
+This class forces XML data into a standard format (usually to UTF-8 or
+Unicode). It is heavily based on code from Mark Pilgrim's Universal
+Feed Parser. It does not rewrite the XML or HTML to reflect a new
+encoding; that's the tree builder's job.
+"""
+
+import codecs
+from html.entities import codepoint2name
+import re
+import logging
+
+# Import a library to autodetect character encodings.
+chardet_type = None
+try:
+ # First try the fast C implementation.
+ # PyPI package: cchardet
+ import cchardet
+ def chardet_dammit(s):
+ return cchardet.detect(s)['encoding']
+except ImportError:
+ try:
+ # Fall back to the pure Python implementation
+ # Debian package: python-chardet
+ # PyPI package: chardet
+ import chardet
+ def chardet_dammit(s):
+ return chardet.detect(s)['encoding']
+ #import chardet.constants
+ #chardet.constants._debug = 1
+ except ImportError:
+ # No chardet available.
+ def chardet_dammit(s):
+ return None
+
+# Available from http://cjkpython.i18n.org/.
+try:
+ import iconv_codec
+except ImportError:
+ pass
+
+xml_encoding_re = re.compile(
+ '^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I)
+html_meta_re = re.compile(
+ '<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I)
+
+class EntitySubstitution(object):
+
+ """Substitute XML or HTML entities for the corresponding characters."""
+
+ def _populate_class_variables():
+ lookup = {}
+ reverse_lookup = {}
+ characters_for_re = []
+ for codepoint, name in list(codepoint2name.items()):
+ character = chr(codepoint)
+ if codepoint != 34:
+ # There's no point in turning the quotation mark into
+ # &quot;, unless it happens within an attribute value, which
+ # is handled elsewhere.
+ characters_for_re.append(character)
+ lookup[character] = name
+ # But we do want to turn &quot; into the quotation mark.
+ reverse_lookup[name] = character
+ re_definition = "[%s]" % "".join(characters_for_re)
+ return lookup, reverse_lookup, re.compile(re_definition)
+ (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER,
+ CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables()
+
+ CHARACTER_TO_XML_ENTITY = {
+ "'": "apos",
+ '"': "quot",
+ "&": "amp",
+ "<": "lt",
+ ">": "gt",
+ }
+
+ BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
+ "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
+ ")")
+
+ @classmethod
+ def _substitute_html_entity(cls, matchobj):
+ entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
+ return "&%s;" % entity
+
+ @classmethod
+ def _substitute_xml_entity(cls, matchobj):
+ """Used with a regular expression to substitute the
+ appropriate XML entity for an XML special character."""
+ entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
+ return "&%s;" % entity
+
+ @classmethod
+ def quoted_attribute_value(self, value):
+ """Make a value into a quoted XML attribute, possibly escaping it.
+
+ Most strings will be quoted using double quotes.
+
+ Bob's Bar -> "Bob's Bar"
+
+ If a string contains double quotes, it will be quoted using
+ single quotes.
+
+ Welcome to "my bar" -> 'Welcome to "my bar"'
+
+ If a string contains both single and double quotes, the
+ double quotes will be escaped, and the string will be quoted
+ using double quotes.
+
+ Welcome to "Bob's Bar" -> "Welcome to &quot;Bob's bar&quot;
+ """
+ quote_with = '"'
+ if '"' in value:
+ if "'" in value:
+ # The string contains both single and double
+ # quotes. Turn the double quotes into
+ # entities. We quote the double quotes rather than
+ # the single quotes because the entity name is
+ # "&quot;" whether this is HTML or XML. If we
+ # quoted the single quotes, we'd have to decide
+ # between &apos; and &squot;.
+ replace_with = "&quot;"
+ value = value.replace('"', replace_with)
+ else:
+ # There are double quotes but no single quotes.
+ # We can use single quotes to quote the attribute.
+ quote_with = "'"
+ return quote_with + value + quote_with
+
+ @classmethod
+ def substitute_xml(cls, value, make_quoted_attribute=False):
+ """Substitute XML entities for special XML characters.
+
+ :param value: A string to be substituted. The less-than sign will
+ become &lt;, the greater-than sign will become &gt;, and any
+ ampersands that are not part of an entity defition will
+ become &amp;.
+
+ :param make_quoted_attribute: If True, then the string will be
+ quoted, as befits an attribute value.
+ """
+ # Escape angle brackets, and ampersands that aren't part of
+ # entities.
+ value = cls.BARE_AMPERSAND_OR_BRACKET.sub(
+ cls._substitute_xml_entity, value)
+
+ if make_quoted_attribute:
+ value = cls.quoted_attribute_value(value)
+ return value
+
+ @classmethod
+ def substitute_html(cls, s):
+ """Replace certain Unicode characters with named HTML entities.
+
+ This differs from data.encode(encoding, 'xmlcharrefreplace')
+ in that the goal is to make the result more readable (to those
+ with ASCII displays) rather than to recover from
+ errors. There's absolutely nothing wrong with a UTF-8 string
+ containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that
+ character with "&eacute;" will make it more readable to some
+ people.
+ """
+ return cls.CHARACTER_TO_HTML_ENTITY_RE.sub(
+ cls._substitute_html_entity, s)
+
+
+class UnicodeDammit:
+ """A class for detecting the encoding of a *ML document and
+ converting it to a Unicode string. If the source encoding is
+ windows-1252, can replace MS smart quotes with their HTML or XML
+ equivalents."""
+
+ # This dictionary maps commonly seen values for "charset" in HTML
+ # meta tags to the corresponding Python codec names. It only covers
+ # values that aren't in Python's aliases and can't be determined
+ # by the heuristics in find_codec.
+ CHARSET_ALIASES = {"macintosh": "mac-roman",
+ "x-sjis": "shift-jis"}
+
+ ENCODINGS_WITH_SMART_QUOTES = [
+ "windows-1252",
+ "iso-8859-1",
+ "iso-8859-2",
+ ]
+
+ def __init__(self, markup, override_encodings=[],
+ smart_quotes_to=None, is_html=False):
+ self.declared_html_encoding = None
+ self.smart_quotes_to = smart_quotes_to
+ self.tried_encodings = []
+ self.contains_replacement_characters = False
+
+ if markup == '' or isinstance(markup, str):
+ self.markup = markup
+ self.unicode_markup = str(markup)
+ self.original_encoding = None
+ return
+
+ new_markup, document_encoding, sniffed_encoding = \
+ self._detectEncoding(markup, is_html)
+ self.markup = new_markup
+
+ u = None
+ if new_markup != markup:
+ # _detectEncoding modified the markup, then converted it to
+ # Unicode and then to UTF-8. So convert it from UTF-8.
+ u = self._convert_from("utf8")
+ self.original_encoding = sniffed_encoding
+
+ if not u:
+ for proposed_encoding in (
+ override_encodings + [document_encoding, sniffed_encoding]):
+ if proposed_encoding is not None:
+ u = self._convert_from(proposed_encoding)
+ if u:
+ break
+
+ # If no luck and we have auto-detection library, try that:
+ if not u and not isinstance(self.markup, str):
+ u = self._convert_from(chardet_dammit(self.markup))
+
+ # As a last resort, try utf-8 and windows-1252:
+ if not u:
+ for proposed_encoding in ("utf-8", "windows-1252"):
+ u = self._convert_from(proposed_encoding)
+ if u:
+ break
+
+ # As an absolute last resort, try the encodings again with
+ # character replacement.
+ if not u:
+ for proposed_encoding in (
+ override_encodings + [
+ document_encoding, sniffed_encoding, "utf-8", "windows-1252"]):
+ if proposed_encoding != "ascii":
+ u = self._convert_from(proposed_encoding, "replace")
+ if u is not None:
+ logging.warning(
+ "Some characters could not be decoded, and were "
+ "replaced with REPLACEMENT CHARACTER.")
+ self.contains_replacement_characters = True
+ break
+
+ # We could at this point force it to ASCII, but that would
+ # destroy so much data that I think giving up is better
+ self.unicode_markup = u
+ if not u:
+ self.original_encoding = None
+
+ def _sub_ms_char(self, match):
+ """Changes a MS smart quote character to an XML or HTML
+ entity, or an ASCII character."""
+ orig = match.group(1)
+ if self.smart_quotes_to == 'ascii':
+ sub = self.MS_CHARS_TO_ASCII.get(orig).encode()
+ else:
+ sub = self.MS_CHARS.get(orig)
+ if type(sub) == tuple:
+ if self.smart_quotes_to == 'xml':
+ sub = '&#x'.encode() + sub[1].encode() + ';'.encode()
+ else:
+ sub = '&'.encode() + sub[0].encode() + ';'.encode()
+ else:
+ sub = sub.encode()
+ return sub
+
+ def _convert_from(self, proposed, errors="strict"):
+ proposed = self.find_codec(proposed)
+ if not proposed or (proposed, errors) in self.tried_encodings:
+ return None
+ self.tried_encodings.append((proposed, errors))
+ markup = self.markup
+
+ # Convert smart quotes to HTML if coming from an encoding
+ # that might have them.
+ if (self.smart_quotes_to is not None
+ and proposed.lower() in self.ENCODINGS_WITH_SMART_QUOTES):
+ smart_quotes_re = b"([\x80-\x9f])"
+ smart_quotes_compiled = re.compile(smart_quotes_re)
+ markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)
+
+ try:
+ #print "Trying to convert document to %s (errors=%s)" % (
+ # proposed, errors)
+ u = self._to_unicode(markup, proposed, errors)
+ self.markup = u
+ self.original_encoding = proposed
+ except Exception as e:
+ #print "That didn't work!"
+ #print e
+ return None
+ #print "Correct encoding: %s" % proposed
+ return self.markup
+
+ def _to_unicode(self, data, encoding, errors="strict"):
+ '''Given a string and its encoding, decodes the string into Unicode.
+ %encoding is a string recognized by encodings.aliases'''
+
+ # strip Byte Order Mark (if present)
+ if (len(data) >= 4) and (data[:2] == '\xfe\xff') \
+ and (data[2:4] != '\x00\x00'):
+ encoding = 'utf-16be'
+ data = data[2:]
+ elif (len(data) >= 4) and (data[:2] == '\xff\xfe') \
+ and (data[2:4] != '\x00\x00'):
+ encoding = 'utf-16le'
+ data = data[2:]
+ elif data[:3] == '\xef\xbb\xbf':
+ encoding = 'utf-8'
+ data = data[3:]
+ elif data[:4] == '\x00\x00\xfe\xff':
+ encoding = 'utf-32be'
+ data = data[4:]
+ elif data[:4] == '\xff\xfe\x00\x00':
+ encoding = 'utf-32le'
+ data = data[4:]
+ newdata = str(data, encoding, errors)
+ return newdata
+
+ def _detectEncoding(self, xml_data, is_html=False):
+ """Given a document, tries to detect its XML encoding."""
+ xml_encoding = sniffed_xml_encoding = None
+ try:
+ if xml_data[:4] == b'\x4c\x6f\xa7\x94':
+ # EBCDIC
+ xml_data = self._ebcdic_to_ascii(xml_data)
+ elif xml_data[:4] == b'\x00\x3c\x00\x3f':
+ # UTF-16BE
+ sniffed_xml_encoding = 'utf-16be'
+ xml_data = str(xml_data, 'utf-16be').encode('utf-8')
+ elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xfe\xff') \
+ and (xml_data[2:4] != b'\x00\x00'):
+ # UTF-16BE with BOM
+ sniffed_xml_encoding = 'utf-16be'
+ xml_data = str(xml_data[2:], 'utf-16be').encode('utf-8')
+ elif xml_data[:4] == b'\x3c\x00\x3f\x00':
+ # UTF-16LE
+ sniffed_xml_encoding = 'utf-16le'
+ xml_data = str(xml_data, 'utf-16le').encode('utf-8')
+ elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xff\xfe') and \
+ (xml_data[2:4] != b'\x00\x00'):
+ # UTF-16LE with BOM
+ sniffed_xml_encoding = 'utf-16le'
+ xml_data = str(xml_data[2:], 'utf-16le').encode('utf-8')
+ elif xml_data[:4] == b'\x00\x00\x00\x3c':
+ # UTF-32BE
+ sniffed_xml_encoding = 'utf-32be'
+ xml_data = str(xml_data, 'utf-32be').encode('utf-8')
+ elif xml_data[:4] == b'\x3c\x00\x00\x00':
+ # UTF-32LE
+ sniffed_xml_encoding = 'utf-32le'
+ xml_data = str(xml_data, 'utf-32le').encode('utf-8')
+ elif xml_data[:4] == b'\x00\x00\xfe\xff':
+ # UTF-32BE with BOM
+ sniffed_xml_encoding = 'utf-32be'
+ xml_data = str(xml_data[4:], 'utf-32be').encode('utf-8')
+ elif xml_data[:4] == b'\xff\xfe\x00\x00':
+ # UTF-32LE with BOM
+ sniffed_xml_encoding = 'utf-32le'
+ xml_data = str(xml_data[4:], 'utf-32le').encode('utf-8')
+ elif xml_data[:3] == b'\xef\xbb\xbf':
+ # UTF-8 with BOM
+ sniffed_xml_encoding = 'utf-8'
+ xml_data = str(xml_data[3:], 'utf-8').encode('utf-8')
+ else:
+ sniffed_xml_encoding = 'ascii'
+ pass
+ except:
+ xml_encoding_match = None
+ xml_encoding_match = xml_encoding_re.match(xml_data)
+ if not xml_encoding_match and is_html:
+ xml_encoding_match = html_meta_re.search(xml_data)
+ if xml_encoding_match is not None:
+ xml_encoding = xml_encoding_match.groups()[0].decode(
+ 'ascii').lower()
+ if is_html:
+ self.declared_html_encoding = xml_encoding
+ if sniffed_xml_encoding and \
+ (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode',
+ 'iso-10646-ucs-4', 'ucs-4', 'csucs4',
+ 'utf-16', 'utf-32', 'utf_16', 'utf_32',
+ 'utf16', 'u16')):
+ xml_encoding = sniffed_xml_encoding
+ return xml_data, xml_encoding, sniffed_xml_encoding
+
+ def find_codec(self, charset):
+ return self._codec(self.CHARSET_ALIASES.get(charset, charset)) \
+ or (charset and self._codec(charset.replace("-", ""))) \
+ or (charset and self._codec(charset.replace("-", "_"))) \
+ or charset
+
+ def _codec(self, charset):
+ if not charset:
+ return charset
+ codec = None
+ try:
+ codecs.lookup(charset)
+ codec = charset
+ except (LookupError, ValueError):
+ pass
+ return codec
+
+ EBCDIC_TO_ASCII_MAP = None
+
+ def _ebcdic_to_ascii(self, s):
+ c = self.__class__
+ if not c.EBCDIC_TO_ASCII_MAP:
+ emap = (0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15,
+ 16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31,
+ 128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7,
+ 144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26,
+ 32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33,
+ 38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94,
+ 45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63,
+ 186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34,
+ 195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,
+ 201,202,106,107,108,109,110,111,112,113,114,203,204,205,
+ 206,207,208,209,126,115,116,117,118,119,120,121,122,210,
+ 211,212,213,214,215,216,217,218,219,220,221,222,223,224,
+ 225,226,227,228,229,230,231,123,65,66,67,68,69,70,71,72,
+ 73,232,233,234,235,236,237,125,74,75,76,77,78,79,80,81,
+ 82,238,239,240,241,242,243,92,159,83,84,85,86,87,88,89,
+ 90,244,245,246,247,248,249,48,49,50,51,52,53,54,55,56,57,
+ 250,251,252,253,254,255)
+ import string
+ c.EBCDIC_TO_ASCII_MAP = string.maketrans(
+ ''.join(map(chr, list(range(256)))), ''.join(map(chr, emap)))
+ return s.translate(c.EBCDIC_TO_ASCII_MAP)
+
+ # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.
+ MS_CHARS = {b'\x80': ('euro', '20AC'),
+ b'\x81': ' ',
+ b'\x82': ('sbquo', '201A'),
+ b'\x83': ('fnof', '192'),
+ b'\x84': ('bdquo', '201E'),
+ b'\x85': ('hellip', '2026'),
+ b'\x86': ('dagger', '2020'),
+ b'\x87': ('Dagger', '2021'),
+ b'\x88': ('circ', '2C6'),
+ b'\x89': ('permil', '2030'),
+ b'\x8A': ('Scaron', '160'),
+ b'\x8B': ('lsaquo', '2039'),
+ b'\x8C': ('OElig', '152'),
+ b'\x8D': '?',
+ b'\x8E': ('#x17D', '17D'),
+ b'\x8F': '?',
+ b'\x90': '?',
+ b'\x91': ('lsquo', '2018'),
+ b'\x92': ('rsquo', '2019'),
+ b'\x93': ('ldquo', '201C'),
+ b'\x94': ('rdquo', '201D'),
+ b'\x95': ('bull', '2022'),
+ b'\x96': ('ndash', '2013'),
+ b'\x97': ('mdash', '2014'),
+ b'\x98': ('tilde', '2DC'),
+ b'\x99': ('trade', '2122'),
+ b'\x9a': ('scaron', '161'),
+ b'\x9b': ('rsaquo', '203A'),
+ b'\x9c': ('oelig', '153'),
+ b'\x9d': '?',
+ b'\x9e': ('#x17E', '17E'),
+ b'\x9f': ('Yuml', ''),}
+
+ # A parochial partial mapping of ISO-Latin-1 to ASCII. Contains
+ # horrors like stripping diacritical marks to turn á into a, but also
+ # contains non-horrors like turning “ into ".
+ MS_CHARS_TO_ASCII = {
+ b'\x80' : 'EUR',
+ b'\x81' : ' ',
+ b'\x82' : ',',
+ b'\x83' : 'f',
+ b'\x84' : ',,',
+ b'\x85' : '...',
+ b'\x86' : '+',
+ b'\x87' : '++',
+ b'\x88' : '^',
+ b'\x89' : '%',
+ b'\x8a' : 'S',
+ b'\x8b' : '<',
+ b'\x8c' : 'OE',
+ b'\x8d' : '?',
+ b'\x8e' : 'Z',
+ b'\x8f' : '?',
+ b'\x90' : '?',
+ b'\x91' : "'",
+ b'\x92' : "'",
+ b'\x93' : '"',
+ b'\x94' : '"',
+ b'\x95' : '*',
+ b'\x96' : '-',
+ b'\x97' : '--',
+ b'\x98' : '~',
+ b'\x99' : '(TM)',
+ b'\x9a' : 's',
+ b'\x9b' : '>',
+ b'\x9c' : 'oe',
+ b'\x9d' : '?',
+ b'\x9e' : 'z',
+ b'\x9f' : 'Y',
+ b'\xa0' : ' ',
+ b'\xa1' : '!',
+ b'\xa2' : 'c',
+ b'\xa3' : 'GBP',
+ b'\xa4' : '$', #This approximation is especially parochial--this is the
+ #generic currency symbol.
+ b'\xa5' : 'YEN',
+ b'\xa6' : '|',
+ b'\xa7' : 'S',
+ b'\xa8' : '..',
+ b'\xa9' : '',
+ b'\xaa' : '(th)',
+ b'\xab' : '<<',
+ b'\xac' : '!',
+ b'\xad' : ' ',
+ b'\xae' : '(R)',
+ b'\xaf' : '-',
+ b'\xb0' : 'o',
+ b'\xb1' : '+-',
+ b'\xb2' : '2',
+ b'\xb3' : '3',
+ b'\xb4' : ("'", 'acute'),
+ b'\xb5' : 'u',
+ b'\xb6' : 'P',
+ b'\xb7' : '*',
+ b'\xb8' : ',',
+ b'\xb9' : '1',
+ b'\xba' : '(th)',
+ b'\xbb' : '>>',
+ b'\xbc' : '1/4',
+ b'\xbd' : '1/2',
+ b'\xbe' : '3/4',
+ b'\xbf' : '?',
+ b'\xc0' : 'A',
+ b'\xc1' : 'A',
+ b'\xc2' : 'A',
+ b'\xc3' : 'A',
+ b'\xc4' : 'A',
+ b'\xc5' : 'A',
+ b'\xc6' : 'AE',
+ b'\xc7' : 'C',
+ b'\xc8' : 'E',
+ b'\xc9' : 'E',
+ b'\xca' : 'E',
+ b'\xcb' : 'E',
+ b'\xcc' : 'I',
+ b'\xcd' : 'I',
+ b'\xce' : 'I',
+ b'\xcf' : 'I',
+ b'\xd0' : 'D',
+ b'\xd1' : 'N',
+ b'\xd2' : 'O',
+ b'\xd3' : 'O',
+ b'\xd4' : 'O',
+ b'\xd5' : 'O',
+ b'\xd6' : 'O',
+ b'\xd7' : '*',
+ b'\xd8' : 'O',
+ b'\xd9' : 'U',
+ b'\xda' : 'U',
+ b'\xdb' : 'U',
+ b'\xdc' : 'U',
+ b'\xdd' : 'Y',
+ b'\xde' : 'b',
+ b'\xdf' : 'B',
+ b'\xe0' : 'a',
+ b'\xe1' : 'a',
+ b'\xe2' : 'a',
+ b'\xe3' : 'a',
+ b'\xe4' : 'a',
+ b'\xe5' : 'a',
+ b'\xe6' : 'ae',
+ b'\xe7' : 'c',
+ b'\xe8' : 'e',
+ b'\xe9' : 'e',
+ b'\xea' : 'e',
+ b'\xeb' : 'e',
+ b'\xec' : 'i',
+ b'\xed' : 'i',
+ b'\xee' : 'i',
+ b'\xef' : 'i',
+ b'\xf0' : 'o',
+ b'\xf1' : 'n',
+ b'\xf2' : 'o',
+ b'\xf3' : 'o',
+ b'\xf4' : 'o',
+ b'\xf5' : 'o',
+ b'\xf6' : 'o',
+ b'\xf7' : '/',
+ b'\xf8' : 'o',
+ b'\xf9' : 'u',
+ b'\xfa' : 'u',
+ b'\xfb' : 'u',
+ b'\xfc' : 'u',
+ b'\xfd' : 'y',
+ b'\xfe' : 'b',
+ b'\xff' : 'y',
+ }
+
+ # A map used when removing rogue Windows-1252/ISO-8859-1
+ # characters in otherwise UTF-8 documents.
+ #
+ # Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in
+ # Windows-1252.
+ WINDOWS_1252_TO_UTF8 = {
+ 0x80 : b'\xe2\x82\xac', # €
+ 0x82 : b'\xe2\x80\x9a', # ‚
+ 0x83 : b'\xc6\x92', # ƒ
+ 0x84 : b'\xe2\x80\x9e', # „
+ 0x85 : b'\xe2\x80\xa6', # …
+ 0x86 : b'\xe2\x80\xa0', # †
+ 0x87 : b'\xe2\x80\xa1', # ‡
+ 0x88 : b'\xcb\x86', # ˆ
+ 0x89 : b'\xe2\x80\xb0', # ‰
+ 0x8a : b'\xc5\xa0', # Š
+ 0x8b : b'\xe2\x80\xb9', # ‹
+ 0x8c : b'\xc5\x92', # Œ
+ 0x8e : b'\xc5\xbd', # Ž
+ 0x91 : b'\xe2\x80\x98', # ‘
+ 0x92 : b'\xe2\x80\x99', # ’
+ 0x93 : b'\xe2\x80\x9c', # “
+ 0x94 : b'\xe2\x80\x9d', # ”
+ 0x95 : b'\xe2\x80\xa2', # •
+ 0x96 : b'\xe2\x80\x93', # –
+ 0x97 : b'\xe2\x80\x94', # —
+ 0x98 : b'\xcb\x9c', # ˜
+ 0x99 : b'\xe2\x84\xa2', # ™
+ 0x9a : b'\xc5\xa1', # š
+ 0x9b : b'\xe2\x80\xba', # ›
+ 0x9c : b'\xc5\x93', # œ
+ 0x9e : b'\xc5\xbe', # ž
+ 0x9f : b'\xc5\xb8', # Ÿ
+ 0xa0 : b'\xc2\xa0', #  
+ 0xa1 : b'\xc2\xa1', # ¡
+ 0xa2 : b'\xc2\xa2', # ¢
+ 0xa3 : b'\xc2\xa3', # £
+ 0xa4 : b'\xc2\xa4', # ¤
+ 0xa5 : b'\xc2\xa5', # ¥
+ 0xa6 : b'\xc2\xa6', # ¦
+ 0xa7 : b'\xc2\xa7', # §
+ 0xa8 : b'\xc2\xa8', # ¨
+ 0xa9 : b'\xc2\xa9', # ©
+ 0xaa : b'\xc2\xaa', # ª
+ 0xab : b'\xc2\xab', # «
+ 0xac : b'\xc2\xac', # ¬
+ 0xad : b'\xc2\xad', # ­
+ 0xae : b'\xc2\xae', # ®
+ 0xaf : b'\xc2\xaf', # ¯
+ 0xb0 : b'\xc2\xb0', # °
+ 0xb1 : b'\xc2\xb1', # ±
+ 0xb2 : b'\xc2\xb2', # ²
+ 0xb3 : b'\xc2\xb3', # ³
+ 0xb4 : b'\xc2\xb4', # ´
+ 0xb5 : b'\xc2\xb5', # µ
+ 0xb6 : b'\xc2\xb6', # ¶
+ 0xb7 : b'\xc2\xb7', # ·
+ 0xb8 : b'\xc2\xb8', # ¸
+ 0xb9 : b'\xc2\xb9', # ¹
+ 0xba : b'\xc2\xba', # º
+ 0xbb : b'\xc2\xbb', # »
+ 0xbc : b'\xc2\xbc', # ¼
+ 0xbd : b'\xc2\xbd', # ½
+ 0xbe : b'\xc2\xbe', # ¾
+ 0xbf : b'\xc2\xbf', # ¿
+ 0xc0 : b'\xc3\x80', # À
+ 0xc1 : b'\xc3\x81', # Á
+ 0xc2 : b'\xc3\x82', # Â
+ 0xc3 : b'\xc3\x83', # Ã
+ 0xc4 : b'\xc3\x84', # Ä
+ 0xc5 : b'\xc3\x85', # Å
+ 0xc6 : b'\xc3\x86', # Æ
+ 0xc7 : b'\xc3\x87', # Ç
+ 0xc8 : b'\xc3\x88', # È
+ 0xc9 : b'\xc3\x89', # É
+ 0xca : b'\xc3\x8a', # Ê
+ 0xcb : b'\xc3\x8b', # Ë
+ 0xcc : b'\xc3\x8c', # Ì
+ 0xcd : b'\xc3\x8d', # Í
+ 0xce : b'\xc3\x8e', # Î
+ 0xcf : b'\xc3\x8f', # Ï
+ 0xd0 : b'\xc3\x90', # Ð
+ 0xd1 : b'\xc3\x91', # Ñ
+ 0xd2 : b'\xc3\x92', # Ò
+ 0xd3 : b'\xc3\x93', # Ó
+ 0xd4 : b'\xc3\x94', # Ô
+ 0xd5 : b'\xc3\x95', # Õ
+ 0xd6 : b'\xc3\x96', # Ö
+ 0xd7 : b'\xc3\x97', # ×
+ 0xd8 : b'\xc3\x98', # Ø
+ 0xd9 : b'\xc3\x99', # Ù
+ 0xda : b'\xc3\x9a', # Ú
+ 0xdb : b'\xc3\x9b', # Û
+ 0xdc : b'\xc3\x9c', # Ü
+ 0xdd : b'\xc3\x9d', # Ý
+ 0xde : b'\xc3\x9e', # Þ
+ 0xdf : b'\xc3\x9f', # ß
+ 0xe0 : b'\xc3\xa0', # à
+ 0xe1 : b'\xa1', # á
+ 0xe2 : b'\xc3\xa2', # â
+ 0xe3 : b'\xc3\xa3', # ã
+ 0xe4 : b'\xc3\xa4', # ä
+ 0xe5 : b'\xc3\xa5', # å
+ 0xe6 : b'\xc3\xa6', # æ
+ 0xe7 : b'\xc3\xa7', # ç
+ 0xe8 : b'\xc3\xa8', # è
+ 0xe9 : b'\xc3\xa9', # é
+ 0xea : b'\xc3\xaa', # ê
+ 0xeb : b'\xc3\xab', # ë
+ 0xec : b'\xc3\xac', # ì
+ 0xed : b'\xc3\xad', # í
+ 0xee : b'\xc3\xae', # î
+ 0xef : b'\xc3\xaf', # ï
+ 0xf0 : b'\xc3\xb0', # ð
+ 0xf1 : b'\xc3\xb1', # ñ
+ 0xf2 : b'\xc3\xb2', # ò
+ 0xf3 : b'\xc3\xb3', # ó
+ 0xf4 : b'\xc3\xb4', # ô
+ 0xf5 : b'\xc3\xb5', # õ
+ 0xf6 : b'\xc3\xb6', # ö
+ 0xf7 : b'\xc3\xb7', # ÷
+ 0xf8 : b'\xc3\xb8', # ø
+ 0xf9 : b'\xc3\xb9', # ù
+ 0xfa : b'\xc3\xba', # ú
+ 0xfb : b'\xc3\xbb', # û
+ 0xfc : b'\xc3\xbc', # ü
+ 0xfd : b'\xc3\xbd', # ý
+ 0xfe : b'\xc3\xbe', # þ
+ }
+
+ MULTIBYTE_MARKERS_AND_SIZES = [
+ (0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF
+ (0xe0, 0xef, 3), # 3-byte characters start with E0-EF
+ (0xf0, 0xf4, 4), # 4-byte characters start with F0-F4
+ ]
+
+ FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0]
+ LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1]
+
+ @classmethod
+ def detwingle(cls, in_bytes, main_encoding="utf8",
+ embedded_encoding="windows-1252"):
+ """Fix characters from one encoding embedded in some other encoding.
+
+ Currently the only situation supported is Windows-1252 (or its
+ subset ISO-8859-1), embedded in UTF-8.
+
+ The input must be a bytestring. If you've already converted
+ the document to Unicode, you're too late.
+
+ The output is a bytestring in which `embedded_encoding`
+ characters have been converted to their `main_encoding`
+ equivalents.
+ """
+ if embedded_encoding.replace('_', '-').lower() not in (
+ 'windows-1252', 'windows_1252'):
+ raise NotImplementedError(
+ "Windows-1252 and ISO-8859-1 are the only currently supported "
+ "embedded encodings.")
+
+ if main_encoding.lower() not in ('utf8', 'utf-8'):
+ raise NotImplementedError(
+ "UTF-8 is the only currently supported main encoding.")
+
+ byte_chunks = []
+
+ chunk_start = 0
+ pos = 0
+ while pos < len(in_bytes):
+ byte = in_bytes[pos]
+ if not isinstance(byte, int):
+ # Python 2.x
+ byte = ord(byte)
+ if (byte >= cls.FIRST_MULTIBYTE_MARKER
+ and byte <= cls.LAST_MULTIBYTE_MARKER):
+ # This is the start of a UTF-8 multibyte character. Skip
+ # to the end.
+ for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
+ if byte >= start and byte <= end:
+ pos += size
+ break
+ elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
+ # We found a Windows-1252 character!
+ # Save the string up to this point as a chunk.
+ byte_chunks.append(in_bytes[chunk_start:pos])
+
+ # Now translate the Windows-1252 character into UTF-8
+ # and add it as another, one-byte chunk.
+ byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
+ pos += 1
+ chunk_start = pos
+ else:
+ # Go on to the next character.
+ pos += 1
+ if chunk_start == 0:
+ # The string is unchanged.
+ return in_bytes
+ else:
+ # Store the final chunk.
+ byte_chunks.append(in_bytes[chunk_start:])
+ return b''.join(byte_chunks)
+
View
84 bs4/element.py
@@ -6,13 +6,9 @@
DEFAULT_OUTPUT_ENCODING = "utf-8"
PY3K = (sys.version_info[0] > 2)
-unicodes = str
-unicode = str
+
whitespace_re = re.compile("\s+")
-str = str
-unicode = str
-bytes = bytes
-basestring = (str,bytes)
+
def _alias(attr):
"""Alias one attribute name to another for backward compatibility"""
@property
@@ -25,19 +21,19 @@ def alias(self):
return alias
-class NamespacedAttribute(unicodes):
+class NamespacedAttribute(str):
def __new__(cls, prefix, name, namespace=None):
if name is None:
- obj = unicode.__new__(cls, prefix)
+ obj = str.__new__(cls, prefix)
else:
- obj = unicode.__new__(cls, prefix + ":" + name)
+ obj = str.__new__(cls, prefix + ":" + name)
obj.prefix = prefix
obj.name = name
obj.namespace = namespace
return obj
-class AttributeValueWithCharsetSubstitution(unicode):
+class AttributeValueWithCharsetSubstitution(str):
"""A stand-in object for a character encoding specified in HTML."""
class CharsetMetaAttributeValue(AttributeValueWithCharsetSubstitution):
@@ -48,7 +44,7 @@ class CharsetMetaAttributeValue(AttributeValueWithCharsetSubstitution):
"""
def __new__(cls, original_value):
- obj = unicode.__new__(cls, original_value)
+ obj = str.__new__(cls, original_value)
obj.original_value = original_value
return obj
@@ -71,9 +67,9 @@ def __new__(cls, original_value):
match = cls.CHARSET_RE.search(original_value)
if match is None:
# No substitution necessary.
- return unicode.__new__(unicode, original_value)
+ return str.__new__(str, original_value)
- obj = unicode.__new__(cls, original_value)
+ obj = str.__new__(cls, original_value)
obj.original_value = original_value
return obj
@@ -108,7 +104,7 @@ class PageElement(object):
@classmethod
def format_string(self, s, formatter='minimal'):
"""Format the given string using the given formatter."""
- if not callable(formatter):
+ if not isinstance(formatter, collections.Callable):
formatter = self.FORMATTERS.get(
formatter, EntitySubstitution.substitute_xml)
if formatter is None:
@@ -199,7 +195,7 @@ def _last_descendant(self):
def insert(self, position, new_child):
if new_child is self:
raise ValueError("Cannot insert a tag into itself.")
- if (isinstance(new_child, basestring)
+ if (isinstance(new_child, str)
and not isinstance(new_child, NavigableString)):
new_child = NavigableString(new_child)
@@ -413,7 +409,7 @@ def _find_all(self, name, attrs, text, limit, generator, **kwargs):
return [element for element in generator
if isinstance(element, Tag)]
# Optimization to find all tags with a given name.
- elif isinstance(name, basestring):
+ elif isinstance(name, str):
return [element for element in generator
if isinstance(element, Tag) and element.name == name]
else:
@@ -639,7 +635,7 @@ def parentGenerator(self):
return self.parents
-class NavigableString(unicode, PageElement):
+class NavigableString(str, PageElement):
PREFIX = ''
SUFFIX = ''
@@ -652,12 +648,12 @@ def __new__(cls, value):
passed in to the superclass's __new__ or the superclass won't know
how to handle non-ASCII characters.
"""
- if isinstance(value, unicode):
- return unicode.__new__(cls, value)
- return unicode.__new__(cls, value, DEFAULT_OUTPUT_ENCODING)
+ if isinstance(value, str):
+ return str.__new__(cls, value)
+ return str.__new__(cls, value, DEFAULT_OUTPUT_ENCODING)
def __getnewargs__(self):
- return (unicode(self),)
+ return (str(self),)
def __getattr__(self, attr):
"""text.string gives you text. This is for backwards
@@ -690,23 +686,23 @@ def output_ready(self, formatter="minimal"):
class CData(PreformattedString):
- PREFIX = u'<![CDATA['
- SUFFIX = u']]>'
+ PREFIX = '<![CDATA['
+ SUFFIX = ']]>'
class ProcessingInstruction(PreformattedString):
- PREFIX = u'<?'
- SUFFIX = u'?>'
+ PREFIX = '<?'