Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Fixed #19237 -- Used HTML parser to strip tags

The regex method used until now for the strip_tags utility is fast,
but subject to flaws and security issues. Consensus and good
practice lead use to use a slower but safer method.
  • Loading branch information...
commit dc51ec8bc214cf60ebb99732363624c23df8005f 1 parent 01948e3
@claudep claudep authored
Showing with 32 additions and 4 deletions.
  1. +26 −2 django/utils/html.py
  2. +6 −2 tests/utils_tests/test_html.py
View
28 django/utils/html.py
@@ -16,6 +16,9 @@
from django.utils import six
from django.utils.text import normalize_newlines
+from .html_parser import HTMLParser
+
+
# Configuration for urlize() function.
TRAILING_PUNCTUATION = ['.', ',', ':', ';', '.)']
WRAPPING_PUNCTUATION = [('(', ')'), ('<', '>'), ('[', ']'), ('&lt;', '&gt;')]
@@ -33,7 +36,6 @@
html_gunk_re = re.compile(r'(?:<br clear="all">|<i><\/i>|<b><\/b>|<em><\/em>|<strong><\/strong>|<\/?smallcaps>|<\/?uppercase>)', re.IGNORECASE)
hard_coded_bullets_re = re.compile(r'((?:<p>(?:%s).*?[a-zA-Z].*?</p>\s*)+)' % '|'.join([re.escape(x) for x in DOTS]), re.DOTALL)
trailing_empty_content_re = re.compile(r'(?:<p>(?:&nbsp;|\s|<br \/>)*?</p>\s*)+\Z')
-strip_tags_re = re.compile(r'</?\S([^=>]*=(\s*"[^"]*"|\s*\'[^\']*\'|\S*)|[^>])*?>', re.IGNORECASE)
def escape(text):
@@ -116,9 +118,31 @@ def linebreaks(value, autoescape=False):
return '\n\n'.join(paras)
linebreaks = allow_lazy(linebreaks, six.text_type)
+
+class MLStripper(HTMLParser):
+ def __init__(self):
+ HTMLParser.__init__(self)
+ self.reset()
+ self.fed = []
+ def handle_data(self, d):
+ self.fed.append(d)
+ def handle_entityref(self, name):
+ self.fed.append('&%s;' % name)
+ def handle_charref(self, name):
+ self.fed.append('&#%s;' % name)
+ def get_data(self):
+ return ''.join(self.fed)
+
def strip_tags(value):
"""Returns the given HTML with all tags stripped."""
- return strip_tags_re.sub('', force_text(value))
+ s = MLStripper()
+ s.feed(value)
+ data = s.get_data()
+ try:
+ res = s.close()
+ except Exception as e:
+ data += s.rawdata
+ return data
strip_tags = allow_lazy(strip_tags)
def remove_tags(html, tags):
View
8 tests/utils_tests/test_html.py
@@ -5,6 +5,7 @@
from django.utils import html
from django.utils._os import upath
+from django.utils.encoding import force_text
from django.utils.unittest import TestCase
@@ -63,10 +64,12 @@ def test_linebreaks(self):
def test_strip_tags(self):
f = html.strip_tags
items = (
+ ('<p>See: &#39;&eacute; is an apostrophe followed by e acute</p>',
+ 'See: &#39;&eacute; is an apostrophe followed by e acute'),
('<adf>a', 'a'),
('</adf>a', 'a'),
('<asdf><asdf>e', 'e'),
- ('<f', '<f'),
+ ('hi, <f x', 'hi, <f x'),
('</fe', '</fe'),
('<x>b<y>', 'b'),
('a<p onclick="alert(\'<test>\')">b</p>c', 'abc'),
@@ -81,8 +84,9 @@ def test_strip_tags(self):
for filename in ('strip_tags1.html', 'strip_tags2.txt'):
path = os.path.join(os.path.dirname(upath(__file__)), 'files', filename)
with open(path, 'r') as fp:
+ content = force_text(fp.read())
start = datetime.now()
- stripped = html.strip_tags(fp.read())
+ stripped = html.strip_tags(content)
elapsed = datetime.now() - start
self.assertEqual(elapsed.seconds, 0)
self.assertIn("Please try again.", stripped)
Please sign in to comment.
Something went wrong with that request. Please try again.