Skip to content
Browse files
Strip quotes when matching meta tags ("modern")
  • Loading branch information
da2x committed Aug 5, 2015
1 parent 682f80d commit 6abfa08361ce6585d288befa18dd78c6b22c73df
Showing 1 changed file with 5 additions and 5 deletions.
@@ -184,11 +184,11 @@ def scan_locs():

""" ignores "html" files with noindex robot directives """
robots_directives = [b'<meta content="noindex" name="robots"',
b'<meta content="none" name="robots"',
b'<meta name="robots" content="noindex"',
b'<meta name="robots" content="none"']
if any([robot_directive in filehead.lower() for robot_directive in robots_directives]):
robots_directives = [b'<meta content=noindex name=robots',
b'<meta content=none name=robots',
b'<meta name=robots content=noindex',
b'<meta name=robots content=none']
if any([robot_directive in filehead.lower().replace('"', '')s for robot_directive in robots_directives]):

# put Atom and RSS in sitemapindex[] instead of in urlset[],

0 comments on commit 6abfa08

Please sign in to comment.