Skip to content
Permalink
Browse files
That only worked in one Py version.
  • Loading branch information
da2x committed Aug 5, 2015
1 parent 62833e9 commit 1d83dca122a65356d0fb3c77e86539f281f23623
Showing 1 changed file with 2 additions and 1 deletion.
@@ -188,7 +188,8 @@ def scan_locs():
b'<meta content=none name=robots',
b'<meta name=robots content=noindex',
b'<meta name=robots content=none']
if any([robot_directive in filehead.lower().replace('"', '') for robot_directive in robots_directives]):
lowquothead = filehead.lower().decode('utf-8').replace('"', '').encode('utf-8')
if any([robot_directive in lowquothead for robot_directive in robots_directives]):
continue

# put Atom and RSS in sitemapindex[] instead of in urlset[],

0 comments on commit 1d83dca

Please sign in to comment.