Skip to content
Browse files
Use BASE_URL to set "Host:" in /robots.txt
Used by Yandex and Google, possibly others.
  • Loading branch information
da2x committed Sep 30, 2015
1 parent 4d2df1c commit e6b1b5ea77a2f78b75431526f607556a0c370506
Showing 1 changed file with 2 additions and 1 deletion.
@@ -64,10 +64,11 @@ def write_robots():

with, 'w+', encoding='utf8') as outf:
outf.write("Sitemap: {0}\n\n".format(sitemapindex_url))
outf.write("User-Agent: *\n")
if kw["robots_exclusions"]:
outf.write("User-Agent: *\n")
for loc in kw["robots_exclusions"]:
outf.write("Disallow: {0}\n".format(loc))
outf.write("Host: {0}\n".format(urlparse(kw["base_url"]).netloc))

yield self.group_task()

0 comments on commit e6b1b5e

Please sign in to comment.