diff --git a/nikola/plugins/task/robots.py b/nikola/plugins/task/robots.py index 555ea36c6b..fbd00e166d 100644 --- a/nikola/plugins/task/robots.py +++ b/nikola/plugins/task/robots.py @@ -64,10 +64,11 @@ def write_robots(): with io.open(robots_path, 'w+', encoding='utf8') as outf: outf.write("Sitemap: {0}\n\n".format(sitemapindex_url)) + outf.write("User-Agent: *\n") if kw["robots_exclusions"]: - outf.write("User-Agent: *\n") for loc in kw["robots_exclusions"]: outf.write("Disallow: {0}\n".format(loc)) + outf.write("Host: {0}\n".format(urlparse(kw["base_url"]).netloc)) yield self.group_task()