diff --git a/robots.txt b/robots.txt index a87c96c98..610fa6bf3 100644 --- a/robots.txt +++ b/robots.txt @@ -6,6 +6,8 @@ User-agent: LinkChecker Crawl-delay: 1 Disallow: /secret +# a comment line inside a rule, followed by non-ascii +Disallow: /umlaut/รถ # The following rule should not match User-agent: *