Add non-ascii values to test robots.txt

git-svn-id: https://linkchecker.svn.sourceforge.net/svnroot/linkchecker/trunk/linkchecker@3851 e7d03fd6-7b0d-0410-9947-9c21f3af8025
linkchecker · Jul 13, 2008 · bb8bca1 · bb8bca1
1 parent 7b2a21c
commit bb8bca1
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/robots.txt b/robots.txt
@@ -6,6 +6,8 @@
 User-agent: LinkChecker
 Crawl-delay: 1
 Disallow: /secret
+# a comment line inside a rule, followed by non-ascii
+Disallow: /umlaut/ö
 
 # The following rule should not match
 User-agent: *