diff --git a/Lib/robotparser.py b/Lib/robotparser.py index a7137a3064ff3c..4e13f7f780ca3c 100644 --- a/Lib/robotparser.py +++ b/Lib/robotparser.py @@ -160,7 +160,10 @@ def can_fetch(self, useragent, url): def __str__(self): - return ''.join([str(entry) + "\n" for entry in self.entries]) + entries = self.entries + if self.default_entry is not None: + entries = entries + [self.default_entry] + return '\n'.join(map(str, entries)) + '\n' class RuleLine: diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 8ed5d89036190d..ba7ccf8b58da24 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -136,6 +136,31 @@ class DefaultEntryTest(BaseRobotTest, unittest.TestCase): bad = ['/cyberworld/map/index.html'] +class StringFormattingTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 +Disallow: /cyberworld/map/ # This is an infinite virtual URL space + +# Cybermapper knows where to go. +User-agent: cybermapper +Disallow: /some/path + """ + + expected_output = """\ +User-agent: cybermapper +Disallow: /some/path + +User-agent: * +Disallow: /cyberworld/map/ + +""" + + def test_string_formatting(self): + self.assertEqual(str(self.parser), self.expected_output) + + class RobotHandler(BaseHTTPRequestHandler): def do_GET(self): @@ -226,6 +251,7 @@ def test_main(): UseFirstUserAgentWildcardTest, EmptyQueryStringTest, DefaultEntryTest, + StringFormattingTest, PasswordProtectedSiteTestCase, NetworkTestCase) diff --git a/Misc/ACKS b/Misc/ACKS index 458f31e6a6b75a..394fbb93f2f184 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -807,6 +807,7 @@ Ben Laurie Simon Law Julia Lawall Chris Lawrence +Michael Lazar Brian Leair Mathieu Leduc-Hamel Amandine Lee diff --git a/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst new file mode 100644 index 00000000000000..13defbb03cff63 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst @@ -0,0 +1,3 @@ +The urllib.robotparser's ``__str__`` representation now includes wildcard +entries and the "Crawl-delay" and "Request-rate" fields. Patch by +Michael Lazar.