From a9c548f44cb9a2fc8d0fc4c1864982432e862459 Mon Sep 17 00:00:00 2001 From: jjact Date: Wed, 25 Sep 2019 11:58:11 +1000 Subject: [PATCH 01/11] Add support for HTTP Only flag in MozillaCookieJar --- Lib/http/cookiejar.py | 19 +++++++++++++++---- Lib/test/test_http_cookiejar.py | 5 +++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 47ed5c3d64ab7d1..ae7c6c21b5cf74f 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -50,7 +50,7 @@ def _debug(*args): logger = logging.getLogger("http.cookiejar") return logger.debug(*args) - +HTTPONLY_ATTR = "HTTPOnly" DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT) MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar " "instance initialised with one)") @@ -2004,6 +2004,7 @@ class MozillaCookieJar(FileCookieJar): header by default (Mozilla can cope with that). """ + httponly_prefix = "#HttpOnly_" magic_re = re.compile("#( Netscape)? HTTP Cookie File") header = """\ # Netscape HTTP Cookie File @@ -2024,8 +2025,15 @@ def _really_load(self, f, filename, ignore_discard, ignore_expires): try: while 1: line = f.readline() + rest = {} + if line == "": break + # detect httponly flag if present + if line.startswith(self.httponly_prefix): + rest[HTTPONLY_ATTR] = "" + line = line[len(self.httponly_prefix):] + # last field may be absent, so keep any trailing tab if line.endswith("\n"): line = line[:-1] @@ -2063,7 +2071,7 @@ def _really_load(self, f, filename, ignore_discard, ignore_expires): discard, None, None, - {}) + rest) if not ignore_discard and c.discard: continue if not ignore_expires and c.is_expired(now): @@ -2086,13 +2094,14 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): f.write(self.header) now = time.time() for cookie in self: + domain = cookie.domain if not ignore_discard and cookie.discard: continue if not ignore_expires and cookie.is_expired(now): continue if cookie.secure: secure = "TRUE" else: secure = "FALSE" - if cookie.domain.startswith("."): initial_dot = "TRUE" + if domain.startswith("."): initial_dot = "TRUE" else: initial_dot = "FALSE" if cookie.expires is not None: expires = str(cookie.expires) @@ -2107,7 +2116,9 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): else: name = cookie.name value = cookie.value + if cookie.has_nonstandard_attr(HTTPONLY_ATTR): + domain = self.httponly_prefix + domain f.write( - "\t".join([cookie.domain, initial_dot, cookie.path, + "\t".join([domain, initial_dot, cookie.path, secure, expires, name, value])+ "\n") diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index 2d7077af6da39e5..79d66724f99f040 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -1771,6 +1771,10 @@ def test_mozilla(self): interact_netscape(c, "http://www.foo.com/", "fooc=bar; Domain=www.foo.com; %s" % expires) + for cookie in c: + if cookie.name == "foo1": + cookie.set_nonstandard_attr("HTTPOnly", "") + def save_and_restore(cj, ignore_discard): try: cj.save(ignore_discard=ignore_discard) @@ -1785,6 +1789,7 @@ def save_and_restore(cj, ignore_discard): new_c = save_and_restore(c, True) self.assertEqual(len(new_c), 6) # none discarded self.assertIn("name='foo1', value='bar'", repr(new_c)) + self.assertIn("rest={'HTTPOnly': ''}", repr(new_c)) new_c = save_and_restore(c, False) self.assertEqual(len(new_c), 4) # 2 of them discarded on save From 96ce45e0c858b52181bd6311960be0d560319667 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2019 05:22:50 +0000 Subject: [PATCH 02/11] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst diff --git a/Misc/NEWS.d/next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst b/Misc/NEWS.d/next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst new file mode 100644 index 000000000000000..7a48943a6c6cc7c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst @@ -0,0 +1,4 @@ +The :mod:`http.cookiejar` module now supports the parsing of cookies in CURL-style cookiejar files through MozillaCookieJar +on all platforms. Previously, such cookie entries would be silently ignored when loading a cookiejar with such entries. + +Additionally, the HTTP Only attribute is persisted in the object, and will be correctly written to file if the MozillaCookieJar object is subsequently dumped. \ No newline at end of file From ef15aba07fab205310ff66cd2e1c9ff6ae4b1a8c Mon Sep 17 00:00:00 2001 From: Jacob Neil Taylor Date: Thu, 2 Apr 2020 19:43:03 +1100 Subject: [PATCH 03/11] Cleanup class variables and update comments --- Lib/http/cookiejar.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index ae7c6c21b5cf74f..46175e25d8c2b4b 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -52,8 +52,15 @@ def _debug(*args): HTTPONLY_ATTR = "HTTPOnly" DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT) +NETSCAPE_MAGIC_RGX = re.compile("#( Netscape)? HTTP Cookie File") MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar " "instance initialised with one)") +NETSCAPE_HEADER_TEXT = """\ +# Netscape HTTP Cookie File +# http://curl.haxx.se/rfc/cookie_spec.html +# This is a generated file! Do not edit. + +""" def _warn_unhandled_exception(): # There are a few catch-all except: statements in this module, for @@ -2004,20 +2011,11 @@ class MozillaCookieJar(FileCookieJar): header by default (Mozilla can cope with that). """ - httponly_prefix = "#HttpOnly_" - magic_re = re.compile("#( Netscape)? HTTP Cookie File") - header = """\ -# Netscape HTTP Cookie File -# http://curl.haxx.se/rfc/cookie_spec.html -# This is a generated file! Do not edit. - -""" def _really_load(self, f, filename, ignore_discard, ignore_expires): now = time.time() - magic = f.readline() - if not self.magic_re.search(magic): + if not NETSCAPE_MAGIC_RGX.match(f.readline()): raise LoadError( "%r does not look like a Netscape format cookies file" % filename) @@ -2029,10 +2027,12 @@ def _really_load(self, f, filename, ignore_discard, ignore_expires): if line == "": break - # detect httponly flag if present - if line.startswith(self.httponly_prefix): + # httponly is a cookie flag as defined in rfc6265 + # when encoded in a netscape cookie file, + # the line is prepended with "#HttpOnly_" + if line.startswith("#HttpOnly_"): rest[HTTPONLY_ATTR] = "" - line = line[len(self.httponly_prefix):] + line = line[10:] # last field may be absent, so keep any trailing tab if line.endswith("\n"): line = line[:-1] From cded0a714909bfc76aca8cba343bfa476acfc9f9 Mon Sep 17 00:00:00 2001 From: Jacob Neil Taylor Date: Thu, 2 Apr 2020 20:07:12 +1100 Subject: [PATCH 04/11] Fix reference to header variable --- Lib/http/cookiejar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 46175e25d8c2b4b..6f1e0033fdd0204 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -2091,7 +2091,7 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): else: raise ValueError(MISSING_FILENAME_TEXT) with open(filename, "w") as f: - f.write(self.header) + f.write(NETSCAPE_HEADER_TEXT) now = time.time() for cookie in self: domain = cookie.domain From a8a0b519e64acaea8b48ce72620d32224096de6f Mon Sep 17 00:00:00 2001 From: Jacob Neil Taylor Date: Thu, 2 Apr 2020 20:31:24 +1100 Subject: [PATCH 05/11] Add the HttpOnly prefix to module constants --- Lib/http/cookiejar.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 6f1e0033fdd0204..cd9f20719652197 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -51,6 +51,7 @@ def _debug(*args): return logger.debug(*args) HTTPONLY_ATTR = "HTTPOnly" +HTTPONLY_PREFIX = "#HttpOnly_" DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT) NETSCAPE_MAGIC_RGX = re.compile("#( Netscape)? HTTP Cookie File") MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar " @@ -2030,9 +2031,9 @@ def _really_load(self, f, filename, ignore_discard, ignore_expires): # httponly is a cookie flag as defined in rfc6265 # when encoded in a netscape cookie file, # the line is prepended with "#HttpOnly_" - if line.startswith("#HttpOnly_"): + if line.startswith(HTTPONLY_PREFIX): rest[HTTPONLY_ATTR] = "" - line = line[10:] + line = line[len(HTTPONLY_PREFIX):] # last field may be absent, so keep any trailing tab if line.endswith("\n"): line = line[:-1] @@ -2117,7 +2118,7 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): name = cookie.name value = cookie.value if cookie.has_nonstandard_attr(HTTPONLY_ATTR): - domain = self.httponly_prefix + domain + domain = HTTPONLY_PREFIX + domain f.write( "\t".join([domain, initial_dot, cookie.path, secure, expires, name, value])+ From 48c2a8d01d0838a8607c4f1ec4585163c94d20b4 Mon Sep 17 00:00:00 2001 From: jjact Date: Wed, 25 Sep 2019 11:58:11 +1000 Subject: [PATCH 06/11] Add support for HTTP Only flag in MozillaCookieJar --- Lib/http/cookiejar.py | 19 +++++++++++++++---- Lib/test/test_http_cookiejar.py | 5 +++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 47ed5c3d64ab7d1..ae7c6c21b5cf74f 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -50,7 +50,7 @@ def _debug(*args): logger = logging.getLogger("http.cookiejar") return logger.debug(*args) - +HTTPONLY_ATTR = "HTTPOnly" DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT) MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar " "instance initialised with one)") @@ -2004,6 +2004,7 @@ class MozillaCookieJar(FileCookieJar): header by default (Mozilla can cope with that). """ + httponly_prefix = "#HttpOnly_" magic_re = re.compile("#( Netscape)? HTTP Cookie File") header = """\ # Netscape HTTP Cookie File @@ -2024,8 +2025,15 @@ def _really_load(self, f, filename, ignore_discard, ignore_expires): try: while 1: line = f.readline() + rest = {} + if line == "": break + # detect httponly flag if present + if line.startswith(self.httponly_prefix): + rest[HTTPONLY_ATTR] = "" + line = line[len(self.httponly_prefix):] + # last field may be absent, so keep any trailing tab if line.endswith("\n"): line = line[:-1] @@ -2063,7 +2071,7 @@ def _really_load(self, f, filename, ignore_discard, ignore_expires): discard, None, None, - {}) + rest) if not ignore_discard and c.discard: continue if not ignore_expires and c.is_expired(now): @@ -2086,13 +2094,14 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): f.write(self.header) now = time.time() for cookie in self: + domain = cookie.domain if not ignore_discard and cookie.discard: continue if not ignore_expires and cookie.is_expired(now): continue if cookie.secure: secure = "TRUE" else: secure = "FALSE" - if cookie.domain.startswith("."): initial_dot = "TRUE" + if domain.startswith("."): initial_dot = "TRUE" else: initial_dot = "FALSE" if cookie.expires is not None: expires = str(cookie.expires) @@ -2107,7 +2116,9 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): else: name = cookie.name value = cookie.value + if cookie.has_nonstandard_attr(HTTPONLY_ATTR): + domain = self.httponly_prefix + domain f.write( - "\t".join([cookie.domain, initial_dot, cookie.path, + "\t".join([domain, initial_dot, cookie.path, secure, expires, name, value])+ "\n") diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index 2d7077af6da39e5..79d66724f99f040 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -1771,6 +1771,10 @@ def test_mozilla(self): interact_netscape(c, "http://www.foo.com/", "fooc=bar; Domain=www.foo.com; %s" % expires) + for cookie in c: + if cookie.name == "foo1": + cookie.set_nonstandard_attr("HTTPOnly", "") + def save_and_restore(cj, ignore_discard): try: cj.save(ignore_discard=ignore_discard) @@ -1785,6 +1789,7 @@ def save_and_restore(cj, ignore_discard): new_c = save_and_restore(c, True) self.assertEqual(len(new_c), 6) # none discarded self.assertIn("name='foo1', value='bar'", repr(new_c)) + self.assertIn("rest={'HTTPOnly': ''}", repr(new_c)) new_c = save_and_restore(c, False) self.assertEqual(len(new_c), 4) # 2 of them discarded on save From 5013677e4b3bc2bb28481930a92e006b32738421 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2019 05:22:50 +0000 Subject: [PATCH 07/11] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst diff --git a/Misc/NEWS.d/next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst b/Misc/NEWS.d/next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst new file mode 100644 index 000000000000000..7a48943a6c6cc7c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst @@ -0,0 +1,4 @@ +The :mod:`http.cookiejar` module now supports the parsing of cookies in CURL-style cookiejar files through MozillaCookieJar +on all platforms. Previously, such cookie entries would be silently ignored when loading a cookiejar with such entries. + +Additionally, the HTTP Only attribute is persisted in the object, and will be correctly written to file if the MozillaCookieJar object is subsequently dumped. \ No newline at end of file From 4de40c1ddb3def161eff7b123401f7ab92b0f51d Mon Sep 17 00:00:00 2001 From: Jacob Neil Taylor Date: Thu, 2 Apr 2020 19:43:03 +1100 Subject: [PATCH 08/11] Cleanup class variables and update comments --- Lib/http/cookiejar.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index ae7c6c21b5cf74f..46175e25d8c2b4b 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -52,8 +52,15 @@ def _debug(*args): HTTPONLY_ATTR = "HTTPOnly" DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT) +NETSCAPE_MAGIC_RGX = re.compile("#( Netscape)? HTTP Cookie File") MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar " "instance initialised with one)") +NETSCAPE_HEADER_TEXT = """\ +# Netscape HTTP Cookie File +# http://curl.haxx.se/rfc/cookie_spec.html +# This is a generated file! Do not edit. + +""" def _warn_unhandled_exception(): # There are a few catch-all except: statements in this module, for @@ -2004,20 +2011,11 @@ class MozillaCookieJar(FileCookieJar): header by default (Mozilla can cope with that). """ - httponly_prefix = "#HttpOnly_" - magic_re = re.compile("#( Netscape)? HTTP Cookie File") - header = """\ -# Netscape HTTP Cookie File -# http://curl.haxx.se/rfc/cookie_spec.html -# This is a generated file! Do not edit. - -""" def _really_load(self, f, filename, ignore_discard, ignore_expires): now = time.time() - magic = f.readline() - if not self.magic_re.search(magic): + if not NETSCAPE_MAGIC_RGX.match(f.readline()): raise LoadError( "%r does not look like a Netscape format cookies file" % filename) @@ -2029,10 +2027,12 @@ def _really_load(self, f, filename, ignore_discard, ignore_expires): if line == "": break - # detect httponly flag if present - if line.startswith(self.httponly_prefix): + # httponly is a cookie flag as defined in rfc6265 + # when encoded in a netscape cookie file, + # the line is prepended with "#HttpOnly_" + if line.startswith("#HttpOnly_"): rest[HTTPONLY_ATTR] = "" - line = line[len(self.httponly_prefix):] + line = line[10:] # last field may be absent, so keep any trailing tab if line.endswith("\n"): line = line[:-1] From a42631ef5bfbe29f069c1cfa8aa0ca3d9a3685a0 Mon Sep 17 00:00:00 2001 From: Jacob Neil Taylor Date: Thu, 2 Apr 2020 20:07:12 +1100 Subject: [PATCH 09/11] Fix reference to header variable --- Lib/http/cookiejar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 46175e25d8c2b4b..6f1e0033fdd0204 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -2091,7 +2091,7 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): else: raise ValueError(MISSING_FILENAME_TEXT) with open(filename, "w") as f: - f.write(self.header) + f.write(NETSCAPE_HEADER_TEXT) now = time.time() for cookie in self: domain = cookie.domain From de66b4a054aa519fbf35678fcc87bd9daa00ea58 Mon Sep 17 00:00:00 2001 From: Jacob Neil Taylor Date: Thu, 2 Apr 2020 20:31:24 +1100 Subject: [PATCH 10/11] Add the HttpOnly prefix to module constants --- Lib/http/cookiejar.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 6f1e0033fdd0204..cd9f20719652197 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -51,6 +51,7 @@ def _debug(*args): return logger.debug(*args) HTTPONLY_ATTR = "HTTPOnly" +HTTPONLY_PREFIX = "#HttpOnly_" DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT) NETSCAPE_MAGIC_RGX = re.compile("#( Netscape)? HTTP Cookie File") MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar " @@ -2030,9 +2031,9 @@ def _really_load(self, f, filename, ignore_discard, ignore_expires): # httponly is a cookie flag as defined in rfc6265 # when encoded in a netscape cookie file, # the line is prepended with "#HttpOnly_" - if line.startswith("#HttpOnly_"): + if line.startswith(HTTPONLY_PREFIX): rest[HTTPONLY_ATTR] = "" - line = line[10:] + line = line[len(HTTPONLY_PREFIX):] # last field may be absent, so keep any trailing tab if line.endswith("\n"): line = line[:-1] @@ -2117,7 +2118,7 @@ def save(self, filename=None, ignore_discard=False, ignore_expires=False): name = cookie.name value = cookie.value if cookie.has_nonstandard_attr(HTTPONLY_ATTR): - domain = self.httponly_prefix + domain + domain = HTTPONLY_PREFIX + domain f.write( "\t".join([domain, initial_dot, cookie.path, secure, expires, name, value])+ From 1dc6baa3f30c79acda55f126a9bde787cb070da9 Mon Sep 17 00:00:00 2001 From: Jacob Neil Taylor Date: Tue, 7 Apr 2020 19:40:10 +1000 Subject: [PATCH 11/11] Fixed whitespace --- Lib/http/cookiejar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index cd9f20719652197..eaa76c26b9c591a 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -2029,7 +2029,7 @@ def _really_load(self, f, filename, ignore_discard, ignore_expires): if line == "": break # httponly is a cookie flag as defined in rfc6265 - # when encoded in a netscape cookie file, + # when encoded in a netscape cookie file, # the line is prepended with "#HttpOnly_" if line.startswith(HTTPONLY_PREFIX): rest[HTTPONLY_ATTR] = ""