Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

sys-apps/portage-2.1.11.9: backport of HTTP compression and header usage

  • Loading branch information...
commit 35bbcd14235b5f7060617ae497b4a6ede811c265 1 parent bf5c64d
W. Mark Kubacki authored
6 sys-apps/portage/Manifest
View
@@ -1,6 +1,4 @@
-AUX 0001-Use-If-Modified-Since-HTTP-header-and-avoid-download.patch 4868 SHA256 a9535d68b4c8b7eb27aa01db6f8bd0204de068b83edf2b8fe81b97c7b5b6ee4e SHA512 08f8cf51d5b531d20ad2383c27c15ed374116935ecf47dcc64ee58a0d50f787663fc82448b8dda95b2c47656951d9a46b1d48b6467f729063dd05af5e7b16208 WHIRLPOOL 313ecd8684e9d7fcea930967c89ede2ad176db96e0c24563834e59623a226e801210b6a25b3ee661f1b287b4e042f5054891c816f26d6e67e33537fa5267d3e2
-AUX 0002-Add-support-for-HTTP-compression-bzip2-gzip-and-defl.patch 2588 SHA256 96e844e5ec7d8ebc84cfe107390c016e642cd5ba45dc21b147ff7984dd163057 SHA512 432fad19a23d90982b7901373df245497afac6efcfa7d99dcf27fc894d50e0de0ccb7e7f2cc7e4b56b7e985541f3fc1a69247856512b9154dd1a4968d616df3b WHIRLPOOL feb697d6d184f8547fa174ceea46efb9b8c25878e85291a36b7aca2b6ef81abdd3dde53e45a96fd87395a862396e1c2855766f2fcd54973bdc9ac046088af0a3
-AUX 0003-Fix-index-file-s-mtime-which-can-differ-from-TIMESTA.patch 1545 SHA256 eb7e595b50a54215587114e2ebeb2bb39156058c5f10b9d5a74f6a9225fc1a7a SHA512 4c2734c0069da38ba6f46e364d1eae031426559a984cde82b4dade396c581c6b3d65be99d40ac790e38587dd0e7715711ae510d54a95ed619783b27c92c19e40 WHIRLPOOL 49bf32bdcc2d7bdcc3cbcd45b392b22e1d6d8ade25330d8c57835f67a53db9e3daaf632271a3bea1f8f8d7c57689329cf48d967ff83f3aae895a71edc78f52cd
+AUX portage-2.1.11.9-compression_and_remote_index.patch 11461 SHA256 3bf08a44e3e80cdd704c730235caeefec2a13c8e4dd0400ed5583b8246c740f4 SHA512 68abe82cd4a0959c61a07d278c322856a06d6152b3815e2c2ee9c6acc8f061ae10661f9ec4325166fcaa401ea493d95ffd1ad8029ac250e38bf34ab263bbb012 WHIRLPOOL 54c0d9a7463c4edd98928d10a55e0fcd894332f8834319eda56e62c76fbdc42346a88c55b4de51a56ad592c2fa451847e3b2e931a49c917f210611ac8f274460
DIST portage-2.1.11.9.tar.bz2 844344 SHA256 590ecdae12581e0ead0eca6e4b8e0ca6bc5f1ecaafb98a5d3b684db5b1af7a05 SHA512 64919b833a719f6f1de5bc3ec8a921dd5fca75ea8c4535ce21faa608ff2bf811632ef627845e9b050fcd1ebf56f5792c7bd471925e75c1264d01277979d704a8 WHIRLPOOL d325c2b5ec50cde40f4b2ce127728b3712321febef58897b3141287776424610bd12701c159f15c5cec41f2eb35569e302a5692045406205e833f8e4cfd3d9fd
DIST portage-man-pl-2.1.2.tar.bz2 53893 SHA256 960eaa7c6f3a2af44bdc665266a8e884628a562373cc477d301597ecc5ef961f SHA512 5f5c8dd1559048546633b1e7291ff8c0f5d637595f1a4c98405424b08c30c089fc9359e0214f78fbe358754f20d327794f4692b9b67639585c29b755975ee853 WHIRLPOOL 756e267b84077501d976f46fa448cda5368cd52aeca6ec43d109c81fe8e2364e0cd0f35630d68a4186c70e3b440b7c870b86ac7ef83cc0afa43e10f9f8e1f422
-EBUILD portage-2.1.11.9.ebuild 10052 SHA256 c6c778e7612f085254fe1f445ab03a93d7a680a80966cc2ae8ec427025a8cf85 SHA512 ec1af74ae1112409dc46c7b93c53b57d40b82f3339ef3350b097a1d95cbb5536f979b894a29660bcabe999dbd9eafa3dcadee648b3f4305772bcf6e9a48c1c78 WHIRLPOOL 7214b21d2142267535cc593d9738d097972dbc0d2c46cd11fc62c2e155d55c97b9fddb5c01c2a68aec79daa48d71d5300a30cc01cc082fa3209dc4f6f83b2895
+EBUILD portage-2.1.11.9.ebuild 9866 SHA256 59ecdfd199849866e524d3c3247ec9b55c5457b00faaf8c4e571a4221624fe5b SHA512 ef1780282537c2f5985924422d074a29a5127456d283f36e3a7b2bfb872d37c6482d087260ba773c6f09df511ccd1b7735c6fabda1c94f94cb21c91e022651f4 WHIRLPOOL d3483e1e201bdcd43ddbf88577b8837fdb2c0951fc6ea82a9f600a687f4d30475914b0b6ee8fea5057943e153018a91bf40b42e2badd3e88cc554986d1d69f28
135 sys-apps/portage/files/0001-Use-If-Modified-Since-HTTP-header-and-avoid-download.patch
View
@@ -1,135 +0,0 @@
-From a52924ce950fd2efbf99959e4dd0452b5feb92da Mon Sep 17 00:00:00 2001
-From: W-Mark Kubacki <wmark@hurrikane.de>
-Date: Wed, 1 Aug 2012 19:49:34 +0200
-Subject: [PATCH 1/3] Use If-Modified-Since HTTP-header and avoid downloading
- a remote index if the local copy is recent enough.
-
----
- pym/portage/dbapi/bintree.py | 24 +++++++++++++++++++++---
- pym/portage/util/_urlopen.py | 33 ++++++++++++++++++++++++++++++---
- 2 files changed, 51 insertions(+), 6 deletions(-)
-
-diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
-index 9527b07..16ae8ec 100644
---- a/pym/portage/dbapi/bintree.py
-+++ b/pym/portage/dbapi/bintree.py
-@@ -54,6 +54,11 @@ if sys.hexversion >= 0x3000000:
- else:
- _unicode = unicode
-
-+class UseCachedCopyOfRemoteIndex(Exception):
-+ # If the local copy is recent enough
-+ # then fetching the remote index can be skipped.
-+ pass
-+
- class bindbapi(fakedbapi):
- _known_keys = frozenset(list(fakedbapi._known_keys) + \
- ["CHOST", "repository", "USE"])
-@@ -852,6 +857,7 @@ class binarytree(object):
- if e.errno != errno.ENOENT:
- raise
- local_timestamp = pkgindex.header.get("TIMESTAMP", None)
-+ remote_timestamp = None
- rmt_idx = self._new_pkgindex()
- proc = None
- tmp_filename = None
-@@ -861,8 +867,13 @@ class binarytree(object):
- # slash, so join manually...
- url = base_url.rstrip("/") + "/Packages"
- try:
-- f = _urlopen(url)
-- except IOError:
-+ f = _urlopen(url, if_modified_since=local_timestamp)
-+ if hasattr(f, 'headers') and f.headers.get('timestamp', ''):
-+ remote_timestamp = f.headers.get('timestamp')
-+ except IOError as err:
-+ if hasattr(err, 'code') and err.code == 304: # not modified (since local_timestamp)
-+ raise UseCachedCopyOfRemoteIndex()
-+
- path = parsed_url.path.rstrip("/") + "/Packages"
-
- if parsed_url.scheme == 'sftp':
-@@ -903,7 +914,8 @@ class binarytree(object):
- _encodings['repo.content'], errors='replace')
- try:
- rmt_idx.readHeader(f_dec)
-- remote_timestamp = rmt_idx.header.get("TIMESTAMP", None)
-+ if not remote_timestamp: # in case it had not been read from HTTP header
-+ remote_timestamp = rmt_idx.header.get("TIMESTAMP", None)
- if not remote_timestamp:
- # no timestamp in the header, something's wrong
- pkgindex = None
-@@ -931,6 +943,12 @@ class binarytree(object):
- writemsg("\n\n!!! %s\n" % \
- _("Timed out while closing connection to binhost"),
- noiselevel=-1)
-+ except UseCachedCopyOfRemoteIndex:
-+ writemsg_stdout("\n")
-+ writemsg_stdout(
-+ colorize("GOOD", _("Local copy of remote index is up-to-date and will be used.")) + \
-+ "\n")
-+ rmt_idx = pkgindex
- except EnvironmentError as e:
- writemsg(_("\n\n!!! Error fetching binhost package" \
- " info from '%s'\n") % _hide_url_passwd(base_url))
-diff --git a/pym/portage/util/_urlopen.py b/pym/portage/util/_urlopen.py
-index 307624b..a5db411 100644
---- a/pym/portage/util/_urlopen.py
-+++ b/pym/portage/util/_urlopen.py
-@@ -2,6 +2,9 @@
- # Distributed under the terms of the GNU General Public License v2
-
- import sys
-+from datetime import datetime
-+from time import mktime
-+from email.utils import formatdate, parsedate
-
- try:
- from urllib.request import urlopen as _urlopen
-@@ -14,12 +17,26 @@ except ImportError:
- import urllib2 as urllib_request
- from urllib import splituser as urllib_parse_splituser
-
--def urlopen(url):
-+# to account for the difference between TIMESTAMP of the index' contents
-+# and the file-'mtime'
-+TIMESTAMP_TOLERANCE=5
-+
-+def urlopen(url, if_modified_since=None):
- try:
-- return _urlopen(url)
-+ request = urllib_request.Request(url)
-+ request.add_header('User-Agent', 'Gentoo Portage')
-+ if if_modified_since:
-+ request.add_header('If-Modified-Since', _timestamp_to_http(if_modified_since))
-+ opener = urllib_request.build_opener()
-+ hdl = opener.open(request)
-+ if hdl.headers.get('last-modified', ''):
-+ hdl.headers.addheader('timestamp', _http_to_timestamp(hdl.headers.get('last-modified')))
-+ return hdl
- except SystemExit:
- raise
-- except Exception:
-+ except Exception as e:
-+ if hasattr(e, 'code') and e.code == 304: # HTTPError 304: not modified
-+ raise
- if sys.hexversion < 0x3000000:
- raise
- parse_result = urllib_parse.urlparse(url)
-@@ -40,3 +57,13 @@ def _new_urlopen(url):
- auth_handler = urllib_request.HTTPBasicAuthHandler(password_manager)
- opener = urllib_request.build_opener(auth_handler)
- return opener.open(url)
-+
-+def _timestamp_to_http(timestamp):
-+ dt = datetime.fromtimestamp(float(long(timestamp)+TIMESTAMP_TOLERANCE))
-+ stamp = mktime(dt.timetuple())
-+ return formatdate(timeval=stamp, localtime=False, usegmt=True)
-+
-+def _http_to_timestamp(http_datetime_string):
-+ tuple = parsedate(http_datetime_string)
-+ timestamp = mktime(tuple)
-+ return str(long(timestamp))
---
-1.7.8.6
-
67 sys-apps/portage/files/0002-Add-support-for-HTTP-compression-bzip2-gzip-and-defl.patch
View
@@ -1,67 +0,0 @@
-From 88a289b07642cb200b83b98f03d508dcbfd2ce64 Mon Sep 17 00:00:00 2001
-From: W-Mark Kubacki <wmark@hurrikane.de>
-Date: Wed, 1 Aug 2012 20:36:31 +0200
-Subject: [PATCH 2/3] Add support for HTTP compression (bzip2, gzip and
- deflate).
-
----
- pym/portage/util/_urlopen.py | 32 +++++++++++++++++++++++++++++++-
- 1 files changed, 31 insertions(+), 1 deletions(-)
-
-diff --git a/pym/portage/util/_urlopen.py b/pym/portage/util/_urlopen.py
-index a5db411..70535c5 100644
---- a/pym/portage/util/_urlopen.py
-+++ b/pym/portage/util/_urlopen.py
-@@ -5,6 +5,7 @@ import sys
- from datetime import datetime
- from time import mktime
- from email.utils import formatdate, parsedate
-+from StringIO import StringIO
-
- try:
- from urllib.request import urlopen as _urlopen
-@@ -27,7 +28,7 @@ def urlopen(url, if_modified_since=None):
- request.add_header('User-Agent', 'Gentoo Portage')
- if if_modified_since:
- request.add_header('If-Modified-Since', _timestamp_to_http(if_modified_since))
-- opener = urllib_request.build_opener()
-+ opener = urllib_request.build_opener(CompressedResponseProcessor)
- hdl = opener.open(request)
- if hdl.headers.get('last-modified', ''):
- hdl.headers.addheader('timestamp', _http_to_timestamp(hdl.headers.get('last-modified')))
-@@ -67,3 +68,32 @@ def _http_to_timestamp(http_datetime_string):
- tuple = parsedate(http_datetime_string)
- timestamp = mktime(tuple)
- return str(long(timestamp))
-+
-+class CompressedResponseProcessor(urllib_request.BaseHandler):
-+ # Handler for compressed responses.
-+
-+ def http_request(self, req):
-+ req.add_header('Accept-Encoding', 'bzip2,gzip,deflate')
-+ return req
-+ https_request = http_request
-+
-+ def http_response(self, req, response):
-+ decompressed = None
-+ if response.headers.get('content-encoding') == 'bzip2':
-+ import bz2
-+ decompressed = StringIO.StringIO(bz2.decompress(response.read()))
-+ elif response.headers.get('content-encoding') == 'gzip':
-+ from gzip import GzipFile
-+ decompressed = GzipFile(fileobj=StringIO(response.read()), mode='r')
-+ elif response.headers.get('content-encoding') == 'deflate':
-+ import zlib
-+ try:
-+ decompressed = StringIO.StringIO(zlib.decompress(response.read()))
-+ except zlib.error: # they ignored RFC1950
-+ decompressed = StringIO.StringIO(zlib.decompress(response.read(), -zlib.MAX_WBITS))
-+ if decompressed:
-+ old_response = response
-+ response = urllib_request.addinfourl(decompressed, old_response.headers, old_response.url, old_response.code)
-+ response.msg = old_response.msg
-+ return response
-+ https_response = http_response
---
-1.7.8.6
-
40 sys-apps/portage/files/0003-Fix-index-file-s-mtime-which-can-differ-from-TIMESTA.patch
View
@@ -1,40 +0,0 @@
-From 2b7ba96c8c6e81541cfba095c113638ac9a847f4 Mon Sep 17 00:00:00 2001
-From: W-Mark Kubacki <wmark@hurrikane.de>
-Date: Wed, 1 Aug 2012 21:12:24 +0200
-Subject: [PATCH 3/3] Fix index file's mtime, which can differ from TIMESTAMP.
-
-This enables Portage to reliably query for remote indices with
-HTTP-header If-Modified-Since.
-
-Without this patch mtime is greater than TIMESTAMP for large
-indices and slow storages - because writing a large file takes
-time. If the difference spans a second (TIMESTAMP 08:00:00, mtime
-08:00:01), then Portage will always fetch the remote index because
-it will appear being modified (mtime is used there) after the copy
-has been made (local copy's TIMESTAMP is used here).
----
- pym/portage/dbapi/bintree.py | 6 +++++-
- 1 files changed, 5 insertions(+), 1 deletions(-)
-
-diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
-index 16ae8ec..0367503 100644
---- a/pym/portage/dbapi/bintree.py
-+++ b/pym/portage/dbapi/bintree.py
-@@ -1186,9 +1186,13 @@ class binarytree(object):
- pkgindex.packages.append(d)
-
- self._update_pkgindex_header(pkgindex.header)
-- f = atomic_ofstream(os.path.join(self.pkgdir, "Packages"))
-+ pkgindex_filename = os.path.join(self.pkgdir, "Packages")
-+ f = atomic_ofstream(pkgindex_filename)
- pkgindex.write(f)
- f.close()
-+ # some seconds might have elapsed since TIMESTAMP
-+ atime = mtime = long(pkgindex.header["TIMESTAMP"])
-+ os.utime(pkgindex_filename, (atime, mtime))
- finally:
- if pkgindex_lock:
- unlockfile(pkgindex_lock)
---
-1.7.8.6
-
303 sys-apps/portage/files/portage-2.1.11.9-compression_and_remote_index.patch
View
@@ -0,0 +1,303 @@
+From c549f2f1b1c742a37a9bbb5f877fc3cb9e01aef4 Mon Sep 17 00:00:00 2001
+From: W-Mark Kubacki <wmark@hurrikane.de>
+Date: Mon, 13 Aug 2012 16:01:48 +0200
+Subject: [PATCH] HTTP compression, gzipped index and If-Modified-Since header
+ usage
+
+---
+ bin/emaint | 8 +---
+ man/make.conf.5 | 7 +++
+ pym/portage/const.py | 2 +-
+ pym/portage/dbapi/bintree.py | 56 ++++++++++++++++++++----
+ pym/portage/util/_urlopen.py | 99 +++++++++++++++++++++++++++++++-----------
+ 5 files changed, 131 insertions(+), 41 deletions(-)
+
+diff --git a/bin/emaint b/bin/emaint
+index 2160451..5a06eb1 100755
+--- a/bin/emaint
++++ b/bin/emaint
+@@ -243,12 +243,8 @@ class BinhostHandler(object):
+
+ del pkgindex.packages[:]
+ pkgindex.packages.extend(metadata.values())
+- from portage.util import atomic_ofstream
+- f = atomic_ofstream(self._pkgindex_file)
+- try:
+- self._pkgindex.write(f)
+- finally:
+- f.close()
++ bintree._pkgindex_write(self._pkgindex)
++
+ finally:
+ locks.unlockfile(pkgindex_lock)
+
+diff --git a/man/make.conf.5 b/man/make.conf.5
+index bb9c424..9b1704e 100644
+--- a/man/make.conf.5
++++ b/man/make.conf.5
+@@ -268,6 +268,13 @@ space. Make sure you have built both binutils and gdb with USE=zlib
+ support for this to work. See \fBsplitdebug\fR for general split debug
+ information (upon which this feature depends).
+ .TP
++.B compress\-index
++If set then a compressed copy of 'Packages' index file will be written.
++This feature is intended for Gentoo binhosts using certain webservers
++(such as, but not limited to, Nginx with gzip_static module) to avoid
++redundant on\-the\-fly compression. The resulting file will be called
++'Packages.gz' and its modification time will match that of 'Packages'.
++.TP
+ .B config\-protect\-if\-modified
+ This causes the \fBCONFIG_PROTECT\fR behavior to be skipped for files
+ that have not been modified since they were installed. This feature is
+diff --git a/pym/portage/const.py b/pym/portage/const.py
+index c7fc7df..444428f 100644
+--- a/pym/portage/const.py
++++ b/pym/portage/const.py
+@@ -89,7 +89,7 @@ SUPPORTED_FEATURES = frozenset([
+ "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy",
+ "ccache", "chflags", "clean-logs",
+ "collision-protect", "compress-build-logs", "compressdebug",
+- "config-protect-if-modified",
++ "compress-index", "config-protect-if-modified",
+ "digest", "distcc", "distcc-pump", "distlocks",
+ "downgrade-backup", "ebuild-locks", "fakeroot",
+ "fail-clean", "force-mirror", "force-prefix", "getbinpkg",
+diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
+index 9527b07..6c01867 100644
+--- a/pym/portage/dbapi/bintree.py
++++ b/pym/portage/dbapi/bintree.py
+@@ -41,6 +41,7 @@ import sys
+ import tempfile
+ import textwrap
+ import warnings
++from gzip import GzipFile
+ from itertools import chain
+ try:
+ from urllib.parse import urlparse
+@@ -54,6 +55,11 @@ if sys.hexversion >= 0x3000000:
+ else:
+ _unicode = unicode
+
++class UseCachedCopyOfRemoteIndex(Exception):
++ # If the local copy is recent enough
++ # then fetching the remote index can be skipped.
++ pass
++
+ class bindbapi(fakedbapi):
+ _known_keys = frozenset(list(fakedbapi._known_keys) + \
+ ["CHOST", "repository", "USE"])
+@@ -807,9 +813,7 @@ class binarytree(object):
+ del pkgindex.packages[:]
+ pkgindex.packages.extend(iter(metadata.values()))
+ self._update_pkgindex_header(pkgindex.header)
+- f = atomic_ofstream(self._pkgindex_file)
+- pkgindex.write(f)
+- f.close()
++ self._pkgindex_write(pkgindex)
+
+ if getbinpkgs and not self.settings["PORTAGE_BINHOST"]:
+ writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
+@@ -852,6 +856,7 @@ class binarytree(object):
+ if e.errno != errno.ENOENT:
+ raise
+ local_timestamp = pkgindex.header.get("TIMESTAMP", None)
++ remote_timestamp = None
+ rmt_idx = self._new_pkgindex()
+ proc = None
+ tmp_filename = None
+@@ -861,8 +866,13 @@ class binarytree(object):
+ # slash, so join manually...
+ url = base_url.rstrip("/") + "/Packages"
+ try:
+- f = _urlopen(url)
+- except IOError:
++ f = _urlopen(url, if_modified_since=local_timestamp)
++ if hasattr(f, 'headers') and f.headers.get('timestamp', ''):
++ remote_timestamp = f.headers.get('timestamp')
++ except IOError as err:
++ if hasattr(err, 'code') and err.code == 304: # not modified (since local_timestamp)
++ raise UseCachedCopyOfRemoteIndex()
++
+ path = parsed_url.path.rstrip("/") + "/Packages"
+
+ if parsed_url.scheme == 'sftp':
+@@ -903,7 +913,8 @@ class binarytree(object):
+ _encodings['repo.content'], errors='replace')
+ try:
+ rmt_idx.readHeader(f_dec)
+- remote_timestamp = rmt_idx.header.get("TIMESTAMP", None)
++ if not remote_timestamp: # in case it had not been read from HTTP header
++ remote_timestamp = rmt_idx.header.get("TIMESTAMP", None)
+ if not remote_timestamp:
+ # no timestamp in the header, something's wrong
+ pkgindex = None
+@@ -931,6 +942,12 @@ class binarytree(object):
+ writemsg("\n\n!!! %s\n" % \
+ _("Timed out while closing connection to binhost"),
+ noiselevel=-1)
++ except UseCachedCopyOfRemoteIndex:
++ writemsg_stdout("\n")
++ writemsg_stdout(
++ colorize("GOOD", _("Local copy of remote index is up-to-date and will be used.")) + \
++ "\n")
++ rmt_idx = pkgindex
+ except EnvironmentError as e:
+ writemsg(_("\n\n!!! Error fetching binhost package" \
+ " info from '%s'\n") % _hide_url_passwd(base_url))
+@@ -1168,13 +1185,34 @@ class binarytree(object):
+ pkgindex.packages.append(d)
+
+ self._update_pkgindex_header(pkgindex.header)
+- f = atomic_ofstream(os.path.join(self.pkgdir, "Packages"))
+- pkgindex.write(f)
+- f.close()
++ self._pkgindex_write(pkgindex)
++
+ finally:
+ if pkgindex_lock:
+ unlockfile(pkgindex_lock)
+
++ def _pkgindex_write(self, pkgindex):
++ contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO())
++ pkgindex.write(contents)
++ contents = contents.getvalue()
++ atime = mtime = long(pkgindex.header["TIMESTAMP"])
++ output_files = [(atomic_ofstream(self._pkgindex_file, mode="wb"),
++ self._pkgindex_file, None)]
++
++ if "compress-index" in self.settings.features:
++ gz_fname = self._pkgindex_file + ".gz"
++ fileobj = atomic_ofstream(gz_fname, mode="wb")
++ output_files.append((GzipFile(filename='', mode="wb",
++ fileobj=fileobj, mtime=mtime), gz_fname, fileobj))
++
++ for f, fname, f_close in output_files:
++ f.write(contents)
++ f.close()
++ if f_close is not None:
++ f_close.close()
++ # some seconds might have elapsed since TIMESTAMP
++ os.utime(fname, (atime, mtime))
++
+ def _pkgindex_entry(self, cpv):
+ """
+ Performs checksums and evaluates USE flag conditionals.
+diff --git a/pym/portage/util/_urlopen.py b/pym/portage/util/_urlopen.py
+index 307624b..768ccb8 100644
+--- a/pym/portage/util/_urlopen.py
++++ b/pym/portage/util/_urlopen.py
+@@ -1,7 +1,11 @@
+ # Copyright 2012 Gentoo Foundation
+ # Distributed under the terms of the GNU General Public License v2
+
++import io
+ import sys
++from datetime import datetime
++from time import mktime
++from email.utils import formatdate, parsedate
+
+ try:
+ from urllib.request import urlopen as _urlopen
+@@ -14,29 +18,74 @@ except ImportError:
+ import urllib2 as urllib_request
+ from urllib import splituser as urllib_parse_splituser
+
+-def urlopen(url):
+- try:
+- return _urlopen(url)
+- except SystemExit:
+- raise
+- except Exception:
+- if sys.hexversion < 0x3000000:
+- raise
+- parse_result = urllib_parse.urlparse(url)
+- if parse_result.scheme not in ("http", "https") or \
+- not parse_result.username:
+- raise
+-
+- return _new_urlopen(url)
+-
+-def _new_urlopen(url):
+- # This is experimental code for bug #413983.
++if sys.hexversion >= 0x3000000:
++ long = int
++
++# to account for the difference between TIMESTAMP of the index' contents
++# and the file-'mtime'
++TIMESTAMP_TOLERANCE=5
++
++def urlopen(url, if_modified_since=None):
+ parse_result = urllib_parse.urlparse(url)
+- netloc = urllib_parse_splituser(parse_result.netloc)[1]
+- url = urllib_parse.urlunparse((parse_result.scheme, netloc, parse_result.path, parse_result.params, parse_result.query, parse_result.fragment))
+- password_manager = urllib_request.HTTPPasswordMgrWithDefaultRealm()
+- if parse_result.username is not None:
+- password_manager.add_password(None, url, parse_result.username, parse_result.password)
+- auth_handler = urllib_request.HTTPBasicAuthHandler(password_manager)
+- opener = urllib_request.build_opener(auth_handler)
+- return opener.open(url)
++ if parse_result.scheme not in ("http", "https"):
++ return _urlopen(url)
++ else:
++ netloc = urllib_parse_splituser(parse_result.netloc)[1]
++ url = urllib_parse.urlunparse((parse_result.scheme, netloc, parse_result.path, parse_result.params, parse_result.query, parse_result.fragment))
++ password_manager = urllib_request.HTTPPasswordMgrWithDefaultRealm()
++ request = urllib_request.Request(url)
++ request.add_header('User-Agent', 'Gentoo Portage')
++ if if_modified_since:
++ request.add_header('If-Modified-Since', _timestamp_to_http(if_modified_since))
++ if parse_result.username is not None:
++ password_manager.add_password(None, url, parse_result.username, parse_result.password)
++ auth_handler = CompressedResponseProcessor(password_manager)
++ opener = urllib_request.build_opener(auth_handler)
++ hdl = opener.open(request)
++ if hdl.headers.get('last-modified', ''):
++ try:
++ add_header = hdl.headers.add_header
++ except AttributeError:
++ # Python 2
++ add_header = hdl.headers.addheader
++ add_header('timestamp', _http_to_timestamp(hdl.headers.get('last-modified')))
++ return hdl
++
++def _timestamp_to_http(timestamp):
++ dt = datetime.fromtimestamp(float(long(timestamp)+TIMESTAMP_TOLERANCE))
++ stamp = mktime(dt.timetuple())
++ return formatdate(timeval=stamp, localtime=False, usegmt=True)
++
++def _http_to_timestamp(http_datetime_string):
++ tuple = parsedate(http_datetime_string)
++ timestamp = mktime(tuple)
++ return str(long(timestamp))
++
++class CompressedResponseProcessor(urllib_request.HTTPBasicAuthHandler):
++ # Handler for compressed responses.
++
++ def http_request(self, req):
++ req.add_header('Accept-Encoding', 'bzip2,gzip,deflate')
++ return req
++ https_request = http_request
++
++ def http_response(self, req, response):
++ decompressed = None
++ if response.headers.get('content-encoding') == 'bzip2':
++ import bz2
++ decompressed = io.BytesIO(bz2.decompress(response.read()))
++ elif response.headers.get('content-encoding') == 'gzip':
++ from gzip import GzipFile
++ decompressed = GzipFile(fileobj=io.BytesIO(response.read()), mode='r')
++ elif response.headers.get('content-encoding') == 'deflate':
++ import zlib
++ try:
++ decompressed = io.BytesIO(zlib.decompress(response.read()))
++ except zlib.error: # they ignored RFC1950
++ decompressed = io.BytesIO(zlib.decompress(response.read(), -zlib.MAX_WBITS))
++ if decompressed:
++ old_response = response
++ response = urllib_request.addinfourl(decompressed, old_response.headers, old_response.url, old_response.code)
++ response.msg = old_response.msg
++ return response
++ https_response = http_response
+--
+1.7.8.6
+
4 sys-apps/portage/portage-2.1.11.9.ebuild
View
@@ -203,9 +203,7 @@ src_prepare() {
# patches for more efficient binhost usage
cd "${S}"
- epatch "${FILESDIR}"/0001-Use-If-Modified-Since-HTTP-header-and-avoid-download.patch
- epatch "${FILESDIR}"/0002-Add-support-for-HTTP-compression-bzip2-gzip-and-defl.patch
- epatch "${FILESDIR}"/0003-Fix-index-file-s-mtime-which-can-differ-from-TIMESTA.patch
+ epatch "${FILESDIR}"/${PN}-2.1.11.9-compression_and_remote_index.patch
}
src_compile() {
Please sign in to comment.
Something went wrong with that request. Please try again.