From 37c805eecf3556ed37ce3e0355d5b66f57557a48 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 16 Mar 2020 08:05:00 +0100 Subject: [PATCH 1/7] censor authorization part of headers before logging ReST API request --- easybuild/base/rest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/easybuild/base/rest.py b/easybuild/base/rest.py index 0aa0b08fcf..842c0bd3c8 100644 --- a/easybuild/base/rest.py +++ b/easybuild/base/rest.py @@ -35,6 +35,7 @@ :author: Jens Timmerman """ import base64 +import copy import json from functools import partial @@ -162,7 +163,13 @@ def request(self, method, url, body, headers, content_type=None): if self.auth_header is not None: headers['Authorization'] = self.auth_header headers['User-Agent'] = self.user_agent - fancylogger.getLogger().debug('cli request: %s, %s, %s, %s', method, url, body, headers) + + # censor contents of 'Authorization' part of header, to avoid leaking tokens or passwords in logs + headers_censored = copy.deepcopy(headers) + headers_censored['Authorization'] = '' + + fancylogger.getLogger().debug('cli request: %s, %s, %s, %s', method, url, body, headers_censored) + # TODO: in recent python: Context manager conn = self.get_connection(method, url, body, headers) status = conn.code From 379e9bab52b0d2e84d0cb3abb7690d25f76494a0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 16 Mar 2020 09:41:55 +0100 Subject: [PATCH 2/7] add test to check that --from-pr doesn't leak token in debug log --- test/framework/options.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index bcb3dcbe09..df35391d1e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1233,6 +1233,42 @@ def test_from_pr(self): print("Ignoring URLError '%s' in test_from_pr" % err) shutil.rmtree(tmpdir) + def test_from_pr_token_log(self): + """Check that --from-pr doesn't leak GitHub token in log.""" + if self.github_token is None: + print("Skipping test_from_pr_token_log, no GitHub token available?") + return + + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + tmpdir = tempfile.mkdtemp() + args = [ + # PR for foss/2018b, see https://github.com/easybuilders/easybuild-easyconfigs/pull/6424/files + '--from-pr=6424', + '--dry-run', + '--debug', + #'--logtostdout', + # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed + '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), + #'--unittest-file=%s' % self.logfile, + '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user + ] + try: + self.mock_stdout(True) + self.mock_stderr(True) + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertFalse(self.github_token in outtxt) + self.assertFalse(self.github_token in stdout) + self.assertFalse(self.github_token in stderr) + + except URLError as err: + print("Ignoring URLError '%s' in test_from_pr" % err) + def test_from_pr_listed_ecs(self): """Test --from-pr in combination with specifying easyconfigs on the command line.""" if self.github_token is None: From a0f41626b28521731f9c057dd92fd755ae62bf0d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Mar 2020 17:06:22 +0100 Subject: [PATCH 3/7] fix broken test for --review-pr by using different PR to test with --- test/framework/options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index df35391d1e..94347b8377 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2746,17 +2746,17 @@ def test_review_pr(self): self.mock_stdout(True) self.mock_stderr(True) - # PR for CMake 3.12.1 easyconfig, see https://github.com/easybuilders/easybuild-easyconfigs/pull/6660 + # PR for gzip 1.10 easyconfig, see https://github.com/easybuilders/easybuild-easyconfigs/pull/9921 args = [ '--color=never', '--github-user=%s' % GITHUB_TEST_ACCOUNT, - '--review-pr=6660', + '--review-pr=9921', ] self.eb_main(args, raise_error=True) txt = self.get_stdout() self.mock_stdout(False) self.mock_stderr(False) - regex = re.compile(r"^Comparing CMake-3.12.1-\S* with CMake-3.12.1-") + regex = re.compile(r"^Comparing gzip-1.10-\S* with gzip-1.10-") self.assertTrue(regex.search(txt), "Pattern '%s' not found in: %s" % (regex.pattern, txt)) def test_set_tmpdir(self): From e80d3b44f2a32856a517b46f1e45f21e7ca9adc9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 16 Mar 2020 10:49:24 +0100 Subject: [PATCH 4/7] appease the Hound --- test/framework/options.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 94347b8377..ee6d789a14 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1242,16 +1242,13 @@ def test_from_pr_token_log(self): fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') os.close(fd) - tmpdir = tempfile.mkdtemp() args = [ # PR for foss/2018b, see https://github.com/easybuilders/easybuild-easyconfigs/pull/6424/files '--from-pr=6424', '--dry-run', '--debug', - #'--logtostdout', # an argument must be specified to --robot, since easybuild-easyconfigs may not be installed '--robot=%s' % os.path.join(os.path.dirname(__file__), 'easyconfigs'), - #'--unittest-file=%s' % self.logfile, '--github-user=%s' % GITHUB_TEST_ACCOUNT, # a GitHub token should be available for this user ] try: From 482d03a23d3dc4d69f7e0f24f89d8a5775670c08 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 16 Mar 2020 11:03:49 +0100 Subject: [PATCH 5/7] bump version to v4.1.2& update release notes --- RELEASE_NOTES | 8 ++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 86d174b768..0031d47b7f 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,6 +3,14 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.1.2 (March 16th 2020) +------------------------ + +bugfix release + +- fix broken test for --review-pr by using different PR to test with (#3226) +- censor authorization part of headers before logging ReST API request (#3248) + v4.1.1 (January 16th 2020) -------------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index ca7d7a8e65..03ea55c176 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.1.1') +VERSION = LooseVersion('4.1.2') UNKNOWN = 'UNKNOWN' From 915782ec2afd7de4794fbedb91ee9042900a635b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 24 Feb 2020 12:00:54 +0100 Subject: [PATCH 6/7] Fix gitdb dependency on Python 2.6 --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index b7aa408d58..3a93ac826f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,10 @@ keyring==5.7.1; python_version < '2.7' keyring<=9.1; python_version >= '2.7' keyrings.alt; python_version >= '2.7' +# GitDB 4.0.1 no longer supports Python 2.6 +gitdb==0.6.4; python_version < '2.7' +gitdb; python_version >= '2.7' + # GitPython 2.1.9 no longer supports Python 2.6 GitPython==2.1.8; python_version < '2.7' GitPython; python_version >= '2.7' From a029a91898703403450ec67a683df130377e3db6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 16 Mar 2020 11:36:18 +0100 Subject: [PATCH 7/7] also include #3212 in release notes for EasyBuild v4.1.2 --- RELEASE_NOTES | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 0031d47b7f..8239f714bc 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -8,6 +8,7 @@ v4.1.2 (March 16th 2020) bugfix release +- fix gitdb dependency on Python 2.6 in test configuration (#3212) - fix broken test for --review-pr by using different PR to test with (#3226) - censor authorization part of headers before logging ReST API request (#3248)