From 5bee0adb0f8987c6f75ceaad400685621d082c81 Mon Sep 17 00:00:00 2001 From: mozillazg Date: Sat, 20 Apr 2019 22:38:30 +0800 Subject: [PATCH 1/7] Support use local file path as value of PEP_ARTIFACT_URL --- .../management/commands/generate_pep_pages.py | 36 +++++++++++-------- peps/tests/test_commands.py | 6 +++- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/peps/management/commands/generate_pep_pages.py b/peps/management/commands/generate_pep_pages.py index e05ddaaf9..6f63f9574 100644 --- a/peps/management/commands/generate_pep_pages.py +++ b/peps/management/commands/generate_pep_pages.py @@ -52,20 +52,7 @@ def verbose(msg): with ExitStack() as stack: verbose(f"== Fetching PEP artifact from {settings.PEP_ARTIFACT_URL}") - peps_last_updated = get_peps_last_updated() - with requests.get(settings.PEP_ARTIFACT_URL, stream=True) as r: - artifact_last_modified = parsedate(r.headers['last-modified']) - if peps_last_updated > artifact_last_modified: - verbose(f"== No update to artifacts, we're done here!") - return - - temp_file = stack.enter_context(TemporaryFile()) - for chunk in r.iter_content(chunk_size=8192): - if chunk: - temp_file.write(chunk) - - temp_file.seek(0) - + temp_file = self.get_artifact_tar_ball(stack, verbose) temp_dir = stack.enter_context(TemporaryDirectory()) tar_ball = stack.enter_context(TarFile.open(fileobj=temp_file, mode='r:gz')) tar_ball.extractall(path=temp_dir, numeric_owner=False) @@ -129,3 +116,24 @@ def verbose(msg): verbose("- Skipping non-PEP related image '{}'".format(img)) verbose("== Finished") + + def get_artifact_tar_ball(self, stack, verbose): + artifact_url = settings.PEP_ARTIFACT_URL + if not (artifact_url.startswith('http://') or + artifact_url.startswith('https://')): + return stack.enter_context(open(artifact_url, 'rb')) + + peps_last_updated = get_peps_last_updated() + with requests.get(artifact_url, stream=True) as r: + artifact_last_modified = parsedate(r.headers['last-modified']) + if peps_last_updated > artifact_last_modified: + verbose(f"== No update to artifacts, we're done here!") + return + + temp_file = stack.enter_context(TemporaryFile()) + for chunk in r.iter_content(chunk_size=8192): + if chunk: + temp_file.write(chunk) + + temp_file.seek(0) + return temp_file diff --git a/peps/tests/test_commands.py b/peps/tests/test_commands.py index 54396b860..2579a5f99 100644 --- a/peps/tests/test_commands.py +++ b/peps/tests/test_commands.py @@ -32,7 +32,11 @@ def setUp(self): ) @responses.activate - def test_generate_pep_pages_real(self): + def test_generate_pep_pages_real_with_remote_artifact(self): + call_command('generate_pep_pages') + + @override_settings(PEP_ARTIFACT_URL=FAKE_PEP_ARTIFACT) + def test_generate_pep_pages_real_with_local_artifact(self): call_command('generate_pep_pages') @responses.activate From 637a1ea5107f1aa47243184b7109b14c7c4a0ad6 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 21 Apr 2019 11:34:49 +0800 Subject: [PATCH 2/7] Apply suggestions from @berkerpeksag 's code review Co-Authored-By: mozillazg --- peps/management/commands/generate_pep_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/peps/management/commands/generate_pep_pages.py b/peps/management/commands/generate_pep_pages.py index 6f63f9574..6c40d78aa 100644 --- a/peps/management/commands/generate_pep_pages.py +++ b/peps/management/commands/generate_pep_pages.py @@ -117,7 +117,7 @@ def verbose(msg): verbose("== Finished") - def get_artifact_tar_ball(self, stack, verbose): + def get_artifact_tarball(self, stack, verbose): artifact_url = settings.PEP_ARTIFACT_URL if not (artifact_url.startswith('http://') or artifact_url.startswith('https://')): From a9275fca8f83fba326dfbfccc9f258308045e55c Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 21 Apr 2019 11:35:45 +0800 Subject: [PATCH 3/7] Apply suggestions from @berkerpeksag 's code review Co-Authored-By: mozillazg --- peps/management/commands/generate_pep_pages.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/peps/management/commands/generate_pep_pages.py b/peps/management/commands/generate_pep_pages.py index 6c40d78aa..4fff05c3a 100644 --- a/peps/management/commands/generate_pep_pages.py +++ b/peps/management/commands/generate_pep_pages.py @@ -119,8 +119,7 @@ def verbose(msg): def get_artifact_tarball(self, stack, verbose): artifact_url = settings.PEP_ARTIFACT_URL - if not (artifact_url.startswith('http://') or - artifact_url.startswith('https://')): + if not artifact_url.startswith(('http://', 'https://')): return stack.enter_context(open(artifact_url, 'rb')) peps_last_updated = get_peps_last_updated() From 2fce20b948b93793ffd0ac274d49ad2a128a3073 Mon Sep 17 00:00:00 2001 From: mozillazg Date: Sun, 21 Apr 2019 11:37:37 +0800 Subject: [PATCH 4/7] Change value of PEP_ARTIFACT_URL to local path in local settings --- pydotorg/settings/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydotorg/settings/local.py b/pydotorg/settings/local.py index a96b6bdce..ec966761d 100644 --- a/pydotorg/settings/local.py +++ b/pydotorg/settings/local.py @@ -25,7 +25,7 @@ EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' # Set the URL to where to fetch PEP artifacts from -PEP_ARTIFACT_URL = 'https://pythondotorg-assets-staging.s3.amazonaws.com/fake-peps.tar.gz' +PEP_ARTIFACT_URL = os.path.join(BASE, 'peps/tests/peps.tar.gz') # Use Dummy SASS compiler to avoid performance issues and remove the need to # have a sass compiler installed at all during local development if you aren't From 5d59a3369759f84c25f694917851f8a12b488fc5 Mon Sep 17 00:00:00 2001 From: mozillazg Date: Sun, 21 Apr 2019 11:39:48 +0800 Subject: [PATCH 5/7] fix typo --- peps/management/commands/generate_pep_pages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/peps/management/commands/generate_pep_pages.py b/peps/management/commands/generate_pep_pages.py index 4fff05c3a..c820411fd 100644 --- a/peps/management/commands/generate_pep_pages.py +++ b/peps/management/commands/generate_pep_pages.py @@ -52,7 +52,7 @@ def verbose(msg): with ExitStack() as stack: verbose(f"== Fetching PEP artifact from {settings.PEP_ARTIFACT_URL}") - temp_file = self.get_artifact_tar_ball(stack, verbose) + temp_file = self.get_artifact_tarball(stack, verbose) temp_dir = stack.enter_context(TemporaryDirectory()) tar_ball = stack.enter_context(TarFile.open(fileobj=temp_file, mode='r:gz')) tar_ball.extractall(path=temp_dir, numeric_owner=False) From e023260c2c723187e09ebdcab6b012fbface8b31 Mon Sep 17 00:00:00 2001 From: mozillazg Date: Sun, 21 Apr 2019 11:44:39 +0800 Subject: [PATCH 6/7] Update comment about PEP_ARTIFACT_URL Noted the value of PEP_ARTIFACT_URL can be local path --- pydotorg/settings/local.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pydotorg/settings/local.py b/pydotorg/settings/local.py index ec966761d..29b89057a 100644 --- a/pydotorg/settings/local.py +++ b/pydotorg/settings/local.py @@ -24,7 +24,8 @@ EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' -# Set the URL to where to fetch PEP artifacts from +# Set the path to where to fetch PEP artifacts from. +# The value can be local path or remote URL PEP_ARTIFACT_URL = os.path.join(BASE, 'peps/tests/peps.tar.gz') # Use Dummy SASS compiler to avoid performance issues and remove the need to From 10be21b2836a686b7a66ec3e30d75a1766ac8ba3 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sun, 21 Apr 2019 17:59:34 +0300 Subject: [PATCH 7/7] Update local.py --- pydotorg/settings/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydotorg/settings/local.py b/pydotorg/settings/local.py index 29b89057a..71df0ed30 100644 --- a/pydotorg/settings/local.py +++ b/pydotorg/settings/local.py @@ -25,7 +25,7 @@ EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' # Set the path to where to fetch PEP artifacts from. -# The value can be local path or remote URL +# The value can be a local path or a remote URL. PEP_ARTIFACT_URL = os.path.join(BASE, 'peps/tests/peps.tar.gz') # Use Dummy SASS compiler to avoid performance issues and remove the need to