From 00df32268c000bbaf996dd91b74b1299c7e4832a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E1=84=89=E1=85=A9=E1=86=BC=E1=84=8B=E1=85=B2=E1=86=AB?= =?UTF-8?q?=E1=84=89=E1=85=A5=E1=86=B8?= Date: Mon, 18 Sep 2017 15:59:37 +0900 Subject: [PATCH 1/3] Fixed cached wheel file raise BadZipfile exception --- zappa/core.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/zappa/core.py b/zappa/core.py index ddf325ba4..831b074c5 100644 --- a/zappa/core.py +++ b/zappa/core.py @@ -541,13 +541,22 @@ def splitpath(path): print(" - %s==%s: Using precompiled lambda package " % (installed_package_name, installed_package_version,)) self.extract_lambda_package(installed_package_name, temp_project_path) else: - cached_wheel_path = self.get_cached_manylinux_wheel(installed_package_name, installed_package_version, disable_progress) + cached_wheel_path = self.get_cached_manylinux_wheel(installed_package_name, installed_package_version, force_install=False, disable_progress=disable_progress) if cached_wheel_path: # Otherwise try to use manylinux packages from PyPi.. # Related: https://github.com/Miserlou/Zappa/issues/398 shutil.rmtree(os.path.join(temp_project_path, installed_package_name), ignore_errors=True) - with zipfile.ZipFile(cached_wheel_path) as zfile: - zfile.extractall(temp_project_path) + try: + with zipfile.ZipFile(cached_wheel_path) as zfile: + zfile.extractall(temp_project_path) + except BadZipfile as e: + # Need to re-install wheel file. + # Related: https://github.com/Miserlou/Zappa/issues/1104 + shutil.rmtree(cached_wheel_path, ignore_errors=True) + cached_wheel_path = self.get_cached_manylinux_wheel(installed_package_name, installed_package_version, force_install=True, disable_progress=disable_progress) + if cached_wheel_path: + with zipfile.ZipFile(cached_wheel_path) as zfile: + zfile.extractall(temp_project_path) elif self.have_any_lambda_package_version(installed_package_name): # Finally see if we may have at least one version of the package in lambda packages @@ -726,7 +735,7 @@ def download_url_with_progress(url, stream, disable_progress): progress.close() - def get_cached_manylinux_wheel(self, package_name, package_version, disable_progress=False): + def get_cached_manylinux_wheel(self, package_name, package_version, force_install, disable_progress=False): """ Gets the locally stored version of a manylinux wheel. If one does not exist, the function downloads it. """ @@ -737,7 +746,7 @@ def get_cached_manylinux_wheel(self, package_name, package_version, disable_prog wheel_file = '{0!s}-{1!s}-{2!s}'.format(package_name, package_version, self.manylinux_wheel_file_suffix) wheel_path = os.path.join(cached_wheels_dir, wheel_file) - if not os.path.exists(wheel_path): + if not os.path.exists(wheel_path) or force_install: # The file is not cached, download it. wheel_url = self.get_manylinux_wheel_url(package_name, package_version) if not wheel_url: From 82a226e3df03baeb7fab85874176a7623d58c1c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E1=84=89=E1=85=A9=E1=86=BC=E1=84=8B=E1=85=B2=E1=86=AB?= =?UTF-8?q?=E1=84=89=E1=85=A5=E1=86=B8?= Date: Mon, 18 Sep 2017 20:33:38 +0900 Subject: [PATCH 2/3] Revert "Fixed cached wheel file raise BadZipfile exception" This reverts commit 00df32268c000bbaf996dd91b74b1299c7e4832a. --- zappa/core.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/zappa/core.py b/zappa/core.py index 831b074c5..ddf325ba4 100644 --- a/zappa/core.py +++ b/zappa/core.py @@ -541,22 +541,13 @@ def splitpath(path): print(" - %s==%s: Using precompiled lambda package " % (installed_package_name, installed_package_version,)) self.extract_lambda_package(installed_package_name, temp_project_path) else: - cached_wheel_path = self.get_cached_manylinux_wheel(installed_package_name, installed_package_version, force_install=False, disable_progress=disable_progress) + cached_wheel_path = self.get_cached_manylinux_wheel(installed_package_name, installed_package_version, disable_progress) if cached_wheel_path: # Otherwise try to use manylinux packages from PyPi.. # Related: https://github.com/Miserlou/Zappa/issues/398 shutil.rmtree(os.path.join(temp_project_path, installed_package_name), ignore_errors=True) - try: - with zipfile.ZipFile(cached_wheel_path) as zfile: - zfile.extractall(temp_project_path) - except BadZipfile as e: - # Need to re-install wheel file. - # Related: https://github.com/Miserlou/Zappa/issues/1104 - shutil.rmtree(cached_wheel_path, ignore_errors=True) - cached_wheel_path = self.get_cached_manylinux_wheel(installed_package_name, installed_package_version, force_install=True, disable_progress=disable_progress) - if cached_wheel_path: - with zipfile.ZipFile(cached_wheel_path) as zfile: - zfile.extractall(temp_project_path) + with zipfile.ZipFile(cached_wheel_path) as zfile: + zfile.extractall(temp_project_path) elif self.have_any_lambda_package_version(installed_package_name): # Finally see if we may have at least one version of the package in lambda packages @@ -735,7 +726,7 @@ def download_url_with_progress(url, stream, disable_progress): progress.close() - def get_cached_manylinux_wheel(self, package_name, package_version, force_install, disable_progress=False): + def get_cached_manylinux_wheel(self, package_name, package_version, disable_progress=False): """ Gets the locally stored version of a manylinux wheel. If one does not exist, the function downloads it. """ @@ -746,7 +737,7 @@ def get_cached_manylinux_wheel(self, package_name, package_version, force_instal wheel_file = '{0!s}-{1!s}-{2!s}'.format(package_name, package_version, self.manylinux_wheel_file_suffix) wheel_path = os.path.join(cached_wheels_dir, wheel_file) - if not os.path.exists(wheel_path) or force_install: + if not os.path.exists(wheel_path): # The file is not cached, download it. wheel_url = self.get_manylinux_wheel_url(package_name, package_version) if not wheel_url: From 87c26b8c1e1dd6fbc08ae5cc45e18dc28389964e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E1=84=89=E1=85=A9=E1=86=BC=E1=84=8B=E1=85=B2=E1=86=AB?= =?UTF-8?q?=E1=84=89=E1=85=A5=E1=86=B8?= Date: Mon, 18 Sep 2017 21:23:58 +0900 Subject: [PATCH 3/3] Check is_zipfile in get_cached_manylinux_wheel --- zappa/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zappa/core.py b/zappa/core.py index ddf325ba4..651b4af53 100644 --- a/zappa/core.py +++ b/zappa/core.py @@ -737,7 +737,7 @@ def get_cached_manylinux_wheel(self, package_name, package_version, disable_prog wheel_file = '{0!s}-{1!s}-{2!s}'.format(package_name, package_version, self.manylinux_wheel_file_suffix) wheel_path = os.path.join(cached_wheels_dir, wheel_file) - if not os.path.exists(wheel_path): + if not os.path.exists(wheel_path) or not zipfile.is_zipfile(wheel_path): # The file is not cached, download it. wheel_url = self.get_manylinux_wheel_url(package_name, package_version) if not wheel_url: @@ -746,6 +746,9 @@ def get_cached_manylinux_wheel(self, package_name, package_version, disable_prog print(" - {}=={}: Downloading".format(package_name, package_version)) with open(wheel_path, 'wb') as f: self.download_url_with_progress(wheel_url, f, disable_progress) + + if not zipfile.is_zipfile(wheel_path): + return None else: print(" - {}=={}: Using locally cached manylinux wheel".format(package_name, package_version))