From 6bc07eae7dea6bce853d59ef471b341057e4f9ef Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 11 Jul 2018 17:15:35 +0200 Subject: [PATCH 1/5] unzip xz --- conans/client/tools/files.py | 6 ++++++ conans/test/util/xz_test.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 conans/test/util/xz_test.py diff --git a/conans/client/tools/files.py b/conans/client/tools/files.py index 5d6ddef970b..0d42fe368f4 100644 --- a/conans/client/tools/files.py +++ b/conans/client/tools/files.py @@ -11,6 +11,7 @@ from conans.errors import ConanException from conans.util.files import (load, save, _generic_algorithm_sum) from conans.unicode import get_cwd +import six _global_output = None @@ -68,6 +69,11 @@ def unzip(filename, destination=".", keep_permissions=False, pattern=None): filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or filename.endswith(".tar")): return untargz(filename, destination, pattern) + if filename.endswith(".tar.xz") or filename.endswith(".txz"): + if six.PY2: + raise ConanException("XZ format not supported in Python 2. Use Python 3 instead") + return untargz(filename, destination, pattern) + import zipfile full_path = os.path.normpath(os.path.join(get_cwd(), destination)) diff --git a/conans/test/util/xz_test.py b/conans/test/util/xz_test.py new file mode 100644 index 00000000000..06d797466bd --- /dev/null +++ b/conans/test/util/xz_test.py @@ -0,0 +1,33 @@ +import os +from unittest import TestCase +import six +import unittest +import tarfile + +from conans.test.utils.test_files import temp_folder +from conans.tools import unzip, save +from conans.util.files import load +from conans.errors import ConanException + + +class XZTest(TestCase): + + @unittest.skipUnless(six.PY3, "only Py3") + def test(self): + tmp_dir = temp_folder() + file_path = os.path.join(tmp_dir, "a_file.txt") + save(file_path, "my content!") + txz = os.path.join(tmp_dir, "sample.tar.xz") + with tarfile.open(txz, "w:xz") as tar: + tar.add(file_path, "a_file.txt") + + dest_folder = temp_folder() + unzip(txz, dest_folder) + content = load(os.path.join(dest_folder, "a_file.txt")) + self.assertEqual(content, "my content!") + + @unittest.skipUnless(six.PY2, "only Py2") + def test_error_python2(self): + with self.assertRaisesRegexp(ConanException, "XZ format not supported in Python 2"): + dest_folder = temp_folder() + unzip("somefile.tar.xz", dest_folder) From 1da63fff9885a3dfae285a09ac65d6edd4208c0b Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 11 Jul 2018 17:38:45 +0200 Subject: [PATCH 2/5] inplace check for compression formats --- conans/client/remote_manager.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index dcecb31902d..a4020afa7bc 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -382,6 +382,11 @@ def unzip_and_get_files(files, destination_dir, tgz_name): """Moves all files from package_files, {relative_name: tmp_abs_path} to destination_dir, unzipping the "tgz_name" if found""" + bare_name = os.path.splitext(tgz_name)[0] + for f in files: + if bare_name == os.path.splitext(f)[0]: + raise ConanException("This Conan version is not prepared to handle '%s' file format. " + "Please upgrade conan client." % f) tgz_file = files.pop(tgz_name, None) if tgz_file: uncompress_file(tgz_file, destination_dir) From b825cccbfd1a6ffb1918af86c2ca7adc3c8df71a Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 11 Jul 2018 23:38:23 +0200 Subject: [PATCH 3/5] fixed order --- conans/client/remote_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index a4020afa7bc..05915ede5a7 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -382,12 +382,12 @@ def unzip_and_get_files(files, destination_dir, tgz_name): """Moves all files from package_files, {relative_name: tmp_abs_path} to destination_dir, unzipping the "tgz_name" if found""" + tgz_file = files.pop(tgz_name, None) bare_name = os.path.splitext(tgz_name)[0] for f in files: if bare_name == os.path.splitext(f)[0]: raise ConanException("This Conan version is not prepared to handle '%s' file format. " "Please upgrade conan client." % f) - tgz_file = files.pop(tgz_name, None) if tgz_file: uncompress_file(tgz_file, destination_dir) os.remove(tgz_file) From 438b50ffceb1273f65716d7ccbaaa7763d956307 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 12 Jul 2018 10:54:21 +0200 Subject: [PATCH 4/5] fix tests --- .ci/jenkins/conf.py | 2 +- conans/client/remote_manager.py | 17 +++--- .../file_upload_download_controller.py | 7 ++- conans/test/util/xz_test.py | 57 ++++++++++++++++++- 4 files changed, 73 insertions(+), 10 deletions(-) diff --git a/.ci/jenkins/conf.py b/.ci/jenkins/conf.py index f59d4441e39..02c9bc1187a 100644 --- a/.ci/jenkins/conf.py +++ b/.ci/jenkins/conf.py @@ -9,7 +9,7 @@ macpylocation = {"py27": "/usr/bin/python", # /Users/jenkins_ci/.pyenv/versions/2.7.11/bin/python", "py34": "/Users/jenkins_ci/.pyenv/versions/3.4.7/bin/python", - "py36": "/Users/jenkins_ci/.pyenv/versions/3.6.3/bin/python"} + "py36": "/Users/jenkins_ci/.pyenv/versions/3.6.5/bin/python"} linuxpylocation = {"py27": "/usr/bin/python2.7", "py34": "/usr/bin/python3.4", diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 05915ede5a7..7b3d1173755 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -202,6 +202,7 @@ def get_recipe_sources(self, conan_reference, export_folder, export_sources_fold t1 = time.time() def filter_function(urls): + check_compressed_files(EXPORT_SOURCES_TGZ_NAME, urls) file_url = urls.get(EXPORT_SOURCES_TGZ_NAME) if file_url: urls = {EXPORT_SOURCES_TGZ_NAME: file_url} @@ -340,8 +341,6 @@ def compress_package_files(files, symlinks, dest_folder, output): def compress_files(files, symlinks, name, dest_dir): - """Compress the package and returns the new dict (name => content) of files, - only with the conanXX files and the compressed file""" t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) @@ -378,16 +377,20 @@ def compress_files(files, symlinks, name, dest_dir): return tgz_path -def unzip_and_get_files(files, destination_dir, tgz_name): - """Moves all files from package_files, {relative_name: tmp_abs_path} - to destination_dir, unzipping the "tgz_name" if found""" - - tgz_file = files.pop(tgz_name, None) +def check_compressed_files(tgz_name, files): bare_name = os.path.splitext(tgz_name)[0] for f in files: if bare_name == os.path.splitext(f)[0]: raise ConanException("This Conan version is not prepared to handle '%s' file format. " "Please upgrade conan client." % f) + + +def unzip_and_get_files(files, destination_dir, tgz_name): + """Moves all files from package_files, {relative_name: tmp_abs_path} + to destination_dir, unzipping the "tgz_name" if found""" + + tgz_file = files.pop(tgz_name, None) + check_compressed_files(tgz_name, files) if tgz_file: uncompress_file(tgz_file, destination_dir) os.remove(tgz_file) diff --git a/conans/server/rest/controllers/file_upload_download_controller.py b/conans/server/rest/controllers/file_upload_download_controller.py index b49091a0798..fe8ec36abed 100644 --- a/conans/server/rest/controllers/file_upload_download_controller.py +++ b/conans/server/rest/controllers/file_upload_download_controller.py @@ -20,7 +20,12 @@ def get(filepath): token = request.query.get("signature", None) file_path = service.get_file_path(filepath, token) # https://github.com/kennethreitz/requests/issues/1586 - mimetype = "x-gzip" if filepath.endswith(".tgz") else "auto" + if filepath.endswith(".tgz"): + mimetype = "x-gzip" + elif filepath.endswith(".txz"): + mimetype = "x-xz" + else: + mimetype = "auto" return static_file(os.path.basename(file_path), root=os.path.dirname(file_path), mimetype=mimetype) diff --git a/conans/test/util/xz_test.py b/conans/test/util/xz_test.py index 06d797466bd..26254973630 100644 --- a/conans/test/util/xz_test.py +++ b/conans/test/util/xz_test.py @@ -6,11 +6,66 @@ from conans.test.utils.test_files import temp_folder from conans.tools import unzip, save -from conans.util.files import load +from conans.util.files import load, save_files from conans.errors import ConanException +from conans.test.utils.tools import TestClient, TestServer +from conans.model.ref import ConanFileReference, PackageReference class XZTest(TestCase): + def test_error_xz(self): + server = TestServer() + ref = ConanFileReference.loads("Pkg/0.1@user/channel") + export = server.paths.export(ref) + save_files(export, {"conanfile.py": "#", + "conanmanifest.txt": "#", + "conan_export.txz": "#"}) + client = TestClient(servers={"default": server}, + users={"default": [("lasote", "mypass")]}) + error = client.run("install Pkg/0.1@user/channel", ignore_error=True) + self.assertTrue(error) + self.assertIn("ERROR: This Conan version is not prepared to handle " + "'conan_export.txz' file format", client.out) + + def test_error_sources_xz(self): + server = TestServer() + ref = ConanFileReference.loads("Pkg/0.1@user/channel") + client = TestClient(servers={"default": server}, + users={"default": [("lasote", "mypass")]}) + export = server.paths.export(ref) + conanfile = """from conans import ConanFile +class Pkg(ConanFile): + exports_sources = "*" +""" + save_files(export, {"conanfile.py": conanfile, + "conanmanifest.txt": "1", + "conan_sources.txz": "#"}) + error = client.run("install Pkg/0.1@user/channel --build", ignore_error=True) + self.assertTrue(error) + self.assertIn("ERROR: This Conan version is not prepared to handle " + "'conan_sources.txz' file format", client.out) + + def test_error_package_xz(self): + server = TestServer() + ref = ConanFileReference.loads("Pkg/0.1@user/channel") + client = TestClient(servers={"default": server}, + users={"default": [("lasote", "mypass")]}) + export = server.paths.export(ref) + conanfile = """from conans import ConanFile +class Pkg(ConanFile): + exports_sources = "*" +""" + save_files(export, {"conanfile.py": conanfile, + "conanmanifest.txt": "1"}) + pkg_ref = PackageReference(ref, "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9") + package = server.paths.package(pkg_ref) + save_files(package, {"conaninfo.txt": "#", + "conanmanifest.txt": "1", + "conan_package.txz": "#"}) + error = client.run("install Pkg/0.1@user/channel", ignore_error=True) + self.assertTrue(error) + self.assertIn("ERROR: This Conan version is not prepared to handle " + "'conan_package.txz' file format", client.out) @unittest.skipUnless(six.PY3, "only Py3") def test(self): From a8bbdd54048f1307b2c83639871d65ca8054f801 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 12 Jul 2018 11:17:59 +0200 Subject: [PATCH 5/5] fix broken tests --- conans/client/remote_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 7b3d1173755..929174655b1 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -202,8 +202,8 @@ def get_recipe_sources(self, conan_reference, export_folder, export_sources_fold t1 = time.time() def filter_function(urls): + file_url = urls.pop(EXPORT_SOURCES_TGZ_NAME, None) check_compressed_files(EXPORT_SOURCES_TGZ_NAME, urls) - file_url = urls.get(EXPORT_SOURCES_TGZ_NAME) if file_url: urls = {EXPORT_SOURCES_TGZ_NAME: file_url} else: