From 61c5de793c756271ec2daf5462b685686d6bd999 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 29 Jan 2022 12:15:13 +0100 Subject: [PATCH 01/14] Only share environment with subprocesses if needed --- libmamba/src/api/install.cpp | 1 + libmamba/src/core/link.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index 2802f9999c..3f6f1d1574 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -58,6 +58,7 @@ namespace mamba auto [wrapped_command, tmpfile] = prepare_wrapped_call(ctx.target_prefix, install_args); reproc::options options; + options.env.behavior = reproc::env::empty; options.redirect.parent = true; options.working_directory = cwd.c_str(); diff --git a/libmamba/src/core/link.cpp b/libmamba/src/core/link.cpp index a9467ea44a..c82fa21967 100644 --- a/libmamba/src/core/link.cpp +++ b/libmamba/src/core/link.cpp @@ -414,7 +414,11 @@ namespace mamba reproc::options options; options.redirect.parent = true; - options.env.behavior = reproc::env::extend; + if (activate) { + options.env.behavior = reproc::env::empty; + } else { + options.env.behavior = reproc::env::extend; + } options.env.extra = envmap; std::string cwd = path.parent_path(); options.working_directory = cwd.c_str(); @@ -703,6 +707,7 @@ namespace mamba if (binary_changed && m_pkg_info.subdir == "osx-arm64") { reproc::options options; + options.env.behavior = reproc::env::empty; if (Context::instance().verbosity <= 1) { reproc::redirect silence; @@ -838,6 +843,7 @@ namespace mamba } reproc::options options; + options.env.behavior = reproc::env::empty; std::string out, err; std::string cwd = m_context->target_prefix; From 336ca3a989457bb95dc0466c4bba01020f276b71 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Fri, 28 Jan 2022 13:25:53 +0100 Subject: [PATCH 02/14] Show a warning if repodata_record.json generation fails in constructor --- micromamba/src/constructor.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/micromamba/src/constructor.cpp b/micromamba/src/constructor.cpp index 9f4617be5d..63b874ecb4 100644 --- a/micromamba/src/constructor.cpp +++ b/micromamba/src/constructor.cpp @@ -97,6 +97,7 @@ construct(const fs::path& prefix, bool extract_conda_pkgs, bool extract_tarball) std::string pkg_name = index["name"]; index["fn"] = entry.path().filename(); + bool found_match = false; for (const auto& pkg_info : package_details) { if (pkg_info.fn == entry.path().filename()) @@ -112,9 +113,14 @@ construct(const fs::path& prefix, bool extract_conda_pkgs, bool extract_tarball) { index["sha256"] = pkg_info.sha256; } + found_match = true; break; } } + if (!found_match) + { + LOG_WARNING << "Failed to add extra info to " << repodata_record_path << std::endl; + } LOG_TRACE << "Writing " << repodata_record_path; std::ofstream repodata_record(repodata_record_path); From 56ad68ee95fe2dd892ac54985d83642fbba03429 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 31 Jan 2022 17:25:55 +0100 Subject: [PATCH 03/14] Optimise compilation of python sources for "noarch: python" --- .../mamba/core/transaction_context.hpp | 14 ++ libmamba/src/core/link.cpp | 82 +------- libmamba/src/core/package_handling.cpp | 7 + libmamba/src/core/transaction.cpp | 2 + libmamba/src/core/transaction_context.cpp | 189 ++++++++++++++++++ 5 files changed, 218 insertions(+), 76 deletions(-) diff --git a/libmamba/include/mamba/core/transaction_context.hpp b/libmamba/include/mamba/core/transaction_context.hpp index 35e192a364..9422445f3c 100644 --- a/libmamba/include/mamba/core/transaction_context.hpp +++ b/libmamba/include/mamba/core/transaction_context.hpp @@ -9,9 +9,12 @@ #include +#include + #include "context.hpp" #include "mamba_fs.hpp" #include "match_spec.hpp" +#include "util.hpp" namespace mamba { @@ -27,9 +30,13 @@ namespace mamba { public: TransactionContext(); + TransactionContext& operator=(const TransactionContext&); TransactionContext(const fs::path& prefix, const std::string& py_version, const std::vector& requested_specs); + ~TransactionContext(); + bool try_pyc_compation(const std::vector& py_files); + void wait_for_pyc_compation(); bool has_python; fs::path target_prefix; @@ -42,6 +49,13 @@ namespace mamba bool always_softlink = false; bool compile_pyc = true; std::vector requested_specs; + + private: + bool start_pyc_compilation_process(); + + std::unique_ptr m_pyc_process = nullptr; + std::unique_ptr m_pyc_script_file = nullptr; + std::unique_ptr m_pyc_compileall = nullptr; }; } // namespace mamba diff --git a/libmamba/src/core/link.cpp b/libmamba/src/core/link.cpp index c82fa21967..f108efd043 100644 --- a/libmamba/src/core/link.cpp +++ b/libmamba/src/core/link.cpp @@ -11,6 +11,8 @@ #include "termcolor/termcolor.hpp" #include +#include +#include #include "mamba/core/environment.hpp" #include "mamba/core/menuinst.hpp" @@ -794,88 +796,16 @@ namespace mamba if (py_files.size() == 0) return {}; - if (!m_context->has_python) - { - LOG_WARNING << "Can't compile pyc: Python not found"; - return {}; - } - std::vector pyc_files; - TemporaryFile all_py_files; - std::ofstream all_py_files_f = open_ofstream(all_py_files.path()); - for (auto& f : py_files) { - all_py_files_f << f.c_str() << '\n'; pyc_files.push_back(pyc_path(f, m_context->short_python_version)); - LOG_TRACE << "Compiling " << pyc_files.back(); } - LOG_INFO << "Compiling " << pyc_files.size() << " python files for " << m_pkg_info.name - << " to pyc"; - - all_py_files_f.close(); - - std::vector command = { m_context->target_prefix / m_context->python_path, - "-Wi", - "-m", - "compileall", - "-q", - "-l", - "-i", - all_py_files.path() }; - - auto py_ver_split = split(m_context->python_version, "."); - - try - { - if (std::stoull(std::string(py_ver_split[0])) >= 3 - && std::stoull(std::string(py_ver_split[1])) > 5) - { - // activate parallel pyc compilation - command.push_back("-j0"); - } - } - catch (const std::exception& e) - { - LOG_ERROR << "Bad conversion of Python version '" << m_context->python_version - << "': " << e.what(); - throw std::runtime_error("Bad conversion. Aborting."); - } - - reproc::options options; - options.env.behavior = reproc::env::empty; - std::string out, err; - - std::string cwd = m_context->target_prefix; - options.working_directory = cwd.c_str(); - - auto [wrapped_command, script_file] - = prepare_wrapped_call(m_context->target_prefix, command); - - LOG_DEBUG << "Running wrapped python compilation command " << join(" ", command); - auto [_, ec] = reproc::run( - wrapped_command, options, reproc::sink::string(out), reproc::sink::string(err)); - - if (ec || !err.empty()) - { - LOG_INFO << "noarch pyc compilation failed (cross-compiling?). " << ec.message(); - LOG_INFO << err; - } - - std::vector final_pyc_files; - for (auto& f : pyc_files) - { - if (fs::exists(m_context->target_prefix / f)) - { - final_pyc_files.push_back(f); - } - else if (!ec) - { - LOG_WARNING << "Python file couldn't be compiled to pyc: " << f; - } + if (m_context->try_pyc_compation(py_files)) { + return pyc_files; + } else { + return {}; } - - return final_pyc_files; } enum class NoarchType diff --git a/libmamba/src/core/package_handling.cpp b/libmamba/src/core/package_handling.cpp index bbe3a52d2d..7e5f98aa2c 100644 --- a/libmamba/src/core/package_handling.cpp +++ b/libmamba/src/core/package_handling.cpp @@ -461,6 +461,13 @@ namespace mamba } } + fs::path extract_subproc(const fs::path& file) + { + fs::path dest_dir = extract_dest_dir(file); + extract_subproc(file, dest_dir); + return dest_dir; + } + bool transmute(const fs::path& pkg_file, const fs::path& target, int compression_level) { TemporaryDirectory extract_dir; diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp index fe6a1ac821..1982860ffb 100644 --- a/libmamba/src/core/transaction.cpp +++ b/libmamba/src/core/transaction.cpp @@ -866,6 +866,8 @@ namespace mamba } else { + LOG_INFO << "Waiting for pyc compilation to finish"; + m_transaction_context.wait_for_pyc_compation(); Console::stream() << "Transaction finished"; prefix.history().add_entry(m_history_entry); } diff --git a/libmamba/src/core/transaction_context.cpp b/libmamba/src/core/transaction_context.cpp index 299c2b054a..bdd51840b2 100644 --- a/libmamba/src/core/transaction_context.cpp +++ b/libmamba/src/core/transaction_context.cpp @@ -3,6 +3,11 @@ // Distributed under the terms of the BSD 3-Clause License. // // The full license is in the file LICENSE, distributed with this software. +#ifndef _WIN32 +#include +#endif + +#include #include "mamba/core/transaction_context.hpp" #include "mamba/core/output.hpp" @@ -10,6 +15,25 @@ namespace mamba { + void compile_python_sources(std::ostream& out) + { + out << "from compileall import compile_file\n"; + out << "from concurrent.futures import ProcessPoolExecutor\n"; + out << "import sys\n"; + + out << "results = []\n"; + out << "with sys.stdin:\n"; + out << " with ProcessPoolExecutor(max_workers=None) as executor:\n"; + out << " while True:\n"; + out << " name = sys.stdin.readline().strip()\n"; + out << " if not name:\n"; + out << " break\n"; + out << " results.append(executor.submit(compile_file, name, quiet=1))\n"; + out << " success = all(r.result() for r in results)\n"; + + out << "sys.exit(int(not success))\n"; + } + std::string compute_short_python_version(const std::string& long_version) { auto sv = split(long_version, "."); @@ -99,4 +123,169 @@ namespace mamba site_packages_path = get_python_site_packages_short_path(short_python_version); } } + + TransactionContext& TransactionContext::operator=(const TransactionContext& other) + { + if (this != &other) { + has_python = other.has_python; + target_prefix = other.target_prefix; + python_version = other.python_version; + requested_specs = other.requested_specs; + + compile_pyc = other.compile_pyc; + allow_softlinks = other.allow_softlinks; + always_copy = other.always_copy; + always_softlink = other.always_softlink; + short_python_version = other.short_python_version; + python_path = other.python_path; + site_packages_path = other.site_packages_path; + } + return *this; + } + + TransactionContext::~TransactionContext() { + wait_for_pyc_compation(); + } + + bool TransactionContext::start_pyc_compilation_process() + { + if (m_pyc_process) { + return true; + } + +#ifndef _WIN32 + std::signal(SIGPIPE, SIG_IGN); +#endif + + std::vector command = { target_prefix / python_path, + "-Wi", + "-m", + "compileall", + "-q", + "-l", + "-i", + "-"}; + + auto py_ver_split = split(python_version, "."); + + try + { + if (std::stoull(std::string(py_ver_split[0])) >= 3 + && std::stoull(std::string(py_ver_split[1])) > 5) + { + m_pyc_compileall = std::make_unique(); + std::ofstream compileall_f = open_ofstream(m_pyc_compileall->path()); + compile_python_sources(compileall_f); + compileall_f.close(); + + command = {target_prefix / python_path, "-Wi", "-u", m_pyc_compileall->path().c_str()}; + } + } + catch (const std::exception& e) + { + LOG_ERROR << "Bad conversion of Python version '" << python_version + << "': " << e.what(); + throw std::runtime_error("Bad conversion. Aborting."); + } + + m_pyc_process = std::make_unique(); + + reproc::options options; + options.env.behavior = reproc::env::empty; + options.stop = { + { reproc::stop::wait, reproc::milliseconds(10000) }, + { reproc::stop::terminate, reproc::milliseconds(5000) }, + { reproc::stop::kill, reproc::milliseconds(2000) }, + }; + + options.redirect.out.type = reproc::redirect::pipe; + options.redirect.err.type = reproc::redirect::pipe; + + std::string cwd = target_prefix; + options.working_directory = cwd.c_str(); + + auto [wrapped_command, script_file] + = prepare_wrapped_call(target_prefix, command); + m_pyc_script_file = std::move(script_file); + + LOG_INFO << "Running wrapped python compilation command " << join(" ", command); + std::error_code ec = m_pyc_process->start(wrapped_command, options); + + if (ec == std::errc::no_such_file_or_directory) { + LOG_ERROR << "Program not found. Make sure it's available from the PATH. " << ec.message(); + throw std::runtime_error("pyc compilation failed with program not found. Aborting."); + } + + return true; + } + + bool TransactionContext::try_pyc_compation(const std::vector& py_files) + { + static std::mutex pyc_compation_mutex; + std::lock_guard lock(pyc_compation_mutex); + + if (!has_python) + { + LOG_WARNING << "Can't compile pyc: Python not found"; + return false; + } + + start_pyc_compilation_process(); + if (!m_pyc_process) + { + return false; + } + + for (auto& f : py_files) + { + LOG_INFO << "Compiling " << f; + auto fs = f.string() + "\n"; + size_t nbytes; + std::error_code ec; + + std::tie(nbytes, ec) = m_pyc_process->write(reinterpret_cast(&fs[0]), fs.size()); + if (ec) { + LOG_INFO << "writing to stdin failed " << ec.message(); + return false; + } + } + + return true; + } + + void TransactionContext::wait_for_pyc_compation() { + if (m_pyc_process) { + std::error_code ec = m_pyc_process->close(reproc::stream::in); + if (ec) { + LOG_INFO << "closing stdin failed " << ec.message(); + } + + std::string output; + std::string err; + reproc::sink::string output_sink(output); + reproc::sink::string err_sink(err); + ec = reproc::drain(*m_pyc_process, output_sink, err_sink); + if (ec) { + LOG_INFO << "draining failed " << ec.message(); + } + + int status = 0; + std::tie(status, ec) = m_pyc_process->stop( + { + { reproc::stop::wait, reproc::milliseconds(100000) }, + { reproc::stop::terminate, reproc::milliseconds(5000) }, + { reproc::stop::kill, reproc::milliseconds(2000) }, + } + ); + if (ec || status != 0) { + LOG_INFO << "noarch pyc compilation failed (cross-compiling?)."; + if (ec) { + LOG_INFO << ec.message(); + } + LOG_INFO << "stdout:" << output; + LOG_INFO << "stdout:" << err; + } + m_pyc_process = nullptr; + } + } } From f58c60b65995ede8226cc8085ce4cdc26a532f29 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 1 Feb 2022 10:21:54 +0100 Subject: [PATCH 04/14] use __main__ function in compile pyc.py --- libmamba/data/compile_pyc.py | 22 ++++++++++++++++++++++ libmamba/src/core/transaction_context.cpp | 19 ++++--------------- 2 files changed, 26 insertions(+), 15 deletions(-) create mode 100644 libmamba/data/compile_pyc.py diff --git a/libmamba/data/compile_pyc.py b/libmamba/data/compile_pyc.py new file mode 100644 index 0000000000..39e0698e3f --- /dev/null +++ b/libmamba/data/compile_pyc.py @@ -0,0 +1,22 @@ +R"MAMBARAW( +from compileall import compile_file +from concurrent.futures import ProcessPoolExecutor +import sys + +def main(): + results = [] + with sys.stdin: + with ProcessPoolExecutor(max_workers=None) as executor: + while True: + name = sys.stdin.readline().strip() + if not name: + break + results.append(executor.submit(compile_file, name, quiet=1)) + success = all(r.result() for r in results) + return success + +if __name__ == "__main__": + success = main() + sys.exit(int(not success)) + +)MAMBARAW" diff --git a/libmamba/src/core/transaction_context.cpp b/libmamba/src/core/transaction_context.cpp index bdd51840b2..c98a3f7431 100644 --- a/libmamba/src/core/transaction_context.cpp +++ b/libmamba/src/core/transaction_context.cpp @@ -17,21 +17,10 @@ namespace mamba { void compile_python_sources(std::ostream& out) { - out << "from compileall import compile_file\n"; - out << "from concurrent.futures import ProcessPoolExecutor\n"; - out << "import sys\n"; - - out << "results = []\n"; - out << "with sys.stdin:\n"; - out << " with ProcessPoolExecutor(max_workers=None) as executor:\n"; - out << " while True:\n"; - out << " name = sys.stdin.readline().strip()\n"; - out << " if not name:\n"; - out << " break\n"; - out << " results.append(executor.submit(compile_file, name, quiet=1))\n"; - out << " success = all(r.result() for r in results)\n"; - - out << "sys.exit(int(not success))\n"; + constexpr const char script[] = +#include "../data/compile_pyc.hpp" + ; + out << script; } std::string compute_short_python_version(const std::string& long_version) From f90fe57bb7418edc1b94a89db399d759cc7a3eec Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 1 Feb 2022 10:30:05 +0100 Subject: [PATCH 05/14] use correct file name for include --- libmamba/src/core/transaction_context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmamba/src/core/transaction_context.cpp b/libmamba/src/core/transaction_context.cpp index c98a3f7431..fd3d69ee60 100644 --- a/libmamba/src/core/transaction_context.cpp +++ b/libmamba/src/core/transaction_context.cpp @@ -18,7 +18,7 @@ namespace mamba void compile_python_sources(std::ostream& out) { constexpr const char script[] = -#include "../data/compile_pyc.hpp" +#include "../data/compile_pyc.py" ; out << script; } From 73271a006d7c46b7098bac43f5e6e663cf9b4848 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Tue, 1 Feb 2022 16:21:13 +0100 Subject: [PATCH 06/14] correct typo, do not use empty env when compiling --- .../{compile_pyc.py => compile_pyc.py.hpp} | 0 .../mamba/core/transaction_context.hpp | 4 +- libmamba/src/core/link.cpp | 14 ++- libmamba/src/core/package_handling.cpp | 7 -- libmamba/src/core/transaction.cpp | 2 +- libmamba/src/core/transaction_context.cpp | 89 ++++++++++--------- micromamba/src/constructor.cpp | 3 +- 7 files changed, 62 insertions(+), 57 deletions(-) rename libmamba/data/{compile_pyc.py => compile_pyc.py.hpp} (100%) diff --git a/libmamba/data/compile_pyc.py b/libmamba/data/compile_pyc.py.hpp similarity index 100% rename from libmamba/data/compile_pyc.py rename to libmamba/data/compile_pyc.py.hpp diff --git a/libmamba/include/mamba/core/transaction_context.hpp b/libmamba/include/mamba/core/transaction_context.hpp index 9422445f3c..266e74a798 100644 --- a/libmamba/include/mamba/core/transaction_context.hpp +++ b/libmamba/include/mamba/core/transaction_context.hpp @@ -35,8 +35,8 @@ namespace mamba const std::string& py_version, const std::vector& requested_specs); ~TransactionContext(); - bool try_pyc_compation(const std::vector& py_files); - void wait_for_pyc_compation(); + bool try_pyc_compilation(const std::vector& py_files); + void wait_for_pyc_compilation(); bool has_python; fs::path target_prefix; diff --git a/libmamba/src/core/link.cpp b/libmamba/src/core/link.cpp index f108efd043..453cc1fb5a 100644 --- a/libmamba/src/core/link.cpp +++ b/libmamba/src/core/link.cpp @@ -416,9 +416,12 @@ namespace mamba reproc::options options; options.redirect.parent = true; - if (activate) { + if (activate) + { options.env.behavior = reproc::env::empty; - } else { + } + else + { options.env.behavior = reproc::env::extend; } options.env.extra = envmap; @@ -801,9 +804,12 @@ namespace mamba { pyc_files.push_back(pyc_path(f, m_context->short_python_version)); } - if (m_context->try_pyc_compation(py_files)) { + if (m_context->try_pyc_compilation(py_files)) + { return pyc_files; - } else { + } + else + { return {}; } } diff --git a/libmamba/src/core/package_handling.cpp b/libmamba/src/core/package_handling.cpp index 7e5f98aa2c..bbe3a52d2d 100644 --- a/libmamba/src/core/package_handling.cpp +++ b/libmamba/src/core/package_handling.cpp @@ -461,13 +461,6 @@ namespace mamba } } - fs::path extract_subproc(const fs::path& file) - { - fs::path dest_dir = extract_dest_dir(file); - extract_subproc(file, dest_dir); - return dest_dir; - } - bool transmute(const fs::path& pkg_file, const fs::path& target, int compression_level) { TemporaryDirectory extract_dir; diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp index 1982860ffb..1c3fd16bca 100644 --- a/libmamba/src/core/transaction.cpp +++ b/libmamba/src/core/transaction.cpp @@ -867,7 +867,7 @@ namespace mamba else { LOG_INFO << "Waiting for pyc compilation to finish"; - m_transaction_context.wait_for_pyc_compation(); + m_transaction_context.wait_for_pyc_compilation(); Console::stream() << "Transaction finished"; prefix.history().add_entry(m_history_entry); } diff --git a/libmamba/src/core/transaction_context.cpp b/libmamba/src/core/transaction_context.cpp index fd3d69ee60..05c7787b15 100644 --- a/libmamba/src/core/transaction_context.cpp +++ b/libmamba/src/core/transaction_context.cpp @@ -18,7 +18,7 @@ namespace mamba void compile_python_sources(std::ostream& out) { constexpr const char script[] = -#include "../data/compile_pyc.py" +#include "../data/compile_pyc.py.hpp" ; out << script; } @@ -115,7 +115,8 @@ namespace mamba TransactionContext& TransactionContext::operator=(const TransactionContext& other) { - if (this != &other) { + if (this != &other) + { has_python = other.has_python; target_prefix = other.target_prefix; python_version = other.python_version; @@ -132,13 +133,15 @@ namespace mamba return *this; } - TransactionContext::~TransactionContext() { - wait_for_pyc_compation(); + TransactionContext::~TransactionContext() + { + wait_for_pyc_compilation(); } bool TransactionContext::start_pyc_compilation_process() { - if (m_pyc_process) { + if (m_pyc_process) + { return true; } @@ -146,14 +149,8 @@ namespace mamba std::signal(SIGPIPE, SIG_IGN); #endif - std::vector command = { target_prefix / python_path, - "-Wi", - "-m", - "compileall", - "-q", - "-l", - "-i", - "-"}; + std::vector command + = { target_prefix / python_path, "-Wi", "-m", "compileall", "-q", "-l", "-i", "-" }; auto py_ver_split = split(python_version, "."); @@ -167,7 +164,9 @@ namespace mamba compile_python_sources(compileall_f); compileall_f.close(); - command = {target_prefix / python_path, "-Wi", "-u", m_pyc_compileall->path().c_str()}; + command = { + target_prefix / python_path, "-Wi", "-u", m_pyc_compileall->path().c_str() + }; } } catch (const std::exception& e) @@ -180,7 +179,7 @@ namespace mamba m_pyc_process = std::make_unique(); reproc::options options; - options.env.behavior = reproc::env::empty; + // options.env.behavior = reproc::env::empty; options.stop = { { reproc::stop::wait, reproc::milliseconds(10000) }, { reproc::stop::terminate, reproc::milliseconds(5000) }, @@ -193,25 +192,26 @@ namespace mamba std::string cwd = target_prefix; options.working_directory = cwd.c_str(); - auto [wrapped_command, script_file] - = prepare_wrapped_call(target_prefix, command); + auto [wrapped_command, script_file] = prepare_wrapped_call(target_prefix, command); m_pyc_script_file = std::move(script_file); LOG_INFO << "Running wrapped python compilation command " << join(" ", command); std::error_code ec = m_pyc_process->start(wrapped_command, options); - if (ec == std::errc::no_such_file_or_directory) { - LOG_ERROR << "Program not found. Make sure it's available from the PATH. " << ec.message(); + if (ec == std::errc::no_such_file_or_directory) + { + LOG_ERROR << "Program not found. Make sure it's available from the PATH. " + << ec.message(); throw std::runtime_error("pyc compilation failed with program not found. Aborting."); } return true; } - bool TransactionContext::try_pyc_compation(const std::vector& py_files) + bool TransactionContext::try_pyc_compilation(const std::vector& py_files) { - static std::mutex pyc_compation_mutex; - std::lock_guard lock(pyc_compation_mutex); + static std::mutex pyc_compilation_mutex; + std::lock_guard lock(pyc_compilation_mutex); if (!has_python) { @@ -229,11 +229,11 @@ namespace mamba { LOG_INFO << "Compiling " << f; auto fs = f.string() + "\n"; - size_t nbytes; - std::error_code ec; - std::tie(nbytes, ec) = m_pyc_process->write(reinterpret_cast(&fs[0]), fs.size()); - if (ec) { + auto [nbytes, ec] + = m_pyc_process->write(reinterpret_cast(&fs[0]), fs.size()); + if (ec) + { LOG_INFO << "writing to stdin failed " << ec.message(); return false; } @@ -242,11 +242,15 @@ namespace mamba return true; } - void TransactionContext::wait_for_pyc_compation() { - if (m_pyc_process) { - std::error_code ec = m_pyc_process->close(reproc::stream::in); - if (ec) { - LOG_INFO << "closing stdin failed " << ec.message(); + void TransactionContext::wait_for_pyc_compilation() + { + if (m_pyc_process) + { + std::error_code ec; + ec = m_pyc_process->close(reproc::stream::in); + if (ec) + { + LOG_WARNING << "closing stdin failed " << ec.message(); } std::string output; @@ -254,21 +258,22 @@ namespace mamba reproc::sink::string output_sink(output); reproc::sink::string err_sink(err); ec = reproc::drain(*m_pyc_process, output_sink, err_sink); - if (ec) { - LOG_INFO << "draining failed " << ec.message(); + if (ec) + { + LOG_WARNING << "draining failed " << ec.message(); } int status = 0; - std::tie(status, ec) = m_pyc_process->stop( - { - { reproc::stop::wait, reproc::milliseconds(100000) }, - { reproc::stop::terminate, reproc::milliseconds(5000) }, - { reproc::stop::kill, reproc::milliseconds(2000) }, - } - ); - if (ec || status != 0) { + std::tie(status, ec) = m_pyc_process->stop({ + { reproc::stop::wait, reproc::milliseconds(100000) }, + { reproc::stop::terminate, reproc::milliseconds(5000) }, + { reproc::stop::kill, reproc::milliseconds(2000) }, + }); + if (ec || status != 0) + { LOG_INFO << "noarch pyc compilation failed (cross-compiling?)."; - if (ec) { + if (ec) + { LOG_INFO << ec.message(); } LOG_INFO << "stdout:" << output; diff --git a/micromamba/src/constructor.cpp b/micromamba/src/constructor.cpp index 63b874ecb4..4bfcc6f792 100644 --- a/micromamba/src/constructor.cpp +++ b/micromamba/src/constructor.cpp @@ -119,7 +119,8 @@ construct(const fs::path& prefix, bool extract_conda_pkgs, bool extract_tarball) } if (!found_match) { - LOG_WARNING << "Failed to add extra info to " << repodata_record_path << std::endl; + LOG_WARNING << "Failed to add extra info to " << repodata_record_path + << std::endl; } LOG_TRACE << "Writing " << repodata_record_path; From 63d9fa91d53fb6e2b06cccaa90e2f30f3951f88a Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Tue, 1 Feb 2022 22:29:02 +0100 Subject: [PATCH 07/14] Add test of pyc compilation --- micromamba/tests/test_create.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/micromamba/tests/test_create.py b/micromamba/tests/test_create.py index a93985b319..98c6cc2580 100644 --- a/micromamba/tests/test_create.py +++ b/micromamba/tests/test_create.py @@ -538,3 +538,35 @@ def test_set_platform(self, existing_cache): assert "__archspec=1=x86" in res["virtual packages"] assert "__win=0=0" in res["virtual packages"] assert res["platform"] == "win-32" + + @pytest.mark.skipif( + dry_run_tests is DryRun.ULTRA_DRY, reason="Running only ultra-dry tests" + ) + @pytest.mark.parametrize( + "version,build,cache_tag", + [ + ["2.7", "*", ""], + ["3.10", "*_cpython", "cpython-310"], + [ "3.7", "*_pypy","pypy37"], + ], + ) + def test_pyc_compilation(self, version, build, cache_tag): + prefix = Path(TestCreate.prefix) + site_packages = prefix / "lib" / f"python{version}" / "site-packages" + if cache_tag: + pyc_fn = Path("__pycache__") / f"six.{cache_tag}.pyc" + else: + pyc_fn = Path(f"six.pyc") + + cmd = ["-n", TestCreate.env_name, f"python={version}.*={build}", "six"] + + # Disable pyc compilation to ensure that files are still registered in conda-meta + create(*cmd, "--no-pyc") + assert not (site_packages / pyc_fn).exists() + six_meta = next((prefix / "conda-meta").glob("six-*.json")).read_text() + assert pyc_fn.name in six_meta + + # Enable pyc compilation to ensure that the pyc files are created + create(*cmd) + assert (site_packages / pyc_fn).exists() + assert pyc_fn.name in six_meta From 389261df77236202bc6a76a02c1a50197f12d73b Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 2 Feb 2022 08:44:40 +0100 Subject: [PATCH 08/14] Always write pyc files to conda-meta/*.json --- libmamba/src/core/link.cpp | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/libmamba/src/core/link.cpp b/libmamba/src/core/link.cpp index 453cc1fb5a..dc59c539d3 100644 --- a/libmamba/src/core/link.cpp +++ b/libmamba/src/core/link.cpp @@ -804,14 +804,11 @@ namespace mamba { pyc_files.push_back(pyc_path(f, m_context->short_python_version)); } - if (m_context->try_pyc_compilation(py_files)) + if (m_context->compile_pyc) { - return pyc_files; - } - else - { - return {}; + m_context->try_pyc_compilation(py_files); } + return pyc_files; } enum class NoarchType @@ -994,17 +991,13 @@ namespace mamba } } - if (m_context->compile_pyc) + std::vector pyc_files = compile_pyc_files(for_compilation); + for (const fs::path& pyc_path : pyc_files) { - std::vector pyc_files = compile_pyc_files(for_compilation); + out_json["paths_data"]["paths"].push_back( + { { "_path", std::string(pyc_path) }, { "path_type", "pyc_file" } }); - for (const fs::path& pyc_path : pyc_files) - { - out_json["paths_data"]["paths"].push_back( - { { "_path", std::string(pyc_path) }, { "path_type", "pyc_file" } }); - - out_json["files"].push_back(pyc_path); - } + out_json["files"].push_back(pyc_path); } if (link_json.find("noarch") != link_json.end() From 92d1eb3df5dc2ffddbbe83166f07d42a0b9d8760 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 2 Feb 2022 08:44:46 +0100 Subject: [PATCH 09/14] Disable testing pypy for test_pyc_compilation as it's expected to fail --- micromamba/tests/test_create.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/micromamba/tests/test_create.py b/micromamba/tests/test_create.py index 98c6cc2580..0bdc130fc5 100644 --- a/micromamba/tests/test_create.py +++ b/micromamba/tests/test_create.py @@ -547,7 +547,8 @@ def test_set_platform(self, existing_cache): [ ["2.7", "*", ""], ["3.10", "*_cpython", "cpython-310"], - [ "3.7", "*_pypy","pypy37"], + # FIXME: https://github.com/mamba-org/mamba/issues/1432 + # [ "3.7", "*_pypy","pypy37"], ], ) def test_pyc_compilation(self, version, build, cache_tag): From eaa4112b877a23b80c8b051c89ffcd6d140022ac Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 2 Feb 2022 08:42:10 +0100 Subject: [PATCH 10/14] Clean environment for compiling pyc files --- libmamba/src/core/transaction_context.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/libmamba/src/core/transaction_context.cpp b/libmamba/src/core/transaction_context.cpp index 05c7787b15..10dde51a6a 100644 --- a/libmamba/src/core/transaction_context.cpp +++ b/libmamba/src/core/transaction_context.cpp @@ -9,6 +9,7 @@ #include +#include "mamba/core/environment.hpp" #include "mamba/core/transaction_context.hpp" #include "mamba/core/output.hpp" #include "mamba/core/util.hpp" @@ -179,7 +180,15 @@ namespace mamba m_pyc_process = std::make_unique(); reproc::options options; - // options.env.behavior = reproc::env::empty; + options.env.behavior = reproc::env::empty; + +#ifdef _WIN32 + std::map envmap; + std::string PATH = env::get("PATH").value_or(""); + envmap["PATH"] = PATH; + options.env.extra = envmap; +#endif + options.stop = { { reproc::stop::wait, reproc::milliseconds(10000) }, { reproc::stop::terminate, reproc::milliseconds(5000) }, From b4d8e5659fe64bc4d13e41c509a0e062d343eda2 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 2 Feb 2022 11:10:22 +0100 Subject: [PATCH 11/14] Fix test_pyc_compilation for Windows --- micromamba/tests/test_create.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/micromamba/tests/test_create.py b/micromamba/tests/test_create.py index 0bdc130fc5..aed2451739 100644 --- a/micromamba/tests/test_create.py +++ b/micromamba/tests/test_create.py @@ -553,7 +553,12 @@ def test_set_platform(self, existing_cache): ) def test_pyc_compilation(self, version, build, cache_tag): prefix = Path(TestCreate.prefix) - site_packages = prefix / "lib" / f"python{version}" / "site-packages" + + if platform.system() == "Windows": + site_packages = prefix / "Lib" / "site-packages" + else: + site_packages = prefix / "lib" / f"python{version}" / "site-packages" + if cache_tag: pyc_fn = Path("__pycache__") / f"six.{cache_tag}.pyc" else: From 59eb6203c227e79429c1eb85fff9803444782dfd Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 2 Feb 2022 11:50:20 +0100 Subject: [PATCH 12/14] Keep the host environment for pyc compilation on Windows --- libmamba/src/core/transaction_context.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/libmamba/src/core/transaction_context.cpp b/libmamba/src/core/transaction_context.cpp index 10dde51a6a..b45d5a585d 100644 --- a/libmamba/src/core/transaction_context.cpp +++ b/libmamba/src/core/transaction_context.cpp @@ -9,7 +9,6 @@ #include -#include "mamba/core/environment.hpp" #include "mamba/core/transaction_context.hpp" #include "mamba/core/output.hpp" #include "mamba/core/util.hpp" @@ -180,13 +179,8 @@ namespace mamba m_pyc_process = std::make_unique(); reproc::options options; +#ifndef _WIN32 options.env.behavior = reproc::env::empty; - -#ifdef _WIN32 - std::map envmap; - std::string PATH = env::get("PATH").value_or(""); - envmap["PATH"] = PATH; - options.env.extra = envmap; #endif options.stop = { From 43b3d21e34a4cdad72d264bd85b2e2166a1678f2 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 2 Feb 2022 12:41:09 +0100 Subject: [PATCH 13/14] Python 2.7 on Windows requires -c default --- micromamba/tests/test_create.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/micromamba/tests/test_create.py b/micromamba/tests/test_create.py index aed2451739..0b9b638fed 100644 --- a/micromamba/tests/test_create.py +++ b/micromamba/tests/test_create.py @@ -553,9 +553,12 @@ def test_set_platform(self, existing_cache): ) def test_pyc_compilation(self, version, build, cache_tag): prefix = Path(TestCreate.prefix) + cmd = ["-n", TestCreate.env_name, f"python={version}.*={build}", "six"] if platform.system() == "Windows": site_packages = prefix / "Lib" / "site-packages" + if version == "2.7": + cmd += ["-c", "defaults"] # for vc=9.* else: site_packages = prefix / "lib" / f"python{version}" / "site-packages" @@ -564,8 +567,6 @@ def test_pyc_compilation(self, version, build, cache_tag): else: pyc_fn = Path(f"six.pyc") - cmd = ["-n", TestCreate.env_name, f"python={version}.*={build}", "six"] - # Disable pyc compilation to ensure that files are still registered in conda-meta create(*cmd, "--no-pyc") assert not (site_packages / pyc_fn).exists() From ea5c0ef12b7a7e20003be1a5e90ee147589346e7 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 2 Feb 2022 15:49:18 +0100 Subject: [PATCH 14/14] Respect MAMBA_EXTRACT_THREADS when compiling pyc sources --- libmamba/data/compile_pyc.py.hpp | 7 ++++++- libmamba/src/core/transaction_context.cpp | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/libmamba/data/compile_pyc.py.hpp b/libmamba/data/compile_pyc.py.hpp index 39e0698e3f..e7cf9f5420 100644 --- a/libmamba/data/compile_pyc.py.hpp +++ b/libmamba/data/compile_pyc.py.hpp @@ -1,12 +1,17 @@ R"MAMBARAW( from compileall import compile_file from concurrent.futures import ProcessPoolExecutor +import os import sys def main(): + max_workers = int(os.environ.get("MAMBA_EXTRACT_THREADS", "0")) + if max_workers <= 0: + max_workers = None + results = [] with sys.stdin: - with ProcessPoolExecutor(max_workers=None) as executor: + with ProcessPoolExecutor(max_workers=max_workers) as executor: while True: name = sys.stdin.readline().strip() if not name: diff --git a/libmamba/src/core/transaction_context.cpp b/libmamba/src/core/transaction_context.cpp index b45d5a585d..e080329fa4 100644 --- a/libmamba/src/core/transaction_context.cpp +++ b/libmamba/src/core/transaction_context.cpp @@ -182,6 +182,10 @@ namespace mamba #ifndef _WIN32 options.env.behavior = reproc::env::empty; #endif + std::map envmap; + auto& ctx = Context::instance(); + envmap["MAMBA_EXTRACT_THREADS"] = std::to_string(ctx.extract_threads); + options.env.extra = envmap; options.stop = { { reproc::stop::wait, reproc::milliseconds(10000) },