diff --git a/libmamba/data/compile_pyc.py.hpp b/libmamba/data/compile_pyc.py.hpp new file mode 100644 index 0000000000..e7cf9f5420 --- /dev/null +++ b/libmamba/data/compile_pyc.py.hpp @@ -0,0 +1,27 @@ +R"MAMBARAW( +from compileall import compile_file +from concurrent.futures import ProcessPoolExecutor +import os +import sys + +def main(): + max_workers = int(os.environ.get("MAMBA_EXTRACT_THREADS", "0")) + if max_workers <= 0: + max_workers = None + + results = [] + with sys.stdin: + with ProcessPoolExecutor(max_workers=max_workers) as executor: + while True: + name = sys.stdin.readline().strip() + if not name: + break + results.append(executor.submit(compile_file, name, quiet=1)) + success = all(r.result() for r in results) + return success + +if __name__ == "__main__": + success = main() + sys.exit(int(not success)) + +)MAMBARAW" diff --git a/libmamba/include/mamba/core/transaction_context.hpp b/libmamba/include/mamba/core/transaction_context.hpp index 35e192a364..266e74a798 100644 --- a/libmamba/include/mamba/core/transaction_context.hpp +++ b/libmamba/include/mamba/core/transaction_context.hpp @@ -9,9 +9,12 @@ #include +#include + #include "context.hpp" #include "mamba_fs.hpp" #include "match_spec.hpp" +#include "util.hpp" namespace mamba { @@ -27,9 +30,13 @@ namespace mamba { public: TransactionContext(); + TransactionContext& operator=(const TransactionContext&); TransactionContext(const fs::path& prefix, const std::string& py_version, const std::vector& requested_specs); + ~TransactionContext(); + bool try_pyc_compilation(const std::vector& py_files); + void wait_for_pyc_compilation(); bool has_python; fs::path target_prefix; @@ -42,6 +49,13 @@ namespace mamba bool always_softlink = false; bool compile_pyc = true; std::vector requested_specs; + + private: + bool start_pyc_compilation_process(); + + std::unique_ptr m_pyc_process = nullptr; + std::unique_ptr m_pyc_script_file = nullptr; + std::unique_ptr m_pyc_compileall = nullptr; }; } // namespace mamba diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index 2802f9999c..3f6f1d1574 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -58,6 +58,7 @@ namespace mamba auto [wrapped_command, tmpfile] = prepare_wrapped_call(ctx.target_prefix, install_args); reproc::options options; + options.env.behavior = reproc::env::empty; options.redirect.parent = true; options.working_directory = cwd.c_str(); diff --git a/libmamba/src/core/link.cpp b/libmamba/src/core/link.cpp index a9467ea44a..dc59c539d3 100644 --- a/libmamba/src/core/link.cpp +++ b/libmamba/src/core/link.cpp @@ -11,6 +11,8 @@ #include "termcolor/termcolor.hpp" #include +#include +#include #include "mamba/core/environment.hpp" #include "mamba/core/menuinst.hpp" @@ -414,7 +416,14 @@ namespace mamba reproc::options options; options.redirect.parent = true; - options.env.behavior = reproc::env::extend; + if (activate) + { + options.env.behavior = reproc::env::empty; + } + else + { + options.env.behavior = reproc::env::extend; + } options.env.extra = envmap; std::string cwd = path.parent_path(); options.working_directory = cwd.c_str(); @@ -703,6 +712,7 @@ namespace mamba if (binary_changed && m_pkg_info.subdir == "osx-arm64") { reproc::options options; + options.env.behavior = reproc::env::empty; if (Context::instance().verbosity <= 1) { reproc::redirect silence; @@ -789,87 +799,16 @@ namespace mamba if (py_files.size() == 0) return {}; - if (!m_context->has_python) - { - LOG_WARNING << "Can't compile pyc: Python not found"; - return {}; - } - std::vector pyc_files; - TemporaryFile all_py_files; - std::ofstream all_py_files_f = open_ofstream(all_py_files.path()); - for (auto& f : py_files) { - all_py_files_f << f.c_str() << '\n'; pyc_files.push_back(pyc_path(f, m_context->short_python_version)); - LOG_TRACE << "Compiling " << pyc_files.back(); } - LOG_INFO << "Compiling " << pyc_files.size() << " python files for " << m_pkg_info.name - << " to pyc"; - - all_py_files_f.close(); - - std::vector command = { m_context->target_prefix / m_context->python_path, - "-Wi", - "-m", - "compileall", - "-q", - "-l", - "-i", - all_py_files.path() }; - - auto py_ver_split = split(m_context->python_version, "."); - - try - { - if (std::stoull(std::string(py_ver_split[0])) >= 3 - && std::stoull(std::string(py_ver_split[1])) > 5) - { - // activate parallel pyc compilation - command.push_back("-j0"); - } - } - catch (const std::exception& e) + if (m_context->compile_pyc) { - LOG_ERROR << "Bad conversion of Python version '" << m_context->python_version - << "': " << e.what(); - throw std::runtime_error("Bad conversion. Aborting."); + m_context->try_pyc_compilation(py_files); } - - reproc::options options; - std::string out, err; - - std::string cwd = m_context->target_prefix; - options.working_directory = cwd.c_str(); - - auto [wrapped_command, script_file] - = prepare_wrapped_call(m_context->target_prefix, command); - - LOG_DEBUG << "Running wrapped python compilation command " << join(" ", command); - auto [_, ec] = reproc::run( - wrapped_command, options, reproc::sink::string(out), reproc::sink::string(err)); - - if (ec || !err.empty()) - { - LOG_INFO << "noarch pyc compilation failed (cross-compiling?). " << ec.message(); - LOG_INFO << err; - } - - std::vector final_pyc_files; - for (auto& f : pyc_files) - { - if (fs::exists(m_context->target_prefix / f)) - { - final_pyc_files.push_back(f); - } - else if (!ec) - { - LOG_WARNING << "Python file couldn't be compiled to pyc: " << f; - } - } - - return final_pyc_files; + return pyc_files; } enum class NoarchType @@ -1052,17 +991,13 @@ namespace mamba } } - if (m_context->compile_pyc) + std::vector pyc_files = compile_pyc_files(for_compilation); + for (const fs::path& pyc_path : pyc_files) { - std::vector pyc_files = compile_pyc_files(for_compilation); + out_json["paths_data"]["paths"].push_back( + { { "_path", std::string(pyc_path) }, { "path_type", "pyc_file" } }); - for (const fs::path& pyc_path : pyc_files) - { - out_json["paths_data"]["paths"].push_back( - { { "_path", std::string(pyc_path) }, { "path_type", "pyc_file" } }); - - out_json["files"].push_back(pyc_path); - } + out_json["files"].push_back(pyc_path); } if (link_json.find("noarch") != link_json.end() diff --git a/libmamba/src/core/transaction.cpp b/libmamba/src/core/transaction.cpp index fe6a1ac821..1c3fd16bca 100644 --- a/libmamba/src/core/transaction.cpp +++ b/libmamba/src/core/transaction.cpp @@ -866,6 +866,8 @@ namespace mamba } else { + LOG_INFO << "Waiting for pyc compilation to finish"; + m_transaction_context.wait_for_pyc_compilation(); Console::stream() << "Transaction finished"; prefix.history().add_entry(m_history_entry); } diff --git a/libmamba/src/core/transaction_context.cpp b/libmamba/src/core/transaction_context.cpp index 299c2b054a..e080329fa4 100644 --- a/libmamba/src/core/transaction_context.cpp +++ b/libmamba/src/core/transaction_context.cpp @@ -3,6 +3,11 @@ // Distributed under the terms of the BSD 3-Clause License. // // The full license is in the file LICENSE, distributed with this software. +#ifndef _WIN32 +#include +#endif + +#include #include "mamba/core/transaction_context.hpp" #include "mamba/core/output.hpp" @@ -10,6 +15,14 @@ namespace mamba { + void compile_python_sources(std::ostream& out) + { + constexpr const char script[] = +#include "../data/compile_pyc.py.hpp" + ; + out << script; + } + std::string compute_short_python_version(const std::string& long_version) { auto sv = split(long_version, "."); @@ -99,4 +112,181 @@ namespace mamba site_packages_path = get_python_site_packages_short_path(short_python_version); } } + + TransactionContext& TransactionContext::operator=(const TransactionContext& other) + { + if (this != &other) + { + has_python = other.has_python; + target_prefix = other.target_prefix; + python_version = other.python_version; + requested_specs = other.requested_specs; + + compile_pyc = other.compile_pyc; + allow_softlinks = other.allow_softlinks; + always_copy = other.always_copy; + always_softlink = other.always_softlink; + short_python_version = other.short_python_version; + python_path = other.python_path; + site_packages_path = other.site_packages_path; + } + return *this; + } + + TransactionContext::~TransactionContext() + { + wait_for_pyc_compilation(); + } + + bool TransactionContext::start_pyc_compilation_process() + { + if (m_pyc_process) + { + return true; + } + +#ifndef _WIN32 + std::signal(SIGPIPE, SIG_IGN); +#endif + + std::vector command + = { target_prefix / python_path, "-Wi", "-m", "compileall", "-q", "-l", "-i", "-" }; + + auto py_ver_split = split(python_version, "."); + + try + { + if (std::stoull(std::string(py_ver_split[0])) >= 3 + && std::stoull(std::string(py_ver_split[1])) > 5) + { + m_pyc_compileall = std::make_unique(); + std::ofstream compileall_f = open_ofstream(m_pyc_compileall->path()); + compile_python_sources(compileall_f); + compileall_f.close(); + + command = { + target_prefix / python_path, "-Wi", "-u", m_pyc_compileall->path().c_str() + }; + } + } + catch (const std::exception& e) + { + LOG_ERROR << "Bad conversion of Python version '" << python_version + << "': " << e.what(); + throw std::runtime_error("Bad conversion. Aborting."); + } + + m_pyc_process = std::make_unique(); + + reproc::options options; +#ifndef _WIN32 + options.env.behavior = reproc::env::empty; +#endif + std::map envmap; + auto& ctx = Context::instance(); + envmap["MAMBA_EXTRACT_THREADS"] = std::to_string(ctx.extract_threads); + options.env.extra = envmap; + + options.stop = { + { reproc::stop::wait, reproc::milliseconds(10000) }, + { reproc::stop::terminate, reproc::milliseconds(5000) }, + { reproc::stop::kill, reproc::milliseconds(2000) }, + }; + + options.redirect.out.type = reproc::redirect::pipe; + options.redirect.err.type = reproc::redirect::pipe; + + std::string cwd = target_prefix; + options.working_directory = cwd.c_str(); + + auto [wrapped_command, script_file] = prepare_wrapped_call(target_prefix, command); + m_pyc_script_file = std::move(script_file); + + LOG_INFO << "Running wrapped python compilation command " << join(" ", command); + std::error_code ec = m_pyc_process->start(wrapped_command, options); + + if (ec == std::errc::no_such_file_or_directory) + { + LOG_ERROR << "Program not found. Make sure it's available from the PATH. " + << ec.message(); + throw std::runtime_error("pyc compilation failed with program not found. Aborting."); + } + + return true; + } + + bool TransactionContext::try_pyc_compilation(const std::vector& py_files) + { + static std::mutex pyc_compilation_mutex; + std::lock_guard lock(pyc_compilation_mutex); + + if (!has_python) + { + LOG_WARNING << "Can't compile pyc: Python not found"; + return false; + } + + start_pyc_compilation_process(); + if (!m_pyc_process) + { + return false; + } + + for (auto& f : py_files) + { + LOG_INFO << "Compiling " << f; + auto fs = f.string() + "\n"; + + auto [nbytes, ec] + = m_pyc_process->write(reinterpret_cast(&fs[0]), fs.size()); + if (ec) + { + LOG_INFO << "writing to stdin failed " << ec.message(); + return false; + } + } + + return true; + } + + void TransactionContext::wait_for_pyc_compilation() + { + if (m_pyc_process) + { + std::error_code ec; + ec = m_pyc_process->close(reproc::stream::in); + if (ec) + { + LOG_WARNING << "closing stdin failed " << ec.message(); + } + + std::string output; + std::string err; + reproc::sink::string output_sink(output); + reproc::sink::string err_sink(err); + ec = reproc::drain(*m_pyc_process, output_sink, err_sink); + if (ec) + { + LOG_WARNING << "draining failed " << ec.message(); + } + + int status = 0; + std::tie(status, ec) = m_pyc_process->stop({ + { reproc::stop::wait, reproc::milliseconds(100000) }, + { reproc::stop::terminate, reproc::milliseconds(5000) }, + { reproc::stop::kill, reproc::milliseconds(2000) }, + }); + if (ec || status != 0) + { + LOG_INFO << "noarch pyc compilation failed (cross-compiling?)."; + if (ec) + { + LOG_INFO << ec.message(); + } + LOG_INFO << "stdout:" << output; + LOG_INFO << "stdout:" << err; + } + m_pyc_process = nullptr; + } + } } diff --git a/micromamba/src/constructor.cpp b/micromamba/src/constructor.cpp index 9f4617be5d..4bfcc6f792 100644 --- a/micromamba/src/constructor.cpp +++ b/micromamba/src/constructor.cpp @@ -97,6 +97,7 @@ construct(const fs::path& prefix, bool extract_conda_pkgs, bool extract_tarball) std::string pkg_name = index["name"]; index["fn"] = entry.path().filename(); + bool found_match = false; for (const auto& pkg_info : package_details) { if (pkg_info.fn == entry.path().filename()) @@ -112,9 +113,15 @@ construct(const fs::path& prefix, bool extract_conda_pkgs, bool extract_tarball) { index["sha256"] = pkg_info.sha256; } + found_match = true; break; } } + if (!found_match) + { + LOG_WARNING << "Failed to add extra info to " << repodata_record_path + << std::endl; + } LOG_TRACE << "Writing " << repodata_record_path; std::ofstream repodata_record(repodata_record_path); diff --git a/micromamba/tests/test_create.py b/micromamba/tests/test_create.py index a93985b319..0b9b638fed 100644 --- a/micromamba/tests/test_create.py +++ b/micromamba/tests/test_create.py @@ -538,3 +538,42 @@ def test_set_platform(self, existing_cache): assert "__archspec=1=x86" in res["virtual packages"] assert "__win=0=0" in res["virtual packages"] assert res["platform"] == "win-32" + + @pytest.mark.skipif( + dry_run_tests is DryRun.ULTRA_DRY, reason="Running only ultra-dry tests" + ) + @pytest.mark.parametrize( + "version,build,cache_tag", + [ + ["2.7", "*", ""], + ["3.10", "*_cpython", "cpython-310"], + # FIXME: https://github.com/mamba-org/mamba/issues/1432 + # [ "3.7", "*_pypy","pypy37"], + ], + ) + def test_pyc_compilation(self, version, build, cache_tag): + prefix = Path(TestCreate.prefix) + cmd = ["-n", TestCreate.env_name, f"python={version}.*={build}", "six"] + + if platform.system() == "Windows": + site_packages = prefix / "Lib" / "site-packages" + if version == "2.7": + cmd += ["-c", "defaults"] # for vc=9.* + else: + site_packages = prefix / "lib" / f"python{version}" / "site-packages" + + if cache_tag: + pyc_fn = Path("__pycache__") / f"six.{cache_tag}.pyc" + else: + pyc_fn = Path(f"six.pyc") + + # Disable pyc compilation to ensure that files are still registered in conda-meta + create(*cmd, "--no-pyc") + assert not (site_packages / pyc_fn).exists() + six_meta = next((prefix / "conda-meta").glob("six-*.json")).read_text() + assert pyc_fn.name in six_meta + + # Enable pyc compilation to ensure that the pyc files are created + create(*cmd) + assert (site_packages / pyc_fn).exists() + assert pyc_fn.name in six_meta