From 114daad5ab9961d969d6a427e83a259d96871005 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Wed, 19 Mar 2025 18:14:59 +1100 Subject: [PATCH 01/38] DEV: ignore dll files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 87df248a..0ea37ac7 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ # piqtree specific ignores src/piqtree/_libiqtree/**/*.a +src/piqtree/_libiqtree/**/*.dll +src/piqtree/_libiqtree/**/*.lib +src/*.dll # docs data From 7bf0d64f6fb23a0ec7e34c8fa9a631f1af3cd3c4 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Wed, 19 Mar 2025 18:16:40 +1100 Subject: [PATCH 02/38] DEV: update libiqtree function signatures and add custom type convertors for structs --- src/piqtree/_libiqtree/_piqtree.cpp | 158 ++++++++++++++-------------- src/piqtree/_libiqtree/_piqtree.h | 113 ++++++++++++++++++++ 2 files changed, 194 insertions(+), 77 deletions(-) create mode 100644 src/piqtree/_libiqtree/_piqtree.h diff --git a/src/piqtree/_libiqtree/_piqtree.cpp b/src/piqtree/_libiqtree/_piqtree.cpp index 52210fee..52697f85 100644 --- a/src/piqtree/_libiqtree/_piqtree.cpp +++ b/src/piqtree/_libiqtree/_piqtree.cpp @@ -1,3 +1,5 @@ +#include "_piqtree.h" +#include #include #include #include @@ -8,82 +10,84 @@ using namespace std; namespace py = pybind11; -/* - * Calculates the robinson fould distance between two trees - */ -extern int robinson_fould(const string& tree1, const string& tree2); - -/* - * Generates a set of random phylogenetic trees - * tree_gen_mode allows:"YULE_HARDING", "UNIFORM", "CATERPILLAR", "BALANCED", - * "BIRTH_DEATH", "STAR_TREE" output: a newick tree (in string format) - */ -extern string random_tree(int num_taxa, - string tree_gen_mode, - int num_trees, - int rand_seed = 0); - -/* - * Perform phylogenetic analysis on the input alignment - * With estimation of the best topology - * output: results in YAML format with the tree and the details of parameters - */ -extern string build_tree(vector& names, - vector& seqs, - string model, - int rand_seed = 0, - int bootstrap_rep = 0, - int num_thres = 1); - -/* - * Perform phylogenetic analysis on the input alignment - * With restriction to the input toplogy - * output: results in YAML format with the details of parameters - */ -extern string fit_tree(vector& names, - vector& seqs, - string model, - string intree, - int rand_seed = 0, - int num_thres = 1); - -/* - * Perform phylogenetic analysis with ModelFinder - * on the input alignment (in string format) - * model_set -- a set of models to consider - * freq_set -- a set of frequency types - * rate_set -- a set of RHAS models - * rand_seed -- random seed, if 0, then will generate a new random seed - * output: modelfinder results in YAML format - */ -extern string modelfinder(vector& names, - vector& seqs, - int rand_seed = 0, - string model_set = "", - string freq_set = "", - string rate_set = "", - int num_thres = 1); - -/* - * Build pairwise JC distance matrix - * output: set of distances - * (n * i + j)-th element of the list represents the distance between i-th and - * j-th sequence, where n is the number of sequences - */ -extern vector build_distmatrix(vector& names, - vector& seqs, - int num_thres); - -/* - * Using Rapid-NJ to build tree from a distance matrix - * output: a newick tree (in string format) - */ -extern string build_njtree(vector& names, vector& distances); - -/* - * verion number - */ -extern string version(); +namespace PYBIND11_NAMESPACE { +namespace detail { +template <> +struct type_caster { + public: + PYBIND11_TYPE_CASTER(StringArray, const_name("StringArray")); + + // Conversion from Python to C++ + bool load(handle src, bool) { + /* Extract PyObject from handle */ + PyObject* source = src.ptr(); + if (!py::isinstance(source)) { + return false; + } + + auto seq = reinterpret_borrow(src); + value.length = seq.size(); + + tmpStrings.reserve(value.length); + tmpCStrs.reserve(value.length); + + for (size_t i = 0; i < seq.size(); ++i) { + auto item = seq[i]; + if (!py::isinstance(item)) { + return false; + } + + tmpStrings.push_back(item.cast()); + tmpCStrs.push_back(tmpStrings[i].c_str()); + } + + value.strings = tmpCStrs.data(); + + return true; + } + + // Conversion from C++ to Python + static handle cast(StringArray src, return_value_policy, handle) { + throw std::runtime_error("Unsupported operation"); + } + + private: + vector tmpStrings; + vector tmpCStrs; +}; + +template <> +struct type_caster { + public: + PYBIND11_TYPE_CASTER(DoubleArray, _("DoubleArray")); + + // Conversion from Python to C++ + bool load(handle src, bool) { + if (!py::isinstance>(src)) { + return false; // Only accept numpy arrays of float64 + } + + auto arr = py::cast>(src); + if (arr.ndim() != 1) { + return false; // Only accept 1D arrays + } + + value.length = arr.size(); + value.doubles = new double[value.length]; + std::memcpy(value.doubles, arr.data(), value.length * sizeof(double)); + return true; + } + + // Conversion from C++ to Python + static handle cast(DoubleArray src, return_value_policy, handle) { + auto result = py::array_t(src.length); + std::memcpy(result.mutable_data(), src.doubles, + src.length * sizeof(double)); + return result.release(); + } +}; +} // namespace detail +} // namespace PYBIND11_NAMESPACE int mine() { return 42; @@ -112,5 +116,5 @@ PYBIND11_MODULE(_piqtree, m) { "Construct pairwise distance matrix for alignment."); m.def("iq_nj_tree", &build_njtree, "Build neighbour-joining tree from distance matrix."); - m.def("mine", &mine, "The meaning of life, the universe (and everything)!"); + m.def("mine", &mine, "The meaning of life, the universe (and everything) !"); } diff --git a/src/piqtree/_libiqtree/_piqtree.h b/src/piqtree/_libiqtree/_piqtree.h new file mode 100644 index 00000000..ed77708b --- /dev/null +++ b/src/piqtree/_libiqtree/_piqtree.h @@ -0,0 +1,113 @@ +#ifndef _PIQTREE_H +#define _PIQTREE_H + +using namespace std; + +#ifdef _MSC_VER +#pragma pack(push, 1) +#else +#pragma pack(1) +#endif + +typedef struct { + const char** strings; + size_t length; +} StringArray; + +typedef struct { + double* doubles; + size_t length; +} DoubleArray; + +#ifdef _MSC_VER +#pragma pack(pop) +#else +#pragma pack() +#endif + +/* + * Calculates the robinson fould distance between two trees + */ +extern "C" int robinson_fould(const char* ctree1, const char* ctree2); + +/* + * Generates a set of random phylogenetic trees + * tree_gen_mode allows:"YULE_HARDING", "UNIFORM", "CATERPILLAR", "BALANCED", + * "BIRTH_DEATH", "STAR_TREE" output: a newick tree (in string format) + */ +extern "C" char* random_tree(int num_taxa, + const char* tree_gen_mode, + int num_trees, + int rand_seed = 0); + +/* + * Perform phylogenetic analysis on the input alignment + * With estimation of the best topology + * num_thres -- number of cpu threads to be used, default: 1; 0 - auto detection + * of the optimal number of cpu threads output: results in YAML format with the + * tree and the details of parameters + */ +extern "C" char* build_tree(StringArray& names, + StringArray& seqs, + const char* model, + int rand_seed = 0, + int bootstrap_rep = 0, + int num_thres = 1); + +/* + * Perform phylogenetic analysis on the input alignment + * With restriction to the input toplogy + * num_thres -- number of cpu threads to be used, default: 1; 0 - auto detection + * of the optimal number of cpu threads output: results in YAML format with the + * details of parameters + */ +extern "C" char* fit_tree(StringArray& names, + StringArray& seqs, + const char* model, + const char* intree, + int rand_seed = 0, + int num_thres = 1); + +/* + * Perform phylogenetic analysis with ModelFinder + * on the input alignment (in string format) + * model_set -- a set of models to consider + * freq_set -- a set of frequency types + * rate_set -- a set of RHAS models + * rand_seed -- random seed, if 0, then will generate a new random seed + * num_thres -- number of cpu threads to be used, default: 1; 0 - auto detection + * of the optimal number of cpu threads output: modelfinder results in YAML + * format + */ +extern "C" char* modelfinder(StringArray& names, + StringArray& seqs, + int rand_seed = 0, + const char* model_set = "", + const char* freq_set = "", + const char* rate_set = "", + int num_thres = 1); + +/* + * Build pairwise JC distance matrix + * output: set of distances + * (n * i + j)-th element of the list represents the distance between i-th and + * j-th sequence, where n is the number of sequences num_thres -- number of cpu + * threads to be used, default: 1; 0 - use all available cpu threads on the + * machine + */ +extern "C" DoubleArray build_distmatrix(StringArray& names, + StringArray& seqs, + int num_thres = 1); + +/* + * Using Rapid-NJ to build tree from a distance matrix + * output: a newick tree (in string format) + */ +extern "C" char* build_njtree(StringArray& names, DoubleArray& distances); + +/* + * verion number + */ +extern "C" char* version(); + +#endif /* LIBIQTREE2_FUN */ From ef252bdac8136effbf8b273fd51a65a8ca633b9c Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Wed, 19 Mar 2025 18:17:48 +1100 Subject: [PATCH 03/38] DEV: ensure DLLs are included in build on windows --- pyproject.toml | 7 +++- setup.py | 80 ++++++++++++++++++++++++++++++++--------- src/piqtree/__init__.py | 13 +++++++ 3 files changed, 82 insertions(+), 18 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 28f4cb7f..19871515 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools >= 61.0", "pybind11 >= 2.12"] +requires = ["setuptools >= 61.0", "pybind11 >= 2.12", "delvewheel >= 1.10"] build-backend = "setuptools.build_meta" [project] @@ -152,6 +152,11 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" "src/piqtree/_app/__init__.py" = [ "N801", # apps follow function naming convention ] +"src/piqtree/__init__.py" = [ + "E402", # handle DLLs before imports + "PTH118", # os operations for DLL path + "PTH120", # os operations for DLL path +] "src/piqtree/model/_substitution_model.py" = ["N815"] # use IQ-TREE naming scheme [tool.ruff.format] diff --git a/setup.py b/setup.py index fcd22a34..62a6b4c4 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,8 @@ from pybind11.setup_helpers import Pybind11Extension, build_ext from setuptools import setup -LIBRARY_DIR = "src/piqtree/_libiqtree" +LIBRARY_DIR = "src/piqtree/_libiqtree/" +IQTREE_LIB_NAME = "iqtree2" def get_brew_prefix(package: str) -> Path: @@ -18,7 +19,36 @@ def get_brew_prefix(package: str) -> Path: ) -if platform.system() == "Darwin": +extra_libs = [] +extra_compile_args = [] +include_dirs = [] +library_dirs = [] + + +def include_dlls() -> None: + import shutil + + from delvewheel._dll_utils import get_all_needed + + needed_dll_paths, _, _, _ = get_all_needed( + LIBRARY_DIR + f"{IQTREE_LIB_NAME}.dll", + set(), + None, + "raise", + False, # noqa: FBT003 + False, # noqa: FBT003 + 0, + ) + + for dll_path in needed_dll_paths: + shutil.copy(dll_path, LIBRARY_DIR) + + +def setup_windows() -> None: + include_dlls() + + +def setup_macos() -> None: brew_prefix_llvm = get_brew_prefix("llvm") brew_prefix_libomp = get_brew_prefix("libomp") @@ -27,20 +57,34 @@ def get_brew_prefix(package: str) -> Path: os.environ["CXX"] = str(brew_prefix_llvm / "bin" / "clang++") # Define OpenMP flags and libraries for macOS - openmp_flags = ["-Xpreprocessor", "-fopenmp"] - openmp_libs = ["omp"] + extra_compile_args.extend(["-Xpreprocessor", "-fopenmp"]) + extra_libs.extend(["z", "omp"]) # Use the paths from Homebrew for libomp - openmp_include = str(brew_prefix_libomp / "include") - library_dirs = [ - str(brew_prefix_libomp / "lib"), - str(brew_prefix_llvm / "lib"), - ] -else: - openmp_flags = ["-fopenmp"] - openmp_libs = ["gomp"] - openmp_include = None - library_dirs = [] + include_dirs.extend([str(brew_prefix_libomp / "include")]) + library_dirs.extend( + [ + str(brew_prefix_libomp / "lib"), + str(brew_prefix_llvm / "lib"), + ], + ) + + +def setup_linux() -> None: + extra_compile_args.extend(["-fopenmp"]) + extra_libs.extend(["z", "gomp"]) + + +match system := platform.system(): + case "Windows": + setup_windows() + case "Darwin": + setup_macos() + case "Linux": + setup_linux() + case _: + msg = f"Unsupported platform: {system}" + raise ValueError(msg) ext_modules = [ Pybind11Extension( @@ -50,14 +94,16 @@ def get_brew_prefix(package: str) -> Path: *library_dirs, LIBRARY_DIR, ], - libraries=["iqtree2", "z", *openmp_libs], - extra_compile_args=openmp_flags, - include_dirs=[openmp_include] if openmp_include else [], + libraries=[IQTREE_LIB_NAME, *extra_libs], + extra_compile_args=extra_compile_args, + include_dirs=include_dirs, ), ] setup( + name="piqtree", ext_modules=ext_modules, cmdclass={"build_ext": build_ext}, zip_safe=False, + package_data={"piqtree": ["_libiqtree/*.dll"]}, ) diff --git a/src/piqtree/__init__.py b/src/piqtree/__init__.py index 5461c950..d5392dbb 100644 --- a/src/piqtree/__init__.py +++ b/src/piqtree/__init__.py @@ -1,5 +1,18 @@ """piqtree - access the power of IQ-TREE within Python.""" + +def _add_dll_path() -> None: + import os + + if "add_dll_directory" in dir(os): + dll_folder = os.path.join(os.path.dirname(__file__), "_libiqtree") + os.add_dll_directory(dll_folder) + + +_add_dll_path() +del _add_dll_path + + from _piqtree import __iqtree_version__ from piqtree._data import dataset_names, download_dataset From c027ffa480ff7f16a3602091d91e0d09e5ee4464 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Wed, 19 Mar 2025 18:18:25 +1100 Subject: [PATCH 04/38] BUG: on windows, change out of tmp directory before cleaning up --- src/piqtree/iqtree/_decorator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/piqtree/iqtree/_decorator.py b/src/piqtree/iqtree/_decorator.py index d7f4fd61..4d59e101 100644 --- a/src/piqtree/iqtree/_decorator.py +++ b/src/piqtree/iqtree/_decorator.py @@ -84,7 +84,7 @@ def wrapper_iqtree_func(*args: Param.args, **kwargs: Param.kwargs) -> RetType: os.close(devnull_fd) if hide_files: - tempdir.cleanup() os.chdir(original_dir) + tempdir.cleanup() return wrapper_iqtree_func From 439b2ea74c42f14f79a434abbd0cf725e94800c7 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Wed, 19 Mar 2025 18:18:56 +1100 Subject: [PATCH 05/38] TST: skip tests which throw error on IQ-TREE side on windows --- tests/test_iqtree/test_build_tree.py | 5 +++++ tests/test_iqtree/test_random_trees.py | 6 ++++++ tests/test_iqtree/test_segmentation_fault.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/tests/test_iqtree/test_build_tree.py b/tests/test_iqtree/test_build_tree.py index 90cbf8ab..d8232d15 100644 --- a/tests/test_iqtree/test_build_tree.py +++ b/tests/test_iqtree/test_build_tree.py @@ -1,3 +1,4 @@ +import platform import re import pytest @@ -87,6 +88,10 @@ def test_rate_model_build_tree( ) +@pytest.mark.skipif( + platform.system() == "Windows", + reason="IQ-TREE errors can't be caught yet on Windows", +) def test_build_tree_inadequate_bootstrapping(four_otu: ArrayAlignment) -> None: with pytest.raises(IqTreeError, match=re.escape("#replicates must be >= 1000")): piqtree.build_tree(four_otu, Model(DnaModel.GTR), bootstrap_replicates=10) diff --git a/tests/test_iqtree/test_random_trees.py b/tests/test_iqtree/test_random_trees.py index 60c09ce1..82ce8f4a 100644 --- a/tests/test_iqtree/test_random_trees.py +++ b/tests/test_iqtree/test_random_trees.py @@ -1,3 +1,5 @@ +import platform + import pytest import piqtree @@ -45,6 +47,10 @@ def test_random_trees_no_seed( @pytest.mark.parametrize("num_taxa", [-1, 0, 1, 2]) @pytest.mark.parametrize("tree_mode", list(piqtree.TreeGenMode)) +@pytest.mark.skipif( + platform.system() == "Windows", + reason="IQ-TREE errors can't be caught yet on Windows", +) def test_invalid_taxa( num_taxa: int, tree_mode: piqtree.TreeGenMode, diff --git a/tests/test_iqtree/test_segmentation_fault.py b/tests/test_iqtree/test_segmentation_fault.py index 701cc406..1fb58575 100644 --- a/tests/test_iqtree/test_segmentation_fault.py +++ b/tests/test_iqtree/test_segmentation_fault.py @@ -1,5 +1,7 @@ """Test combinations of calls which under previous versions resulted in a segmentation fault.""" +import platform + import pytest from cogent3 import make_aligned_seqs, make_tree @@ -8,6 +10,10 @@ from piqtree.model import DiscreteGammaModel, DnaModel, FreeRateModel, Model +@pytest.mark.skipif( + platform.system() == "Windows", + reason="IQ-TREE errors can't be caught yet on Windows", +) def test_two_build_random_trees() -> None: """ Calling build tree twice followed by random trees with a bad input @@ -22,6 +28,10 @@ def test_two_build_random_trees() -> None: random_trees(3, 2, TreeGenMode.BALANCED, 1) +@pytest.mark.skipif( + platform.system() == "Windows", + reason="IQ-TREE errors can't be caught yet on Windows", +) def test_two_fit_random_trees() -> None: """ Calling fit tree twice followed by random trees with a bad input @@ -39,6 +49,10 @@ def test_two_fit_random_trees() -> None: @pytest.mark.parametrize("rate_model_class", [DiscreteGammaModel, FreeRateModel]) @pytest.mark.parametrize("categories", [0, -4]) +@pytest.mark.skipif( + platform.system() == "Windows", + reason="IQ-TREE errors can't be caught yet on Windows", +) def test_two_invalid_models( rate_model_class: type[DiscreteGammaModel] | type[FreeRateModel], categories: int, From 48df261e24b9b9eb50f688bfa6c5b6c95f847807 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Wed, 19 Mar 2025 18:21:38 +1100 Subject: [PATCH 06/38] ENH: include cstddef for other platforms --- src/piqtree/_libiqtree/_piqtree.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/piqtree/_libiqtree/_piqtree.h b/src/piqtree/_libiqtree/_piqtree.h index ed77708b..070c621b 100644 --- a/src/piqtree/_libiqtree/_piqtree.h +++ b/src/piqtree/_libiqtree/_piqtree.h @@ -1,6 +1,8 @@ #ifndef _PIQTREE_H #define _PIQTREE_H +#include + using namespace std; #ifdef _MSC_VER From a631b0fba6735c5f06b7c98cb7753c5878790c3e Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Wed, 19 Mar 2025 18:25:48 +1100 Subject: [PATCH 07/38] MAINT: ignore attr-defined typing error for non-windows --- src/piqtree/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/piqtree/__init__.py b/src/piqtree/__init__.py index d5392dbb..5a91c957 100644 --- a/src/piqtree/__init__.py +++ b/src/piqtree/__init__.py @@ -6,7 +6,7 @@ def _add_dll_path() -> None: if "add_dll_directory" in dir(os): dll_folder = os.path.join(os.path.dirname(__file__), "_libiqtree") - os.add_dll_directory(dll_folder) + os.add_dll_directory(dll_folder) # type: ignore[attr-defined] _add_dll_path() From 7d6b6b7adf7a28baecc3edbcd34bea47202b16c8 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 14:06:34 +1100 Subject: [PATCH 08/38] DEV: add windows build scripts --- build_tools/before_all_windows.sh | 10 ++++++++++ build_tools/build_iqtree.sh | 19 +++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 build_tools/before_all_windows.sh diff --git a/build_tools/before_all_windows.sh b/build_tools/before_all_windows.sh new file mode 100644 index 00000000..4eac69f3 --- /dev/null +++ b/build_tools/before_all_windows.sh @@ -0,0 +1,10 @@ +# Install dependencies using choco +choco install -y llvm eigen boost libomp make + +# Set environment variables for LLVM +export CMAKE_C_COMPILER="clang" +export CMAKE_CXX_COMPILER="clang++" +export CMAKE_MAKE_PROGRAM="make" + +# Build IQ-TREE +bash build_tools/build_iqtree.sh \ No newline at end of file diff --git a/build_tools/build_iqtree.sh b/build_tools/build_iqtree.sh index 38bb7afe..e5048783 100755 --- a/build_tools/build_iqtree.sh +++ b/build_tools/build_iqtree.sh @@ -9,11 +9,26 @@ if [[ "$OSTYPE" == "darwin"* ]]; then echo $CXXFLAGS cmake -DBUILD_LIB=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ .. gmake -j +elif [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* ]]; then + echo "Building for Windows." + cmake -G "MinGW Makefiles" \ + -DCMAKE_C_COMPILER=$CMAKE_C_COMPILER \ + -DCMAKE_CXX_COMPILER=$CMAKE_CXX_COMPILER \ + -DCMAKE_MAKE_PROGRAM=$CMAKE_MAKE_PROGRAM \ + -DBUILD_LIB=ON \ + .. + make -j else - echo "Building for linux." + echo "Building for Linux." cmake -DBUILD_LIB=ON .. make -j fi cd ../.. -mv iqtree2/build/libiqtree2.a src/piqtree/_libiqtree/ \ No newline at end of file + +if [[ "$OSTYPE" == "darwin"* || "$OSTYPE" == "linux"* ]]; then + mv iqtree2/build/libiqtree2.a src/piqtree/_libiqtree/ +elif [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* ]]; then + mv iqtree2/build/iqtree2.lib src/piqtree/_libiqtree/ + mv iqtree2/build/iqtree2.dll src/piqtree/_libiqtree/ +fi From d94890a7ef5a34505d00f6996a5b4cab07e6b3d4 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 14:07:41 +1100 Subject: [PATCH 09/38] DEV: build windows when building iqtree and cache correct files. --- .github/actions/build-iqtree/action.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index 6ce03727..d0fc1b7e 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -23,7 +23,8 @@ runs: id: cache with: key: libiqtree-${{ inputs.os }}-${{ steps.iqtree2-sha.outputs.iqtree2-sha }} - path: src/piqtree/_libiqtree/libiqtree2.a + path: | + ${{ inputs.os == 'windows-latest' && 'src/piqtree/_libiqtree/iqtree2.lib\nsrc/piqtree/_libiqtree/iqtree2.dll' || 'src/piqtree/_libiqtree/libiqtree2.a' }} lookup-only: true - name: Build IQ-TREE @@ -31,7 +32,12 @@ runs: if: steps.cache.outputs.cache-hit != 'true' run: | if [[ "${{ inputs.os }}" == "ubuntu-latest" ]]; then - sudo ./build_tools/before_all_linux.sh + sudo ./build_tools/before_all_linux.sh + elif [[ "${{ inputs.os }}" == "macOS-latest" ]]; then + ./build_tools/before_all_mac.sh + elif [[ "${{ inputs.os }}" == "windows-latest" ]]; then + ./build_tools/before_all_windows.sh else - ./build_tools/before_all_mac.sh - fi + echo "Unrecognized OS: '${{ inputs.os }}'." + exit 1 + fi \ No newline at end of file From 177b4adfa3c1ec23abbae590cd3b6c6e78bf46fe Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 14:07:52 +1100 Subject: [PATCH 10/38] DEV: test build process on windows --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71e763b9..e795b246 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-13, macos-14] # Intel linux, Intel Mac, ARM Mac + os: [windows-latest] + # os: [ubuntu-latest, macos-13, macos-14, windows-latest] # Intel linux, Intel Mac, ARM Mac, Windows steps: - uses: "actions/checkout@v4" From 30515d2c8fde3ecc682cb9f496c21a456015bd22 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 14:17:25 +1100 Subject: [PATCH 11/38] DEV: install boost through action on windows, remove non-available choco packages --- .github/actions/build-iqtree/action.yml | 9 +++++++++ build_tools/before_all_windows.sh | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index d0fc1b7e..23c8e6f9 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -27,6 +27,15 @@ runs: ${{ inputs.os == 'windows-latest' && 'src/piqtree/_libiqtree/iqtree2.lib\nsrc/piqtree/_libiqtree/iqtree2.dll' || 'src/piqtree/_libiqtree/libiqtree2.a' }} lookup-only: true + - name: Install Boost + if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' + uses: MarkusJx/install-boost@v2.4.5 + id: install-boost + with: + boost_version: 1.84.0 + platform_version: 2022 + toolset: mingw + - name: Build IQ-TREE shell: bash if: steps.cache.outputs.cache-hit != 'true' diff --git a/build_tools/before_all_windows.sh b/build_tools/before_all_windows.sh index 4eac69f3..6cff4465 100644 --- a/build_tools/before_all_windows.sh +++ b/build_tools/before_all_windows.sh @@ -1,5 +1,5 @@ # Install dependencies using choco -choco install -y llvm eigen boost libomp make +choco install -y llvm eigen make # Set environment variables for LLVM export CMAKE_C_COMPILER="clang" From 5c993c5c16a4108c7602f74270c7fb0c084d3caf Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 14:27:12 +1100 Subject: [PATCH 12/38] DEV: set boost env variables --- .github/actions/build-iqtree/action.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index 23c8e6f9..5e036857 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -36,6 +36,13 @@ runs: platform_version: 2022 toolset: mingw + - name: Set Boost Environment Variables + if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' + shell: bash + run: | + echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" + echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" + - name: Build IQ-TREE shell: bash if: steps.cache.outputs.cache-hit != 'true' From ed9c4b73a1c5074a7ea15dbfdf2de7b23cf80564 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 14:33:27 +1100 Subject: [PATCH 13/38] DEV: debug boost env vars --- .github/actions/build-iqtree/action.yml | 9 +++++++++ build_tools/before_all_windows.sh | 5 +++++ build_tools/build_iqtree.sh | 5 +++++ 3 files changed, 19 insertions(+) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index 5e036857..e016c0cc 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -42,6 +42,15 @@ runs: run: | echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" + + - name: Debug Boost Variables + if: runner.os == 'Windows' + shell: bash + run: | + echo "BOOST_ROOT=$BOOST_ROOT" + echo "Boost_INCLUDE_DIR=$Boost_INCLUDE_DIR" + echo "Boost_LIBRARY_DIRS=$Boost_LIBRARY_DIRS" + echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" - name: Build IQ-TREE shell: bash diff --git a/build_tools/before_all_windows.sh b/build_tools/before_all_windows.sh index 6cff4465..8bf78859 100644 --- a/build_tools/before_all_windows.sh +++ b/build_tools/before_all_windows.sh @@ -1,3 +1,8 @@ +echo "BOOST_ROOT=$BOOST_ROOT" +echo "Boost_INCLUDE_DIR=$Boost_INCLUDE_DIR" +echo "Boost_LIBRARY_DIRS=$Boost_LIBRARY_DIRS" +echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" + # Install dependencies using choco choco install -y llvm eigen make diff --git a/build_tools/build_iqtree.sh b/build_tools/build_iqtree.sh index e5048783..6ae2ee8a 100755 --- a/build_tools/build_iqtree.sh +++ b/build_tools/build_iqtree.sh @@ -1,3 +1,8 @@ +echo "BOOST_ROOT=$BOOST_ROOT" +echo "Boost_INCLUDE_DIR=$Boost_INCLUDE_DIR" +echo "Boost_LIBRARY_DIRS=$Boost_LIBRARY_DIRS" +echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" + cd iqtree2 rm -rf build mkdir build && cd build From f4439cd45b98bfc29b155545463aba4abf9cec58 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 14:42:52 +1100 Subject: [PATCH 14/38] DEV: removed path debugs, manually specify boost dirs --- .github/actions/build-iqtree/action.yml | 9 --------- build_tools/before_all_windows.sh | 5 ----- build_tools/build_iqtree.sh | 7 ++----- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index e016c0cc..5e036857 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -42,15 +42,6 @@ runs: run: | echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" - - - name: Debug Boost Variables - if: runner.os == 'Windows' - shell: bash - run: | - echo "BOOST_ROOT=$BOOST_ROOT" - echo "Boost_INCLUDE_DIR=$Boost_INCLUDE_DIR" - echo "Boost_LIBRARY_DIRS=$Boost_LIBRARY_DIRS" - echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" - name: Build IQ-TREE shell: bash diff --git a/build_tools/before_all_windows.sh b/build_tools/before_all_windows.sh index 8bf78859..6cff4465 100644 --- a/build_tools/before_all_windows.sh +++ b/build_tools/before_all_windows.sh @@ -1,8 +1,3 @@ -echo "BOOST_ROOT=$BOOST_ROOT" -echo "Boost_INCLUDE_DIR=$Boost_INCLUDE_DIR" -echo "Boost_LIBRARY_DIRS=$Boost_LIBRARY_DIRS" -echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" - # Install dependencies using choco choco install -y llvm eigen make diff --git a/build_tools/build_iqtree.sh b/build_tools/build_iqtree.sh index 6ae2ee8a..d1d882c7 100755 --- a/build_tools/build_iqtree.sh +++ b/build_tools/build_iqtree.sh @@ -1,8 +1,3 @@ -echo "BOOST_ROOT=$BOOST_ROOT" -echo "Boost_INCLUDE_DIR=$Boost_INCLUDE_DIR" -echo "Boost_LIBRARY_DIRS=$Boost_LIBRARY_DIRS" -echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" - cd iqtree2 rm -rf build mkdir build && cd build @@ -20,6 +15,8 @@ elif [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* ]]; then -DCMAKE_C_COMPILER=$CMAKE_C_COMPILER \ -DCMAKE_CXX_COMPILER=$CMAKE_CXX_COMPILER \ -DCMAKE_MAKE_PROGRAM=$CMAKE_MAKE_PROGRAM \ + -DBoost_INCLUDE_DIR=$Boost_INCLUDE_DIR \ + -DBoost_LIBRARY_DIRS=$Boost_LIBRARY_DIRS \ -DBUILD_LIB=ON \ .. make -j From 994bafe546c15c20f9d390a839d9377998aaef97 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 14:50:42 +1100 Subject: [PATCH 15/38] DEV: manually specify toolchain, downgrade llvm --- build_tools/before_all_windows.sh | 8 ++------ build_tools/build_iqtree.sh | 9 ++++++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/build_tools/before_all_windows.sh b/build_tools/before_all_windows.sh index 6cff4465..763d3b2d 100644 --- a/build_tools/before_all_windows.sh +++ b/build_tools/before_all_windows.sh @@ -1,10 +1,6 @@ # Install dependencies using choco -choco install -y llvm eigen make - -# Set environment variables for LLVM -export CMAKE_C_COMPILER="clang" -export CMAKE_CXX_COMPILER="clang++" -export CMAKE_MAKE_PROGRAM="make" +choco install -y llvm --version=14.0.6 --allow-downgrade +choco install -y eigen # Build IQ-TREE bash build_tools/build_iqtree.sh \ No newline at end of file diff --git a/build_tools/build_iqtree.sh b/build_tools/build_iqtree.sh index d1d882c7..60a36afc 100755 --- a/build_tools/build_iqtree.sh +++ b/build_tools/build_iqtree.sh @@ -12,11 +12,14 @@ if [[ "$OSTYPE" == "darwin"* ]]; then elif [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* ]]; then echo "Building for Windows." cmake -G "MinGW Makefiles" \ - -DCMAKE_C_COMPILER=$CMAKE_C_COMPILER \ - -DCMAKE_CXX_COMPILER=$CMAKE_CXX_COMPILER \ - -DCMAKE_MAKE_PROGRAM=$CMAKE_MAKE_PROGRAM \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_FLAGS=--target=x86_64-pc-windows-gnu \ + -DCMAKE_CXX_FLAGS=--target=x86_64-pc-windows-gnu \ + -DCMAKE_MAKE_PROGRAM=mingw32-make \ -DBoost_INCLUDE_DIR=$Boost_INCLUDE_DIR \ -DBoost_LIBRARY_DIRS=$Boost_LIBRARY_DIRS \ + -DIQTREE_FLAGS="cpp14" \ -DBUILD_LIB=ON \ .. make -j From f971e7b21afd5d0517d0b1072f1bc00d8af95f8f Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 15:07:32 +1100 Subject: [PATCH 16/38] DEV: add lib.exe to path --- .github/actions/build-iqtree/action.yml | 17 +++++++++++++++++ .github/workflows/ci.yml | 3 ++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index 5e036857..da23170f 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -43,6 +43,23 @@ runs: echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" + - name: Set Up MSVC for lib.exe + if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' + shell: bash + run: | + MSVC_BIN_PATH="C:\\Program Files\\Microsoft Visual Studio\\2022\\BuildTools\\VC\\Tools\\MSVC" + MSVC_VERSION=$(ls "$MSVC_BIN_PATH" | sort -V | tail -n 1) + LIB_PATH="$MSVC_BIN_PATH\\$MSVC_VERSION\\bin\\Hostx64\\x64" + + echo "Adding $LIB_PATH to PATH" + echo "PATH=$LIB_PATH:$PATH" >> "$GITHUB_ENV" + + - name: Verify lib.exe + if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' + shell: bash + run: | + which lib.exe || echo "lib.exe not found" + - name: Build IQ-TREE shell: bash if: steps.cache.outputs.cache-hit != 'true' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e795b246..bd46719a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-13, macos-14] # Intel linux, Intel Mac, ARM Mac + os: [windows-latest] + # os: [ubuntu-latest, macos-13, macos-14] # Intel linux, Intel Mac, ARM Mac python-version: ["3.10", "3.11", "3.12"] steps: - uses: "actions/checkout@v4" From 0ff89f26cd167dfc4e2fadbd686aa70837d76379 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 15:10:35 +1100 Subject: [PATCH 17/38] DEV: debug msvc path --- .github/actions/build-iqtree/action.yml | 29 ++++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index da23170f..a4a53fde 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -47,12 +47,29 @@ runs: if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' shell: bash run: | - MSVC_BIN_PATH="C:\\Program Files\\Microsoft Visual Studio\\2022\\BuildTools\\VC\\Tools\\MSVC" - MSVC_VERSION=$(ls "$MSVC_BIN_PATH" | sort -V | tail -n 1) - LIB_PATH="$MSVC_BIN_PATH\\$MSVC_VERSION\\bin\\Hostx64\\x64" - - echo "Adding $LIB_PATH to PATH" - echo "PATH=$LIB_PATH:$PATH" >> "$GITHUB_ENV" + # Check default VS installation paths + VSWHERE_PATH="/c/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe" + + if [[ -f "$VSWHERE_PATH" ]]; then + echo "Finding latest MSVC installation..." + + MSVC_PATH=$("$VSWHERE_PATH" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath) + + if [[ -d "$MSVC_PATH" ]]; then + MSVC_BIN_PATH="$MSVC_PATH/VC/Tools/MSVC" + MSVC_VERSION=$(ls "$MSVC_BIN_PATH" | sort -V | tail -n 1) + LIB_PATH="$MSVC_BIN_PATH/$MSVC_VERSION/bin/Hostx64/x64" + + echo "MSVC found at: $LIB_PATH" + echo "PATH=$LIB_PATH:$PATH" >> "$GITHUB_ENV" + else + echo "MSVC installation not found." + exit 1 + fi + else + echo "vswhere.exe not found, unable to locate MSVC." + exit 1 + fi - name: Verify lib.exe if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' From b76cb6125c7063cce9812fac14c2c0cf11ed49ff Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 15:13:35 +1100 Subject: [PATCH 18/38] DEV: try running wswhere through cmd --- .github/actions/build-iqtree/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index a4a53fde..788adc01 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -43,6 +43,12 @@ runs: echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" + - name: Debug MSVC Installation + if: runner.os == 'Windows' + shell: cmd + run: | + "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -all + - name: Set Up MSVC for lib.exe if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' shell: bash From f25ff0687b39547f27166b8db12597fe6515d73f Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 15:20:41 +1100 Subject: [PATCH 19/38] DEV: try configuring MSVC --- .github/actions/build-iqtree/action.yml | 34 +++---------------------- 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index 788adc01..5801e2c1 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -43,45 +43,19 @@ runs: echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" - - name: Debug MSVC Installation - if: runner.os == 'Windows' - shell: cmd - run: | - "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -all - - - name: Set Up MSVC for lib.exe + - name: Setup Visual Studio Environment if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' shell: bash run: | - # Check default VS installation paths - VSWHERE_PATH="/c/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe" - - if [[ -f "$VSWHERE_PATH" ]]; then - echo "Finding latest MSVC installation..." - - MSVC_PATH=$("$VSWHERE_PATH" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath) - - if [[ -d "$MSVC_PATH" ]]; then - MSVC_BIN_PATH="$MSVC_PATH/VC/Tools/MSVC" - MSVC_VERSION=$(ls "$MSVC_BIN_PATH" | sort -V | tail -n 1) - LIB_PATH="$MSVC_BIN_PATH/$MSVC_VERSION/bin/Hostx64/x64" - - echo "MSVC found at: $LIB_PATH" - echo "PATH=$LIB_PATH:$PATH" >> "$GITHUB_ENV" - else - echo "MSVC installation not found." - exit 1 - fi - else - echo "vswhere.exe not found, unable to locate MSVC." - exit 1 - fi + source "C:/Program Files (x86)/Microsoft Visual Studio/2022/Enterprise/Common7/Tools/VsDevCmd.bat" + echo "Visual Studio environment configured" - name: Verify lib.exe if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' shell: bash run: | which lib.exe || echo "lib.exe not found" + exit 1 - name: Build IQ-TREE shell: bash From 04b814bb49008acde3bc8b4d94b3423065921155 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 15:27:21 +1100 Subject: [PATCH 20/38] DEV: use cmd to setup VS env --- .github/actions/build-iqtree/action.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index 5801e2c1..6cacd215 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -44,10 +44,9 @@ runs: echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" - name: Setup Visual Studio Environment - if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' - shell: bash + shell: cmd run: | - source "C:/Program Files (x86)/Microsoft Visual Studio/2022/Enterprise/Common7/Tools/VsDevCmd.bat" + "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\Tools\VsDevCmd.bat" echo "Visual Studio environment configured" - name: Verify lib.exe From 1df70b61e0e3eee814e1f0dffadbfe19d995ce71 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 15:32:58 +1100 Subject: [PATCH 21/38] DEV: use action to setup msvc --- .github/actions/build-iqtree/action.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index 6cacd215..694c98b9 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -43,11 +43,9 @@ runs: echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" - - name: Setup Visual Studio Environment - shell: cmd - run: | - "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\Tools\VsDevCmd.bat" - echo "Visual Studio environment configured" + - name: Setup MSVC Developer Command Prompt + if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' + uses: ilammy/msvc-dev-cmd@v1 - name: Verify lib.exe if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' From e25cf8666bc9531b9a8c324ec1a02a4430b0eade Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 15:38:50 +1100 Subject: [PATCH 22/38] DEV: continue build process with found lib --- .github/actions/build-iqtree/action.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index 694c98b9..b252ecda 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -47,13 +47,6 @@ runs: if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' uses: ilammy/msvc-dev-cmd@v1 - - name: Verify lib.exe - if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' - shell: bash - run: | - which lib.exe || echo "lib.exe not found" - exit 1 - - name: Build IQ-TREE shell: bash if: steps.cache.outputs.cache-hit != 'true' From a6f9c1d586ac5651579c2652a1d0e90216f6a881 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 16:05:02 +1100 Subject: [PATCH 23/38] DEV: only brew install on mac --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd46719a..292f26de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: cache-key: libiqtree-${{ matrix.os }}-${{ needs.build-iqtree.outputs.iqtree2-sha }} - name: Install llvm - if: matrix.os != 'ubuntu-latest' + if: runner.os == 'macOS' run: | brew install llvm From 8dd29934ddebaf6c751d2b95cd119c0830fbdbee Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 16:05:32 +1100 Subject: [PATCH 24/38] DEV: use different caches on unix/windows --- .github/actions/build-iqtree/action.yml | 29 +++++++++++++++++++++--- .github/actions/setup-piqtree/action.yml | 25 ++++++++++++++++---- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index b252ecda..f0ca238d 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -19,14 +19,37 @@ runs: IQ_TREE_2_SHA=$(git rev-parse HEAD) echo "iqtree2-sha=${IQ_TREE_2_SHA}" >> "$GITHUB_OUTPUT" - - uses: actions/cache@v4 - id: cache + - name: Cache IQ-TREE 2 (Windows) + if: runner.os == 'Windows' + uses: actions/cache@v4 + id: cache-windows + with: + key: libiqtree-windows-${{ steps.iqtree2-sha.outputs.iqtree2-sha }} + path: | + src/piqtree/_libiqtree/iqtree2.lib + src/piqtree/_libiqtree/iqtree2.dll + lookup-only: true + + - name: Cache IQ-TREE 2 (Linux/macOS) + if: runner.os != 'Windows' + uses: actions/cache@v4 + id: cache-unix with: key: libiqtree-${{ inputs.os }}-${{ steps.iqtree2-sha.outputs.iqtree2-sha }} path: | - ${{ inputs.os == 'windows-latest' && 'src/piqtree/_libiqtree/iqtree2.lib\nsrc/piqtree/_libiqtree/iqtree2.dll' || 'src/piqtree/_libiqtree/libiqtree2.a' }} + src/piqtree/_libiqtree/libiqtree2.a lookup-only: true + - name: Combine Cache Hits + id: cache + shell: bash + run: | + if [[ "${{ steps.cache-windows.outputs.cache-hit }}" == 'true' || "${{ steps.cache-unix.outputs.cache-hit }}" == 'true' ]]; then + echo "cache-hit=true" >> "$GITHUB_OUTPUT" + else + echo "cache-hit=false" >> "$GITHUB_OUTPUT" + fi + - name: Install Boost if: runner.os == 'Windows' && steps.cache.outputs.cache-hit != 'true' uses: MarkusJx/install-boost@v2.4.5 diff --git a/.github/actions/setup-piqtree/action.yml b/.github/actions/setup-piqtree/action.yml index 00ae89b0..01dffc5d 100644 --- a/.github/actions/setup-piqtree/action.yml +++ b/.github/actions/setup-piqtree/action.yml @@ -13,9 +13,24 @@ runs: - uses: "actions/setup-python@v5" with: python-version: ${{ inputs.python-version }} - - - uses: actions/cache/restore@v4 + + - name: Cache IQ-TREE 2 (Windows) + if: runner.os == 'Windows' + uses: actions/cache/restore@v4 + id: cache-windows with: - key: ${{ inputs.cache-key }} - path: src/piqtree/_libiqtree/libiqtree2.a - fail-on-cache-miss: true \ No newline at end of file + key: libiqtree-windows-${{ steps.iqtree2-sha.outputs.iqtree2-sha }} + path: | + src/piqtree/_libiqtree/iqtree2.lib + src/piqtree/_libiqtree/iqtree2.dll + fail-on-cache-miss: true + + - name: Cache IQ-TREE 2 (Linux/macOS) + if: runner.os != 'Windows' + uses: actions/cache/restore@v4 + id: cache-unix + with: + key: libiqtree-${{ inputs.os }}-${{ steps.iqtree2-sha.outputs.iqtree2-sha }} + path: | + src/piqtree/_libiqtree/libiqtree2.a + fail-on-cache-miss: true From 59728e3703060f0d3815983c794ed408074f1a73 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 16:48:39 +1100 Subject: [PATCH 25/38] DEV: Install llvm when testing windows ci, make build_iqtree compatible on local windows --- .github/workflows/ci.yml | 7 ++++++- build_tools/build_iqtree.sh | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 292f26de..4fce96f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,10 +50,15 @@ jobs: python-version: ${{ matrix.python-version }} cache-key: libiqtree-${{ matrix.os }}-${{ needs.build-iqtree.outputs.iqtree2-sha }} - - name: Install llvm + - name: Install llvm (macOS) if: runner.os == 'macOS' run: | brew install llvm + + - name: Install llvm (Windows) + if: runner.os == 'Windows' + run: | + choco install -y llvm --version=14.0.6 --allow-downgrade - name: Run Nox Testing run: | diff --git a/build_tools/build_iqtree.sh b/build_tools/build_iqtree.sh index 60a36afc..5aedccd6 100755 --- a/build_tools/build_iqtree.sh +++ b/build_tools/build_iqtree.sh @@ -11,12 +11,18 @@ if [[ "$OSTYPE" == "darwin"* ]]; then gmake -j elif [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* ]]; then echo "Building for Windows." + + if [[ -n "$BOOST_ROOT" ]]; then + export Boost_INCLUDE_DIR="${BOOST_ROOT}" + export Boost_LIBRARY_DIRS="${BOOST_ROOT}" + fi + cmake -G "MinGW Makefiles" \ -DCMAKE_C_COMPILER=clang \ -DCMAKE_CXX_COMPILER=clang++ \ -DCMAKE_C_FLAGS=--target=x86_64-pc-windows-gnu \ -DCMAKE_CXX_FLAGS=--target=x86_64-pc-windows-gnu \ - -DCMAKE_MAKE_PROGRAM=mingw32-make \ + -DCMAKE_MAKE_PROGRAM=make \ -DBoost_INCLUDE_DIR=$Boost_INCLUDE_DIR \ -DBoost_LIBRARY_DIRS=$Boost_LIBRARY_DIRS \ -DIQTREE_FLAGS="cpp14" \ From a97897d7dc856aea27f6eb36dcbe65b3828e3bc8 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 17:17:17 +1100 Subject: [PATCH 26/38] DEV: re-enable full CI matrix --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fce96f8..acf50784 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest] - # os: [ubuntu-latest, macos-13, macos-14, windows-latest] # Intel linux, Intel Mac, ARM Mac, Windows + os: [ubuntu-latest, macos-13, macos-14, windows-latest] # Intel linux, Intel Mac, ARM Mac, Windows steps: - uses: "actions/checkout@v4" @@ -36,8 +35,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest] - # os: [ubuntu-latest, macos-13, macos-14] # Intel linux, Intel Mac, ARM Mac + os: [ubuntu-latest, macos-13, macos-14, windows-latest] # Intel linux, Intel Mac, ARM Mac, Windows python-version: ["3.10", "3.11", "3.12"] steps: - uses: "actions/checkout@v4" From 1a3b064736c084e394d1896cf3307ecf737a924a Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 17:22:55 +1100 Subject: [PATCH 27/38] DEV: use runner.os for choosing before_all script --- .github/actions/build-iqtree/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index f0ca238d..0a26c3aa 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -74,11 +74,11 @@ runs: shell: bash if: steps.cache.outputs.cache-hit != 'true' run: | - if [[ "${{ inputs.os }}" == "ubuntu-latest" ]]; then + if [[ "${{ runner.os }}" == "Linux" ]]; then sudo ./build_tools/before_all_linux.sh - elif [[ "${{ inputs.os }}" == "macOS-latest" ]]; then + elif [[ "${{ runner.os }}" == "macOS" ]]; then ./build_tools/before_all_mac.sh - elif [[ "${{ inputs.os }}" == "windows-latest" ]]; then + elif [[ "${{ runner.os }}" == "Windows" ]]; then ./build_tools/before_all_windows.sh else echo "Unrecognized OS: '${{ inputs.os }}'." From 477cff66ec5699b20910796cecc02beef38fcb2d Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 17:40:11 +1100 Subject: [PATCH 28/38] DEV: standardise cache names --- .github/actions/build-iqtree/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-iqtree/action.yml b/.github/actions/build-iqtree/action.yml index 0a26c3aa..9f457639 100644 --- a/.github/actions/build-iqtree/action.yml +++ b/.github/actions/build-iqtree/action.yml @@ -24,7 +24,7 @@ runs: uses: actions/cache@v4 id: cache-windows with: - key: libiqtree-windows-${{ steps.iqtree2-sha.outputs.iqtree2-sha }} + key: libiqtree-${{ inputs.os }}-${{ steps.iqtree2-sha.outputs.iqtree2-sha }} path: | src/piqtree/_libiqtree/iqtree2.lib src/piqtree/_libiqtree/iqtree2.dll From ee18040ddf23941dbe7082ffc3ce3071a7daca0d Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 17:40:25 +1100 Subject: [PATCH 29/38] DEV: use correct input for cache key --- .github/actions/setup-piqtree/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-piqtree/action.yml b/.github/actions/setup-piqtree/action.yml index 01dffc5d..fb72ec8a 100644 --- a/.github/actions/setup-piqtree/action.yml +++ b/.github/actions/setup-piqtree/action.yml @@ -19,7 +19,7 @@ runs: uses: actions/cache/restore@v4 id: cache-windows with: - key: libiqtree-windows-${{ steps.iqtree2-sha.outputs.iqtree2-sha }} + key: ${{ inputs.cache-key }} path: | src/piqtree/_libiqtree/iqtree2.lib src/piqtree/_libiqtree/iqtree2.dll @@ -30,7 +30,7 @@ runs: uses: actions/cache/restore@v4 id: cache-unix with: - key: libiqtree-${{ inputs.os }}-${{ steps.iqtree2-sha.outputs.iqtree2-sha }} + key: ${{ inputs.cache-key }} path: | src/piqtree/_libiqtree/libiqtree2.a fail-on-cache-miss: true From e080e88b5ecf3e9eec19105b8a81bb4d13d6f0aa Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 17:56:51 +1100 Subject: [PATCH 30/38] DEV: add windows to cibuildwheel --- .github/workflows/build_wheels.yml | 30 ++++++++++++++++++++++++-- .github/workflows/release.yml | 34 +++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 66dc0819..8b7537a2 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -10,7 +10,7 @@ jobs: fail-fast: false matrix: include: - # manylinux (x86) + # manylinux x86_64 - os: ubuntu-latest platform_id: manylinux_x86_64 @@ -22,6 +22,10 @@ jobs: - os: macos-14 platform_id: macosx_arm64 + # Windows x86_64 + - os: windows-latest + platform_id: win_amd64 + steps: - uses: "actions/checkout@v4" with: @@ -35,7 +39,7 @@ jobs: platforms: arm64 - name: Set macOS Deployment Target - if: ${{startsWith(matrix.os, 'macos')}} + if: runner.os == 'macOS' run: | if [[ "${{ matrix.os }}" == "macos-13" ]]; then echo "MACOSX_DEPLOYMENT_TARGET=13.0" >> $GITHUB_ENV @@ -43,13 +47,35 @@ jobs: echo "MACOSX_DEPLOYMENT_TARGET=14.0" >> $GITHUB_ENV fi + - name: Install Boost + if: runner.os == 'Windows' + uses: MarkusJx/install-boost@v2.4.5 + id: install-boost + with: + boost_version: 1.84.0 + platform_version: 2022 + toolset: mingw + + - name: Set Boost Environment Variables + if: runner.os == 'Windows' + shell: bash + run: | + echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" + echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" + + - name: Setup MSVC Developer Command Prompt + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + - name: Build wheels uses: pypa/cibuildwheel@v2.23.0 env: # Can specify per os - e.g. CIBW_BEFORE_ALL_LINUX, CIBW_BEFORE_ALL_MACOS, CIBW_BEFORE_ALL_WINDOWS CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh + CIBW_BEFORE_ALL_WINDOWS: ./build_tools/before_all_windows.sh CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} + CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} CIBW_BUILD: "*${{matrix.platform_id}}" CIBW_TEST_REQUIRES: pytest CIBW_TEST_COMMAND: pytest {package}/tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 17158bca..15c5a8e8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ jobs: fail-fast: false matrix: include: - # manylinux (x86) + # manylinux x86_64 - os: ubuntu-latest platform_id: manylinux_x86_64 @@ -22,6 +22,10 @@ jobs: - os: macos-14 platform_id: macosx_arm64 + # Windows x86_64 + - os: windows-latest + platform_id: win_amd64 + steps: - uses: "actions/checkout@v4" with: @@ -35,23 +39,47 @@ jobs: platforms: arm64 - name: Set macOS Deployment Target - if: ${{startsWith(matrix.os, 'macos')}} + if: runner.os == 'macOS' run: | if [[ "${{ matrix.os }}" == "macos-13" ]]; then echo "MACOSX_DEPLOYMENT_TARGET=13.0" >> $GITHUB_ENV elif [[ "${{ matrix.os }}" == "macos-14" ]]; then echo "MACOSX_DEPLOYMENT_TARGET=14.0" >> $GITHUB_ENV fi - + + - name: Install Boost + if: runner.os == 'Windows' + uses: MarkusJx/install-boost@v2.4.5 + id: install-boost + with: + boost_version: 1.84.0 + platform_version: 2022 + toolset: mingw + + - name: Set Boost Environment Variables + if: runner.os == 'Windows' + shell: bash + run: | + echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" + echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" + + - name: Setup MSVC Developer Command Prompt + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + - name: Build wheels uses: pypa/cibuildwheel@v2.23.0 env: # Can specify per os - e.g. CIBW_BEFORE_ALL_LINUX, CIBW_BEFORE_ALL_MACOS, CIBW_BEFORE_ALL_WINDOWS CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh + CIBW_BEFORE_ALL_WINDOWS: ./build_tools/before_all_windows.sh CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} + CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} + CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} CIBW_BUILD: "*${{matrix.platform_id}}" CIBW_TEST_REQUIRES: pytest CIBW_TEST_COMMAND: pytest {package}/tests + CIBW_TEST_SKIP: "*-macosx_universal2:x86_64" # skip x86 on m1 mac CIBW_SKIP: pp* # Disable building PyPy wheels on all platforms - name: Upload wheels From acbb19289738f2c3c24e2a156d071220fedfdeed Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 18:08:42 +1100 Subject: [PATCH 31/38] DEV: run before_all_windows script with bash --- .github/workflows/build_wheels.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 8b7537a2..c078cd7c 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -72,7 +72,7 @@ jobs: env: # Can specify per os - e.g. CIBW_BEFORE_ALL_LINUX, CIBW_BEFORE_ALL_MACOS, CIBW_BEFORE_ALL_WINDOWS CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh - CIBW_BEFORE_ALL_WINDOWS: ./build_tools/before_all_windows.sh + CIBW_BEFORE_ALL_WINDOWS: bash ./build_tools/before_all_windows.sh CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 15c5a8e8..4b9fc0a5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,7 +72,7 @@ jobs: env: # Can specify per os - e.g. CIBW_BEFORE_ALL_LINUX, CIBW_BEFORE_ALL_MACOS, CIBW_BEFORE_ALL_WINDOWS CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh - CIBW_BEFORE_ALL_WINDOWS: ./build_tools/before_all_windows.sh + CIBW_BEFORE_ALL_WINDOWS: bash ./build_tools/before_all_windows.sh CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} From acf96f70c823efcdbe69718736748096980a38c9 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 18:32:37 +1100 Subject: [PATCH 32/38] DEV: debug BOOST env vars --- .github/workflows/release.yml | 1 + build_tools/before_all_windows.sh | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4b9fc0a5..2158af18 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,6 +73,7 @@ jobs: CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh CIBW_BEFORE_ALL_WINDOWS: bash ./build_tools/before_all_windows.sh + CIBW_ENVIRONMENT_WINDOWS: BOOST_ROOT='${{ steps.install-boost.outputs.BOOST_ROOT }}' CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} diff --git a/build_tools/before_all_windows.sh b/build_tools/before_all_windows.sh index 763d3b2d..5f81b9aa 100644 --- a/build_tools/before_all_windows.sh +++ b/build_tools/before_all_windows.sh @@ -1,4 +1,9 @@ # Install dependencies using choco + +echo "Boost_INCLUDE_DIR: " $Boost_INCLUDE_DIR +echo "Boost_LIBRARY_DIRS: " $Boost_LIBRARY_DIRS +echo "BOOST_ROOT: " $BOOST_ROOT + choco install -y llvm --version=14.0.6 --allow-downgrade choco install -y eigen From 4e622616aa68d24415e23af9a5531e6a95983a09 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 18:45:55 +1100 Subject: [PATCH 33/38] DEV: attempt hardcoding boost paths --- .github/workflows/build_wheels.yml | 1 + .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index c078cd7c..41903b1c 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -73,6 +73,7 @@ jobs: CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh CIBW_BEFORE_ALL_WINDOWS: bash ./build_tools/before_all_windows.sh + CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='D:/a/piqtree/piqtree/boost/boost/include' Boost_LIBRARY_DIRS='D:/a/piqtree/piqtree/boost/boost/include' CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2158af18..ff52508b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,7 +73,7 @@ jobs: CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh CIBW_BEFORE_ALL_WINDOWS: bash ./build_tools/before_all_windows.sh - CIBW_ENVIRONMENT_WINDOWS: BOOST_ROOT='${{ steps.install-boost.outputs.BOOST_ROOT }}' + CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='D:/a/piqtree/piqtree/boost/boost/include' Boost_LIBRARY_DIRS='D:/a/piqtree/piqtree/boost/boost/include' CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} From a0e29ec06eb629f57326b889934d5dadf711bab0 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 18:50:21 +1100 Subject: [PATCH 34/38] DEV: fix BOOST lib path --- .github/workflows/build_wheels.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 41903b1c..28856edc 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -73,7 +73,7 @@ jobs: CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh CIBW_BEFORE_ALL_WINDOWS: bash ./build_tools/before_all_windows.sh - CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='D:/a/piqtree/piqtree/boost/boost/include' Boost_LIBRARY_DIRS='D:/a/piqtree/piqtree/boost/boost/include' + CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='D:/a/piqtree/piqtree/boost/boost/include' Boost_LIBRARY_DIRS='D:/a/piqtree/piqtree/boost/boost/lib' CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff52508b..6ae48e8b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,7 +73,7 @@ jobs: CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh CIBW_BEFORE_ALL_WINDOWS: bash ./build_tools/before_all_windows.sh - CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='D:/a/piqtree/piqtree/boost/boost/include' Boost_LIBRARY_DIRS='D:/a/piqtree/piqtree/boost/boost/include' + CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='D:/a/piqtree/piqtree/boost/boost/include' Boost_LIBRARY_DIRS='D:/a/piqtree/piqtree/boost/boost/lib' CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} From c1bfbcccaf610e180ec645c1d90a9a99c8c281c1 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 18:53:32 +1100 Subject: [PATCH 35/38] DEV: remove hardcoding of BOOST paths --- .github/workflows/build_wheels.yml | 9 +-------- .github/workflows/release.yml | 9 +-------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 28856edc..a9676115 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -56,13 +56,6 @@ jobs: platform_version: 2022 toolset: mingw - - name: Set Boost Environment Variables - if: runner.os == 'Windows' - shell: bash - run: | - echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" - echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" - - name: Setup MSVC Developer Command Prompt if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 @@ -73,7 +66,7 @@ jobs: CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh CIBW_BEFORE_ALL_WINDOWS: bash ./build_tools/before_all_windows.sh - CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='D:/a/piqtree/piqtree/boost/boost/include' Boost_LIBRARY_DIRS='D:/a/piqtree/piqtree/boost/boost/lib' + CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='${{ steps.install-boost.outputs.BOOST_ROOT }}/include"' Boost_LIBRARY_DIRS='${{ steps.install-boost.outputs.BOOST_ROOT }}/lib' CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6ae48e8b..89e4c5d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,13 +56,6 @@ jobs: platform_version: 2022 toolset: mingw - - name: Set Boost Environment Variables - if: runner.os == 'Windows' - shell: bash - run: | - echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" - echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" - - name: Setup MSVC Developer Command Prompt if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 @@ -73,7 +66,7 @@ jobs: CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh CIBW_BEFORE_ALL_WINDOWS: bash ./build_tools/before_all_windows.sh - CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='D:/a/piqtree/piqtree/boost/boost/include' Boost_LIBRARY_DIRS='D:/a/piqtree/piqtree/boost/boost/lib' + CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='${{ steps.install-boost.outputs.BOOST_ROOT }}/include"' Boost_LIBRARY_DIRS='${{ steps.install-boost.outputs.BOOST_ROOT }}/lib' CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} From 14d05a79d93ca61ad5d12186920d0c313289faff Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 19:04:23 +1100 Subject: [PATCH 36/38] DEV: hardcode paths again --- .github/workflows/build_wheels.yml | 11 ++++++++++- .github/workflows/release.yml | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index a9676115..f89bea4b 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -56,6 +56,15 @@ jobs: platform_version: 2022 toolset: mingw + - name: Set Boost Environment Variables + if: runner.os == 'Windows' + shell: bash + run: | + echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" + echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" + echo $Boost_INCLUDE_DIR + echo $Boost_LIBRARY_DIRS + - name: Setup MSVC Developer Command Prompt if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 @@ -66,7 +75,7 @@ jobs: CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh CIBW_BEFORE_ALL_WINDOWS: bash ./build_tools/before_all_windows.sh - CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='${{ steps.install-boost.outputs.BOOST_ROOT }}/include"' Boost_LIBRARY_DIRS='${{ steps.install-boost.outputs.BOOST_ROOT }}/lib' + CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='D:/a/piqtree/piqtree/boost/boost/include' Boost_LIBRARY_DIRS='D:/a/piqtree/piqtree/boost/boost/lib' CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 89e4c5d9..09657f31 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,6 +56,15 @@ jobs: platform_version: 2022 toolset: mingw + - name: Set Boost Environment Variables + if: runner.os == 'Windows' + shell: bash + run: | + echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" + echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" + echo $Boost_INCLUDE_DIR + echo $Boost_LIBRARY_DIRS + - name: Setup MSVC Developer Command Prompt if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 @@ -66,7 +75,7 @@ jobs: CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh CIBW_BEFORE_ALL_WINDOWS: bash ./build_tools/before_all_windows.sh - CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='${{ steps.install-boost.outputs.BOOST_ROOT }}/include"' Boost_LIBRARY_DIRS='${{ steps.install-boost.outputs.BOOST_ROOT }}/lib' + CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='D:/a/piqtree/piqtree/boost/boost/include' Boost_LIBRARY_DIRS='D:/a/piqtree/piqtree/boost/boost/lib' CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} From b058c5e9f85d8b74ac595033d5b2eae4af4b1eb5 Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 19:16:15 +1100 Subject: [PATCH 37/38] DEV: normalise slashes in before_all_windows --- .github/workflows/build_wheels.yml | 11 +---------- .github/workflows/release.yml | 11 +---------- build_tools/before_all_windows.sh | 8 +++++--- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f89bea4b..5d56787d 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -56,15 +56,6 @@ jobs: platform_version: 2022 toolset: mingw - - name: Set Boost Environment Variables - if: runner.os == 'Windows' - shell: bash - run: | - echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" - echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" - echo $Boost_INCLUDE_DIR - echo $Boost_LIBRARY_DIRS - - name: Setup MSVC Developer Command Prompt if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 @@ -75,7 +66,7 @@ jobs: CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh CIBW_BEFORE_ALL_WINDOWS: bash ./build_tools/before_all_windows.sh - CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='D:/a/piqtree/piqtree/boost/boost/include' Boost_LIBRARY_DIRS='D:/a/piqtree/piqtree/boost/boost/lib' + CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='${{ steps.install-boost.outputs.BOOST_ROOT }}/include' Boost_LIBRARY_DIRS='${{ steps.install-boost.outputs.BOOST_ROOT }}/lib' CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 09657f31..500c3127 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,15 +56,6 @@ jobs: platform_version: 2022 toolset: mingw - - name: Set Boost Environment Variables - if: runner.os == 'Windows' - shell: bash - run: | - echo "Boost_INCLUDE_DIR=${{ steps.install-boost.outputs.BOOST_ROOT }}/include" >> "$GITHUB_ENV" - echo "Boost_LIBRARY_DIRS=${{ steps.install-boost.outputs.BOOST_ROOT }}/lib" >> "$GITHUB_ENV" - echo $Boost_INCLUDE_DIR - echo $Boost_LIBRARY_DIRS - - name: Setup MSVC Developer Command Prompt if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 @@ -75,7 +66,7 @@ jobs: CIBW_BEFORE_ALL_LINUX: ./build_tools/before_all_linux.sh CIBW_BEFORE_ALL_MACOS: ./build_tools/before_all_mac.sh CIBW_BEFORE_ALL_WINDOWS: bash ./build_tools/before_all_windows.sh - CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='D:/a/piqtree/piqtree/boost/boost/include' Boost_LIBRARY_DIRS='D:/a/piqtree/piqtree/boost/boost/lib' + CIBW_ENVIRONMENT_WINDOWS: Boost_INCLUDE_DIR='${{ steps.install-boost.outputs.BOOST_ROOT }}/include' Boost_LIBRARY_DIRS='${{ steps.install-boost.outputs.BOOST_ROOT }}/lib' CIBW_ARCHS_LINUX: ${{endsWith(matrix.platform_id, '_x86_64') && 'x86_64' || 'aarch64'}} CIBW_ARCHS_MACOS: ${{endsWith(matrix.platform_id, 'universal2') && 'universal2' || 'auto'}} CIBW_ARCHS_WINDOWS: ${{endsWith(matrix.platform_id, '_amd64') && 'AMD64' || 'ARM64'}} diff --git a/build_tools/before_all_windows.sh b/build_tools/before_all_windows.sh index 5f81b9aa..763536ea 100644 --- a/build_tools/before_all_windows.sh +++ b/build_tools/before_all_windows.sh @@ -1,8 +1,10 @@ # Install dependencies using choco -echo "Boost_INCLUDE_DIR: " $Boost_INCLUDE_DIR -echo "Boost_LIBRARY_DIRS: " $Boost_LIBRARY_DIRS -echo "BOOST_ROOT: " $BOOST_ROOT +export Boost_INCLUDE_DIR=$(echo $Boost_INCLUDE_DIR | sed 's|\\|/|g') +export Boost_LIBRARY_DIRS=$(echo $Boost_LIBRARY_DIRS | sed 's|\\|/|g') + +echo "Boost_INCLUDE_DIR: $Boost_INCLUDE_DIR" +echo "Boost_LIBRARY_DIRS: $Boost_LIBRARY_DIRS" choco install -y llvm --version=14.0.6 --allow-downgrade choco install -y eigen From 52ae29391af2519bf4ee9400883b3b26f7472fdb Mon Sep 17 00:00:00 2001 From: Robert McArthur Date: Thu, 20 Mar 2025 19:54:50 +1100 Subject: [PATCH 38/38] MAINT: remove extra space --- src/piqtree/_libiqtree/_piqtree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/piqtree/_libiqtree/_piqtree.cpp b/src/piqtree/_libiqtree/_piqtree.cpp index 52697f85..5101edd1 100644 --- a/src/piqtree/_libiqtree/_piqtree.cpp +++ b/src/piqtree/_libiqtree/_piqtree.cpp @@ -116,5 +116,5 @@ PYBIND11_MODULE(_piqtree, m) { "Construct pairwise distance matrix for alignment."); m.def("iq_nj_tree", &build_njtree, "Build neighbour-joining tree from distance matrix."); - m.def("mine", &mine, "The meaning of life, the universe (and everything) !"); + m.def("mine", &mine, "The meaning of life, the universe (and everything)!"); }