Streamline the processes of building libbls, testing it, and creating…

… the blspy wheels (#367)

* Speedup the process of creating wheels by skipping the compiling and linking of libbls' tests and benchmarks.
* We no longer need Windows-specific CMake version checking in setup.py.
* We no longer need Windows-specific architecture flag passing in setup.py.
* We no longer treat Windows in a special way w.r.t. compiling and linking in setup.py.
* We no longer need Windows-specific preparations before starting the process of creating wheels.
* We no longer need Windows-specific repairs after wheels creation.
* We no longer need the relic_ietf_64 repository and its related tasks.

Dedicated to @wjblanke

.github/workflows/build-wheels.yml

@@ -154,26 +154,6 @@ jobs:
         CIBW_BEFORE_BUILD_MACOS: >
           python -m pip install --upgrade pip
         CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=10.14"
-        CIBW_BEFORE_ALL_WINDOWS: >
-          curl -L https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable-msvc.zip > libsodium-1.0.18-stable-msvc.zip
-          && 7z x libsodium-1.0.18-stable-msvc.zip
-          && git clone https://github.com/Chia-Network/relic_ietf_64.git
-          && ls -l relic_ietf_64
-          && git clone https://github.com/Chia-Network/mpir_gc_x64.git
-          && ls -l mpir_gc_x64
-        CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >
-          ls -l mpir_gc_x64 && pip uninstall -y delocate
-          && pip install git+https://github.com/Chia-Network/delocate.git
-          && delocate-wheel -v -i mpir_gc_x64/mpir.dll {wheel}
-          && delocate-wheel -v -i mpir_gc_x64/mpir_gc.dll {wheel}
-          && delocate-wheel -v -i mpir_gc_x64/mpir_broadwell.dll {wheel}
-          && delocate-wheel -v -i mpir_gc_x64/mpir_broadwell_avx.dll {wheel}
-          && delocate-wheel -v -i mpir_gc_x64/mpir_bulldozer.dll {wheel}
-          && delocate-wheel -v -i mpir_gc_x64/mpir_haswell.dll {wheel}
-          && delocate-wheel -v -i mpir_gc_x64/mpir_piledriver.dll {wheel}
-          && delocate-wheel -v -i mpir_gc_x64/mpir_sandybridge.dll {wheel}
-          && delocate-wheel -v -i mpir_gc_x64/mpir_skylake_avx.dll {wheel}
-          && cp {wheel} {dest_dir}
         CIBW_TEST_REQUIRES: pytest
         CIBW_TEST_COMMAND: py.test -v {project}/python-bindings/test.py
       run:

CMakeLists.txt

@@ -47,7 +47,7 @@ if(DEFINED ENV{RELIC_MAIN})
   set(RELIC_REPOSITORY "https://github.com/relic-toolkit/relic.git")
 else()
   # This is currently anchored to upstream aecdcae7956f542fbee2392c1f0feb0a8ac41dc5
-  set(RELIC_GIT_TAG "215c69966cb78b255995f0ee9c86bbbb41c3c42b")
+  set(RELIC_GIT_TAG "2a4ed8ded3a9a5ac5e39f3877f357264b2e16f11")
   set(RELIC_REPOSITORY "https://github.com/Chia-Network/relic.git")
 endif()
 
@@ -63,12 +63,25 @@ FetchContent_Declare(
 
 set(STBIN "off" CACHE STRING "Relic - Build static binaries")
 
-find_package(gmp)
-if (GMP_FOUND)
-  message(STATUS "Found libgmp")
+if(MSVC)
   set(ARITH "gmp" CACHE STRING "")
+  # We'll be using our custom MPIR build instead of GMP for MSVC builds
+  FetchContent_Declare(
+    chia_mpir
+    GIT_REPOSITORY https://github.com/Chia-Network/mpir_gc_x64.git
+    GIT_SHALLOW true
+  )
+  FetchContent_MakeAvailable(chia_mpir)
+  set(MPIR_INCLUDE_DIR ${chia_mpir_SOURCE_DIR} CACHE STRING "")
+  set(MPIR_IMPLIB "${chia_mpir_SOURCE_DIR}/mpir.lib" CACHE STRING "")
 else()
-  set(ARITH "easy" CACHE STRING "")
+  find_package(gmp)
+  if(GMP_FOUND)
+    message(STATUS "Found libgmp")
+    set(ARITH "gmp" CACHE STRING "")
+  else()
+    set(ARITH "easy" CACHE STRING "")
+  endif()
 endif()
 
 if(CMAKE_SIZEOF_VOID_P EQUAL 4)
@@ -141,6 +154,17 @@ set(PP_METHD "LAZYR;OATEP" CACHE STRING "")
 
 FetchContent_MakeAvailable(relic)
 
+function(handle_mpir_dlls _target)
+  file(GLOB chia_mpir_dlls ${chia_mpir_SOURCE_DIR}/*.dll)
+  foreach(mpir_dll IN LISTS chia_mpir_dlls)
+    add_custom_command(
+      TARGET ${_target}
+      POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different ${mpir_dll} $<TARGET_FILE_DIR:${_target}>
+    )
+  endforeach()
+endfunction()
+
 add_subdirectory(src)
 
 if(EMSCRIPTEN)

python-bindings/CMakeLists.txt

@@ -4,7 +4,8 @@ FetchContent_Declare(
   GIT_REPOSITORY https://github.com/pybind/pybind11.git
   GIT_TAG        v2.10.0
 )
-FetchContent_MakeAvailable(pybind11 relic)
+FetchContent_MakeAvailable(pybind11)
 
 pybind11_add_module(blspy ${CMAKE_CURRENT_SOURCE_DIR}/pythonbindings.cpp)
 target_link_libraries(blspy PRIVATE bls)
+handle_mpir_dlls(blspy)

setup.py

@@ -1,12 +1,9 @@
 #!/usr/bin/python3
 import os
 import platform
-import re
 import subprocess
 import sys
-from distutils.version import LooseVersion
-
-from setuptools import Extension, setup, setuptools
+from setuptools import Extension, setup
 from setuptools.command.build_ext import build_ext
 
 
@@ -19,21 +16,14 @@ def __init__(self, name, sourcedir=""):
 class CMakeBuild(build_ext):
     def run(self):
         try:
-            out = subprocess.check_output(["cmake", "--version"])
+            subprocess.check_output(["cmake", "--version"])
         except OSError:
             raise RuntimeError(
                 "CMake must be installed to build"
                 + " the following extensions: "
                 + ", ".join(e.name for e in self.extensions)
             )
 
-        if platform.system() == "Windows":
-            cmake_version = LooseVersion(
-                re.search(r"version\s*([\d.]+)", out.decode()).group(1)
-            )
-            if cmake_version < "3.1.0":
-                raise RuntimeError("CMake >= 3.1.0 is required on Windows")
-
         for ext in self.extensions:
             self.build_extension(ext)
 
@@ -42,6 +32,8 @@ def build_extension(self, ext):
         cmake_args = [
             "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=" + extdir,
             "-DPYTHON_EXECUTABLE=" + sys.executable,
+            "-DBUILD_BLS_TESTS=0",
+            "-DBUILD_BLS_BENCHMARKS=0",
         ]
 
         cfg = "Debug" if self.debug else "Release"
@@ -51,9 +43,6 @@ def build_extension(self, ext):
             cmake_args += [
                 "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format(cfg.upper(), extdir)
             ]
-            if sys.maxsize > 2 ** 32:
-                cmake_args += ["-A", "x64"]
-            build_args += ["--", "/m"]
         else:
             cmake_args += ["-DCMAKE_BUILD_TYPE=" + cfg]
             build_args += ["--", "-j", "6"]
@@ -72,148 +61,17 @@ def build_extension(self, ext):
         )
 
 
-class get_pybind_include(object):
-    """Helper class to determine the pybind11 include path
-
-    The purpose of this class is to postpone importing pybind11
-    until it is actually installed, so that the ``get_include()``
-    method can be invoked."""
-
-    def __init__(self, user=False):
-        self.user = user
-
-    def __str__(self):
-        import pybind11
-
-        return pybind11.get_include(self.user)
-
-
-ext_modules = [
-    Extension(
-        "blspy",
-        [
-            "src/elements.cpp",
-            "src/schemes.cpp",
-            "src/privatekey.cpp",
-            "src/bls.cpp",
-            "python-bindings/pythonbindings.cpp",
-        ],
-        include_dirs=[
-            # Path to pybind11 headers
-            get_pybind_include(),
-            get_pybind_include(user=True),
-            "relic_ietf_64/include",
-            "mpir_gc_x64",
-            "libsodium/include",
-        ],
-        library_dirs=[
-            "relic_ietf_64",
-            "mpir_gc_x64",
-            "libsodium/x64/Release/v142/static",
-        ],
-        libraries=["relic_s", "Advapi32", "mpir", "libsodium"],
-        language="c++",
-    ),
-]
-
-
-# As of Python 3.6, CCompiler has a `has_flag` method.
-# cf http://bugs.python.org/issue26689
-def has_flag(compiler, flagname):
-    """Return a boolean indicating whether a flag name is supported on
-    the specified compiler.
-    """
-    import tempfile
-
-    with tempfile.NamedTemporaryFile("w", suffix=".cpp") as f:
-        f.write("int main (int argc, char **argv) { return 0; }")
-        try:
-            compiler.compile([f.name], extra_postargs=[flagname])
-        except setuptools.distutils.errors.CompileError:
-            return False
-    return True
-
-
-def cpp_flag(compiler):
-    """Return the -std=c++[11/14/17] compiler flag.
-
-    The newer version is prefered over c++11 (when it is available).
-    """
-    flags = ["-std=c++17", "-std=c++14", "-std=c++11"]
-
-    for flag in flags:
-        if has_flag(compiler, flag):
-            return flag
-
-    raise RuntimeError("Unsupported compiler -- at least C++11 support " "is needed!")
-
-
-class BuildExt(build_ext):
-    """A custom build extension for adding compiler-specific options."""
-
-    c_opts = {
-        "msvc": ["/EHsc", "/std:c++17", "/DBLSALLOC_SODIUM=1", "/DSODIUM_STATIC"],
-        "unix": [],
-    }
-    l_opts = {
-        "msvc": [],
-        "unix": [],
-    }
-
-    if sys.platform == "darwin":
-        darwin_opts = ["-stdlib=libc++", "-mmacosx-version-min=10.14"]
-        c_opts["unix"] += darwin_opts
-        l_opts["unix"] += darwin_opts
-
-    def build_extensions(self):
-        ct = self.compiler.compiler_type
-        opts = self.c_opts.get(ct, [])
-        link_opts = self.l_opts.get(ct, [])
-        if ct == "unix":
-            opts.append('-DVERSION_INFO="%s"' % self.distribution.get_version())
-            opts.append(cpp_flag(self.compiler))
-            if has_flag(self.compiler, "-fvisibility=hidden"):
-                opts.append("-fvisibility=hidden")
-        elif ct == "msvc":
-            if sys.version_info < (3, 9):
-                ver_flag = '/DVERSION_INFO=\"%s\"'
-            else:
-                ver_flag = '-DVERSION_INFO="%s"'
-            opts.append(ver_flag % self.distribution.get_version())
-        for ext in self.extensions:
-            ext.extra_compile_args = opts
-            ext.extra_link_args = link_opts
-        build_ext.build_extensions(self)
-
-
-if platform.system() == "Windows":
-    setup(
-        name="blspy",
-        author="Mariano Sorgente",
-        author_email="mariano@chia.net",
-        description="BLS signatures in c++ (with python bindings)",
-        long_description=open("README.md").read(),
-        long_description_content_type="text/markdown",
-        url="https://github.com/Chia-Network/bls-signatures",
-        python_requires=">=3.7",
-        setup_requires=["pybind11>=2.10.0"],
-        install_requires=["pybind11>=2.10.0"],
-        ext_modules=ext_modules,
-        cmdclass={"build_ext": BuildExt},
-        zip_safe=False,
-    )
-else:
-    setup(
-        name="blspy",
-        author="Mariano Sorgente",
-        author_email="mariano@chia.net",
-        description="BLS signatures in c++ (python bindings)",
-        python_requires=">=3.7",
-        install_requires=["wheel"],
-        long_description=open("README.md").read(),
-        long_description_content_type="text/markdown",
-        url="https://github.com/Chia-Network/bls-signatures",
-        ext_modules=[CMakeExtension("blspy", ".")],
-        cmdclass=dict(build_ext=CMakeBuild),
-        zip_safe=False,
-    )
+setup(
+    name="blspy",
+    author="Mariano Sorgente",
+    author_email="mariano@chia.net",
+    description="BLS signatures in c++ (python bindings)",
+    python_requires=">=3.7",
+    install_requires=["wheel"],
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    url="https://github.com/Chia-Network/bls-signatures",
+    ext_modules=[CMakeExtension("blspy", ".")],
+    cmdclass=dict(build_ext=CMakeBuild),
+    zip_safe=False,
+)

src/CMakeLists.txt

@@ -39,6 +39,10 @@ install(DIRECTORY ${relic_BINARY_DIR}/include/ DESTINATION include/chiabls)
 install(FILES ${HEADERS} DESTINATION include/chiabls)
 install(FILES $<TARGET_FILE:bls> DESTINATION lib)
 
+if(MSVC AND (BUILD_BLS_TESTS OR BUILD_BLS_BENCHMARKS))
+  handle_mpir_dlls(bls)
+endif()
+
 if(BUILD_BLS_TESTS)
   FetchContent_Declare(
     Catch2