diff --git a/TODO.md b/TODO.md index e27ad99..33b9e71 100644 --- a/TODO.md +++ b/TODO.md @@ -12,6 +12,7 @@ Interface changing * refactor PtrArrFactoryMixIn into factory object * .sizeof and .offsetof() shall be available at runtime only and done by the compiler (seems to be compiler dependent) + * replace packing by libclangs sizeof/offsetof to support #pragma pack Small (can be done by occassion) -------------------------------- @@ -99,6 +100,7 @@ Small (can be done by occassion) * comparison shall also work if python type is different. I.e. ts.struct.x(1, 2) == (1, 2) does not work yet, as the .val of struct returns a dict, which is not equal to the tuple. +* Add Support for 64bit MinGW (is stored in different directory than 32bit) Medium @@ -144,7 +146,9 @@ Major/Investigation necessary + if a macro is set in the TestSetup it must not be overwritten by the parser (= allow overwriting macros) * test C instrumentalization (address sanitizer, see -f... flags) -* packing pragma (attribute?) is not supported +* Switch from access via ABI to access via API. This would require less + compiler adaptions +* C++ Support Needed for publishing diff --git a/docs/about.rst b/docs/about.rst index 619a8aa..e90d9ff 100644 --- a/docs/about.rst +++ b/docs/about.rst @@ -17,7 +17,8 @@ C/Python bridges like ctypes, cffi, swig, cython, ...: python file. These steps done by headlock include: - * No need to create Makefiles/Buildscripts. + * No need to create extra Makefiles/Buildscripts for unit testing a single + C module * No need to run extra build steps before using the C code. * No need to rewrite the C moduels interface definition (the header file) in Python. @@ -70,13 +71,11 @@ C/Python bridges like ctypes, cffi, swig, cython, ...: Explicitly Non-Goals Are: - * Support for C++ - * High Performance (This does not mean that it is slow. But if speed conflicts with one of the goals of this project, there will be no compromises in favour of speed). * Being self-contained - (At least A C-compiler will always be required to be installed). + (At least LLVM and a C-compiler will always be required to be installed). * Support for Python < 3.6 diff --git a/headlock/bridge_gen.py b/headlock/bridge_gen.py index 7e292ad..a4d66e3 100644 --- a/headlock/bridge_gen.py +++ b/headlock/bridge_gen.py @@ -82,11 +82,13 @@ def write_c2py_bridge(output:TextIO, required_funcptrs:Iterable[CFuncType], for instance_ndx in range(max_instances): write_c2py_bridge_func(output, bridge_ndx, instance_ndx, cfunc) output.write( - f'void (* _c2py_bridge_[][{max_instances}])(void) = {{\n') + f'typedef void (* _c2py_bridge_func_t)(void);\n' + f'_c2py_bridge_func_t _c2py_bridge_[][{max_instances}] = {{\n') for bridge_ndx in range(len(bridge_ndxs)): output.write('\t{ ') for instance_ndx in range(max_instances): - output.write(f'_c2py_bridge_{bridge_ndx}_{instance_ndx}, ') + output.write(f'(_c2py_bridge_func_t) ' + f'_c2py_bridge_{bridge_ndx}_{instance_ndx}, ') output.write(' },\n') output.write('};\n\n') return bridge_ndxs diff --git a/headlock/buildsys_drvs/__init__.py b/headlock/buildsys_drvs/__init__.py new file mode 100644 index 0000000..53083d4 --- /dev/null +++ b/headlock/buildsys_drvs/__init__.py @@ -0,0 +1,110 @@ +import abc +import hashlib +import os +from pathlib import Path +from typing import List, Dict + + +class BuildError(Exception): + + def __init__(self, msg, path=None): + super().__init__(msg) + self.path = path + + def __str__(self): + return (f'building {self.path} failed: ' if self.path + else '') \ + + super().__str__() + + +class BuildDescription: + """ + This is an abstract base class for all classes that allow to specify + C projects + """ + + def __init__(self, name:str, build_dir:Path, unique_name=True): + """ + Abstract Base Class for Descriptions how to build a bunch of C files + :param name: Descriptive Name of BuildDescription + :param build_dir: Directory where all generated files shall be stored + :param unique_name: If True, a BuildDescription with the same name + cannot exist + """ + self.name = name + self.__build_dir = build_dir + self.unique_name = unique_name + + @property + def build_dir(self): + if self.unique_name: + return self.__build_dir + else: + hash = hashlib.md5() + incl_dirs = self.incl_dirs() + predef_macros = self.predef_macros() + for c_source in self.c_sources(): + hash.update(os.fsencode(c_source)) + for incl_dir in incl_dirs[c_source]: + hash.update(os.fsencode(incl_dir)) + for mname, mvalue in predef_macros[c_source].items(): + hash.update((mname + '=' + mvalue).encode('ascii')) + return self.__build_dir.parent \ + / (self.__build_dir.name + '_' + hash.hexdigest()[:8]) + + @abc.abstractmethod + def clang_target(self) -> str: + """ + returns a string that represents the "target" parameter of clang + """ + raise NotImplementedError() + + @abc.abstractmethod + def sys_incl_dirs(self) -> List[Path]: + """ + retrieves a list of all system include directories + """ + + def sys_predef_macros(self) -> Dict[str, str]: + """ + A dictionary of toolchain-inhernt macros, that shall be always + predefined additionally to the predefined macros provided by clang + """ + return {} + + def c_sources(self) -> List[Path]: + """ + returns all C source files + """ + + @abc.abstractmethod + def incl_dirs(self) -> Dict[Path, List[Path]]: + """ + returns a list of all source code files. + :return: + """ + + @abc.abstractmethod + def predef_macros(self) -> Dict[Path, Dict[str, str]]: + """ + returns all predefined macros per source file a list of all source code files. + :return: + """ + + @abc.abstractmethod + def exe_path(self) -> Path: + """ + returns name of executable image/shared object library/dll + """ + + @abc.abstractmethod + def build(self, additonal_c_sources:List[Path]=None): + """ + builds executable image + """ + + def is_header_file(self, src_path:Path) -> bool: + """ + Returns True, if this is a header file + """ + return src_path.suffix.lower() == '.h' diff --git a/headlock/buildsys_drvs/gcc.py b/headlock/buildsys_drvs/gcc.py new file mode 100644 index 0000000..c32c737 --- /dev/null +++ b/headlock/buildsys_drvs/gcc.py @@ -0,0 +1,141 @@ +import subprocess +import os +from pathlib import Path + +from typing import Dict, List, Any +from . import BuildDescription, BuildError + + +BUILD_CACHE = set() + + +class GccBuildDescription(BuildDescription): + + SYS_INCL_DIR_CACHE:Dict[Path, List[Path]] = {} + + ADDITIONAL_COMPILE_OPTIONS = [] + ADDITIONAL_LINK_OPTIONS = [] + + def __init__(self, name:str, build_dir:Path, unique_name=True, *, + c_sources:List[Path]=None, predef_macros:Dict[str, Any]=None, + incl_dirs:List[Path]=None, lib_dirs:List[Path]=None, + req_libs:List[str]=None, gcc_executable='gcc'): + super().__init__(name, build_dir, unique_name) + self.gcc_executable = gcc_executable + self.__c_sources = c_sources or [] + self.__incl_dirs = incl_dirs or [] + self.__req_libs = req_libs or [] + self.__lib_dirs = lib_dirs or [] + self.__predef_macros = {} + self.add_predef_macros(predef_macros or {}) + + def add_c_source(self, c_filename:Path): + if c_filename in self.__c_sources: + raise ValueError(f'Translation Unit {c_filename!r} is already part ' + f'of BuildDescription') + self.__c_sources.append(c_filename) + + def add_predef_macros(self, predef_macros:Dict[str, Any]): + for name, val in predef_macros.items(): + self.__predef_macros[name] = str(val) if val is not None else None + + def add_incl_dir(self, incl_dir:Path): + self.__incl_dirs.append(incl_dir) + + def add_lib_dir(self, lib_dir:Path): + self.__lib_dirs.append(lib_dir) + + def add_req_lib(self, lib_name:str): + self.__req_libs.append(lib_name) + + def sys_incl_dirs(self): + if self.gcc_executable not in self.SYS_INCL_DIR_CACHE: + try: + gcc_info = subprocess.check_output( + [self.gcc_executable, + '-v', '-xc', '-c', '/dev/null', '-o', '/dev/null'], + stderr=subprocess.STDOUT, + encoding='utf8') + except (subprocess.CalledProcessError, FileNotFoundError) as e: + raise BuildError('failed to retrieve SYS include path from gcc') + else: + incl_dirs = self.SYS_INCL_DIR_CACHE[self.gcc_executable] = [] + collecting = False + for line in gcc_info.splitlines(): + if line.startswith('#include <...> search starts here'): + collecting = True + elif line.startswith('End of search list.'): + collecting = False + elif collecting: + incl_dirs.append(Path(line.strip())) + return self.SYS_INCL_DIR_CACHE[self.gcc_executable] + + def c_sources(self): + return self.__c_sources + + def predef_macros(self): + return dict.fromkeys(self.__c_sources, self.__predef_macros) + + def incl_dirs(self): + return dict.fromkeys(self.__c_sources, self.__incl_dirs) + + def _run_gcc(self, call_params, dest_file): + try: + completed_proc = subprocess.run( + [self.gcc_executable] + call_params, + encoding='utf8', + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except (subprocess.CalledProcessError, FileNotFoundError) as e: + raise BuildError(f'failed to call gcc: {e}', dest_file) + else: + if completed_proc.returncode != 0: + raise BuildError(completed_proc.stderr, dest_file) + + def exe_path(self): + return self.build_dir / '__headlock__.dll' + + def build(self, additonal_c_sources=None): + transunits = (tuple(self.__c_sources), + tuple(self.__predef_macros.items()), + tuple(self.__incl_dirs), + tuple(self.__req_libs), + tuple(self.__lib_dirs)) + if (tuple(transunits), self.build_dir) in BUILD_CACHE: + return + total_sources = self.__c_sources + (additonal_c_sources or []) + for c_src in total_sources: + if self.is_header_file(c_src): + continue + obj_file_path = self.build_dir / (c_src.stem + '.o') + self._run_gcc(['-c', os.fspath(c_src)] + + ['-o', os.fspath(obj_file_path)] + + ['-I' + os.fspath(incl_dir) + for incl_dir in self.__incl_dirs] + + [f'-D{mname}={mval or ""}' + for mname, mval in self.__predef_macros.items()] + + ['-Werror'] + + self.ADDITIONAL_COMPILE_OPTIONS, + obj_file_path) + exe_file_path = self.exe_path() + self._run_gcc([str(self.build_dir / (c_src.stem + '.o')) + for c_src in total_sources + if not self.is_header_file(c_src)] + + ['-shared', '-o', os.fspath(exe_file_path)] + + ['-l' + str(req_lib) for req_lib in self.__req_libs] + + ['-L' + str(lib_dir) for lib_dir in self.__lib_dirs] + + ['-Werror'] + + self.ADDITIONAL_LINK_OPTIONS, + exe_file_path) + BUILD_CACHE.add((tuple(transunits), self.build_dir)) + + +class Gcc32BuildDescription(GccBuildDescription): + def clang_target(self): + return 'i386-pc-linux-gnu' + + +class Gcc64BuildDescription(GccBuildDescription): + ADDITIONAL_COMPILE_OPTIONS = ['-fPIC'] + def clang_target(self): + return 'x86_64-pc-linux-gnu' diff --git a/headlock/toolchains/mingw.py b/headlock/buildsys_drvs/mingw.py similarity index 60% rename from headlock/toolchains/mingw.py rename to headlock/buildsys_drvs/mingw.py index 722b4ae..524e2af 100644 --- a/headlock/toolchains/mingw.py +++ b/headlock/buildsys_drvs/mingw.py @@ -1,33 +1,48 @@ from pathlib import Path -import winreg +import sys +import platform import itertools import fnmatch +from typing import List, Dict, Any +try: + import winreg +except ImportError: + pass -from . import BuildError -from .gcc import GccToolChain +from . import BuildError, gcc BUILD_CACHE = set() -class MinGWToolChain(GccToolChain): +class MinGWBuildDescription(gcc.GccBuildDescription): ADDITIONAL_COMPILE_OPTIONS = [] ADDITIONAL_LINK_OPTIONS = ['-static-libgcc', '-static-libstdc++'] ARCHITECTURE:str = None - def __init__(self, version=None, thread_model=None, - exception_model=None, rev=None, mingw_install_dir=None): - super().__init__() + def __init__(self, name, build_dir, unique_name=True, *, + c_sources:List[Path]=None, predef_macros:Dict[str, Any]=None, + incl_dirs:List[Path]=None, lib_dirs:List[Path]=None, + req_libs:List[str]=None, version:str=None, + thread_model:str=None, exception_model:str=None, rev:str=None, + mingw_install_dir:Path=None): if mingw_install_dir is None: + arch, plat, compiler = self.clang_target().split('-') self.mingw_install_dir = self._autodetect_mingw_dir( - version, thread_model, exception_model, rev) + version, thread_model, exception_model, rev) / compiler else: self.mingw_install_dir = mingw_install_dir - self.gcc_executable = str(self.mingw_install_dir / 'bin' / 'gcc') - - def _autodetect_mingw_dir(self,version, thread_model, exception_model, rev): + gcc_executable = str(self.mingw_install_dir / 'bin' / 'gcc') + super().__init__( + name, build_dir, unique_name, + c_sources=c_sources, predef_macros=predef_macros, + incl_dirs=incl_dirs, lib_dirs=lib_dirs, req_libs=req_libs, + gcc_executable=gcc_executable) + + @classmethod + def _autodetect_mingw_dir(cls, version, thread_model, exception_model, rev): def iter_uninstall_progkeys(): for uninstall_regkey in [r"SOFTWARE\WOW6432Node\Microsoft\Windows" r"\CurrentVersion\Uninstall", @@ -45,7 +60,7 @@ def iter_uninstall_progkeys(): subkey_name) as subkey: yield subkey_name, subkey - mingw_filter = f"{self.ARCHITECTURE}-{version or '*'}" \ + mingw_filter = f"{cls.ARCHITECTURE}-{version or '*'}" \ f"-{thread_model or '*'}-{exception_model or '*'}" \ f"-*-{rev or '*'}" mingw_install_dirs = {} @@ -70,9 +85,7 @@ def iter_uninstall_progkeys(): f'Requested MinGW Version ({mingw_filter}) was not found ' f'on this system') _, mingw_install_dir = max(mingw_install_dirs.items()) - - arch, plat, compiler = self.CLANG_TARGET.split('-') - return mingw_install_dir / compiler + return mingw_install_dir def sys_incl_dirs(self): ext_incl_dir_base = self.mingw_install_dir \ @@ -82,11 +95,28 @@ def sys_incl_dirs(self): + list(ext_incl_dir_base.glob('*.*.*/include'))[:1] -class MinGW32ToolChain(MinGWToolChain): - CLANG_TARGET = 'i386-pc-mingw32' +class MinGW32BuildDescription(MinGWBuildDescription): ARCHITECTURE = 'i686' + def clang_target(self): + return 'i386-pc-mingw32' -class MinGW64ToolChain(MinGWToolChain): - CLANG_TARGET = 'x86_64-pc-mingw64' +class MinGW64BuildDescription(MinGWBuildDescription): ARCHITECTURE = 'x86_64' + def clang_target(self): + return 'x86_64-pc-mingw64' + + +def get_default_builddesc_cls(): + if sys.platform == 'win32': + if platform.architecture()[0] == '32bit': + return MinGW32BuildDescription + else: + return MinGW64BuildDescription + elif sys.platform == 'linux': + if platform.architecture()[0] == '32bit': + return gcc.Gcc32BuildDescription + else: + return gcc.Gcc64BuildDescription + else: + raise NotImplementedError('This OS is not supported') diff --git a/headlock/buildsys_drvs/test_buildsys_drvs.py b/headlock/buildsys_drvs/test_buildsys_drvs.py new file mode 100644 index 0000000..aaef05d --- /dev/null +++ b/headlock/buildsys_drvs/test_buildsys_drvs.py @@ -0,0 +1,24 @@ +import pytest +from unittest.mock import Mock +from pathlib import Path + +from headlock.buildsys_drvs import BuildDescription + + +class TestBuildDescription: + + def test_init_onUniqueNamedObj(self): + builddesc = BuildDescription('PrjName', Path('path/to/prj')) + assert builddesc.name == 'PrjName' + assert builddesc.build_dir == Path('path/to/prj') + + def test_init_onNonUniqueNamedObj_addsId(self): + builddesc = BuildDescription('PrjName', Path('path/to/prj'), + unique_name=False) + builddesc.c_sources = Mock(return_value=[Path('src.c')]) + builddesc.incl_dirs = Mock(return_value={Path('src.c'): [Path('dir')]}) + builddesc.predef_macros = Mock(return_value={Path('src.c'): {'m': 'v'}}) + assert builddesc.name == 'PrjName' + assert builddesc.build_dir.parent == Path('path/to') + assert builddesc.build_dir.name[:-8] == 'prj_' + assert all(c.isalnum() for c in builddesc.build_dir.name[-8:]) diff --git a/headlock/integrations/pytest/debug_failed.py b/headlock/integrations/pytest/debug_failed.py index 178a96e..36cdaf9 100644 --- a/headlock/integrations/pytest/debug_failed.py +++ b/headlock/integrations/pytest/debug_failed.py @@ -20,9 +20,9 @@ if platform.architecture()[0] == '32bit': - from headlock.toolchains.mingw import MinGW32ToolChain as MinGWxxToolChain + from headlock.buildsys_drvs.mingw import MinGW32BuildDescription as MinGWxxToolChain else: - from headlock.toolchains.mingw import MinGW64ToolChain as MinGWxxToolChain + from headlock.buildsys_drvs.mingw import MinGW64BuildDescription as MinGWxxToolChain class DummyToolChain(MinGWxxToolChain): diff --git a/headlock/integrations/pytest/plugin_headlock_debug.py b/headlock/integrations/pytest/plugin_headlock_debug.py index 6762069..fc75ff4 100644 --- a/headlock/integrations/pytest/plugin_headlock_debug.py +++ b/headlock/integrations/pytest/plugin_headlock_debug.py @@ -17,7 +17,9 @@ """ import os from pathlib import Path -from headlock.testsetup import TestSetup, ToolChainDriver +from collections import defaultdict +from headlock.testsetup import TestSetup +from headlock.buildsys_drvs import BuildDescription from .common import PYTEST_HEADLOCK_DIR @@ -109,29 +111,32 @@ def pytest_runtest_logreport(report): finish_test(report.nodeid, report.failed) -class CMakeToolChain(ToolChainDriver): +class CMakeFileGenerator(BuildDescription): - ADDITIONAL_COMPILE_OPTIONS = ['-Werror'] - ADDITIONAL_LINK_OPTIONS = ['-Werror'] + def __init__(self, base_builddesc): + super().__init__(base_builddesc.name, base_builddesc.build_dir) + self.base_builddesc = base_builddesc - def __init__(self, base_toolchain): - self.base_toolchain = base_toolchain - self.ADDITIONAL_COMPILE_OPTIONS = \ - base_toolchain.ADDITIONAL_COMPILE_OPTIONS + \ - self.ADDITIONAL_COMPILE_OPTIONS - self.ADDITIONAL_LINK_OPTIONS = \ - base_toolchain.ADDITIONAL_LINK_OPTIONS + \ - self.ADDITIONAL_LINK_OPTIONS - self.CLANG_TARGET = base_toolchain.CLANG_TARGET + def clang_target(self): + return self.base_builddesc.clang_target() + + def c_sources(self): + return self.base_builddesc.c_sources() def sys_predef_macros(self): - return self.base_toolchain.sys_predef_macros() + return self.base_builddesc.sys_predef_macros() def sys_incl_dirs(self): - return self.base_toolchain.sys_incl_dirs() + return self.base_builddesc.sys_incl_dirs() + + def predef_macros(self): + return self.base_builddesc.predef_macros() - def exe_path(self, name, build_dir): - return self.base_toolchain.exe_path(name, build_dir) + def incl_dirs(self): + return self.base_builddesc.incl_dirs() + + def exe_path(self): + return self.base_builddesc.exe_path() @staticmethod def escape(str): @@ -140,73 +145,93 @@ def escape(str): else: return str - def generate_cmakelists(self, prj_name, build_dir, transunits, - req_libs, lib_dirs): + def group_c_sources_by_paramset(self): + param_sets = defaultdict(list) + predef_macros = self.predef_macros() + incl_dirs = self.incl_dirs() + for c_src in self.c_sources(): + param_set = tuple(sorted(predef_macros[c_src].items())) + \ + tuple(sorted(incl_dirs[c_src])) + param_sets[param_set].append(c_src) + return list(param_sets.values()) + + def generate_cmakelists(self): + def add_lib_desc(lib_name, lib_type, c_srcs): + yield f'add_library({lib_name} {lib_type}' + for c_src in c_srcs: + rel_c_src_path = os.path.relpath(c_src, self.build_dir) + yield ' ' + rel_c_src_path.replace('\\', '/') + yield ')\n' + predef_macros = self.base_builddesc.predef_macros()[c_srcs[0]] + if predef_macros: + yield f'target_compile_definitions({lib_name} PUBLIC' + for mname, mval in predef_macros.items(): + yield ' ' + yield self.escape(mname + + ('' if mval is None else f'={mval}')) + yield ')\n' + incl_dirs = self.base_builddesc.incl_dirs()[c_srcs[0]] + if incl_dirs: + yield f'target_include_directories({lib_name} PUBLIC' + for incl_dir in incl_dirs: + rel_incl_dir = os.path.relpath(incl_dir, self.build_dir) + yield ' ' + rel_incl_dir.replace('\\', '/') + yield ')\n' + main_lib_name = 'TS_' + self.name yield f'# This file was generated by CMakeToolChain ' \ f'automaticially.\n' \ f'# Do not modify it manually!\n' \ f'\n' \ f'cmake_minimum_required(VERSION 3.6)\n' \ - f'project({prj_name} C)\n' \ + f'project({self.name} C)\n' \ f'set(CMAKE_C_STANDARD 99)\n' \ - f'\n' \ - f'add_library(TS_{prj_name} SHARED' - subsys_names = sorted({tu.subsys_name for tu in transunits}) - shortend_prj_name, *_ = prj_name.split('.') - for subsys_name in subsys_names: - yield f' $' - yield ')\n' - compile_options = self.base_toolchain.ADDITIONAL_COMPILE_OPTIONS \ - + self.ADDITIONAL_COMPILE_OPTIONS - link_options = self.base_toolchain.ADDITIONAL_LINK_OPTIONS \ - + self.ADDITIONAL_LINK_OPTIONS - yield f"add_compile_options({' '.join(compile_options)})\n" - yield f"set(CMAKE_EXE_LINKER_FLAGS \"{' '.join(link_options)}\")\n" - yield '\n' + f'\n' + grouped_c_sources = self.group_c_sources_by_paramset() + if len(grouped_c_sources) == 1: + c_srcs = grouped_c_sources[0] + yield from add_lib_desc(main_lib_name, 'SHARED', c_srcs) + else: + yield f'add_library({main_lib_name} SHARED' + for cmod_ndx, c_srcs in enumerate(grouped_c_sources): + yield f' $' + yield ')\n' + compile_opts = getattr(self.base_builddesc, + 'ADDITIONAL_COMPILE_OPTIONS', []) + link_opts = getattr(self.base_builddesc, + 'ADDITIONAL_LINK_OPTIONS', []) + lib_dirs = getattr(self.base_builddesc, 'lib_dirs', []) + req_libs = getattr(self.base_builddesc, 'req_libs', []) + if compile_opts: + yield f"add_compile_options({' '.join(compile_opts)})\n" + if link_opts: + yield f"set(CMAKE_EXE_LINKER_FLAGS \"{' '.join(link_opts)}\")\n" if lib_dirs: yield f'link_directories({" ".join(lib_dirs)})\n' - yield f'set_target_properties(TS_{prj_name} PROPERTIES\n' \ + if req_libs: + req_libs_str = ' '.join(req_libs) + yield f'target_link_libraries({main_lib_name} {req_libs_str})\n' + yield f'set_target_properties({main_lib_name} PROPERTIES\n' \ f' RUNTIME_OUTPUT_DIRECTORY ${{CMAKE_CURRENT_SOURCE_DIR}}\n' \ f' OUTPUT_NAME __headlock_dbg__\n' \ f' PREFIX "")\n' - if req_libs: - yield f'target_link_libraries(TS_{prj_name} {" ".join(req_libs)})\n' yield '\n' - for subsys_name in subsys_names: - yield f'add_library(CMOD_{subsys_name}_{shortend_prj_name} OBJECT' - abs_incl_dirs = [] - predef_macros = {} - transunits_of_subsys = list(filter( - lambda tu: tu.subsys_name == subsys_name, transunits)) - for transunit in sorted(transunits_of_subsys): - rel_path = os.path.relpath(transunit.abs_src_filename,build_dir) - yield ' ' + str(rel_path).replace('\\', '/') - abs_incl_dirs = transunit.abs_incl_dirs - predef_macros = transunit.predef_macros - yield ')\n' - yield f'target_compile_definitions(CMOD_{subsys_name}_{shortend_prj_name} PUBLIC' - for mname, mval in predef_macros.items(): - yield ' ' - yield self.escape(mname + ('' if mval is None else f'={mval}')) - yield ')\n' - yield f'target_include_directories(CMOD_{subsys_name}_{shortend_prj_name} PUBLIC' - for incl_dir in abs_incl_dirs: - relative_path = os.path.relpath(incl_dir, build_dir) - yield ' ' + relative_path.replace('\\', '/') - yield ')\n' - yield '\n' - - def build(self, name, build_dir, transunits, req_libs, lib_dirs): - cmakelists_path = build_dir / 'CMakeLists.txt' + if len(grouped_c_sources) > 1: + for cmod_ndx, c_srcs in enumerate(grouped_c_sources): + cmod_name = f'CMOD_{self.name}_{cmod_ndx}' + yield from add_lib_desc(cmod_name, 'OBJECT', c_srcs) + yield '\n' + + def build(self): + cmakelists_path = self.build_dir / 'CMakeLists.txt' cmakelists_content = ''.join( - self.generate_cmakelists(name, build_dir, transunits, - req_libs, lib_dirs)) + self.generate_cmakelists()) cmakelists_path.write_text(cmakelists_content) if master_cmakelist: master_cmakelist_path = Path(master_cmakelist) master_cmakelist_dir = master_cmakelist_path.parent.resolve() - rel_build_dir = os.path.relpath(build_dir,str(master_cmakelist_dir)) + rel_build_dir = os.path.relpath(self.build_dir, + str(master_cmakelist_dir)) rel_build_dir_str = str(rel_build_dir).replace('\\', '/') if master_cmakelist_path.exists(): lines = master_cmakelist_path.open().readlines() @@ -216,10 +241,25 @@ def build(self, name, build_dir, transunits, req_libs, lib_dirs): with master_cmakelist_path.open('a') as cmfile: cmfile.write( f'add_subdirectory(' - f'{rel_build_dir_str} {name})\n') + f'{rel_build_dir_str} {self.name})\n') + + self.base_builddesc.build() + - self.base_toolchain.build(name, build_dir, transunits, - req_libs, lib_dirs) +###!!! TestSetup.__TOOLCHAIN__ = CMakeFileGenerator(TestSetup.__TOOLCHAIN__) -TestSetup.__TOOLCHAIN__ = CMakeToolChain(TestSetup.__TOOLCHAIN__) +if __name__ == '__main__': + from unittest.mock import Mock + base_builddesc = Mock( + spec=BuildDescription, + c_sources=Mock(return_value=['src1.c', 'src2.c', 'src3.c']), + predef_macros=Mock(return_value={'src1.c': dict(A='1', B='2'), + 'src2.c': dict(), 'src3.c': dict()}), + incl_dirs=Mock(return_value={'src1.c': ['/incl/dir1', '/incl/dir2'], + 'src2.c': [], 'src3.c': []}), + req_libs=['a', 'b'], + build_dir=Path('/build/dir')) + base_builddesc.name = 'PrjName' + cmakefilegen = CMakeFileGenerator(base_builddesc) + print(''.join(cmakefilegen.generate_cmakelists())) diff --git a/headlock/testsetup.py b/headlock/testsetup.py index fa86a5d..a0a804e 100644 --- a/headlock/testsetup.py +++ b/headlock/testsetup.py @@ -1,12 +1,9 @@ -import ctypes as ct import functools, itertools import os import sys import weakref -import abc -import hashlib import platform -from typing import List, Iterator, Dict, Any, Tuple, Union, Set +from typing import List, Dict, Any, Union, Set from pathlib import Path from .c_data_model import BuildInDefs, CStructType, CEnumType, CFuncType, \ @@ -14,7 +11,7 @@ from .address_space import AddressSpace from .address_space.inprocess import InprocessAddressSpace from .c_parser import CParser, ParseError -from .toolchains import ToolChainDriver, BuildError, TransUnit +from .buildsys_drvs import BuildDescription, BuildError, gcc, mingw from . import bridge_gen @@ -61,89 +58,6 @@ def __init__(self, addrspace:AddressSpace): self.__addrspace__ = addrspace -class CModuleDecoratorBase: - """ - This is a class-decorator, that creates a derived class from a TestSetup - which is extended by a C-Module. - """ - - def __call__(self, parent_cls): - class SubCls(parent_cls): - pass - SubCls.__name__ = parent_cls.__name__ - SubCls.__qualname__ = parent_cls.__qualname__ - SubCls.__module__ = parent_cls.__module__ - for transunit in self.iter_transunits(deco_cls=parent_cls): - SubCls.__extend_by_transunit__(transunit) - req_libs, lib_dirs = self.get_lib_search_params(deco_cls=parent_cls) - SubCls.__extend_by_lib_search_params__(req_libs, lib_dirs) - return SubCls - - @abc.abstractmethod - def iter_transunits(self, deco_cls: 'TestSetup') -> Iterator[TransUnit]: - return iter([]) - - def get_lib_search_params(self, deco_cls: 'TestSetup') \ - -> Tuple[List[str], List[str]]: - return [], [] - - -class CModule(CModuleDecoratorBase): - """ - This is a standard implementation for CModuleDecoratorBase, that allows - multiple source-filenames and parametersets to be combined into one module. - """ - - def __init__(self, *src_filenames:Union[str, Path], - include_dirs:List[Union[os.PathLike, str]]=(), - library_dirs:List[Union[os.PathLike, str]]=(), - required_libs:List[str]=(), - predef_macros:Dict[str, Any]=None, - **kw_predef_macros:Any): - if len(src_filenames) == 0: - raise ValueError('expect at least one positional argument as C ' - 'source filename') - self.src_filenames = list(map(Path, src_filenames)) - self.include_dirs = list(map(Path, include_dirs)) - self.library_dirs = list(map(Path, library_dirs)) - self.required_libs = required_libs - self.predef_macros = kw_predef_macros - if predef_macros: - self.predef_macros.update(predef_macros) - - def iter_transunits(self, deco_cls): - srcs = self._resolve_and_check_list('C Sourcefile', self.src_filenames, - Path.is_file, deco_cls) - for src in srcs: - yield TransUnit( - self.src_filenames[0].stem, - src, - self._resolve_and_check_list( - 'Include directorie', self.include_dirs, Path.is_dir, - deco_cls), - self.predef_macros) - - def get_lib_search_params(self, deco_cls): - return (list(self.required_libs), - self._resolve_and_check_list('Library Directorie', - self.library_dirs, Path.is_dir, - deco_cls)) - - @classmethod - def _resolve_and_check_list(cls, name, paths, valid_check, deco_cls): - abs_paths = [cls.resolve_path(p, deco_cls) for p in paths] - invalid_paths = [str(p) for p in abs_paths if not valid_check(p)] - if invalid_paths: - raise IOError('Cannot find ' + name + '(s): ' - + ', '.join(invalid_paths)) - return abs_paths - - @staticmethod - def resolve_path(filename, deco_cls:'TestSetup') -> Path: - mod = sys.modules[deco_cls.__module__] - return (Path(mod.__file__).parent / filename).resolve() - - SYS_WHITELIST = [ 'NULL', 'EOF', 'int8_t', 'uint8_t', 'int16_t', 'uint16_t', 'int32_t', 'uint32_t', @@ -161,14 +75,12 @@ class struct(StructUnionEnumCTypeCollection): pass __test__ = False # avoid that pytest/nose/... collect this as test __parser_factory__ = functools.partial(CParser, sys_whitelist=SYS_WHITELIST) - __TOOLCHAIN__:ToolChainDriver = None + __builddesc__:BuildDescription = None __globals:Dict[str, CProxyType] = {} __implementations:Set[str] = set() __required_funcptrs:Dict[str, CFuncType] = {} - __transunits__ = frozenset() - __required_libraries = [] __library_directories = [] @@ -176,72 +88,47 @@ class struct(StructUnionEnumCTypeCollection): pass def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) - bases = list(reversed([base for base in cls.__bases__ - if issubclass(base, TestSetup)])) - cls.__globals = bases[0].__globals.copy() - cls.__implementations = bases[0].__implementations.copy() - cls.__required_funcptrs = bases[0].__required_funcptrs.copy() - cls.struct = type(cls.__name__ + '_struct', - tuple(base.struct for base in bases), - {}) - cls.union = cls.struct - cls.enum = cls.struct - cls.__transunits__ = bases[0].__transunits__ - for base in reversed(bases[1:]): - cls.__globals.update(base.__globals) - cls.__implementations.update(base.__implementations) - cls.__transunits__ |= base.__transunits__ + cls.__builddesc__ = cls.__builddesc_factory__() + cls.__add_c_interface__(cls.__builddesc__) @classmethod - def get_ts_abspath(cls): + def __builddesc_factory__(cls) -> BuildDescription: + # This is a preliminary workaround until there is a clean solution on + # how to configure builddescs. + builddesc_cls = mingw.get_default_builddesc_cls() src_filename = sys.modules[cls.__module__].__file__ - return Path(src_filename).resolve() - - @classmethod - def get_src_dir(cls): - return cls.get_ts_abspath().parent - - @classmethod - def get_build_dir(cls): - return cls.get_src_dir() / cls._BUILD_DIR_ / cls.get_ts_abspath().stem \ - / cls.get_ts_name() - - @classmethod - def get_ts_name(cls): + ts_abspath = Path(src_filename).resolve() + src_dir = ts_abspath.parent static_qualname = cls.__qualname__.replace('..', '.') shortend_name_parts = [nm[:32] for nm in static_qualname.split('.')] rev_static_qualname = '.'.join(reversed(shortend_name_parts)) - if '..' not in cls.__qualname__: - return rev_static_qualname - else: - # this is a dynamicially generated class - # => require separate directory for every generated variant - hash = hashlib.md5() - for tu in sorted(cls.__transunits__): - hash.update(bytes(tu.abs_src_filename)) - hash.update(repr(sorted(tu.abs_incl_dirs)).encode('utf8')) - hash.update(repr(sorted(tu.predef_macros.items())) - .encode('utf8')) - return rev_static_qualname + '_' + hash.hexdigest()[:8] + build_dir = src_dir / cls._BUILD_DIR_ / ts_abspath.stem \ + / rev_static_qualname + return builddesc_cls(rev_static_qualname, build_dir, + unique_name='..' not in cls.__qualname__) @classmethod - def __extend_by_transunit__(cls, transunit:TransUnit): - predef_macros = cls.__TOOLCHAIN__.sys_predef_macros() - predef_macros.update(transunit.predef_macros) - parser = cls.__parser_factory__( - predef_macros, - transunit.abs_incl_dirs, - cls.__TOOLCHAIN__.sys_incl_dirs(), - target_compiler=cls.__TOOLCHAIN__.CLANG_TARGET) - try: - parser.read(str(transunit.abs_src_filename)) - except ParseError as exc: - exc = CompileError(exc.errors, transunit.abs_src_filename) - raise exc - else: + def __add_c_interface__(cls, builddesc:BuildDescription): + cls.__globals = {} + cls.__implementations = set() + cls.__required_funcptrs = {} + for c_src in builddesc.c_sources(): + predef_macros = builddesc.sys_predef_macros() + predef_macros.update(builddesc.predef_macros()[c_src]) + parser = cls.__parser_factory__( + predef_macros, + builddesc.incl_dirs()[c_src], + builddesc.sys_incl_dirs(), + target_compiler=builddesc.clang_target()) + try: + parser.read(c_src) + except ParseError as exc: + exc = CompileError(exc.errors, c_src) + raise exc + cls.__globals.update(parser.funcs) cls.__globals.update(parser.vars) - cls.__implementations |= parser.implementations + cls.__implementations.update(parser.implementations) cls.__required_funcptrs.update({ subtype.base_type.sig_id: subtype.base_type for typedict in [parser.funcs, parser.vars, @@ -249,6 +136,7 @@ def __extend_by_transunit__(cls, transunit:TransUnit): for ctype in typedict.values() for subtype in ctype.iter_subtypes() if isinstance(subtype, CFuncPointerType)}) + for name, ctype in parser.funcs.items(): descr = GlobalCProxyDescriptor(name, ctype) setattr(cls, name, descr) @@ -265,23 +153,9 @@ def __extend_by_transunit__(cls, transunit:TransUnit): setattr(cls.struct, typedef.struct_name, typedef) elif isinstance(typedef, CEnumType): setattr(cls.enum, name, typedef) - for name, macro_def in parser.macros.items(): setattr(cls, name, macro_def) - if transunit.abs_src_filename.suffix != '.h': - cls.__transunits__ |= {transunit} - - @classmethod - def __extend_by_lib_search_params__(cls, req_libs:List[str], - lib_dirs:List[Path]=()): - cls.__required_libraries = cls.__required_libraries[:] + \ - [req_lib for req_lib in req_libs - if req_lib not in cls.__required_libraries] - cls.__library_directories = cls.__library_directories[:] + \ - [libdir for libdir in lib_dirs - if libdir not in cls.__library_directories] - def __init__(self): super(TestSetup, self).__init__() self.__unload_events = [] @@ -326,22 +200,15 @@ def __write_bridge__(self, bridge_path): self.__c2py_bridge_ndxs = bridge_gen.write_c2py_bridge( output, c2py_funcs, MAX_C2PY_BRIDGE_INSTANCES) - def __build__(self): - if not self.get_build_dir().exists(): - self.get_build_dir().mkdir(parents=True) - bridge_file_path = self.get_build_dir() / '__headlock_bridge__.c' + if not self.__builddesc__.build_dir.exists(): + self.__builddesc__.build_dir.mkdir(parents=True) + bridge_file_path = self.__builddesc__.build_dir/'__headlock_bridge__.c' self.__write_bridge__(bridge_file_path) - mock_tu = TransUnit('mocks', bridge_file_path, [], {}) - self.__TOOLCHAIN__.build(self.get_ts_name(), - self.get_build_dir(), - sorted(self.__transunits__ | {mock_tu}), - self.__required_libraries, - self.__library_directories) + self.__builddesc__.build([bridge_file_path]) def __load__(self): - exepath = self.__TOOLCHAIN__.exe_path(self.get_ts_name(), - self.get_build_dir()) + exepath = self.__builddesc__.exe_path() self.__addrspace__ = InprocessAddressSpace(os.fspath(exepath), self.__py2c_bridge_ndxs, self.__c2py_bridge_ndxs, @@ -416,19 +283,60 @@ def register_unload_event(self, func, *args): self.__unload_events.append((func, args)) -# This is a preliminary workaround until there is a clean solution on -# how to configure toolchains. -if sys.platform == 'win32': - if platform.architecture()[0] == '32bit': - from .toolchains.mingw import MinGW32ToolChain - TestSetup.__TOOLCHAIN__ = MinGW32ToolChain() - else: - from .toolchains.mingw import MinGW64ToolChain - TestSetup.__TOOLCHAIN__ = MinGW64ToolChain() -elif sys.platform == 'linux': - if platform.architecture()[0] == '32bit': - from .toolchains.gcc import Gcc32ToolChain - TestSetup.__TOOLCHAIN__ = Gcc32ToolChain() - else: - from .toolchains.gcc import Gcc64ToolChain - TestSetup.__TOOLCHAIN__ = Gcc64ToolChain() +class CModule: + """ + This is a decorator for classes derived from TestSetup, that allows + multiple source-filenames and parametersets to be combined into one module. + """ + + def __init__(self, *src_filenames:Union[str, Path], + include_dirs:List[Union[os.PathLike, str]]=(), + library_dirs:List[Union[os.PathLike, str]]=(), + required_libs:List[str]=(), + predef_macros:Dict[str, Any]=None, + **kw_predef_macros:Any): + self.src_filenames = list(map(Path, src_filenames)) + self.include_dirs = list(map(Path, include_dirs)) + self.library_dirs = list(map(Path, library_dirs)) + self.required_libs = required_libs + self.predef_macros = kw_predef_macros + if predef_macros: + self.predef_macros.update(predef_macros) + + def __call__(self, ts): + @classmethod + def __builddesc_factory__(cls): + builddesc = super(ts, cls).__builddesc_factory__() + if not isinstance(builddesc, gcc.GccBuildDescription): + raise ValueError('currently only GCC based BuildDescriptions ' + 'are allowed') + for c_src in self.src_filenames: + abs_c_src = self.resolve_and_check(c_src, Path.is_file, ts) + builddesc.add_c_source(abs_c_src) + for incl_dir in self.include_dirs: + abs_incl_dir = self.resolve_and_check(incl_dir, Path.is_dir, ts) + builddesc.add_incl_dir(abs_incl_dir) + for lib_dir in self.library_dirs: + abs_lib_dir = self.resolve_and_check(lib_dir, Path.is_dir, ts) + builddesc.add_lib_dir(abs_lib_dir) + for req_lib in self.required_libs: + builddesc.add_req_lib(req_lib) + builddesc.add_predef_macros(self.predef_macros) + return builddesc + if not hasattr(ts, '__builddesc_factory__'): + raise TypeError('CModule can only decorate classes that are ' + 'derived from classes that provide ' + '"__builddesc_factory__" method') + if '__builddesc_factory__' in ts.__dict__: + raise TypeError('CModule cannot be used for classes that overwrite ' + '"__builddesc_factory__" explicitly.') + ts.__builddesc_factory__ = __builddesc_factory__ + return ts + + @classmethod + def resolve_and_check(cls, path, valid_check, ts): + module_path = Path(sys.modules[ts.__module__].__file__).parent + abs_path = (module_path / path).resolve() + if not valid_check(abs_path): + raise IOError(f'Cannot find {abs_path}') + return abs_path diff --git a/headlock/toolchains/__init__.py b/headlock/toolchains/__init__.py deleted file mode 100644 index 9c248c5..0000000 --- a/headlock/toolchains/__init__.py +++ /dev/null @@ -1,71 +0,0 @@ -import abc -from pathlib import Path -from typing import List, Dict, Any, NamedTuple - - -class BuildError(Exception): - - def __init__(self, msg, path=None): - super().__init__(msg) - self.path = path - - def __str__(self): - return (f'building {self.path} failed: ' if self.path - else '') \ - + super().__str__() - - -class TransUnit(NamedTuple): - """ - Represents a reference to a "translation unit" which is a unique - translation of C file. As the preprocessor allows a lot of different - translations of the same code base (depending on the macros passed by - command line and the include files) this object provides all information - to get unique preprocessor runs. - """ - subsys_name:str - abs_src_filename:Path - abs_incl_dirs:List[Path] = [] - predef_macros:Dict[str, Any] = {} - - def __hash__(self): - return sum(map(hash, [self.subsys_name, - self.abs_src_filename, - tuple(self.abs_incl_dirs), - tuple(sorted(self.predef_macros.items()))])) - - -class ToolChainDriver: - """ - This is an abstract base class for all ToolChain-Drivers. - a toolchain is the compiler, linker, libraries and header files. - """ - - CLANG_TARGET = '' - - def sys_predef_macros(self): - """ - A dictionary of toolchain-inhernt macros, that shall be always - predefined additionally to the predefined macros provided by clang - """ - return {} - - @abc.abstractmethod - def sys_incl_dirs(self): - """ - retrieves a list of all system include directories - """ - - @abc.abstractmethod - def exe_path(self, name:str, build_dir:Path): - """ - returns name of executable image/shared object library/dll - """ - - @abc.abstractmethod - def build(self, name:str, build_dir:Path, - transunits:List[TransUnit], req_libs:List[str], - lib_dirs:List[Path]): - """ - builds executable image from translation units 'transunits' - """ diff --git a/headlock/toolchains/gcc.py b/headlock/toolchains/gcc.py deleted file mode 100644 index a3aded1..0000000 --- a/headlock/toolchains/gcc.py +++ /dev/null @@ -1,90 +0,0 @@ -import subprocess -import os -from pathlib import Path - - -from . import ToolChainDriver, BuildError - - -BUILD_CACHE = set() - - -class GccToolChain(ToolChainDriver): - - ADDITIONAL_COMPILE_OPTIONS = [] - ADDITIONAL_LINK_OPTIONS = [] - - def __init__(self): - super().__init__() - self.gcc_executable = 'gcc' - self.sys_incl_dir_cache = None - - def sys_incl_dirs(self): - if self.sys_incl_dir_cache is None: - try: - gcc_info = subprocess.check_output( - [self.gcc_executable, - '-v', '-xc', '-c', '/dev/null', '-o', '/dev/null'], - stderr=subprocess.STDOUT, - encoding='utf8') - except (subprocess.CalledProcessError, FileNotFoundError) as e: - raise BuildError('failed to retrieve SYS include path from gcc') - else: - self.sys_incl_dir_cache = [] - collecting = False - for line in gcc_info.splitlines(): - if line.startswith('#include <...> search starts here'): - collecting = True - elif line.startswith('End of search list.'): - collecting = False - elif collecting: - self.sys_incl_dir_cache.append(Path(line.strip())) - return self.sys_incl_dir_cache - - def _run_gcc(self, call_params, dest_file): - try: - completed_proc = subprocess.run( - [self.gcc_executable] + call_params, - encoding='utf8', - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except (subprocess.CalledProcessError, FileNotFoundError) as e: - raise BuildError(f'failed to call gcc: {e}', dest_file) - else: - if completed_proc.returncode != 0: - raise BuildError(completed_proc.stderr, dest_file) - - def exe_path(self, name, build_dir): - return build_dir / '__headlock__.dll' - - def build(self, name, build_dir, transunits, req_libs, lib_dirs): - if (tuple(transunits), build_dir) in BUILD_CACHE: - return - for tu in transunits: - obj_file_path = build_dir / (tu.abs_src_filename.stem + '.o') - self._run_gcc(['-c', os.fspath(tu.abs_src_filename)] - + ['-o', os.fspath(obj_file_path)] - + ['-I' + os.fspath(incl_dir) - for incl_dir in tu.abs_incl_dirs] - + [f'-D{mname}={mval or ""}' - for mname, mval in tu.predef_macros.items()] - + self.ADDITIONAL_COMPILE_OPTIONS, - obj_file_path) - exe_file_path = self.exe_path(name, build_dir) - self._run_gcc([str(build_dir / (tu.abs_src_filename.stem + '.o')) - for tu in transunits] - + ['-shared', '-o', os.fspath(exe_file_path)] - + ['-l' + req_lib for req_lib in req_libs] - + ['-L' + str(lib_dir) for lib_dir in lib_dirs] - + self.ADDITIONAL_LINK_OPTIONS, - exe_file_path) - BUILD_CACHE.add((tuple(transunits), build_dir)) - - -class Gcc32ToolChain(GccToolChain): - CLANG_TARGET = 'i386-pc-linux-gnu' - - -class Gcc64ToolChain(GccToolChain): - CLANG_TARGET = 'x86_64-pc-linux-gnu' - ADDITIONAL_COMPILE_OPTIONS = ['-fPIC'] diff --git a/setup.py b/setup.py index e26b96f..2ecdc36 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ 'headlock.c_data_model', 'headlock.integrations.pytest', 'headlock.libclang', - 'headlock.toolchains'], + 'headlock.buildsys_drvs'], classifiers=[ 'Development Status :: 3 - Alpha', 'Operating System :: Microsoft :: Windows', diff --git a/tests/test_address_space/test_inprocess_address_space.py b/tests/test_address_space/test_inprocess_address_space.py index 3f5a5dd..15061ab 100644 --- a/tests/test_address_space/test_inprocess_address_space.py +++ b/tests/test_address_space/test_inprocess_address_space.py @@ -1,11 +1,11 @@ import pytest import ctypes as ct from headlock.address_space.inprocess import InprocessAddressSpace -from headlock.toolchains import TransUnit from pathlib import Path from contextlib import contextmanager from tempfile import TemporaryDirectory from unittest.mock import Mock +from headlock.buildsys_drvs.mingw import get_default_builddesc_cls MAX_C2PY_BRIDGE_INSTANCES = 4 @@ -22,37 +22,25 @@ def bridge_src(bridge_array=None, py2c_retval='1'): b'}\n' b'void (* _c2py_bridge_handler)' b'(int, int, unsigned char *, unsigned char *);\n' - b'void (* _c2py_bridge_[][' + max_bridge_inst_str + b'])(void) = {\n' + + b'typedef void (* _c2py_bridge_t)(void);\n' + b'_c2py_bridge_t _c2py_bridge_[][' + max_bridge_inst_str + b'] = {\n' + b''.join( - b'\t{ ' + b', '.join(str(v).encode('ascii') for v in insts) +b'},\n' + b'\t{ ' + b', '.join(b'(_c2py_bridge_t) ' + str(v).encode('ascii') + for v in insts) +b'},\n' for insts in bridge_array) + b'};') -def create_tool_chain(): - import sys, platform - if sys.platform == 'win32': - if platform.architecture()[0] == '32bit': - from headlock.toolchains.mingw import MinGW32ToolChain as ToolChain - else: - from headlock.toolchains.mingw import MinGW64ToolChain as ToolChain - else: - if platform.architecture()[0] == '32bit': - from headlock.toolchains.gcc import Gcc32ToolChain as ToolChain - else: - from headlock.toolchains.gcc import Gcc64ToolChain as ToolChain - return ToolChain() - @contextmanager def addrspace_for(content): with TemporaryDirectory() as tempdir: dummy_dll_dir = Path(tempdir) c_file = Path(dummy_dll_dir) / 'source.c' c_file.write_bytes(content) - tool_chain = create_tool_chain() - trans_unit = TransUnit('dummy', c_file) - tool_chain.build('dummy', dummy_dll_dir, [trans_unit], [], []) + builddesc_cls = get_default_builddesc_cls() + builddesc = builddesc_cls('dummy', dummy_dll_dir) + builddesc.build([c_file]) addrspace = InprocessAddressSpace( - str(tool_chain.exe_path('dummy', dummy_dll_dir)), + str(builddesc.exe_path()), {f'py2c_sigid{ndx}': ndx for ndx in range(MAX_SIG_CNT)}, {f'c2py_sigid{ndx}': ndx for ndx in range(MAX_SIG_CNT)}, MAX_C2PY_BRIDGE_INSTANCES) @@ -110,8 +98,10 @@ def test_invokeCCode_onBridgeReturnsFalse_raisesValueError(): inproc_addrspace.invoke_c_code(0, 'py2c_sigid0', 0, 0) def test_invokeCCode_onBridgeReturnsTrue_passesAllParameters(): - src = bridge_src(py2c_retval='bridge_ndx == 2 && (int) func_ptr == 123 && ' - '(int) params == 456 && (int) retval == 789') + src = bridge_src(py2c_retval='bridge_ndx == 2 && ' + '(void*) func_ptr == (void*) 123 && ' + '(void*) params == (void*) 456 && ' + '(void*) retval == (void*) 789') with addrspace_for(src) as inproc_addrspace: inproc_addrspace.invoke_c_code(123, 'py2c_sigid2', 456, 789) diff --git a/tests/test_toolchains/__init__.py b/tests/test_buildsys_drvs/__init__.py similarity index 100% rename from tests/test_toolchains/__init__.py rename to tests/test_buildsys_drvs/__init__.py diff --git a/tests/test_buildsys_drvs/test_gcc.py b/tests/test_buildsys_drvs/test_gcc.py new file mode 100644 index 0000000..1ac7742 --- /dev/null +++ b/tests/test_buildsys_drvs/test_gcc.py @@ -0,0 +1,136 @@ +import pytest +import sys +import subprocess +from unittest.mock import patch +from headlock.buildsys_drvs.gcc import GccBuildDescription +import platform +from pathlib import Path +import ctypes as ct + +from ..helpers import build_tree +if platform.architecture()[0] == '32bit': + from headlock.buildsys_drvs.gcc import Gcc32BuildDescription as GccXXBuildDescription +else: + from headlock.buildsys_drvs.gcc import Gcc64BuildDescription as GccXXBuildDescription + + +class TestGccBuildDescription: + + def test_init_withoutParams_createsEmptyBuilddesc(self): + builddesc = GccBuildDescription('dummy', Path('.')) + assert builddesc.c_sources() == [] + assert builddesc.predef_macros() == {} + assert builddesc.incl_dirs() == {} + + def test_init_withParams_createsPreinitializedBuilddesc(self): + predef_macros = {'m1': 'v1', 'm2': 'v2'} + incl_dirs = [Path('dir/1'), Path('dir/2')] + builddesc = GccBuildDescription( + 'dummy', Path('.'), + c_sources=[Path('src1.c'), Path('src2.c')], + predef_macros=predef_macros, incl_dirs=incl_dirs) + assert builddesc.c_sources() == [Path('src1.c'), Path('src2.c')] + assert builddesc.predef_macros() \ + == {Path('src1.c'): predef_macros, Path('src2.c'): predef_macros} + assert builddesc.incl_dirs() \ + == {Path('src1.c'): incl_dirs, Path('src2.c'): incl_dirs} + + @patch('subprocess.check_output', return_value= + 'Using built-in specs.\n' + 'COLLECT_GCC=gcc\n' + '...\n' + 'ignoring nonexistent directory "/usr/lib/gcc/x86_64-linux-gnu/7/../../../../x86_64-linux-gnu/include"\n' + '#include "..." search starts here:\n' + '#include <...> search starts here:\n' + ' /usr/lib/gcc/x86_64-linux-gnu/7/include\n' + ' /usr/local/include\n' + ' /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed\n' + ' /usr/include/x86_64-linux-gnu\n' + ' /usr/include\n' + 'End of search list.\n' + '...\n') + def test_sysInclDirs_returnsSysInclDirsReturnedByGcc(self, check_output): + GccBuildDescription.SYS_INCL_DIR_CACHE = {} + assert GccBuildDescription('dummy', Path('.')).sys_incl_dirs() == [ + Path('/usr/lib/gcc/x86_64-linux-gnu/7/include'), + Path('/usr/local/include'), + Path('/usr/lib/gcc/x86_64-linux-gnu/7/include-fixed'), + Path('/usr/include/x86_64-linux-gnu'), + Path('/usr/include')] + check_output.assert_called_with( + ['gcc', '-v', '-xc', '-c', '/dev/null', '-o', '/dev/null'], + encoding='utf8', stderr=subprocess.STDOUT) + + def test_addCSources_addsCSources(self): + builddesc = GccBuildDescription('dummy', Path('.'), + c_sources=[Path('src1.c')]) + builddesc.add_c_source(Path('src2.c')) + assert set(builddesc.c_sources()) == {Path('src1.c'), Path('src2.c')} + + def test_addPredefMacros_addsMacros(self): + builddesc = GccBuildDescription( + 'dummy', Path('.'), c_sources=[Path('src.c')], + predef_macros={'m1': 'old', 'm2': 'old'}) + builddesc.add_predef_macros({'m2': 'new', 'm3': 'new'}) + assert builddesc.predef_macros() \ + == {Path('src.c'): {'m1': 'old', 'm2': 'new', 'm3': 'new'}} + + def test_addInclDirs_addsInclDirs(self): + builddesc = GccBuildDescription( + 'dummy', Path('.'), c_sources=[Path('src.c')], + incl_dirs=[Path('dir/1')]) + builddesc.add_incl_dir(Path('dir/2')) + assert builddesc.incl_dirs() \ + == {Path('src.c'): [Path('dir/1'), Path('dir/2')]} + + @patch('subprocess.run') + @patch.object(GccXXBuildDescription, 'ADDITIONAL_COMPILE_OPTIONS', ['-O1', '-Cx']) + @patch.object(GccXXBuildDescription, 'ADDITIONAL_LINK_OPTIONS', ['-O2', '-Lx']) + def test_build_passesParametersToGcc(self, subprocess_run): + subprocess_run.return_value.returncode = 0 + builddesc = GccXXBuildDescription('dummy', Path('.')) + builddesc.add_c_source(Path('src.c')) + builddesc.add_predef_macros({'MACRO1': 1, 'MACRO2':''}) + builddesc.add_incl_dir(Path('incl_dir')) + builddesc.add_lib_dir(Path('lib_dir')) + builddesc.add_req_lib('lib_name') + builddesc.build() + ((gcc_call, *_), *_), ((linker_call, *_), *_) = \ + subprocess_run.call_args_list + assert '-Cx' in gcc_call + assert '-O1' in gcc_call + assert '-O2' not in gcc_call + assert '-DMACRO1=1' in gcc_call + assert '-DMACRO2=' in gcc_call + assert '-Iincl_dir' in gcc_call + assert 'src.c' in gcc_call + assert '-Lx' in linker_call + assert '-O2' in linker_call + assert '-llib_name' in linker_call + assert '-Llib_dir' in linker_call + assert '-O1' not in linker_call + assert 'src.o' in linker_call + + @patch('subprocess.run') + def test_build_passesAdditonalSourcesToGcc(self, subprocess_run): + subprocess_run.return_value.returncode = 0 + builddesc = GccXXBuildDescription('dummy', Path('.')) + builddesc.add_c_source(Path('src.c')) + builddesc.build([Path('additional_src.c')]) + ((gcc1_call, *_), *_), ((gcc2_call, *_), *_), ((linker_call, *_), *_) =\ + subprocess_run.call_args_list + assert 'src.c' in gcc1_call + assert 'additional_src.c' in gcc2_call + assert 'src.o' in linker_call + assert 'additional_src.o' in linker_call + + @pytest.mark.skipif(sys.platform == 'win32', + reason='works only on non-win platforms') + def test_build_createsDll(self, tmpdir): + basedir = build_tree(tmpdir, {'src.c': b'int func(void) { return 22; }', + 'build': {}}) + builddesc = GccXXBuildDescription('dummy', basedir / 'build') + builddesc.add_c_source(basedir / 'src.c') + builddesc.build() + c_dll = ct.CDLL(str(builddesc.exe_path())) + assert c_dll.func() == 22 diff --git a/tests/test_buildsys_drvs/test_mingw.py b/tests/test_buildsys_drvs/test_mingw.py new file mode 100644 index 0000000..4f61d6b --- /dev/null +++ b/tests/test_buildsys_drvs/test_mingw.py @@ -0,0 +1,27 @@ +import pytest +import sys +import ctypes +import platform +from ..helpers import build_tree + +if sys.platform == 'win32': + + if platform.architecture()[0] == '32bit': + from headlock.buildsys_drvs.mingw import \ + MinGW32BuildDescription as MinGWxxBuildDescription + else: + from headlock.buildsys_drvs.mingw import \ + MinGW64BuildDescription as MinGWxxBuildDescription + + + class TestMinGW32ToolChain: + + def test_build_createsDll(self, tmpdir): + basedir = build_tree(tmpdir, { + 'src.c': b'int func(void) { return 22; }', + 'build': {}}) + builddesc = MinGWxxBuildDescription('dummy', basedir / 'build') + builddesc.add_c_source(basedir / 'src.c') + builddesc.build() + dll = ctypes.CDLL(str(builddesc.exe_path())) + assert dll.func() == 22 diff --git a/tests/test_testsetup.py b/tests/test_testsetup.py index e1a084e..21c2493 100644 --- a/tests/test_testsetup.py +++ b/tests/test_testsetup.py @@ -1,13 +1,15 @@ import contextlib import sys -import os from pathlib import Path -from unittest.mock import patch, Mock, MagicMock, call, ANY +from unittest.mock import patch, Mock, call import pytest from .helpers import build_tree from headlock.testsetup import TestSetup, MethodNotMockedError, \ - BuildError, CompileError, CModuleDecoratorBase, CModule, TransUnit + BuildError, CompileError, CModule +from headlock.buildsys_drvs.mingw import get_default_builddesc_cls +from headlock.buildsys_drvs.gcc import GccBuildDescription, \ + Gcc32BuildDescription import headlock.c_data_model as cdm @@ -72,127 +74,40 @@ def test_iter_iteratesErrors(self): assert list(exc) == errlist -class TestCModuleDecoratorBase: - - def test_call_onCls_returnsDerivedClassWithSameNameAndModule(self, TSDummy): - c_mod_decorator = CModuleDecoratorBase() - TSDecorated = c_mod_decorator(TSDummy) - assert issubclass(TSDecorated, TSDummy) \ - and TSDecorated is not TSDummy - assert TSDecorated.__name__ == 'TSDummy' - assert TSDecorated.__qualname__ == 'Container.TSDummy' - assert TSDecorated.__module__ == 'testsetup_dummy' - - def test_call_onCls_createsDerivedCls(self, TSDummy): - deco = CModuleDecoratorBase() - TSDerived = deco(TSDummy) - assert issubclass(TSDerived, TSDummy) - - def test_call_onCls_callsExtendByTransUnit(self, TSDummy): - with patch.object(TSDummy, '__extend_by_transunit__'): - deco = CModuleDecoratorBase() - tu1, tu2 = Mock(), Mock() - deco.iter_transunits = Mock(return_value=iter([tu1, tu2])) - deco(TSDummy) - assert TSDummy.__extend_by_transunit__.call_args_list \ - == [call(tu1), call(tu2)] - - def test_call_onCls_extendsLibrSearchParams(self, TSDummy): - with patch.object(TSDummy, '__extend_by_lib_search_params__'): - deco = CModuleDecoratorBase() - deco.iter_transunits = MagicMock() - deco.get_lib_search_params = Mock(return_value=([Path('dir')], - ['lib'])) - deco(TSDummy) - TSDummy.__extend_by_lib_search_params__.assert_called_once_with( - [Path('dir')], ['lib']) - - -class TestCModule: - - def test_iterTransunits_onRelSrcFilename_resolves(self, tmpdir): - with sim_tsdummy_tree(tmpdir, {'dir': {'src': b''}}) as base_dir: - c_mod = CModule('dir/src') - [transunit] = c_mod.iter_transunits(TSDummy) - assert transunit.abs_src_filename == base_dir / 'dir/src' - - def test_iterTransunits_onAbsSrcFilename_resolves(self, tmpdir): - base_dir = build_tree(tmpdir, {'dir': {'src': b''}}) - abs_path = str(base_dir / 'dir/src') - c_mod = CModule(abs_path) - [transunit] = c_mod.iter_transunits(TSDummy) - assert transunit.abs_src_filename == Path(abs_path) - - class TSSubClass: - pass - - def test_iterTransunits_setsSubsysName(self, tmpdir): - with sim_tsdummy_tree(tmpdir, {'t1.c': b'', 't2.c': b''}): - c_mod = CModule('t1.c', 't2.c') - [transunit, *_] = c_mod.iter_transunits(self.TSSubClass) - assert transunit.subsys_name == 't1' - - def test_iterTransunits_retrievesAndResolvesIncludeDirs(self, tmpdir): - with sim_tsdummy_tree(tmpdir, {'src': b'', 'd1': {}, 'd2': {}}) \ - as base_dir: - c_mod = CModule('src', include_dirs=['d1', 'd2']) - [transunit] = c_mod.iter_transunits(TSDummy) - assert transunit.abs_incl_dirs == [base_dir / 'd1', base_dir / 'd2'] - - def test_iterTransunits_createsOneUnitTestPerSrcFilename(self, tmpdir): - with sim_tsdummy_tree(tmpdir, {'t1.c': b'', 't2.c': b''}) as base_dir: - c_mod = CModule('t1.c', 't2.c') - assert [tu.abs_src_filename - for tu in c_mod.iter_transunits(TSDummy)] \ - == [base_dir / 't1.c', base_dir / 't2.c'] - - def test_iterTransunits_onPredefMacros_passesMacrosToCompParams(self, tmpdir): - with sim_tsdummy_tree(tmpdir, {'src': b''}): - c_mod = CModule('src', MACRO1=11, MACRO2=22) - [transunit] = c_mod.iter_transunits(TSDummy) - assert transunit.predef_macros == {'MACRO1':11, 'MACRO2':22} - - def test_iterTransunits_onInvalidPath_raiseIOError(self): - c_mod = CModule('test_invalid.c') - with pytest.raises(IOError): - list(c_mod.iter_transunits(TSDummy)) - - def test_getLibSearchParams_retrievesAndResolvesLibsDirs(self, tmpdir, TSDummy): - with sim_tsdummy_tree(tmpdir, {'src':b'', 'd1':{}, 'd2':{}}) \ - as base_dir: - c_mod = CModule('src', library_dirs=['d1', 'd2']) - [_, lib_dirs] = c_mod.get_lib_search_params(TSDummy) - assert lib_dirs == [base_dir / 'd1', base_dir / 'd2'] - - def test_iterTransunits_retrievesRequiredLibs(self, tmpdir, TSDummy): - with sim_tsdummy_tree(tmpdir, {'src': b''}) as base_dir: - c_mod = CModule('src', required_libs=['lib1', 'lib2']) - [req_libs, _] = c_mod.get_lib_search_params(TSDummy) - assert req_libs == ['lib1', 'lib2'] - - def test_resolvePath_onRelativeFileAndRelativeModulePath_returnsAbsolutePath(self, TSDummy, tmpdir): - module = sys.modules[TSDummy.__module__] - with build_tree(tmpdir, {'file.py': b'', 'file.c': b''}) as dir: - os.chdir(dir) - with patch.object(module, '__file__', 'file.py'): - assert CModule.resolve_path('file.c', TSDummy) == dir / 'file.c' - - class TestTestSetup(object): """ This is an integration test, that tests the testsetup class and the collaboration of the headlock components """ - def extend_by_ccode(self, cls, src, filename, **macros): - sourcefile = (Path(__file__).parent / 'c_files' / filename).resolve() - sourcefile.write_bytes(src) - transunit = TransUnit('test_tu', sourcefile, [], macros) - cls.__extend_by_transunit__(transunit) - - def cls_from_ccode(self, src, filename, **macros): - class TSDummy(TestSetup): pass - self.extend_by_ccode(TSDummy, src, filename, **macros) + def abs_dir(self): + return (Path(__file__).parent / 'c_files').resolve() + + def extend_builddesc(self, builddesc:GccBuildDescription, + source_code, filename): + abs_filename = self.abs_dir() / filename + abs_filename.write_bytes(source_code) + builddesc.add_c_source(abs_filename) + + def create_builddesc(self, source_code, filename, *, unique_name=True, + **macros): + builddesc_cls = get_default_builddesc_cls() + builddesc = builddesc_cls(Path(filename).name, + self.abs_dir() / (filename + '.build'), + unique_name) + builddesc.add_predef_macros(macros) + self.extend_builddesc(builddesc, source_code, filename) + return builddesc + + def cls_from_ccode(self, src, filename, + src_2=None, filename_2=None, **macros): + class TSDummy(TestSetup): + @classmethod + def __builddesc_factory__(cls): + builddesc = self.create_builddesc(src, filename, **macros) + if src_2 is not None: + self.extend_builddesc(builddesc, src_2, filename_2) + return builddesc return TSDummy @pytest.fixture @@ -200,67 +115,34 @@ def ts_dummy(self): TSDummy = self.cls_from_ccode(b'', 'test.c') return TSDummy - def test_extendByTransunit_addsModulesToGetCModules(self): - class TSDummy(TestSetup): - __parser_factory__ = MagicMock() - transunits = [MagicMock(), MagicMock()] - TSDummy.__extend_by_transunit__(transunits[0]) - TSDummy.__extend_by_transunit__(transunits[1]) - assert TSDummy.__transunits__ == frozenset(transunits) - - def test_extendByTransunit_doesNotModifyParentCls(self): - transunits = [MagicMock(), MagicMock()] - class TSParent(TestSetup): - __parser_factory__ = MagicMock() - TSParent.__extend_by_transunit__(transunits[0]) - class TSChild(TSParent): - pass - TSChild.__extend_by_transunit__(transunits[1]) - assert list(TSParent.__transunits__) == transunits[:1] - assert TSChild.__transunits__ == set(transunits[:2]) - - def test_extendByTransunit_onInvalidSourceCode_raisesCompileErrorDuringParsing(self): - with pytest.raises(CompileError) as comp_err: - self.cls_from_ccode(b'#error invalid c src', 'compile_err.c') - assert len(comp_err.value.errors) == 1 - assert comp_err.value.path.name == 'compile_err.c' - - def test_getTsAbspath_returnsAbsPathOfFile(self): - TSDummy = self.cls_from_ccode(b'', 'test.c') - assert TSDummy.get_ts_abspath() == Path(__file__).resolve() - - def test_getSrcDir_returnsAbsDirOfFile(self): - TSDummy = self.cls_from_ccode(b'', 'test.c') - assert TSDummy.get_src_dir() == Path(__file__).resolve().parent - - class TSEmpty(TestSetup): - __transunits__ = frozenset([ - TransUnit('empty', Path(__file__, 'c_files/empty.c'), [], {})]) - - def test_getTsName_onStaticTSCls_returnsReversedQualifiedClassName(self): - assert self.TSEmpty.get_ts_name() == 'TSEmpty.TestTestSetup' - - def test_getTsName_onDynamicGeneratedTSCls_returnsReversedQualifiedClassNameAndUid(self): - TSDummy = self.cls_from_ccode(b'', 'test.c') - assert TSDummy.get_ts_name()[:-8] \ - == 'TSDummy.cls_from_ccode.TestTestSetup_' - int(str(TSDummy.get_ts_name())[-8:], 16) # expect hexnumber at the end - - def test_getTsName_onDynamicGeneratedTSClsWithSameParams_returnsSameStr(self): - TSDummy1 = self.cls_from_ccode(b'', 'test.c', MACRO=1) - TSDummy2 = self.cls_from_ccode(b'', 'test.c', MACRO=1) - assert TSDummy1.get_ts_name() == TSDummy2.get_ts_name() - - def test_getTsName_onDynamicGeneratedTSClsWithDifferentParams_returnsDifferentStr(self): - TSDummy1 = self.cls_from_ccode(b'', 'test.c', A=1, B=222, C=3) - TSDummy2 = self.cls_from_ccode(b'', 'test.c', A=1, B=2, C=3) - assert TSDummy1.get_ts_name() != TSDummy2.get_ts_name() - - def test_getBuildDir_returnsCorrectPath(self, ts_dummy): - this_file = Path(__file__).resolve() - assert ts_dummy.get_build_dir() \ - == this_file.parent / TestSetup._BUILD_DIR_ / this_file.stem \ - / ts_dummy.get_ts_name() + class TSBuildDescFactory(TestSetup): pass + + def test_builddescFactory_returnsBuildDescWithGlobalBuildDirAndName(self): + builddesc = self.TSBuildDescFactory.__builddesc_factory__() + assert builddesc.name == 'TSBuildDescFactory.TestTestSetup' + assert builddesc.build_dir \ + == Path(__file__).parent.resolve() / \ + '.headlock/test_testsetup' / builddesc.name + + def test_builddescFactory_onLocalTestSetupDefinition_returnsBuildDescWithHashInBuilDir(self): + class TSBuildDescFactory(TestSetup): pass + builddesc = TSBuildDescFactory.__builddesc_factory__() + int(builddesc.build_dir.name[-8:], 16) # expect hexnumber at the end + assert builddesc.build_dir.name[-9] == '_' + + def test_builddescFactory_onDynamicGeneratedTSClsWithSameParams_returnsBuildDescWithSameBuildDir(self): + builddesc1 = self.create_builddesc(b'', 'test.c', unique_name=False, + MACRO=1) + builddesc2 = self.create_builddesc(b'', 'test.c', unique_name=False, + MACRO=1) + assert builddesc1.build_dir == builddesc2.build_dir + + def test_builddescFactory_onDynamicGeneratedTSClsWithDifferentParams_returnsBuildDescWithDifferentBuildDir(self): + builddesc1 = self.create_builddesc(b'', 'test.c', unique_name=False, + A=1, B=222, C=3) + builddesc2 = self.create_builddesc(b'', 'test.c', unique_name=False, + A=1, B=2, C=3) + assert builddesc1.build_dir != builddesc2.build_dir def test_macroWrapper_ok(self): TS = self.cls_from_ccode(b'#define MACRONAME 123', 'macro.c') @@ -308,6 +190,12 @@ def test_init_doesNotCallStartup(self, __startup__): __startup__.assert_not_called() ts.__unload__() + def test_build_onMultipleFilesWithReferences_ok(self): + TSMock = self.cls_from_ccode( + b'void callee(void) { return; }', 'prev.c', + b'void callee(void); void caller(void) { callee(); }', 'refprev.c') + TSMock() + def test_build_onPredefinedMacros_passesMacrosToCompiler(self): TSMock = self.cls_from_ccode(b'int a = A;\n' b'int b = B 22;\n' @@ -321,41 +209,20 @@ def test_build_onPredefinedMacros_passesMacrosToCompiler(self): assert ts.b.val == 22 assert ts.c.val == 33 - def test_build_onExtendByLibsSearchParams_passesMergedLibsAndSearchDirectories(self): - class TSChkLibDirs(TestSetup): - __TOOLCHAIN__ = Mock() - __load__ = __unload__ = Mock() - TSChkLibDirs.__extend_by_lib_search_params__(['lib1'], [Path('dir1')], ) - TSChkLibDirs.__extend_by_lib_search_params__(['lib2'], [Path('dir2')]) - ts = TSChkLibDirs() - TSChkLibDirs.__TOOLCHAIN__.build.assert_called_once_with( - ANY, ANY, ANY, ['lib1', 'lib2'], [Path('dir1'), Path('dir2')]) - - def test_build_onExtendByLibsSearchParams_doesNotModifyParentCls(self): - class TSChkLibDirs(TestSetup): - __TOOLCHAIN__ = Mock() - __load__ = __unload__ = Mock() - class TSChkLibDirsChild(TSChkLibDirs): - pass - TSChkLibDirsChild.__extend_by_lib_search_params__(['l'], [Path('d')]) - TSChkLibDirs() - TSChkLibDirs.__TOOLCHAIN__.build.assert_called_once_with( - ANY, ANY, ANY, [], []) - @patch('headlock.testsetup.TestSetup.__shutdown__') def test_unload_onStarted_callsShutdown(self, __shutdown__): TS = self.cls_from_ccode(b'int var;', 'unload_calls_shutdown.c') ts = TS() ts.__startup__() ts.__unload__() - ts.__shutdown__.assert_called_once() + __shutdown__.assert_called_once() @patch('headlock.testsetup.TestSetup.__shutdown__') def test_unload_onNotStarted_doesNotCallsShutdown(self, __shutdown__): TS = self.cls_from_ccode(b'int var;', 'unload_calls_shutdown.c') ts = TS() ts.__unload__() - ts.__shutdown__.assert_not_called() + __shutdown__.assert_not_called() def test_unload_calledTwice_ignoresSecondCall(self): TS = self.cls_from_ccode(b'', 'unload_run_twice.c') @@ -507,7 +374,10 @@ def test_headerFileOnly_createsMockOnly(self): def test_mockFuncWrapper_ok(self): class TSMock(TestSetup): func_mock = Mock(return_value=33) - self.extend_by_ccode(TSMock, b'int func(int * a, int * b);', 'mocked.c') + @classmethod + def __builddesc_factory__(cls): + return self.create_builddesc( + b'int func(int * a, int * b);', 'mocked.c') with TSMock() as ts: assert ts.func(11, 22) == 33 TSMock.func_mock.assert_called_with(ts.int.ptr(11), ts.int.ptr(22)) @@ -515,8 +385,10 @@ class TSMock(TestSetup): def test_mockFuncWrapper_onNotExistingMockFunc_forwardsToMockFallbackFunc(self): class TSMock(TestSetup): mock_fallback = Mock(return_value=33) - self.extend_by_ccode(TSMock, b'int func(int * a, int * b);', - 'mocked_func_fallback.c') + @classmethod + def __builddesc_factory__(cls): + return self.create_builddesc(b'int func(int * a, int * b);', + 'mocked_func_fallback.c') with TSMock() as ts: assert ts.func(11, 22) == 33 TSMock.mock_fallback.assert_called_with('func', ts.int.ptr(11), @@ -525,10 +397,13 @@ class TSMock(TestSetup): def test_mockFuncWrapper_createsCWrapperCode(self): class TSMock(TestSetup): mocked_func_mock = Mock(return_value=22) - self.extend_by_ccode(TSMock, b'int mocked_func(int p);' - b'int func(int p) { ' - b' return mocked_func(p); }', - 'mocked_func_cwrapper.c') + @classmethod + def __builddesc_factory__(cls): + return self.create_builddesc( + b'int mocked_func(int p);' + b'int func(int p) { ' + b' return mocked_func(p); }', + 'mocked_func_cwrapper.c') with TSMock() as ts: assert ts.func(11) == 22 TSMock.mocked_func_mock.assert_called_once_with(11) @@ -541,30 +416,6 @@ def test_mockFuncWrapper_onUnmockedFunc_raisesMethodNotMockedError(self): assert ts.mock_fallback('unmocked_func', 11, 22) assert "unmocked_func" in str(excinfo.value) - def test_mockFuncWrapper_onRefersToPrevTransUnit_isGenerated(self): - TSMock = self.cls_from_ccode(b'void callee(void) { return; }', - 'prev.c') - self.extend_by_ccode(TSMock, b'void callee(void); ' - b'void caller(void) { callee(); }', - 'refprev.c') - TSMock() - - def test_mockFuncWrapper_onRefersToNextTransUnit_isGenerated(self): - TSMock = self.cls_from_ccode(b'void callee(void);' - b'void caller(void) { callee(); }', - 'refnext.c') - self.extend_by_ccode(TSMock, b'void callee(void) { return; }', - 'next.c') - TSMock() - - def test_mockFunc_onLastTransUnitDoesNotReferToMocks_isGenerated(self): - TSMock = self.cls_from_ccode(b'void mock(void);' - b'void func1(void) { mock(); }', - 'first_with_mock.c') - self.extend_by_ccode(TSMock, b'void func2(void) { return; }', - 'last_with_no_refererence_to_mock.c') - TSMock() - def test_typedefWrapper_storesTypeDefInTypedefCls(self): TSMock = self.cls_from_ccode(b'typedef int td_t;', 'typedef.c') with TSMock() as ts: @@ -635,42 +486,19 @@ def test_enumWrapper_storesEnumDefInEnumCls(self): with TSMock() as ts: assert isinstance(ts.enum.enum_t, cdm.CEnumType) - def test_onTestSetupComposedOfDifferentCModules_parseAndCompileCModulesIndependently(self): - class TSDummy(TestSetup): pass - self.extend_by_ccode(TSDummy, b'#if defined(A)\n' - b'#error A not allowed\n' - b'#endif\n' - b'extern int a;' - b'int b = B;', - 'diff_params_mod_b.c', - B=2) - self.extend_by_ccode(TSDummy, b'#if defined(B)\n' - b'#error B not allowed\n' - b'#endif\n' - b'int a = A;' - b'extern int b;', - 'diff_params_mod_a.c', - A=1) - with TSDummy() as ts: - assert ts.a.val == 1 - assert ts.b.val == 2 - def test_onSameStructWithAnonymousChildInDifferentModules_generateCorrectMockWrapper(self): - class TSDummy(TestSetup): pass - self.extend_by_ccode(TSDummy, b'struct s { struct { int mm; } m; };\n' - b'int func1(struct s p);\n', - 'anonymstruct_mod1.c') - self.extend_by_ccode(TSDummy, b'struct s { struct { int mm; } m; };\n' - b'int func2(struct s p);\n', - 'anonymstruct_mod2.c') + TSDummy = self.cls_from_ccode( + b'struct s { struct { int mm; } m; };\n' + b'int func1(struct s p);\n', 'anonymstruct_mod1.c', + b'struct s { struct { int mm; } m; };\n' + b'int func2(struct s p);\n', 'anonymstruct_mod2.c') with TSDummy() as ts: pass def test_onPointerToArrayOfStruct_generatesCorrectMockWrapper(self): - class TSDummy(TestSetup): pass - self.extend_by_ccode(TSDummy, b'typedef struct strct {} (*type)[1];\n' + TSDummy = self.cls_from_ccode(b'typedef struct strct {} (*type)[1];\n' b'void func(type param);', - 'ptr_to_arr_of_strct.c') + 'ptr_to_arr_of_strct.c') with TSDummy() as ts: pass @@ -710,68 +538,82 @@ def test_attributeAnnotationSupport_onStdIntIncluded_ok(self): with TSDummy() as ts: assert '__cdecl' in ts.cdecl_func.ctype.__c_attribs__ - def test_subclassing_addsAttributesToDerivedClassButDoesNotModifyParentClass(self): - TSDummy = self.cls_from_ccode(b'int func(void);\n' - b'int var;\n' - b'struct strct {};\n' - b'typedef int typedf;', - 'parentcls.c') - class TSDummy2(TSDummy): pass - self.extend_by_ccode(TSDummy2, b'int func2(void);\n' - b'int var2;\n' - b'struct strct2 {};\n' - b'typedef int typedf2;', - 'derivedcls.c') - with TSDummy() as ts: - assert all(hasattr(ts, attr) for attr in ('func', 'var', 'typedf')) - assert not any(hasattr(ts, attr) - for attr in ('func2', 'var2', 'typedf2')) - assert hasattr(ts.struct, 'strct') - assert not hasattr(ts.struct, 'strct2') - with TSDummy2() as ts: - assert all(hasattr(ts, attr) - for attr in ('func', 'var', 'typedf', - 'func2', 'var2', 'typedf2')) - assert hasattr(ts.struct, 'strct') - assert hasattr(ts.struct, 'strct2') - - def test_subclassing_onChildClsImplementsMockedMethodFromParentCls_ok(self): - TSDummy = self.cls_from_ccode(b'int impl_by_subclass_func(void);\n' - b'int not_impl_func(void);', - 'mocked_parentcls.c') - class TSDummy2(TSDummy): pass - self.extend_by_ccode(TSDummy2, - b'int impl_by_subclass_func(void) { return 2; }', - 'impl_derivedcls.c') - with TSDummy() as ts: - with pytest.raises(MethodNotMockedError): - ts.impl_by_subclass_func() - with TSDummy2() as ts: - assert ts.impl_by_subclass_func() == 2 - with pytest.raises(MethodNotMockedError): - ts.not_impl_func() - - def test_subclassing_onMultipleInheritance_mergesBaseClsItems(self): - TSParent1 = self.cls_from_ccode(b'void func1(void) { return; }\n' - b'void mock1(void);\n' - b'struct strct1 {};\n' - b'enum enm1 { a };\n' - b'int var1;\n' - b'#define MACRO1\n', - 'base1.c') - TSParent2 = self.cls_from_ccode(b'void func2(void) { return; }\n' - b'void mock2(void);\n' - b'struct strct2 {};\n' - b'enum enm2 { a };\n' - b'int var2;\n' - b'#define MACRO2\n', - 'base2.c') - class TSMerged(TSParent1, TSParent2): - pass - with TSMerged() as ts: - assert hasattr(ts, 'func1') and hasattr(ts, 'func2') - assert hasattr(ts, 'mock1') and hasattr(ts, 'mock2') - assert hasattr(ts, 'var1') and hasattr(ts, 'var2') - assert hasattr(ts, 'MACRO1') and hasattr(ts, 'MACRO2') - assert hasattr(ts.struct, 'strct1') and hasattr(ts.struct, 'strct2') - assert hasattr(ts.enum, 'enm1') and hasattr(ts.enum, 'enm2') + +class TestCModule: + + def test_call_onBuildDescFactoryAvailableCls_raisesTypeError(self): + class NonTSCls: pass + cmod = CModule() + with pytest.raises(TypeError): + cmod(NonTSCls) + + class BuilddescFactoryImplemented: + @classmethod + def __builddesc_factory__(cls): + return Gcc32BuildDescription('', Path('')) + + def test_call_onBuilddescFactoryImplemented_raisesTypeError(self): + class BuilddescFactoryImplemented: + @classmethod + def __builddesc_factory__(cls): + return Gcc32BuildDescription('', Path('')) + cmod = CModule() + with pytest.raises(TypeError): + cmod(BuilddescFactoryImplemented) + + @patch.object(Path, 'is_file', return_value=True) + def test_call_onSrcPath_derivesBuilddescFactoryToAddAbsSrcPath(self, is_file): + @CModule('rel_src.c') + class TSRelSrc(self.BuilddescFactoryImplemented): pass + builddesc = TSRelSrc.__builddesc_factory__() + abs_c_src = Path(__file__).resolve().parent / 'rel_src.c' + assert builddesc.c_sources() == [abs_c_src] + is_file.assert_called_with(abs_c_src) + + @patch.object(Path, 'is_file', return_value=False) + def test_call_onInvalidSrcPath_raisesOSError(self, is_file): + @CModule('rel_src.c') + class TSInvalidSrc(self.BuilddescFactoryImplemented): pass + with pytest.raises(OSError): + TSInvalidSrc.__builddesc_factory__() + + @patch.object(Path, 'is_file', return_value=True) + def test_call_onPredefMacros_derivsBuilddescFactoryToAddPredefMacros(self, is_file): + @CModule('src.c', MACRO1=1, MACRO2='') + class TSPredefMacros(self.BuilddescFactoryImplemented): pass + builddesc = TSPredefMacros.__builddesc_factory__() + abs_c_src = Path(__file__).resolve().parent / 'src.c' + assert builddesc.predef_macros() \ + == {abs_c_src: dict(MACRO1='1', MACRO2='')} + + @patch.object(Path, 'is_file', return_value=True) + @patch.object(Path, 'is_dir', return_value=True) + def test_call_onInclOrLibDir_derivesBuilddescFactoryToSetAbsDirPath(self, is_dir, is_file): + @CModule('src.c', include_dirs=['rel/dir']) + class TSRelDir(self.BuilddescFactoryImplemented): pass + builddesc = TSRelDir.__builddesc_factory__() + abs_src = Path(__file__).resolve().parent / 'src.c' + abs_path = Path(__file__).resolve().parent / 'rel/dir' + assert builddesc.incl_dirs() == {abs_src: [abs_path]} + is_dir.assert_called_with(abs_path) + + @patch.object(Path, 'is_file', return_value=True) + @patch.object(Path, 'is_dir', return_value=False) + @pytest.mark.parametrize('dir_name', ['library_dirs', 'include_dirs']) + def test_call_onInvalidInclOrLibDir_raisesOSError(self, is_dir, is_file, dir_name): + @CModule('src.c', **{dir_name: 'invalid/dir'}) + class TSInvalidDir(self.BuilddescFactoryImplemented): pass + with pytest.raises(OSError): + TSInvalidDir.__builddesc_factory__() + + @patch.object(Path, 'is_file', return_value=True) + def test_call_onDerivedClass_doesNotModifyBaseClassesBuildDesc(self, is_file): + @CModule('src_base.c') + class TSBase(self.BuilddescFactoryImplemented): pass + @CModule('src_derived.c') + class TSDerived(TSBase): pass + builddesc_base = TSBase.__builddesc_factory__() + builddesc_derived = TSDerived.__builddesc_factory__() + assert {p.name for p in builddesc_base.c_sources()} == {'src_base.c'} + assert {p.name for p in builddesc_derived.c_sources()} \ + == {'src_base.c', 'src_derived.c'} diff --git a/tests/test_toolchains/test_gcc.py b/tests/test_toolchains/test_gcc.py deleted file mode 100644 index 7fa42e1..0000000 --- a/tests/test_toolchains/test_gcc.py +++ /dev/null @@ -1,73 +0,0 @@ -import pytest -import sys -import subprocess -from unittest.mock import patch -from headlock.toolchains.gcc import GccToolChain -import platform -from pathlib import Path -import ctypes as ct - -from ..helpers import build_tree -from headlock.testsetup import TransUnit -if platform.architecture()[0] == '32bit': - from headlock.toolchains.gcc import Gcc32ToolChain as GccXXToolChain -else: - from headlock.toolchains.gcc import Gcc64ToolChain as GccXXToolChain - - -class TestGccToolChain: - - @patch('subprocess.check_output', return_value= - 'Using built-in specs.\n' - 'COLLECT_GCC=gcc\n' - '...\n' - 'ignoring nonexistent directory "/usr/lib/gcc/x86_64-linux-gnu/7/../../../../x86_64-linux-gnu/include"\n' - '#include "..." search starts here:\n' - '#include <...> search starts here:\n' - ' /usr/lib/gcc/x86_64-linux-gnu/7/include\n' - ' /usr/local/include\n' - ' /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed\n' - ' /usr/include/x86_64-linux-gnu\n' - ' /usr/include\n' - 'End of search list.\n' - '...\n') - def test_sysInclDirs_returnsSysInclDirsReturnedByGcc(self, check_output): - assert GccToolChain().sys_incl_dirs() == [ - Path('/usr/lib/gcc/x86_64-linux-gnu/7/include'), - Path('/usr/local/include'), - Path('/usr/lib/gcc/x86_64-linux-gnu/7/include-fixed'), - Path('/usr/include/x86_64-linux-gnu'), - Path('/usr/include')] - check_output.assert_called_with( - ['gcc', '-v', '-xc', '-c', '/dev/null', '-o', '/dev/null'], - encoding='utf8', stderr=subprocess.STDOUT) - - @patch('subprocess.run') - @patch.object(GccXXToolChain, 'ADDITIONAL_COMPILE_OPTIONS', ['-O1','-Cx']) - @patch.object(GccXXToolChain, 'ADDITIONAL_LINK_OPTIONS', ['-O2','-Lx']) - def test_build_passesParametersToGcc(self, subprocess_run): - subprocess_run.return_value.returncode = 0 - toolchain = GccXXToolChain() - toolchain.build('name', Path('dir'), [TransUnit('', Path('src.c'))], - ['lib_name'], [Path('lib_dir')]) - first_call_pos_args, *_ = subprocess_run.call_args_list[0] - assert '-Cx' in first_call_pos_args[0] - assert '-O1' in first_call_pos_args[0] - assert '-O2' not in first_call_pos_args[0] - second_call_pos_args, *_ = subprocess_run.call_args_list[1] - assert '-Lx' in second_call_pos_args[0] - assert '-O2' in second_call_pos_args[0] - assert '-llib_name' in second_call_pos_args[0] - assert '-Llib_dir' in second_call_pos_args[0] - assert '-O1' not in second_call_pos_args[0] - - @pytest.mark.skipif(sys.platform == 'win32', - reason='works only on non-win platforms') - def test_build_createsDll(self, tmpdir): - basedir = build_tree(tmpdir, {'src.c': b'int func(void) { return 22; }', - 'build': {}}) - toolchain = GccXXToolChain() - toolchain.build('xyz', basedir / 'build', - [TransUnit('', basedir / 'src.c')], [], []) - dll = ct.CDLL(str(toolchain.exe_path('xyz', basedir / 'build'))) - assert dll.func() == 22 diff --git a/tests/test_toolchains/test_mingw.py b/tests/test_toolchains/test_mingw.py deleted file mode 100644 index 19332f1..0000000 --- a/tests/test_toolchains/test_mingw.py +++ /dev/null @@ -1,28 +0,0 @@ -import pytest -import sys -import ctypes -import platform -from ..helpers import build_tree - -if sys.platform == 'win32': - - from headlock.testsetup import TransUnit - if platform.architecture()[0] == '32bit': - from headlock.toolchains.mingw import \ - MinGW32ToolChain as MinGWxxToolChain - else: - from headlock.toolchains.mingw import \ - MinGW64ToolChain as MinGWxxToolChain - - - class TestMinGW32ToolChain: - - def test_build_createsDll(self, tmpdir): - basedir = build_tree(tmpdir, { - 'src.c': b'int func(void) { return 22; }', - 'build': {}}) - toolchain = MinGWxxToolChain() - toolchain.build('xyz', basedir / 'build', - [TransUnit('', basedir / 'src.c')], [], []) - dll = ctypes.CDLL(str(toolchain.exe_path('xyz', basedir / 'build'))) - assert dll.func() == 22