Refactor to allow Buildsystem integration

Parsing, creation of bridge and building of C sources is moved out
of TestSetup into BuildDescription.

TODO.md

@@ -12,6 +12,7 @@ Interface changing
  * refactor PtrArrFactoryMixIn into factory object
  * .sizeof and .offsetof() shall be available at runtime only and
    done by the compiler (seems to be compiler dependent)
+ * replace packing by libclangs sizeof/offsetof to support #pragma pack
 
 Small (can be done by occassion)
 --------------------------------
@@ -99,6 +100,7 @@ Small (can be done by occassion)
 * comparison shall also work if python type is different.
   I.e. ts.struct.x(1, 2) == (1, 2) does not work yet, as the .val of
   struct returns a dict, which is not equal to the tuple.
+* Add Support for 64bit MinGW (is stored in different directory than 32bit)
 
 
 Medium
@@ -144,7 +146,9 @@ Major/Investigation necessary
   + if a macro is set in the TestSetup it must not be overwritten by the parser
     (= allow overwriting macros)
 * test C instrumentalization (address sanitizer, see -f... flags)
-* packing pragma (attribute?) is not supported
+* Switch from access via ABI to access via API. This would require less
+  compiler adaptions
+* C++ Support
 
 
 Needed for publishing

docs/about.rst

@@ -17,7 +17,8 @@ C/Python bridges like ctypes, cffi, swig, cython, ...:
    python file.
    These steps done by headlock include:
 
-    * No need to create Makefiles/Buildscripts.
+    * No need to create extra Makefiles/Buildscripts for unit testing a single
+      C module
     * No need to run extra build steps before using the C code.
     * No need to rewrite the C moduels interface definition (the header file)
       in Python.
@@ -70,13 +71,11 @@ C/Python bridges like ctypes, cffi, swig, cython, ...:
 
 Explicitly Non-Goals Are:
 
-   * Support for C++
-
    * High Performance (This does not mean that it is slow.
      But if speed conflicts with one of the goals of this project,
      there will be no compromises in favour of speed).
 
    * Being self-contained
-     (At least A C-compiler will always be required to be installed).
+     (At least LLVM and a C-compiler will always be required to be installed).
 
    * Support for Python < 3.6

headlock/bridge_gen.py

@@ -82,11 +82,13 @@ def write_c2py_bridge(output:TextIO, required_funcptrs:Iterable[CFuncType],
             for instance_ndx in range(max_instances):
                 write_c2py_bridge_func(output, bridge_ndx, instance_ndx, cfunc)
     output.write(
-        f'void (* _c2py_bridge_[][{max_instances}])(void) = {{\n')
+        f'typedef void (* _c2py_bridge_func_t)(void);\n'
+        f'_c2py_bridge_func_t _c2py_bridge_[][{max_instances}] = {{\n')
     for bridge_ndx in range(len(bridge_ndxs)):
         output.write('\t{ ')
         for instance_ndx in range(max_instances):
-            output.write(f'_c2py_bridge_{bridge_ndx}_{instance_ndx}, ')
+            output.write(f'(_c2py_bridge_func_t) '
+                         f'_c2py_bridge_{bridge_ndx}_{instance_ndx}, ')
         output.write(' },\n')
     output.write('};\n\n')
     return bridge_ndxs

headlock/buildsys_drvs/__init__.py

@@ -0,0 +1,110 @@
+import abc
+import hashlib
+import os
+from pathlib import Path
+from typing import List, Dict
+
+
+class BuildError(Exception):
+
+    def __init__(self, msg, path=None):
+        super().__init__(msg)
+        self.path = path
+
+    def __str__(self):
+        return (f'building {self.path} failed: ' if self.path
+                else '') \
+               + super().__str__()
+
+
+class BuildDescription:
+    """
+    This is an abstract base class for all classes that allow to specify
+    C projects
+    """
+
+    def __init__(self, name:str, build_dir:Path, unique_name=True):
+        """
+        Abstract Base Class for Descriptions how to build a bunch of C files
+        :param name: Descriptive Name of BuildDescription
+        :param build_dir: Directory where all generated files shall be stored
+        :param unique_name: If True, a BuildDescription with the same name
+                          cannot exist
+        """
+        self.name = name
+        self.__build_dir = build_dir
+        self.unique_name = unique_name
+
+    @property
+    def build_dir(self):
+        if self.unique_name:
+            return self.__build_dir
+        else:
+            hash = hashlib.md5()
+            incl_dirs = self.incl_dirs()
+            predef_macros = self.predef_macros()
+            for c_source in self.c_sources():
+                hash.update(os.fsencode(c_source))
+                for incl_dir in incl_dirs[c_source]:
+                    hash.update(os.fsencode(incl_dir))
+                for mname, mvalue in predef_macros[c_source].items():
+                    hash.update((mname + '=' + mvalue).encode('ascii'))
+            return self.__build_dir.parent \
+                   / (self.__build_dir.name + '_' + hash.hexdigest()[:8])
+
+    @abc.abstractmethod
+    def clang_target(self) -> str:
+        """
+        returns a string that represents the "target" parameter of clang
+        """
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def sys_incl_dirs(self) -> List[Path]:
+        """
+        retrieves a list of all system include directories
+        """
+
+    def sys_predef_macros(self) -> Dict[str, str]:
+        """
+        A dictionary of toolchain-inhernt macros, that shall be always
+        predefined additionally to the predefined macros provided by clang
+        """
+        return {}
+
+    def c_sources(self) -> List[Path]:
+        """
+        returns all C source files
+        """
+
+    @abc.abstractmethod
+    def incl_dirs(self) -> Dict[Path, List[Path]]:
+        """
+        returns a list of all source code files.
+        :return:
+        """
+
+    @abc.abstractmethod
+    def predef_macros(self) -> Dict[Path, Dict[str, str]]:
+        """
+        returns all predefined macros per source file a list of all source code files.
+        :return:
+        """
+
+    @abc.abstractmethod
+    def exe_path(self) -> Path:
+        """
+        returns name of executable image/shared object library/dll
+        """
+
+    @abc.abstractmethod
+    def build(self, additonal_c_sources:List[Path]=None):
+        """
+        builds executable image
+        """
+
+    def is_header_file(self, src_path:Path) -> bool:
+        """
+        Returns True, if this is a header file
+        """
+        return src_path.suffix.lower() == '.h'

headlock/buildsys_drvs/gcc.py

@@ -0,0 +1,142 @@
+import subprocess
+import os
+from pathlib import Path
+
+from typing import Dict, List, Any
+from . import BuildDescription, BuildError
+
+
+BUILD_CACHE = set()
+
+
+class GccBuildDescription(BuildDescription):
+
+    SYS_INCL_DIR_CACHE:Dict[Path, List[Path]] = {}
+
+    ADDITIONAL_COMPILE_OPTIONS = []
+    ADDITIONAL_LINK_OPTIONS = []
+
+    def __init__(self, name:str, build_dir:Path, unique_name=True, *,
+                 c_sources:List[Path]=None, predef_macros:Dict[str, Any]=None,
+                 incl_dirs:List[Path]=None, lib_dirs:List[Path]=None,
+                 req_libs:List[str]=None, gcc_executable='gcc'):
+        super().__init__(name, build_dir, unique_name)
+        self.gcc_executable = gcc_executable
+        self.__c_sources = c_sources or []
+        self.__incl_dirs = incl_dirs or []
+        self.__req_libs = req_libs or []
+        self.__lib_dirs = lib_dirs or []
+        self.__predef_macros = {}
+        self.add_predef_macros(predef_macros or {})
+
+    def add_c_source(self, c_filename:Path):
+        if c_filename in self.__c_sources:
+            raise ValueError(f'Translation Unit {c_filename!r} is already part '
+                             f'of BuildDescription')
+        self.__c_sources.append(c_filename)
+
+    def add_predef_macros(self, predef_macros:Dict[str, Any]):
+        for name, val in predef_macros.items():
+            self.__predef_macros[name] = str(val) if val is not None else None
+
+    def add_incl_dir(self, incl_dir:Path):
+        self.__incl_dirs.append(incl_dir)
+
+    def add_lib_dir(self, lib_dir:Path):
+        self.__lib_dirs.append(lib_dir)
+
+    def add_req_lib(self, lib_name:str):
+        self.__req_libs.append(lib_name)
+
+    def sys_incl_dirs(self):
+        if self.gcc_executable not in self.SYS_INCL_DIR_CACHE:
+            try:
+                gcc_info = subprocess.check_output(
+                    [self.gcc_executable,
+                     '-v', '-xc', '-c', '/dev/null', '-o', '/dev/null'],
+                    stderr=subprocess.STDOUT,
+                    encoding='utf8')
+            except (subprocess.CalledProcessError, FileNotFoundError) as e:
+                raise BuildError('failed to retrieve SYS include path from gcc')
+            else:
+                incl_dirs = self.SYS_INCL_DIR_CACHE[self.gcc_executable] = []
+                print("gcc_info")
+                collecting = False
+                for line in gcc_info.splitlines():
+                    if line.startswith('#include <...> search starts here'):
+                        collecting = True
+                    elif line.startswith('End of search list.'):
+                        collecting = False
+                    elif collecting:
+                        incl_dirs.append(Path(line.strip()))
+        return self.SYS_INCL_DIR_CACHE[self.gcc_executable]
+
+    def c_sources(self):
+        return self.__c_sources
+
+    def predef_macros(self):
+        return dict.fromkeys(self.__c_sources, self.__predef_macros)
+
+    def incl_dirs(self):
+        return dict.fromkeys(self.__c_sources, self.__incl_dirs)
+
+    def _run_gcc(self, call_params, dest_file):
+        try:
+            completed_proc = subprocess.run(
+                [self.gcc_executable] + call_params,
+                encoding='utf8',
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+        except (subprocess.CalledProcessError, FileNotFoundError) as e:
+            raise BuildError(f'failed to call gcc: {e}', dest_file)
+        else:
+            if completed_proc.returncode != 0:
+                raise BuildError(completed_proc.stderr, dest_file)
+
+    def exe_path(self):
+        return self.build_dir / '__headlock__.dll'
+
+    def build(self, additonal_c_sources=None):
+        transunits = (tuple(self.__c_sources),
+                      tuple(self.__predef_macros.items()),
+                      tuple(self.__incl_dirs),
+                      tuple(self.__req_libs),
+                      tuple(self.__lib_dirs))
+        if (tuple(transunits), self.build_dir) in BUILD_CACHE:
+            return
+        total_sources = self.__c_sources + (additonal_c_sources or [])
+        for c_src in total_sources:
+            if self.is_header_file(c_src):
+                continue
+            obj_file_path = self.build_dir / (c_src.stem + '.o')
+            self._run_gcc(['-c', os.fspath(c_src)]
+                          + ['-o', os.fspath(obj_file_path)]
+                          + ['-I' + os.fspath(incl_dir)
+                             for incl_dir in self.__incl_dirs]
+                          + [f'-D{mname}={mval or ""}'
+                             for mname, mval in self.__predef_macros.items()]
+                          + ['-Werror']
+                          + self.ADDITIONAL_COMPILE_OPTIONS,
+                          obj_file_path)
+        exe_file_path = self.exe_path()
+        self._run_gcc([str(self.build_dir / (c_src.stem + '.o'))
+                       for c_src in total_sources
+                       if not self.is_header_file(c_src)]
+                      + ['-shared', '-o', os.fspath(exe_file_path)]
+                      + ['-l' + str(req_lib) for req_lib in self.__req_libs]
+                      + ['-L' + str(lib_dir) for lib_dir in self.__lib_dirs]
+                      + ['-Werror']
+                      + self.ADDITIONAL_LINK_OPTIONS,
+                      exe_file_path)
+        BUILD_CACHE.add((tuple(transunits), self.build_dir))
+
+
+class Gcc32BuildDescription(GccBuildDescription):
+    def clang_target(self):
+        return 'i386-pc-linux-gnu'
+
+
+class Gcc64BuildDescription(GccBuildDescription):
+    ADDITIONAL_COMPILE_OPTIONS = ['-fPIC']
+    def clang_target(self):
+        return 'x86_64-pc-linux-gnu'