From acf0a2fa3c1b4fc77e38d238867b81997d350dfc Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 23 Apr 2020 22:54:00 +0200 Subject: [PATCH 01/50] prefer interpolating strings --- src/c4/cmany/err.py | 34 ++++++++++++++++++---------------- src/c4/cmany/vsinfo.py | 2 +- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/c4/cmany/err.py b/src/c4/cmany/err.py index 1864026..5d61d54 100644 --- a/src/c4/cmany/err.py +++ b/src/c4/cmany/err.py @@ -6,54 +6,56 @@ def __init__(self, msg, *args, **kwargs): msg = msg.format(*args, **kwargs) if not msg: msg = "unknown error" - super().__init__("ERROR: {}".format(msg)) + super().__init__(f"ERROR: {msg}") + + +class NotImplemented(Error): + def __init__(self, msg): + super().__init__(f"not implemented: {msg}") class VSNotFound(Error): def __init__(self, vs_spec, msg=None): - msg = "" if msg is None else ". {}.".format(msg) - super().__init__("Visual Studio not found: {}{}", vs_spec, msg) + msg = "" if msg is None else f". {msg}." + super().__init__(f"Visual Studio not found: {vs_spec}{msg}") class CompilerNotFound(Error): def __init__(self, compiler_spec, msg=None): - msg = "" if msg is None else ". {}.".format(msg) - super().__init__("compiler not found: {}{}", compiler_spec, msg) + msg = "" if msg is None else f". {msg}." + super().__init__(f"compiler not found: {compiler_spec}{msg}") class InvalidGenerator(Error): def __init__(self, gen_spec, msg=None): - msg = "" if msg is None else ". {}.".format(msg) - super().__init__("invalid generator: {}{}", gen_spec, msg) + msg = "" if msg is None else f". {msg}." + super().__init__(f"invalid generator: {gen_spec}{msg}") class NoSupport(Error): def __init__(self, feature): - super().__init__("feature not supported: {}", feature) + super().__init__(f"feature not supported: {feature}") class SubcommandNotFound(Error): def __init__(self, cmds, args): - super().__init__("could not find subcommand {} in args {}", cmds, args) + super().__init__(f"could not find subcommand {cmds} in args {args}") class UnknownCombinationArg(Error): def __init__(self, a): - super().__init__("unknown combination argument:", a) + super().__init__(f"unknown combination argument: {a}") class ProjDirNotFound(Error): def __init__(self, proj_dir): - if proj_dir is None: - super().__init__("project dir was not given") - else: - msg = "project dir was not found: '{}' (curr dir is '{}')" - super().__init__(msg, proj_dir, os.getcwd()) + cwd = os.getcwd() + super().__init__(f"project dir was not found: '{proj_dir}' (curr dir is '{cwd}')") class CMakeListsNotFound(Error): def __init__(self, proj_dir): - super().__init__("CMakeLists.txt not found at dir: {}", proj_dir) + super().__init__(f"CMakeLists.txt or CMakeCache.txt not found at dir: {proj_dir}") class BuildDirNotFound(Error): diff --git a/src/c4/cmany/vsinfo.py b/src/c4/cmany/vsinfo.py index 3184bc6..87c92c8 100644 --- a/src/c4/cmany/vsinfo.py +++ b/src/c4/cmany/vsinfo.py @@ -196,7 +196,7 @@ def to_name(name_or_gen_or_ver): n = _names.get(name_or_gen_or_ver) if n is not None: return n - raise Exception("could not find '{}'".format(name_or_gen_or_ver)) + raise Exception(f"could not find '{name_or_gen_or_ver}'") def to_ver(name_or_gen_or_ver): From b327a78a950d4c873705b67a0099da16c3b1b10e Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 23 Apr 2020 22:59:20 +0200 Subject: [PATCH 02/50] [wip] add command to build individual file --- src/c4/cmany/build.py | 15 ++++++++--- src/c4/cmany/generator.py | 53 ++++++++++++++++++++------------------- src/c4/cmany/main.py | 25 +++++++++++++----- src/c4/cmany/project.py | 5 ++++ 4 files changed, 62 insertions(+), 36 deletions(-) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index f728327..c2d3c5d 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -284,10 +284,7 @@ def build(self, targets=[]): self.configure() self.handle_deps() if len(targets) == 0: - if self.compiler.is_msvc: - targets = ["ALL_BUILD"] - else: - targets = ["all"] + targets = [self.generator.target_all] # cmake --build and visual studio won't handle # multiple targets at once, so loop over them. for t in targets: @@ -300,6 +297,16 @@ def build(self, targets=[]): # it can come to fail in some corner cases. self.mark_build_done(cmd) + def build_files(self, target, files): + self.create_dir() + with util.setcwd(self.builddir, silent=False): + self.handle_deps() + try: + cmd = self.generator.cmd([t]) + util.runsyscmd(cmd) + except Exception as e: + raise err.CompileFailed(self, cmd, e) + def rebuild(self, targets=[]): self._check_successful_configure('rebuild') with util.setcwd(self.builddir, silent=False): diff --git a/src/c4/cmany/generator.py b/src/c4/cmany/generator.py index 952b29c..b8bfc0c 100644 --- a/src/c4/cmany/generator.py +++ b/src/c4/cmany/generator.py @@ -1,5 +1,6 @@ from . import cmake from . import vsinfo +from . import err from .build_item import BuildItem from .err import TooManyTargets @@ -67,7 +68,12 @@ def configure_args(self, for_json=False, export_compile_commands=True): pass return args - def cmd(self, targets, override_build_type=None, override_num_jobs=None): + @property + def target_all(self): + "return the name of the ALL target" + return "ALL_BUILD" if self.is_msvc else "all" + + def cmd(self, targets): if self.is_makefile: return ['make', '-j', str(self.num_jobs)] + targets elif self.is_ninja: @@ -76,35 +82,30 @@ def cmd(self, targets, override_build_type=None, override_num_jobs=None): bt = str(self.build.build_type) if len(targets) > 1: raise TooManyTargets(self) - if not self.is_msvc: - cmd = ['cmake', '--build', '.', '--target', targets[0], '--config', bt] - else: - # # if a target has a . in the name, it must be substituted for _ - # targets_safe = [re.sub(r'\.', r'_', t) for t in targets] - # if len(targets_safe) != 1: - # raise Exception("msbuild can only build one target at a time: was " + str(targets_safe)) - # t = targets_safe[0] - # pat = os.path.join(self.build.builddir, t + '*.vcxproj') - # projs = glob.glob(pat) - # if len(projs) == 0: - # msg = "could not find vcx project for this target: {} (glob={}, got={})".format(t, pat, projs) - # raise Exception(msg) - # elif len(projs) > 1: - # msg = "multiple vcx projects for this target: {} (glob={}, got={})".format(t, pat, projs) - # raise Exception(msg) - # proj = projs[0] - # cmd = [self.build.compiler.vs.msbuild, proj, - # '/property:Configuration='+bt, - # '/maxcpucount:' + str(self.num_jobs)] - cmd = ['cmake', '--build', '.', '--target', targets[0], '--config', bt, - '--', - #'/property:Configuration='+bt, - '/maxcpucount:' + str(self.num_jobs)] + cmd = ['cmake', '--build', '.', '--config', bt, '--target', targets[0], + '--parallel', str(self.num_jobs)] + return cmd + + def cmd_source_file(self, target, source_file): + """get a command to build a source file + https://stackoverflow.com/questions/38271387/compile-a-single-file-under-cmake-project + """ + relpath = os.path.relpath(source_file, os.path.abspath(self.build.builddir)) + if self.is_makefile: + return ['make', relpath] + elif self.is_ninja: + raise err.NotImplemented() + # https://ninja-build.org/manual.html#_running_ninja + return ['ninja', f'{relpath}^'] + else: + bt = str(self.build.build_type) + cmd = ['cmake', '--build', '.', '--target', targets[0], '--config', bt] return cmd def install(self): bt = str(self.build.build_type) - return ['cmake', '--build', '.', '--config', bt, '--target', 'install'] + return ['cmake', '--build', '.', '--config', bt, '--target', 'install', + '--parallel', str(self.num_jobs)] """ generators: https://cmake.org/cmake/help/v3.7/manual/cmake-generators.7.html diff --git a/src/c4/cmany/main.py b/src/c4/cmany/main.py index 6fde210..2ec07b0 100644 --- a/src/c4/cmany/main.py +++ b/src/c4/cmany/main.py @@ -16,6 +16,7 @@ ('configure', ['c']), ('reconfigure', ['rc']), ('build', ['b']), + ('build_file', ['bf']), ('rebuild', ['rb']), ('install', ['i']), ('reinstall', ['ri']), @@ -172,6 +173,24 @@ def _exec(self, proj, args): proj.build() +class install(selectcmd): + """install the selected builds, building before if necessary""" + def _exec(self, proj, args): + proj.install() + + +class build_file(globcmd): + """compile the given source files""" + def add_args(self, parser): + super().add_args(parser) + parser.add_argument('target', nargs=1, + help="""the target to which the source files belong""") + parser.add_argument('files', default=[], nargs='*', + help="""the files to compile, relative to the CMakeLists.txt root dir""") + def _exec(self, proj, args): + proj.build_files(args.target, args.files) + + class rebuild(globcmd): """rebuild the selected builds, selecting by name using a python glob pattern""" def add_args(self, parser): @@ -182,12 +201,6 @@ def _exec(self, proj, args): proj.rebuild() -class install(selectcmd): - """install the selected builds, building before if necessary""" - def _exec(self, proj, args): - proj.install() - - class reinstall(globcmd): """rebuild the selected builds, selecting by name using a python glob pattern""" def _exec(self, proj, args): diff --git a/src/c4/cmany/project.py b/src/c4/cmany/project.py index 1c486f1..d7eef83 100644 --- a/src/c4/cmany/project.py +++ b/src/c4/cmany/project.py @@ -307,6 +307,11 @@ def do_build(build): build.build(self.targets) self._execute(do_build, "Build", silent=False, **restrict_to) + def build_files(self, target, files): + def do_build_files(build): + build.build_files(target, files) + self._execute(do_build_files, "BuildFiles", silent=False) + def rebuild(self, **restrict_to): def do_rebuild(build): build.rebuild(self.targets) From b534b64575fb96c76287b3dc6ee5690eb7952ae4 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 23 Apr 2020 22:59:59 +0200 Subject: [PATCH 03/50] [wip] improving detection of proj dir and build dir --- src/c4/cmany/args.py | 20 ++++++++++++-------- src/c4/cmany/project.py | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/src/c4/cmany/args.py b/src/c4/cmany/args.py index 6b5384f..dfdf545 100644 --- a/src/c4/cmany/args.py +++ b/src/c4/cmany/args.py @@ -155,14 +155,18 @@ def _handle_hidden_args__skip_rest(args): # ----------------------------------------------------------------------------- def add_basic(parser): - parser.add_argument("proj_dir", nargs="?", default=".", - help="""the directory where the project's CMakeLists.txt - is located. An empty argument will default to the - current directory ie, \".\". Passing a directory which - does not contain a CMakeLists.txt will cause an error.""") - parser.add_argument("--build-dir", default="./build", - help="set the build root (defaults to ./build)") - parser.add_argument("--install-dir", default="./install", + parser.add_argument("-P", "--proj-dir", default=None, + help="""the directory where the project's + CMakeLists.txt is located. An empty argument will + default to the current directory ie, "." or, if + the current directory is a build directory containing + CMakeCache.txt, the source directory for that + build directory. Passing a directory which + does not contain a CMakeLists.txt or CMakeCache.txt + will cause an error.""") + parser.add_argument("-B", "--build-root", default="./build", + help="set the build root (defaults to ./build). ") + parser.add_argument("-I", "--install-root", default="./install", help="set the install root (defaults to ./install)") parser.add_argument("-j", "--jobs", default=cpu_count(), help="""use the given number of parallel jobs diff --git a/src/c4/cmany/project.py b/src/c4/cmany/project.py index d7eef83..8c2b2d9 100644 --- a/src/c4/cmany/project.py +++ b/src/c4/cmany/project.py @@ -42,6 +42,25 @@ def _getdir(attr_name, default, kwargs, cwd): return d +def _is_cmake_folder(folder): + if folder is None: + return False + dbg("exists?", folder) + if not _pexists(folder): + dbg("does not exist:", folder) + return None + dbg("exists:", folder) + if _pexists(folder, "CMakeLists.txt"): + dbg("found CMakeLists.txt in", folder) + return folder + dbg("CMakeLists.txt not found in", folder) + if _pexists(folder, "CMakeCache.txt"): + dbg("found CMakeCache.txt in", folder) + return folder + dbg("CMakeCache.txt not found in", folder) + return None + + # ----------------------------------------------------------------------------- class Project: @@ -56,10 +75,18 @@ def __init__(self, **kwargs): pdir = kwargs.get('proj_dir') dbg("cwd:", cwd) dbg("proj_dir:", pdir) - if pdir is None: - raise err.ProjDirNotFound(None) if pdir == ".": pdir = cwd + elif pdir is None: + dbg("proj_dir not given") + if pdir is None and self.targets: + dbg("is the first target a cmake directory?", self.targets[0]) + pdir = _is_cmake_folder(self.targets[0]) + if pdir is not None: + self.targets = self.targets[1:] + if pdir is None: + dbg("picking current directory", cwd) + pdir = cwd pdir = util.abspath(pdir) dbg("proj_dir, abs:", pdir) # @@ -81,6 +108,10 @@ def __init__(self, **kwargs): self.install_dir = os.path.dirname(ch['CMAKE_INSTALL_PREFIX'].val) self.root_dir = ch['CMAKE_HOME_DIRECTORY'].val self.cmakelists = os.path.join(self.root_dir, "CMakeLists.txt") + else: + self.build_dir = None + self.install_dir = None + self.root_dir = pdir # self.root_dir = os.path.realpath(self.root_dir) self.build_dir = os.path.realpath(self.build_dir) From 340ea34072a2272f2ce01ac2d87182bdacecff59 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 17 Nov 2020 17:27:40 +0000 Subject: [PATCH 04/50] chg: usr: require cmake and ninja --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index a2e66b2..d150d7b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,5 @@ colorama argcomplete dill python-dateutil +cmake +ninja From 692553792208143f91870aaea033a0df64fc303d Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 17 Nov 2020 17:28:25 +0000 Subject: [PATCH 05/50] chg: dev: move comment from place --- src/c4/cmany/generator.py | 102 ++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 49 deletions(-) diff --git a/src/c4/cmany/generator.py b/src/c4/cmany/generator.py index b8bfc0c..5df6c42 100644 --- a/src/c4/cmany/generator.py +++ b/src/c4/cmany/generator.py @@ -5,6 +5,53 @@ from .build_item import BuildItem from .err import TooManyTargets +""" +generators: https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html + +Unix Makefiles +MSYS Makefiles +MinGW Makefiles +NMake Makefiles +Ninja +Watcom WMake +CodeBlocks - Ninja +CodeBlocks - Unix Makefiles +CodeBlocks - MinGW Makefiles +CodeBlocks - NMake Makefiles +CodeLite - Ninja +CodeLite - Unix Makefiles +CodeLite - MinGW Makefiles +CodeLite - NMake Makefiles +Eclipse CDT4 - Ninja +Eclipse CDT4 - Unix Makefiles +Eclipse CDT4 - MinGW Makefiles +Eclipse CDT4 - NMake Makefiles +KDevelop3 +KDevelop3 - Unix Makefiles +Kate - Ninja +Kate - Unix Makefiles +Kate - MinGW Makefiles +Kate - NMake Makefiles +Sublime Text 2 - Ninja +Sublime Text 2 - Unix Makefiles +Sublime Text 2 - MinGW Makefiles +Sublime Text 2 - NMake Makefiles + +Visual Studio 6 +Visual Studio 7 +Visual Studio 7 .NET 2003 +Visual Studio 8 2005 [Win64|IA64] +Visual Studio 9 2008 [Win64|IA64] +Visual Studio 10 2010 [Win64|IA64] +Visual Studio 11 2012 [Win64|ARM] +Visual Studio 12 2013 [Win64|ARM] +Visual Studio 14 2015 [Win64|ARM] +Visual Studio 15 2017 [Win64|ARM] +Visual Studio 16 2019 -A [Win32|x64|ARM|ARM64] + +Green Hills MULTI +Xcode +""" # ----------------------------------------------------------------------------- class Generator(BuildItem): @@ -104,53 +151,10 @@ def cmd_source_file(self, target, source_file): def install(self): bt = str(self.build.build_type) - return ['cmake', '--build', '.', '--config', bt, '--target', 'install', - '--parallel', str(self.num_jobs)] + return ['cmake', '--build', '.', '--config', bt, '--target', 'install'] + + def clean_msbuild_target_name(self, target_name): + # if a target has a . in the name, it must be substituted for _ + target_safe = re.sub(r'\.', r'_', target_name) + - """ - generators: https://cmake.org/cmake/help/v3.7/manual/cmake-generators.7.html - - Unix Makefiles - MSYS Makefiles - MinGW Makefiles - NMake Makefiles - Ninja - Watcom WMake - CodeBlocks - Ninja - CodeBlocks - Unix Makefiles - CodeBlocks - MinGW Makefiles - CodeBlocks - NMake Makefiles - CodeLite - Ninja - CodeLite - Unix Makefiles - CodeLite - MinGW Makefiles - CodeLite - NMake Makefiles - Eclipse CDT4 - Ninja - Eclipse CDT4 - Unix Makefiles - Eclipse CDT4 - MinGW Makefiles - Eclipse CDT4 - NMake Makefiles - KDevelop3 - KDevelop3 - Unix Makefiles - Kate - Ninja - Kate - Unix Makefiles - Kate - MinGW Makefiles - Kate - NMake Makefiles - Sublime Text 2 - Ninja - Sublime Text 2 - Unix Makefiles - Sublime Text 2 - MinGW Makefiles - Sublime Text 2 - NMake Makefiles - - Visual Studio 6 - Visual Studio 7 - Visual Studio 7 .NET 2003 - Visual Studio 8 2005 [Win64|IA64] - Visual Studio 9 2008 [Win64|IA64] - Visual Studio 10 2010 [Win64|IA64] - Visual Studio 11 2012 [Win64|ARM] - Visual Studio 12 2013 [Win64|ARM] - Visual Studio 14 2015 [Win64|ARM] - Visual Studio 15 2017 [Win64|ARM] - Visual Studio 16 2019 -A [Win32|x64|ARM|ARM64] - - Green Hills MULTI - Xcode - """ From 97b846049112e4e3c0853ccf3e4fd6c354ced5f8 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 17 Nov 2020 17:37:09 +0000 Subject: [PATCH 06/50] fix: usr: apparently cmake --build --parallel is broken in VS --- src/c4/cmany/generator.py | 44 +++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/src/c4/cmany/generator.py b/src/c4/cmany/generator.py index 5df6c42..2cc5763 100644 --- a/src/c4/cmany/generator.py +++ b/src/c4/cmany/generator.py @@ -126,11 +126,43 @@ def cmd(self, targets): elif self.is_ninja: return ['ninja', '-j', str(self.num_jobs)] + targets else: - bt = str(self.build.build_type) if len(targets) > 1: raise TooManyTargets(self) - cmd = ['cmake', '--build', '.', '--config', bt, '--target', targets[0], - '--parallel', str(self.num_jobs)] + target = targets[0] + bt = str(self.build.build_type) + # NOTE: + # the `--parallel` flag to `cmake --build` is broken in VS and XCode: + # https://discourse.cmake.org/t/parallel-does-not-really-enable-parallel-compiles-with-msbuild/964/10 + # https://gitlab.kitware.com/cmake/cmake/-/issues/20564 + if not self.is_msvc: + cmd = ['cmake', '--build', '.', '--target', target, '--config', bt, + '--parallel', str(self.num_jobs)] + # TODO XCode is also broken; the flag is this: + # -IDEBuildOperationMaxNumberOfConcurrentCompileTasks=$NUM_JOBS_BUILD + # see: + # https://stackoverflow.com/questions/5417835/how-to-modify-the-number-of-parallel-compilation-with-xcode + # https://gist.github.com/nlutsenko/ee245fbd239087d22137 + else: + cmd = ['cmake', '--build', '.', '--target', target, '--config', bt, + '--', '/maxcpucount:' + str(self.num_jobs)] + # old code for building through msbuild: + # # if a target has a . in the name, it must be substituted for _ + # targets_safe = [re.sub(r'\.', r'_', t) for t in targets] + # if len(targets_safe) != 1: + # raise TooManyTargets("msbuild can only build one target at a time: was " + str(targets_safe)) + # t = targets_safe[0] + # pat = os.path.join(self.build.builddir, t + '*.vcxproj') + # projs = glob.glob(pat) + # if len(projs) == 0: + # msg = "could not find vcx project for this target: {} (glob={}, got={})".format(t, pat, projs) + # raise Exception(msg) + # elif len(projs) > 1: + # msg = "multiple vcx projects for this target: {} (glob={}, got={})".format(t, pat, projs) + # raise Exception(msg) + # proj = projs[0] + # cmd = [self.build.compiler.vs.msbuild, proj, + # '/property:Configuration='+bt, + # '/maxcpucount:' + str(self.num_jobs)] return cmd def cmd_source_file(self, target, source_file): @@ -152,9 +184,3 @@ def cmd_source_file(self, target, source_file): def install(self): bt = str(self.build.build_type) return ['cmake', '--build', '.', '--config', bt, '--target', 'install'] - - def clean_msbuild_target_name(self, target_name): - # if a target has a . in the name, it must be substituted for _ - target_safe = re.sub(r'\.', r'_', target_name) - - From f10fee756d180089b3f44d4dfcb8d393a904c867 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 17 Nov 2020 20:53:27 +0000 Subject: [PATCH 07/50] fix: make sure the proj paths are not null --- src/c4/cmany/project.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/c4/cmany/project.py b/src/c4/cmany/project.py index 8c2b2d9..3cda760 100644 --- a/src/c4/cmany/project.py +++ b/src/c4/cmany/project.py @@ -113,10 +113,11 @@ def __init__(self, **kwargs): self.install_dir = None self.root_dir = pdir # - self.root_dir = os.path.realpath(self.root_dir) - self.build_dir = os.path.realpath(self.build_dir) - self.install_dir = os.path.realpath(self.install_dir) - self.cmakelists = os.path.realpath(self.cmakelists) + _saferealpath = lambda p: p if p is None else os.path.realpath(p) + self.root_dir = _saferealpath(self.root_dir) + self.build_dir = _saferealpath(self.build_dir) + self.install_dir = _saferealpath(self.install_dir) + self.cmakelists = _saferealpath(self.cmakelists) # dbg("root_dir:", self.root_dir) dbg("build_dir:", self.build_dir) From 534acdc6458b4c7681cc62fc1170aad17936678e Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 17 Nov 2020 20:56:15 +0000 Subject: [PATCH 08/50] new: usr: add command build_files --- src/c4/cmany/build.py | 16 ++++++++++------ src/c4/cmany/generator.py | 27 +++++++++++++++++---------- src/c4/cmany/main.py | 28 ++++++++++++++-------------- src/c4/cmany/project.py | 4 ++-- 4 files changed, 43 insertions(+), 32 deletions(-) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index c2d3c5d..2142146 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -297,15 +297,19 @@ def build(self, targets=[]): # it can come to fail in some corner cases. self.mark_build_done(cmd) - def build_files(self, target, files): + def build_files(self, files, target): self.create_dir() with util.setcwd(self.builddir, silent=False): self.handle_deps() - try: - cmd = self.generator.cmd([t]) - util.runsyscmd(cmd) - except Exception as e: - raise err.CompileFailed(self, cmd, e) + for f in files: + try: + cmd = self.generator.cmd_source_file(f, target) + dbg(f"building file {f}, target {target}. cmd={cmd}") + util.runsyscmd(cmd) + dbg(f"building file {f}, target {target}. success!") + except Exception as e: + dbg(f"building file {f}, target {target}. exception: {e}!") + raise err.CompileFailed(self, cmd, e) def rebuild(self, targets=[]): self._check_successful_configure('rebuild') diff --git a/src/c4/cmany/generator.py b/src/c4/cmany/generator.py index 2cc5763..240bf61 100644 --- a/src/c4/cmany/generator.py +++ b/src/c4/cmany/generator.py @@ -4,6 +4,9 @@ from .build_item import BuildItem from .err import TooManyTargets +from .util import logdbg + +import os.path """ generators: https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html @@ -165,21 +168,25 @@ def cmd(self, targets): # '/maxcpucount:' + str(self.num_jobs)] return cmd - def cmd_source_file(self, target, source_file): - """get a command to build a source file - https://stackoverflow.com/questions/38271387/compile-a-single-file-under-cmake-project - """ + def cmd_source_file(self, source_file, target): + """get a command to build a source file""" + logdbg("building source file:", source_file, "from target", target) relpath = os.path.relpath(source_file, os.path.abspath(self.build.builddir)) + logdbg("building source file:", source_file, "from target", target) + basecmd = ['cmake', '--build', '.', '--target', target, '--verbose'] if self.is_makefile: - return ['make', relpath] + # https://stackoverflow.com/questions/38271387/compile-a-single-file-under-cmake-project + return basecmd + ['--', relpath] elif self.is_ninja: - raise err.NotImplemented() # https://ninja-build.org/manual.html#_running_ninja - return ['ninja', f'{relpath}^'] + #return ['ninja', f'{relpath}^'] + return basecmd + ['--', f'{relpath}^'] else: - bt = str(self.build.build_type) - cmd = ['cmake', '--build', '.', '--target', targets[0], '--config', bt] - return cmd + if not self.is_msvc: + raise err.NotImplemented("don't know how to build source files in this generator") + # https://stackoverflow.com/questions/4172438/using-msbuild-to-compile-a-single-cpp-file + return basecmd + ['--config', str(self.build.build_type), '--', + '-t:ClCompile', f'-p:SelectedFiles={relpath}'] def install(self): bt = str(self.build.build_type) diff --git a/src/c4/cmany/main.py b/src/c4/cmany/main.py index 2ec07b0..11b4967 100644 --- a/src/c4/cmany/main.py +++ b/src/c4/cmany/main.py @@ -173,32 +173,32 @@ def _exec(self, proj, args): proj.build() -class install(selectcmd): - """install the selected builds, building before if necessary""" +class rebuild(globcmd): + """rebuild the selected builds, selecting by name using a python glob pattern""" + def add_args(self, parser): + super().add_args(parser) + parser.add_argument('target', default=[], nargs='*', + help="""specify a subset of targets to build""") def _exec(self, proj, args): - proj.install() + proj.rebuild() -class build_file(globcmd): - """compile the given source files""" +class build_file(selectcmd): + """[EXPERIMENTAL] compile the given source files""" def add_args(self, parser): super().add_args(parser) - parser.add_argument('target', nargs=1, + parser.add_argument('target', type=str, help="""the target to which the source files belong""") parser.add_argument('files', default=[], nargs='*', help="""the files to compile, relative to the CMakeLists.txt root dir""") def _exec(self, proj, args): - proj.build_files(args.target, args.files) + proj.build_files(args.files, args.target) -class rebuild(globcmd): - """rebuild the selected builds, selecting by name using a python glob pattern""" - def add_args(self, parser): - super().add_args(parser) - parser.add_argument('target', default=[], nargs='*', - help="""specify a subset of targets to build""") +class install(selectcmd): + """install the selected builds, building before if necessary""" def _exec(self, proj, args): - proj.rebuild() + proj.install() class reinstall(globcmd): diff --git a/src/c4/cmany/project.py b/src/c4/cmany/project.py index 3cda760..b7d468e 100644 --- a/src/c4/cmany/project.py +++ b/src/c4/cmany/project.py @@ -339,9 +339,9 @@ def do_build(build): build.build(self.targets) self._execute(do_build, "Build", silent=False, **restrict_to) - def build_files(self, target, files): + def build_files(self, files, target): def do_build_files(build): - build.build_files(target, files) + build.build_files(files, target) self._execute(do_build_files, "BuildFiles", silent=False) def rebuild(self, **restrict_to): From ce0cc4621f1f88ce0b655a1282e6ce305480bd06 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 17 Nov 2020 21:23:21 +0000 Subject: [PATCH 09/50] chg: usr: disambiguate: use --glob instead of positional arguments --- src/c4/cmany/args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c4/cmany/args.py b/src/c4/cmany/args.py index dfdf545..3543f60 100644 --- a/src/c4/cmany/args.py +++ b/src/c4/cmany/args.py @@ -180,7 +180,7 @@ def add_glob(parser): # add_basic(parser) # - parser.add_argument("glob", nargs="*", default="*", + parser.add_argument("--glob", nargs="*", default="*", help="""glob pattern(s) matching build names (NOTE: not paths to the build dirs)""") From 147da5b9e930dd92deb24401b763dac9ea7d5bc4 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Wed, 18 Nov 2020 19:29:42 +0000 Subject: [PATCH 10/50] new: usr: add -vb/--verbose flag --- src/c4/cmany/args.py | 9 +++++++++ src/c4/cmany/build.py | 7 ++++--- src/c4/cmany/main.py | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/c4/cmany/args.py b/src/c4/cmany/args.py index 3543f60..0488d13 100644 --- a/src/c4/cmany/args.py +++ b/src/c4/cmany/args.py @@ -32,6 +32,7 @@ def setup(subcommands, module): formatter_class=argparse.RawDescriptionHelpFormatter, epilog=help.epilog, ) + add_verbose(p) # gather the list of visible subcommands # https://stackoverflow.com/questions/21692395/hiding-selected-subcommands-using-argparse visible_metavar = ",".join(["{},{}".format(cmd, ",".join(aliases)) @@ -153,6 +154,13 @@ def _handle_hidden_args__skip_rest(args): return False +# ----------------------------------------------------------------------------- +def add_verbose(parser): + parser.add_argument("-vb", "--verbose", default=False, action="store_true", + help="""use verbose commands in the build tools. does not apply to configure step; + build_file is always verbose""") + + # ----------------------------------------------------------------------------- def add_basic(parser): parser.add_argument("-P", "--proj-dir", default=None, @@ -173,6 +181,7 @@ def add_basic(parser): (defaults to %(default)s on this machine).""") parser.add_argument("--continue", default=False, action="store_true", help="attempt to continue when a build fails") + add_verbose(parser) # ----------------------------------------------------------------------------- diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index 2142146..991efba 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -105,18 +105,19 @@ def create_generator(self, num_jobs, fallback_generator="Unix Makefiles"): # toolchain_cache = cmake.get_toolchain_cache(self.toolchain_file) # print(toolchain_cache) # self.adjust(compiler=toolchain_cache['CMAKE_CXX_COMPILER']) + verbose = self.kwargs['verbose'] if self.compiler.is_msvc: vsi = vsinfo.VisualStudioInfo(self.compiler.name) - g = Generator(vsi.gen, self, num_jobs) + g = Generator(vsi.gen, self, num_jobs, verbose) arch = Architecture(vsi.architecture) self.adjust(architecture=arch) self.vsinfo = vsi return g else: if self.system.name == "windows": - return Generator(fallback_generator, self, num_jobs) + return Generator(fallback_generator, self, num_jobs, verbose) else: - return Generator(Generator.default_str(), self, num_jobs) + return Generator(Generator.default_str(), self, num_jobs, verbose) def adjust(self, **kwargs): for k, _ in kwargs.items(): diff --git a/src/c4/cmany/main.py b/src/c4/cmany/main.py index 11b4967..4a05ff4 100644 --- a/src/c4/cmany/main.py +++ b/src/c4/cmany/main.py @@ -184,7 +184,7 @@ def _exec(self, proj, args): class build_file(selectcmd): - """[EXPERIMENTAL] compile the given source files""" + """[EXPERIMENTAL] compile the given source files.""" def add_args(self, parser): super().add_args(parser) parser.add_argument('target', type=str, From 0a5bc01e33fc376df69864e2c4d9b8a85efeefab Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Wed, 18 Nov 2020 22:49:07 +0000 Subject: [PATCH 11/50] fix: usr: build_files now works in Visual Studio!!!! --- requirements.txt | 2 +- src/c4/cmany/architecture.py | 16 +++- src/c4/cmany/build.py | 28 +++++- src/c4/cmany/cmake.py | 21 ++++- src/c4/cmany/generator.py | 86 ++++++++++++++++--- src/c4/cmany/util.py | 11 +++ src/c4/cmany/vsinfo.py | 162 ++++++++++++++++++++++++++++++----- 7 files changed, 282 insertions(+), 44 deletions(-) diff --git a/requirements.txt b/requirements.txt index d150d7b..e68813d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ colorama argcomplete dill python-dateutil -cmake ninja +lxml diff --git a/src/c4/cmany/architecture.py b/src/c4/cmany/architecture.py index f4d5fbc..53b4203 100644 --- a/src/c4/cmany/architecture.py +++ b/src/c4/cmany/architecture.py @@ -26,10 +26,8 @@ def default_str(): @property def is64(self): - def fn(): - s = re.search('64', self.name) - return s is not None - return util.cacheattr(self, "_is64", fn) + return util.cacheattr(self, "_is64", + lambda: re.search('64', self.name) is not None) @property def is32(self): @@ -38,3 +36,13 @@ def is32(self): @property def is_arm(self): return "arm" in self.name.lower() + + @property + def vs_dev_env_name(self): + # [arch]: x86 | amd64 | x86_amd64 | x86_arm | x86_arm64 | amd64_x86 | amd64_arm | amd64_arm64 + if self.is_arm: + raise Exception("not implemented") + if self.is64: + return "x64" + else: + return "x86" diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index 991efba..14674fd 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -220,7 +220,9 @@ def export_compile_commands(self): if not self.compiler.is_msvc: util.runsyscmd(cmd) else: - self.vsinfo.runsyscmd(cmd) + dev_cmd = " ".join(self.vs_dev_cmd("ALL_BUILD")) + cmd = " ".join(cmd) + util.runsyscmd(f"{dev_cmd} {cmd}") except Exception as e: raise err.ConfigureFailed(self, cmd, e) src = os.path.join(trickdir, "compile_commands.json") @@ -233,11 +235,28 @@ def export_compile_commands(self): util.loginfo("exported compile_commands.json:", dst) def run_custom_cmd(self, cmd, **subprocess_args): + if self.needs_configure(): + self.configure() try: util.runcmd(cmd, **subprocess_args, cwd=self.builddir) except subprocess.CalledProcessError as exc: raise err.RunCmdFailed(self, cmd, exc) + @property + def cxx_compiler(self): + return util.cacheattr(self, "_cxx_compiler", lambda: cmake.get_cxx_compiler(self.builddir)) + + def vs_dev_cmd(self, target): + cl_exe = self.cxx_compiler + dbg("cl.exe:", cl_exe) + cl_exe_version = re.sub(".*/MSVC/(.*?)/.*", r"\1", cl_exe) + dbg("cl.exe version:", cl_exe_version) + return vsinfo.dev_env( + vcvarsall=f'"{self.vsinfo.vcvarsall}"', + arch=str(self.architecture.vs_dev_env_name), + winsdk=str(self.generator.vs_get_vcxproj(target).winsdk), + vc_version=cl_exe_version) + def reconfigure(self): """reconfigure a build directory, without touching any cache entry""" self._check_successful_configure('reconfigure') @@ -299,14 +318,17 @@ def build(self, targets=[]): self.mark_build_done(cmd) def build_files(self, files, target): - self.create_dir() + if self.needs_configure(): + self.configure() + self.build([target]) + return with util.setcwd(self.builddir, silent=False): self.handle_deps() for f in files: try: cmd = self.generator.cmd_source_file(f, target) dbg(f"building file {f}, target {target}. cmd={cmd}") - util.runsyscmd(cmd) + util.runsyscmd_str(cmd, cwd=self.builddir) dbg(f"building file {f}, target {target}. success!") except Exception as e: dbg(f"building file {f}, target {target}. exception: {e}!") diff --git a/src/c4/cmany/cmake.py b/src/c4/cmany/cmake.py index ef59c84..47909e0 100644 --- a/src/c4/cmany/cmake.py +++ b/src/c4/cmany/cmake.py @@ -1,10 +1,11 @@ import re import os +from glob import glob from collections import OrderedDict as odict from .conf import USER_DIR -from .util import cacheattr, setcwd, runsyscmd, logdbg +from .util import cacheattr, setcwd, runsyscmd, logdbg, chkf from . import util from . import err @@ -78,6 +79,24 @@ def loadvars(builddir): return v +def get_cxx_compiler(builddir): + cmkf = chkf(builddir, "CMakeFiles") + expr = f"{cmkf}/*/CMakeCXXCompiler.cmake" + files = glob(expr) + if len(files) != 1: + raise Exception(f"could not find compiler settings: {expr}") + cxxfile = files[0] + logdbg("") + with open(cxxfile) as f: + lines = f.readlines() + lookup = "set(CMAKE_CXX_COMPILER " + for line in [l.strip() for l in lines]: + if not line.startswith(lookup): + continue + return line[(len(lookup)+1):-2] + raise Exception("could not find compiler spec") + + # ----------------------------------------------------------------------------- class CMakeCache(odict): diff --git a/src/c4/cmany/generator.py b/src/c4/cmany/generator.py index 240bf61..71be427 100644 --- a/src/c4/cmany/generator.py +++ b/src/c4/cmany/generator.py @@ -1,12 +1,16 @@ from . import cmake from . import vsinfo from . import err +from . import util from .build_item import BuildItem from .err import TooManyTargets from .util import logdbg -import os.path +import fnmatch +import os +import shlex + """ generators: https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html @@ -71,7 +75,7 @@ def default_str(): s = cmake.CMakeSysInfo.generator() return s - def __init__(self, name, build, num_jobs): + def __init__(self, name, build, num_jobs, verbose): if isinstance(name, list): #more_args = name[1:] name = name[0] @@ -84,6 +88,7 @@ def __init__(self, name, build, num_jobs): self.is_ninja = name.endswith("Ninja") self.is_msvc = name.startswith("Visual Studio") self.build = build + self.verbose = verbose # self.sysinfo_name = self.name if self.is_msvc: @@ -125,9 +130,15 @@ def target_all(self): def cmd(self, targets): if self.is_makefile: - return ['make', '-j', str(self.num_jobs)] + targets + if self.verbose: + return ['make', 'VERBOSE=1', '-j', str(self.num_jobs)] + targets + else: + return ['make', '-j', str(self.num_jobs)] + targets elif self.is_ninja: - return ['ninja', '-j', str(self.num_jobs)] + targets + if self.verbose: + return ['ninja', '--verbose', '-j', str(self.num_jobs)] + targets + else: + return ['ninja', '-j', str(self.num_jobs)] + targets else: if len(targets) > 1: raise TooManyTargets(self) @@ -140,14 +151,18 @@ def cmd(self, targets): if not self.is_msvc: cmd = ['cmake', '--build', '.', '--target', target, '--config', bt, '--parallel', str(self.num_jobs)] + if self.verbose: + cmd.append('--verbose') # TODO XCode is also broken; the flag is this: # -IDEBuildOperationMaxNumberOfConcurrentCompileTasks=$NUM_JOBS_BUILD # see: # https://stackoverflow.com/questions/5417835/how-to-modify-the-number-of-parallel-compilation-with-xcode # https://gist.github.com/nlutsenko/ee245fbd239087d22137 else: - cmd = ['cmake', '--build', '.', '--target', target, '--config', bt, - '--', '/maxcpucount:' + str(self.num_jobs)] + cmd = ['cmake', '--build', '.', '--target', target, '--config', bt] + if self.verbose: + cmd.append('--verbose') + cmd += ['--', '/maxcpucount:' + str(self.num_jobs)] # old code for building through msbuild: # # if a target has a . in the name, it must be substituted for _ # targets_safe = [re.sub(r'\.', r'_', t) for t in targets] @@ -172,8 +187,8 @@ def cmd_source_file(self, source_file, target): """get a command to build a source file""" logdbg("building source file:", source_file, "from target", target) relpath = os.path.relpath(source_file, os.path.abspath(self.build.builddir)) - logdbg("building source file:", source_file, "from target", target) - basecmd = ['cmake', '--build', '.', '--target', target, '--verbose'] + logdbg("building source file: relpath:", relpath) + basecmd = ['cmake', '--build', '.', '--target', target, '--verbose'] # always verbose if self.is_makefile: # https://stackoverflow.com/questions/38271387/compile-a-single-file-under-cmake-project return basecmd + ['--', relpath] @@ -181,13 +196,60 @@ def cmd_source_file(self, source_file, target): # https://ninja-build.org/manual.html#_running_ninja #return ['ninja', f'{relpath}^'] return basecmd + ['--', f'{relpath}^'] + elif not self.is_msvc: + raise err.NotImplemented("don't know how to build source files in this generator") else: - if not self.is_msvc: - raise err.NotImplemented("don't know how to build source files in this generator") + # msvc: # https://stackoverflow.com/questions/4172438/using-msbuild-to-compile-a-single-cpp-file - return basecmd + ['--config', str(self.build.build_type), '--', - '-t:ClCompile', f'-p:SelectedFiles={relpath}'] + #return basecmd + ['--config', str(self.build.build_type), '--', + # '-t:ClCompile', f'-p:SelectedFiles={relpath}'] + # + # the command does not result in compiling ONLY the requested file. + # It fails because -p:SelectedFiles is ignored, so the command + # builds the whole target. + # + # So we have to get really down and dirty... + logdbg("generator is vs -- need to discover the cl command line") + cl_cmd = self.vs_cl_cmd(source_file, target) + logdbg("cl_cmd=", cl_cmd) + return cl_cmd def install(self): bt = str(self.build.build_type) return ['cmake', '--build', '.', '--config', bt, '--target', 'install'] + + def vs_cl_cmd(self, source_file, target): + tlog = self.vs_find_cl_cmd_tlog(target) + abspath = os.path.join(self.build.projdir, source_file) + cl_cmd = tlog.get_cmd_line(abspath) + cxx_compiler = self.build.cxx_compiler + cl_cmd = " ".join(shlex.split(f'"{cxx_compiler}" {cl_cmd}', posix=False)) + dev_cmd = " ".join(self.build.vs_dev_cmd(target)) + return f"{dev_cmd} {cl_cmd}" + + def vs_find_cl_cmd_tlog(self, target): + vcxproj = self.vs_get_vcxproj(target) + variant = vcxproj.get_variant(str(self.build.build_type), self.build.architecture) + tlog_dir = util.chkf(f"{variant.intdir}/{target}.tlog") + tlog = util.chkf(f"{tlog_dir}/CL.command.1.tlog") + return vsinfo.ClCommandTlog(tlog) + + def vs_get_vcxproj(self, target): + class Foo: pass + store = util.cacheattr(self, 'vs_target_vcxproj', lambda: Foo()) + vcxproj = util.cacheattr(store, target, lambda: vsinfo.VcxProj(self.vs_find_vcxproj(target))) + return vcxproj + + def vs_find_vcxproj(self, target): + logdbg("searching for vcxproj:", target) + matches = [] + for root, dirnames, filenames in os.walk(self.build.builddir): + for filename in fnmatch.filter(filenames, f"{target}.vcxproj"): + matches.append(os.path.join(root, filename)) + if len(matches) == 0: + raise err.NotImplemented(f"could not find vcxproj: {target}") + if len(matches) > 1: + raise err.NotImplemented("cannot deal with multiple targets with the same name") + vcxproj = os.path.realpath(matches[0]) + logdbg("found vcxproj:", vcxproj) + return vcxproj diff --git a/src/c4/cmany/util.py b/src/c4/cmany/util.py index 851c43a..c07d06d 100644 --- a/src/c4/cmany/util.py +++ b/src/c4/cmany/util.py @@ -599,6 +599,17 @@ def runsyscmd(cmd, echo_cmd=True, echo_output=True, capture_output=False, as_byt result.check_returncode() +# batch commands using vcvarsall have to be passed as a full string +def runsyscmd_str(cmd_str, echo_cmd=True, **run_args): + if echo_cmd: + cwd = os.path.realpath(run_args.get('cwd', os.getcwd())) + logcmd(f'$ cd {os.path.realpath(cwd)} && {cmd_str}') + sp = subprocess.Popen(cmd_str, **run_args) + sp.wait() + if sp.returncode != 0: + raise Exception("failed") + + def runcmd_nocheck(cmd, *cmd_args, **run_args): # TODO: https://stackoverflow.com/questions/17742789/running-multiple-bash-commands-with-subprocess logdbg(f"running command: {cmd} '{cmd_args}'") diff --git a/src/c4/cmany/vsinfo.py b/src/c4/cmany/vsinfo.py index 87c92c8..ddfa085 100644 --- a/src/c4/cmany/vsinfo.py +++ b/src/c4/cmany/vsinfo.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 - import os import re import glob import json import sys +from lxml import etree from collections import OrderedDict as odict @@ -39,27 +39,143 @@ def __init__(self, name): self.is_clang = re.search(r'clang', self.toolset) is not None else: self.is_clang = False - def runsyscmd(self, cmd, **kwargs): - """ - vcvarsall.bat usage: - Syntax: - vcvarsall.bat [arch] [platform_type] [winsdk_version] [-vcvars_ver=vc_version] [-vcvars_spectre_libs=spectre_mode] - where : - [arch]: x86 | amd64 | x86_amd64 | x86_arm | x86_arm64 | amd64_x86 | amd64_arm | amd64_arm64 - [platform_type]: {empty} | store | uwp - [winsdk_version] : full Windows 10 SDK number (e.g. 10.0.10240.0) or "8.1" to use the Windows 8.1 SDK. - [vc_version] : {none} for default VS 2017 VC++ compiler toolset | - "14.0" for VC++ 2015 Compiler Toolset | - "14.1x" for the latest 14.1x.yyyyy toolset installed (e.g. "14.11") | - "14.1x.yyyyy" for a specific full version number (e.g. 14.11.25503) - [spectre_mode] : {none} for default VS 2017 libraries without spectre mitigations | - "spectre" for VS 2017 libraries with spectre mitigations - """ - a = str(self.architecture) - if "64" in a: - a = "x64" - cmd = ["cmd", "/C", "call", self.vcvarsall, a, "8.1", "&"] + cmd - runsyscmd(cmd, **kwargs) + + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- + +def dev_env(vcvarsall, arch, winsdk, vc_version): + """ + vcvarsall.bat usage: + Syntax: + vcvarsall.bat [arch] [platform_type] [winsdk_version] [-vcvars_ver=vc_version] [-vcvars_spectre_libs=spectre_mode] + where : + [arch]: x86 | amd64 | x86_amd64 | x86_arm | x86_arm64 | amd64_x86 | amd64_arm | amd64_arm64 + [platform_type]: {empty} | store | uwp + [winsdk_version] : full Windows 10 SDK number (e.g. 10.0.10240.0) or "8.1" to use the Windows 8.1 SDK. + [vc_version] : {none} for latest installed VC++ compiler toolset | + "14.0" for VC++ 2015 Compiler Toolset | + "14.xx" for the latest 14.xx.yyyyy toolset installed (e.g. "14.11") | + "14.xx.yyyyy" for a specific full version number (e.g. "14.11.25503") + [spectre_mode] : {none} for libraries without spectre mitigations | + "spectre" for libraries with spectre mitigations + The store parameter sets environment variables to support Universal Windows Platform application + development and is an alias for 'uwp'. + For example: + vcvarsall.bat x86_amd64 + vcvarsall.bat x86_amd64 10.0.10240.0 + vcvarsall.bat x86_arm uwp 10.0.10240.0 + vcvarsall.bat x86_arm onecore 10.0.10240.0 -vcvars_ver=14.0 + vcvarsall.bat x64 8.1 + vcvarsall.bat x64 store 8.1 + """ + return ["cmd", "/C", "call", vcvarsall, arch, winsdk, f"-vcvars_ver={vc_version}", "&"] + + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- + +class VcxProj: + + def __init__(self, vcxproj): + self._vcxproj = vcxproj + with open(self._vcxproj, "rb") as f: # the rb is needed on windows + logdbg("parsing", self._vcxproj) + self._tree = etree.parse(f) + xp = lambda *args: xp__(self._tree, *args) + # https://stackoverflow.com/questions/14248063/xpath-to-select-element-by-attribute-value + self.winsdk = xp("PropertyGroup[@Label='Globals']", "WindowsTargetPlatformVersion")[0].text + self.platform = xp("PropertyGroup[@Label='Globals']", "Platform")[0].text + # + self._variants = {} + for node in xp("ItemGroup[@Label='ProjectConfigurations']")[0]: + bv = VcxProj.BuildVariant(self._tree, node) + self._variants[bv.name.lower()] = bv + + def get_variant(self, build_type, platform): + if str(platform) == "x86_64": + platform = "x64" + else: + raise Exception("crl" + platform) + logdbg(self._variants.keys()) + name = f"{build_type}|{platform}".lower() + return self._variants[name] + + class BuildVariant: + def __init__(self, tree, node): + def xp(*args): return xp__(tree, *args) + self.name = node.get("Include") + self.configuration = child__(node, 'Configuration').text + self.platform = child__(node, 'Platform').text + # + condition = f"contains(@Condition, '{self.name}')" + pgnode = xp(f"PropertyGroup[{condition} and @Label='Configuration']")[0] + assert self.name in pgnode.attrib["Condition"] + self.configuration_type = child__(pgnode, "ConfigurationType").text + self.character_set = child__(pgnode, "CharacterSet").text + self.platform_toolset = child__(pgnode, "PlatformToolset").text + # + self.intdir = xp("PropertyGroup", f"IntDir[{condition}]")[0].text + self.outdir = xp("PropertyGroup", f"OutDir[{condition}]")[0].text + self.target_name = xp("PropertyGroup", f"TargetName[{condition}]")[0].text + self.target_ext = xp("PropertyGroup", f"TargetExt[{condition}]")[0].text + + +wtfns__ = 'http://schemas.microsoft.com/developer/msbuild/2003' + +def child__(node, childname): + # WTF?? + # is there not a node[childname] equivalent? + for child in node: + cleantag = child.tag.replace(f"{{{wtfns__}}}", "") + logdbg("asdaldskjlaskdj", childname, child.tag, cleantag, cleantag == childname) + if cleantag == childname: + return child + return None + + +def xp__(tree, *args): # having to do this sucks. + # https://stackoverflow.com/questions/37327774/parsing-vcxproj-with-python-and-lxml + p = f"/wtf:Project" + ("/wtf:" if len(args) > 0 else "") + "/wtf:".join(args) + logdbg("search:", p) + return tree.xpath(p, namespaces={'wtf': wtfns__}) + + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- + +class ClCommandTlog: + + def __init__(self, filename): + def pairwise(iterable): + "s -> (s0, s1), (s2, s3), (s4, s5), ..." + a = iter(iterable) + return zip(a, a) + filename = os.path.abspath(filename) + logdbg("tlog: opening file:", filename) + with open(filename, encoding="utf_16_le") as f: # the encoding is needed + # example + # ^D:\PROJ\C4CORE\SRC\C4\BASE64.CPP + # /c /ID:\PROJ\C4CORE\SRC /Zi /nologo /W4 /WX /diagnostics:column /Od /Ob0 /D WIN32 /D _WINDOWS /D C4_SUBSTR_NO_OSTREAM_LSHIFT /D "CMAKE_INTDIR=\"Debug\"" /D _MBCS /Gm- /EHsc /RTC1 /MDd /GS /fp:precise /Zc:wchar_t /Zc:forScope /Zc:inline /GR /std:c++17 /Fo"C4CORE.DIR\DEBUG\\" /Fd"C4CORE.DIR\DEBUG\C4CORE.PDB" /Gd /TP /we4289 /w14242 /w14254 /w14263 /w14265 /w14287 /w14296 /w14311 /w14545 /w14546 /w14547 /w14549 /w14555 /w14619 /w14640 /w14826 /w14905 /w14906 /w14928 D:\PROJ\C4CORE\SRC\C4\BASE64.CPP + # ^D:\PROJ\C4CORE\SRC\C4\CHAR_TRAITS.CPP + # /c /ID:\PROJ\C4CORE\SRC /Zi /nologo /W4 /WX /diagnostics:column /Od /Ob0 /D WIN32 /D _WINDOWS /D C4_SUBSTR_NO_OSTREAM_LSHIFT /D "CMAKE_INTDIR=\"Debug\"" /D _MBCS /Gm- /EHsc /RTC1 /MDd /GS /fp:precise /Zc:wchar_t /Zc:forScope /Zc:inline /GR /std:c++17 /Fo"C4CORE.DIR\DEBUG\\" /Fd"C4CORE.DIR\DEBUG\C4CORE.PDB" /Gd /TP /we4289 /w14242 /w14254 /w14263 /w14265 /w14287 /w14296 /w14311 /w14545 /w14546 /w14547 /w14549 /w14555 /w14619 /w14640 /w14826 /w14905 /w14906 /w14928 D:\PROJ\C4CORE\SRC\C4\CHAR_TRAITS.CPP + lines = [line.strip() for line in f.readlines()] + self._cmds = {} + for key, v in pairwise(lines): + k = re.sub(r".*?\^(.*)", r"\1", key) + k = self.safe_key(k) + self._cmds[k] = v + logdbg(f"tlog: '{key}' -> '{k}'") + assert len(lines) % 2 == 0, f"len={len(lines)}" + str(lines) + + def safe_key(self, filename): + return os.path.abspath(filename).upper().replace("\\", "/") + + def get_cmd_line(self, filename): + return self._cmds[self.safe_key(filename)] # ----------------------------------------------------------------------------- @@ -386,7 +502,7 @@ def msbuild(name_or_gen_or_ver): pass # default to some probable value if no registry key was found if msbuild is None: - val = 'C:\\Windows\Microsoft.NET\Framework{}\\v{}\\MSBuild.exe' + val = 'C:\\Windows\\Microsoft.NET\\Framework{}\\v{}\\MSBuild.exe' for v in msbvers: msbuild = val.format('64' if util.in_64bit() else '', v)#'3.5') if os.path.exists(msbuild): From 46c71f8b40a698b0691d36d393f3e0f997747d40 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Wed, 18 Nov 2020 23:34:40 +0000 Subject: [PATCH 12/50] fix: usr: mark experimental flags --- src/c4/cmany/args.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/c4/cmany/args.py b/src/c4/cmany/args.py index 0488d13..d4c7546 100644 --- a/src/c4/cmany/args.py +++ b/src/c4/cmany/args.py @@ -207,31 +207,31 @@ def add_proj(parser): # g = parser.add_argument_group('Configuration files') g.add_argument("--config-file", default=[], action="append", - help="""Specify a file containing configurations. Relative + help="""[EXPERIMENTAL] Specify a file containing configurations. Relative paths are taken from the project's CMakeLists.txt directory. Run `cmany help flags` to get help about flag aliases. Multiple invokations are possible, in which case flags given in latter files will prevail over those of earlier files.""") g.add_argument("--no-default-config", default=False, action="store_true", - help="""Do not read the default config file. Run + help="""[EXPERIMENTAL] Do not read the default config file. Run `cmany help flags` to get help about this.""") # d = parser.add_argument_group('Dependencies') d.add_argument('--deps', default='', type=str, metavar='path/to/extern/CMakeLists.txt', - help="""Before configuring, process (ie, configure, build + help="""[EXPERIMENTAL] Before configuring, process (ie, configure, build and install) the given CMakeLists.txt project containing needed external project dependencies. This will be done separately for each build, using the same parameters. The main project will be configured such that the built dependencies are found by cmake.""") d.add_argument('--deps-prefix', default="", type=str, - metavar='path/to/install/directory', + metavar='[EXPERIMENTAL] path/to/install/directory', help="""When using --deps set the install directory for external dependencies to the given dir.""") d.add_argument('--with-conan', action='store_true', default=False, - help="""(WIP)""") + help=argparse.SUPPRESS) # ----------------------------------------------------------------------------- From 4ff618d29aca4a777fd798d940ed01c09566c8d7 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Wed, 18 Nov 2020 23:35:17 +0000 Subject: [PATCH 13/50] fix: usr: dangling references to --build-dir and --install-dir --- doc/quick_tour.rst | 2 +- src/c4/cmany/doc/quick_tour.txt | 2 +- test/test05cmany.py | 16 ++++++++-------- test/test06xcompile.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/quick_tour.rst b/doc/quick_tour.rst index 3da27b9..8414693 100644 --- a/doc/quick_tour.rst +++ b/doc/quick_tour.rst @@ -152,7 +152,7 @@ the install trees are nested under the ``install`` directory. However, you don't have to use these defaults. The following command will use ``foo`` for building and ``bar`` for installing:: - $ cmany i -c clang++,g++ --build-dir foo --install-dir bar + $ cmany i -c clang++,g++ --build-root foo --install-root bar $ ls -1 foo/ bar/ bar/linux-x86_64-clang++3.9-Release/ diff --git a/src/c4/cmany/doc/quick_tour.txt b/src/c4/cmany/doc/quick_tour.txt index 59a0b4c..b3ec8fa 100644 --- a/src/c4/cmany/doc/quick_tour.txt +++ b/src/c4/cmany/doc/quick_tour.txt @@ -157,7 +157,7 @@ directory. However, you don't have to use these defaults. The following command will use "foo" for building and "bar" for installing: - $ cmany i -c clang++,g++ --build-dir foo --install-dir bar + $ cmany i -c clang++,g++ --build-root foo --install-root bar $ ls -1 foo/ bar/ bar/linux-x86_64-clang++3.9-Release/ diff --git a/test/test05cmany.py b/test/test05cmany.py index fc059cf..ef1d90e 100644 --- a/test/test05cmany.py +++ b/test/test05cmany.py @@ -164,7 +164,7 @@ def run_projs(testobj, args, check_fn=None): id = '.test/0--default--install' for p in projs: with testobj.subTest(msg="default parameters", proj=p.proj): - p.run(args + ['--build-dir', bd, '--install-dir', id]) + p.run(args + ['--build-root', bd, '--install-root', id]) if check_fn: tb = TestBuild(proj=p, buildroot=bd, installroot=id, compiler=cmany.Compiler.default(), @@ -194,8 +194,8 @@ def run_projs(testobj, args, check_fn=None): id = '.test/2.1--comps{}--types{}--variants{}--install'.format(len(compiler_set), len(build_types), len(variant_set)) for p in projs: with testobj.subTest(msg="run all combinations at once", proj=p.proj): - p.run(args + ['--build-dir', bd, - '--install-dir', id, + p.run(args + ['--build-root', bd, + '--install-root', id, '-c', ','.join([c.name if c.is_msvc else c.path for c in compiler_set]), '-t', ','.join([str(b) for b in build_types]), '-v', ','.join([v.full_specs for v in variant_set]) @@ -220,8 +220,8 @@ def run_projs(testobj, args, check_fn=None): ','.join([v.full_specs for v in variant_set]) ) #util.logwarn('export CMANY_ARGS={}'.format(os.environ['CMANY_ARGS'])) - p.run(args + ['--build-dir', bd, - '--install-dir', id, + p.run(args + ['--build-root', bd, + '--install-root', id, ]) os.environ['CMANY_ARGS'] = '' if check_fn: @@ -242,8 +242,8 @@ def run_projs(testobj, args, check_fn=None): proj=p.proj, compiler=c, build_type=t, variant=v): bd = '.test/3.1--{}--{}--{}--build'.format(c, t, v.name) id = '.test/3.1--{}--{}--{}--install'.format(c, t, v.name) - p.run(args + ['--build-dir', bd, - '--install-dir', id, + p.run(args + ['--build-root', bd, + '--install-root', id, '-c', c.name if c.is_msvc else c.path, '-t', str(t), '-v', v.full_specs, @@ -268,7 +268,7 @@ def run_projs(testobj, args, check_fn=None): str(t), v.full_specs) #util.logwarn('export CMANY_ARGS={}'.format(os.environ['CMANY_ARGS'])) - p.run(args + ['--build-dir', bd, '--install-dir', id]) + p.run(args + ['--build-root', bd, '--install-root', id]) os.environ['CMANY_ARGS'] = '' if check_fn: tb = TestBuild(proj=p, buildroot=bd, installroot=id, diff --git a/test/test06xcompile.py b/test/test06xcompile.py index bb5bb04..7790298 100644 --- a/test/test06xcompile.py +++ b/test/test06xcompile.py @@ -37,7 +37,7 @@ def run_with_args(testdir, args_in): shutil.rmtree(idir) # args = ['--show-args', 'build'] - args += ['--build-dir', bdir, '--install-dir', idir] + args += ['--build-root', bdir, '--install-root', idir] args += args_in cmany_main(args) # From 172bf50266d7241340e77471387ab515c2525fca Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 19 Nov 2020 21:49:58 +0000 Subject: [PATCH 14/50] fix: usr: build_files: dev_env is different for vs2015 --- src/c4/cmany/build.py | 11 ++++++----- src/c4/cmany/vsinfo.py | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index 14674fd..b0a1198 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -244,18 +244,19 @@ def run_custom_cmd(self, cmd, **subprocess_args): @property def cxx_compiler(self): - return util.cacheattr(self, "_cxx_compiler", lambda: cmake.get_cxx_compiler(self.builddir)) + return util.cacheattr(self, "_cxx_compiler", + lambda: cmake.get_cxx_compiler(self.builddir)) def vs_dev_cmd(self, target): - cl_exe = self.cxx_compiler + cl_exe = self.vsinfo.cxx_compiler + cl_version = self.vsinfo.cl_version dbg("cl.exe:", cl_exe) - cl_exe_version = re.sub(".*/MSVC/(.*?)/.*", r"\1", cl_exe) - dbg("cl.exe version:", cl_exe_version) + dbg("cl.exe version:", cl_version) return vsinfo.dev_env( vcvarsall=f'"{self.vsinfo.vcvarsall}"', arch=str(self.architecture.vs_dev_env_name), winsdk=str(self.generator.vs_get_vcxproj(target).winsdk), - vc_version=cl_exe_version) + vc_version=cl_version) def reconfigure(self): """reconfigure a build directory, without touching any cache entry""" diff --git a/src/c4/cmany/vsinfo.py b/src/c4/cmany/vsinfo.py index ddfa085..2bcbd00 100644 --- a/src/c4/cmany/vsinfo.py +++ b/src/c4/cmany/vsinfo.py @@ -40,6 +40,13 @@ def __init__(self, name): else: self.is_clang = False + @property + def cl_version(self): + if self.ver == 15 or self.ver == 16: + return re.sub(".*/MSVC/(.*?)/.*", r"\1", self.cxx_compiler) + else: + return None + # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- @@ -47,7 +54,7 @@ def __init__(self, name): def dev_env(vcvarsall, arch, winsdk, vc_version): """ - vcvarsall.bat usage: + vcvarsall.bat usage: ##### VS2019, VS2017 Syntax: vcvarsall.bat [arch] [platform_type] [winsdk_version] [-vcvars_ver=vc_version] [-vcvars_spectre_libs=spectre_mode] where : @@ -69,8 +76,31 @@ def dev_env(vcvarsall, arch, winsdk, vc_version): vcvarsall.bat x86_arm onecore 10.0.10240.0 -vcvars_ver=14.0 vcvarsall.bat x64 8.1 vcvarsall.bat x64 store 8.1 + ------------------------------------------------------------ + Syntax ## VS2015 and earlier + vcvarsall.bat [option] + or + vcvarsall.bat [option] store + or + vcvarsall.bat [option] [version number] + or + vcvarsall.bat [option] store [version number] + where [option] is: x86 | amd64 | arm | x86_amd64 | x86_arm | amd64_x86 | amd64_arm + where [version number] is either the full Windows 10 SDK version number or "8.1" to use the windows 8.1 SDK + The store parameter sets environment variables to support store (rather than desktop) development. + For example: + vcvarsall.bat x86_amd64 + vcvarsall.bat x86_arm store + vcvarsall.bat x86_amd64 10.0.10240.0 + vcvarsall.bat x86_arm store 10.0.10240.0 + vcvarsall.bat x64 8.1 + vcvarsall.bat x64 store 8.1 """ - return ["cmd", "/C", "call", vcvarsall, arch, winsdk, f"-vcvars_ver={vc_version}", "&"] + cmd = ["cmd", "/C", "call", vcvarsall, arch, winsdk] + if vc_version is not None: + cmd.append(f"-vcvars_ver={vc_version}") + cmd.append("&") + return cmd # ----------------------------------------------------------------------------- From 548c51ed682b2f213120b41a0a459d4fc9ccd0d9 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 19 Nov 2020 23:26:57 +0000 Subject: [PATCH 15/50] fix: usr: build_files must accept empty targets --- src/c4/cmany/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/c4/cmany/main.py b/src/c4/cmany/main.py index 4a05ff4..e7a19e0 100644 --- a/src/c4/cmany/main.py +++ b/src/c4/cmany/main.py @@ -214,10 +214,10 @@ def add_args(self, parser): parser.add_argument('command', nargs='+', help="""command to be run in each build directory""") parser.add_argument('-np', '--not-posix', action="store_true", - help="""do not use posix mode if splitting the initial string""") + help="""do not use posix mode if splitting the initial command string""") parser.add_argument('-nc', '--no-check', action="store_true", help="""do not use check the error status of the command""") - parser.add_argument('-tg', '--target', nargs="+", + parser.add_argument('-tg', '--target', nargs="+", default=[], help="""build these targets before running the command""") def _exec(self, proj, args): if len(args.target) > 0: From 3102c98aaf0f4b6632cf4a221dc356a446f4d147 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 21 Nov 2020 11:37:02 +0100 Subject: [PATCH 16/50] new: show_targets descends into all directories --- src/c4/cmany/build.py | 24 ++++++++++++++++-------- src/c4/cmany/compiler.py | 6 +----- src/c4/cmany/util.py | 15 ++++++++++++++- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index b0a1198..83a07ad 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -3,6 +3,7 @@ import re import dill import subprocess +import glob from datetime import datetime from collections import OrderedDict as odict @@ -578,28 +579,35 @@ def json_data(self): ]) def get_targets(self): + ret = [] with util.setcwd(self.builddir): + for sd in util.rglob(self.builddir, "CMakeFiles"): + util.logdbg(f"found {sd}...") + sd = os.path.dirname(sd) + rel = os.path.relpath(sd, self.builddir) + util.logdbg(f"descending into {sd}...") + ret += [f"[{rel}] {tg}" for tg in self._get_targets(sd)] + return ret + + def _get_targets(self, subdirectory): + with util.setcwd(subdirectory): if self.generator.is_msvc: # each target in MSVC has a corresponding vcxproj file - files = list(util.find_files_with_ext(self.builddir, ".vcxproj")) + files = list(glob.glob("*.vcxproj")) files = [os.path.basename(f) for f in files] files = [os.path.splitext(f)[0] for f in files] return files elif self.generator.is_makefile: - output = util.runsyscmd(["make", "help"], echo_cmd=False, - echo_output=False, capture_output=True) + output = util.get_output(["make", "help"]) output = output.split("\n") output = output[1:] # The following are some of the valid targets.... output = [o[4:] for o in output] # take off the initial "... " output = [re.sub(r'(.*)\ \(the default if no target.*\)', r'\1', o) for o in output] output = sorted(output) - result = [] - for o in output: - if o: - result.append(o) + result = [o for o in output if o] return result else: - util.logerr("sorry, feature not implemented for this generator: " + + util.logerr("sorry, target extraction not implemented for this generator: " + str(self.generator)) def show_properties(self): diff --git a/src/c4/cmany/compiler.py b/src/c4/cmany/compiler.py index 439ea54..fa21a4e 100644 --- a/src/c4/cmany/compiler.py +++ b/src/c4/cmany/compiler.py @@ -113,11 +113,7 @@ def get_c_compiler(shortname, cxx_compiler): def get_version(self, path): # a function to silently run a system command - def slntout(cmd): - out = util.runsyscmd(cmd, echo_cmd=False, - echo_output=False, capture_output=True) - out = out.strip("\n") - return out + slntout = util.get_output # is this visual studio? if hasattr(self, "vs"): return self.vs.name, str(self.vs.year), self.vs.name diff --git a/src/c4/cmany/util.py b/src/c4/cmany/util.py index c07d06d..63f3e84 100644 --- a/src/c4/cmany/util.py +++ b/src/c4/cmany/util.py @@ -5,9 +5,10 @@ import subprocess import platform import copy +import shlex import datetime from dateutil.relativedelta import relativedelta -import shlex +from pathlib import Path import colorama #from colorama import Fore, Back, Style, init colorama.init() @@ -430,6 +431,11 @@ def find_files_with_ext(folder, ext): yield os.path.join(root, file) +def rglob(directory, glob_pattern): + # Path('src').rglob('*.c'): + return Path(directory).rglob(glob_pattern) + + # ----------------------------------------------------------------------------- def nested_lookup(dictionary, *entry): """get a nested entry from a dictionary""" @@ -610,6 +616,13 @@ def runsyscmd_str(cmd_str, echo_cmd=True, **run_args): raise Exception("failed") +def get_output(cmd): + # a function to silently run a system command + out = runsyscmd(cmd, echo_cmd=False, echo_output=False, capture_output=True) + out = out.strip("\n") + return out + + def runcmd_nocheck(cmd, *cmd_args, **run_args): # TODO: https://stackoverflow.com/questions/17742789/running-multiple-bash-commands-with-subprocess logdbg(f"running command: {cmd} '{cmd_args}'") From cdc70e63b736381124f816cc0a1b8727bf6f779c Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 21 Nov 2020 22:02:18 +0000 Subject: [PATCH 17/50] new: usr: added run_target --- src/c4/cmany/build.py | 61 ++++++++++++++++++--------- src/c4/cmany/main.py | 31 ++++++++++++-- src/c4/cmany/project.py | 7 +++- src/c4/cmany/target.py | 92 +++++++++++++++++++++++++++++++++++++++++ src/c4/cmany/vsinfo.py | 6 +-- 5 files changed, 170 insertions(+), 27 deletions(-) create mode 100644 src/c4/cmany/target.py diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index 83a07ad..ced9b12 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -16,6 +16,7 @@ from .architecture import Architecture from . import err from .util import logdbg as dbg +from .target import Target # experimental. I don't think it will stay unless conan starts accepting args from .conan import Conan @@ -243,6 +244,18 @@ def run_custom_cmd(self, cmd, **subprocess_args): except subprocess.CalledProcessError as exc: raise err.RunCmdFailed(self, cmd, exc) + def run_targets(self, targets, target_args, workdir=None): + if self.needs_configure(): + self.configure() + try: + for tgt_name in targets: + t = self.get_target(tgt_name) + cmd = [t.output_file] + target_args + cwd = workdir if workdir is not None else t.subdir_abs + util.runcmd(cmd, cwd=cwd) + except subprocess.CalledProcessError as exc: + raise err.RunCmdFailed(self, cmd, exc) + @property def cxx_compiler(self): return util.cacheattr(self, "_cxx_compiler", @@ -578,37 +591,47 @@ def json_data(self): # ('variables', []), # this is not needed since the vars are set in the preload file ]) + def get_target(self, tgt_name): + tgts = util.cacheattr(self, "_targets", self.get_targets) + for t in tgts: + if t.name == tgt_name: + return t + raise Exception(f"target not found: {tgt_name}") + def get_targets(self): ret = [] with util.setcwd(self.builddir): - for sd in util.rglob(self.builddir, "CMakeFiles"): + for sd in sorted(util.rglob(self.builddir, "CMakeFiles")): util.logdbg(f"found {sd}...") sd = os.path.dirname(sd) - rel = os.path.relpath(sd, self.builddir) - util.logdbg(f"descending into {sd}...") - ret += [f"[{rel}] {tg}" for tg in self._get_targets(sd)] + util.logdbg(f"descending into {sd}/ ...") + ret += self._dir_targets(sd) return ret - def _get_targets(self, subdirectory): + def _dir_targets(self, subdirectory): + rel = os.path.relpath(subdirectory, self.builddir) with util.setcwd(subdirectory): + targets = [] if self.generator.is_msvc: # each target in MSVC has a corresponding vcxproj file - files = list(glob.glob("*.vcxproj")) - files = [os.path.basename(f) for f in files] - files = [os.path.splitext(f)[0] for f in files] - return files + vcxproj = list(glob.glob("*.vcxproj")) + for p in vcxproj: + tg = os.path.splitext(os.path.basename(p))[0] + targets.append(Target(tg, self, rel, p)) elif self.generator.is_makefile: - output = util.get_output(["make", "help"]) - output = output.split("\n") - output = output[1:] # The following are some of the valid targets.... - output = [o[4:] for o in output] # take off the initial "... " - output = [re.sub(r'(.*)\ \(the default if no target.*\)', r'\1', o) for o in output] - output = sorted(output) - result = [o for o in output if o] - return result + #output = util.get_output(["make", "help"]) + #output = output.split("\n") + #output = output[1:] # The following are some of the valid targets.... + #output = [o[4:] for o in output] # take off the initial "... " + #output = [re.sub(r'(.*)\ \(the default if no target.*\)', r'\1', o) for o in output] + with util.setcwd("CMakeFiles"): + output = [o[:-4] for o in list(glob.glob("*.dir"))] + mkf = os.path.join(subdirectory, "Makefile") + for tg in sorted(output): + targets.append(Target(tg, self, rel, mkf)) else: - util.logerr("sorry, target extraction not implemented for this generator: " + - str(self.generator)) + raise Exception(f"{self.generator}: sorry, target extraction not implemented for this generator") + return targets def show_properties(self): util.logcmd(self.name) diff --git a/src/c4/cmany/main.py b/src/c4/cmany/main.py index e7a19e0..e7bba6f 100644 --- a/src/c4/cmany/main.py +++ b/src/c4/cmany/main.py @@ -20,7 +20,8 @@ ('rebuild', ['rb']), ('install', ['i']), ('reinstall', ['ri']), - ('run', ['r']), + ('run_cmd', ['rcm']), + ('run_target', ['rtg']), ('show_vars', ['sv']), ('show_builds', ['sb']), ('show_build_names', ['sn']), @@ -207,16 +208,16 @@ def _exec(self, proj, args): proj.reinstall() -class run(selectcmd): +class run_cmd(selectcmd): """run a command in each build directory""" def add_args(self, parser): super().add_args(parser) parser.add_argument('command', nargs='+', help="""command to be run in each build directory""") - parser.add_argument('-np', '--not-posix', action="store_true", + parser.add_argument('-np', '--no-posix', action="store_true", help="""do not use posix mode if splitting the initial command string""") parser.add_argument('-nc', '--no-check', action="store_true", - help="""do not use check the error status of the command""") + help="""do not check the error status of the command""") parser.add_argument('-tg', '--target', nargs="+", default=[], help="""build these targets before running the command""") def _exec(self, proj, args): @@ -225,6 +226,28 @@ def _exec(self, proj, args): proj.run_cmd(args.command, posix_mode=not args.not_posix, check=not args.no_check) +class run_target(selectcmd): + """run targets in each build directory""" + def add_args(self, parser): + super().add_args(parser) + parser.add_argument('target', default=[], nargs='*', + help="""specify a subset of targets to run""") + parser.add_argument('-nb', '--no-build', action="store_true", + help="""do not build the target even if it is out of date""") + parser.add_argument('-ta', '--target-args', type=str, default="", + help="""command line arguments to pass to the target invokation""") + parser.add_argument('-np', '--no-posix', action="store_true", + help="""do not use posix mode when splitting the target arguments""") + parser.add_argument('-wd', '--work-dir', type=str, default=None, + help="""the working directory. Defaults to each target file's directory""") + def _exec(self, proj, args): + import shlex + if not args.no_build: + proj.build() + target_args = shlex.split(args.target_args, posix=not args.no_posix) + proj.run_targets(args.target, target_args, args.work_dir) + + class show_vars(selectcmd): """show the value of certain CMake cache vars""" def add_args(self, parser): diff --git a/src/c4/cmany/project.py b/src/c4/cmany/project.py index b7d468e..ea26be2 100644 --- a/src/c4/cmany/project.py +++ b/src/c4/cmany/project.py @@ -363,6 +363,11 @@ def run_it(build): build.run_custom_cmd(cmd, **subprocess_args) self._execute(run_it, "Run cmd", silent=False) + def run_targets(self, targets, target_args, workdir=None): + def run_it(build): + build.run_targets(targets, target_args, workdir) + self._execute(run_it, "Run target", silent=False) + def export_vs(self): confs = [] for b in self.builds: @@ -408,7 +413,7 @@ def show_builds(self): def show_targets(self): for t in self.builds[0].get_targets(): - print(t) + print(t.desc) def _execute(self, fn, msg, silent, **restrict_to): builds = self.select(**restrict_to) diff --git a/src/c4/cmany/target.py b/src/c4/cmany/target.py new file mode 100644 index 0000000..e4ced81 --- /dev/null +++ b/src/c4/cmany/target.py @@ -0,0 +1,92 @@ +import os +from enum import IntFlag + +from . import util +from . import vsinfo + + +class TargetType(IntFlag): + custom = 0 # a custom target + binary = 1 # a library or executable + interm = 2 # intermediate target from source code file (eg .o, .s, .i) + def __str__(self): + return self.name + + +class Target: + + def __init__(self, name, build, subdir, projfile): + assert os.path.exists(projfile) + util.logdbg(f"[{subdir}] {name}: creating target...") + self.name = name + self.build = build + self.subdir = subdir + self.subdir_abs = os.path.join(os.path.abspath(build.builddir), self.subdir) + self.projfile = projfile # Makefile or {name}.vcxproj + self.cmake_private_dir = util.chkf(self.subdir_abs, "CMakeFiles") + # find the target type + self._type = TargetType.custom + gen = self.build.generator + if gen.is_msvc: + tgt_dir = os.path.join(self.subdir_abs, f"{name}.dir") + if os.path.exists(tgt_dir): + self._type |= TargetType.binary + elif gen.is_makefile: + tgt_dir = os.path.join(self.cmake_private_dir, f"{name}.dir") + link_file = os.path.join(tgt_dir, "link.txt") + if os.path.exists(link_file): + util.logdbg(f"{name}: found {link_file}") + self._type |= TargetType.binary + else: + raise Exception(f"unknown generator: {gen}") + + @property + def desc(self): + return f"[{self.subdir}] {self.name} [{self._type}]" + + @property + def is_compiled(self): + return (self._type & TargetType.binary) != 0 + + @property + def is_intermediate(self): + return (self._type & TargetType.interm) != 0 + + @property + def output_file(self): + # TODO this assumes default ouput path. + # if the project's cmake scripts change the output path, + # then this logic will fail + gen = self.build.generator + if gen.is_msvc: + return f"{self.subdir_abs}/{self.build.build_type}/{self.name}.exe" + elif gen.is_makefile: + return f"{self.subdir_abs}/{self.name}" + else: + raise Exception(f"unknown generator: {gen}") + + @property + def vcxproj(self): + assert self.build.generator.is_msvc + return util.cacheattr(self, "_vcxproj", lambda: vsinfo.VcxProj(self.projfile)) + + @property + def tlog(self): + "get the arguments for compiling the given source file with Visual Studio" + assert self.build.generator.is_msvc + assert self.is_binary + def get_tlog(): + variant = self.vcxproj.get_variant(str(self.build.build_type), self.build.architecture) + tlog_dir = util.chkf(f"{variant.intdir}/{target}.tlog") + tlog = util.chkf(f"{tlog_dir}/CL.command.1.tlog") + return vsinfo.ClCommandTlog(tlog) + return util.cacheattr(self, "_tlog", get_tlog) + + def vs_cl_cmd(self, source_file): + "get the arguments for compiling the given source file with Visual Studio" + abspath = os.path.join(self.build.projdir, source_file) + cl_cmd = self.tlog.get_cmd_line(abspath) + cxx_compiler = self.build.cxx_compiler + cl_cmd = " ".join(shlex.split(f'"{cxx_compiler}" {cl_cmd}', posix=False)) + dev_cmd = " ".join(self.build.vs_dev_cmd(target)) + return f"{dev_cmd} {cl_cmd}" diff --git a/src/c4/cmany/vsinfo.py b/src/c4/cmany/vsinfo.py index 2bcbd00..081cef9 100644 --- a/src/c4/cmany/vsinfo.py +++ b/src/c4/cmany/vsinfo.py @@ -114,7 +114,7 @@ def __init__(self, vcxproj): with open(self._vcxproj, "rb") as f: # the rb is needed on windows logdbg("parsing", self._vcxproj) self._tree = etree.parse(f) - xp = lambda *args: xp__(self._tree, *args) + xp = lambda *args: xpath__(self._tree, *args) # https://stackoverflow.com/questions/14248063/xpath-to-select-element-by-attribute-value self.winsdk = xp("PropertyGroup[@Label='Globals']", "WindowsTargetPlatformVersion")[0].text self.platform = xp("PropertyGroup[@Label='Globals']", "Platform")[0].text @@ -135,7 +135,7 @@ def get_variant(self, build_type, platform): class BuildVariant: def __init__(self, tree, node): - def xp(*args): return xp__(tree, *args) + def xp(*args): return xpath__(tree, *args) self.name = node.get("Include") self.configuration = child__(node, 'Configuration').text self.platform = child__(node, 'Platform').text @@ -166,7 +166,7 @@ def child__(node, childname): return None -def xp__(tree, *args): # having to do this sucks. +def xpath__(tree, *args): # having to do this sucks. # https://stackoverflow.com/questions/37327774/parsing-vcxproj-with-python-and-lxml p = f"/wtf:Project" + ("/wtf:" if len(args) > 0 else "") + "/wtf:".join(args) logdbg("search:", p) From aa49fd85340745dd75d55d2cdba8cac4a1c64030 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sun, 22 Nov 2020 02:24:54 +0100 Subject: [PATCH 18/50] new: usr: added run_test --- src/c4/cmany/build.py | 12 ++++++++++++ src/c4/cmany/main.py | 28 +++++++++++++++++++++++++++- src/c4/cmany/project.py | 9 +++++++-- src/c4/cmany/util.py | 8 +++++++- 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index ced9b12..92b3f7a 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -256,6 +256,18 @@ def run_targets(self, targets, target_args, workdir=None): except subprocess.CalledProcessError as exc: raise err.RunCmdFailed(self, cmd, exc) + def run_tests(self, test_selection, ctest_args, workdir, check): + if self.needs_configure(): + self.configure() + try: + for t in test_selection: + cwd = workdir if workdir is not None else "." + cwd = os.path.abspath(os.path.join(self.builddir, cwd)) + args = ctest_args + ["-R", t] + util.runcmd("ctest", *args, cwd=cwd, check=check) + except subprocess.CalledProcessError as exc: + raise err.RunCmdFailed(self, cmd, exc) + @property def cxx_compiler(self): return util.cacheattr(self, "_cxx_compiler", diff --git a/src/c4/cmany/main.py b/src/c4/cmany/main.py index e7bba6f..7562075 100644 --- a/src/c4/cmany/main.py +++ b/src/c4/cmany/main.py @@ -22,6 +22,7 @@ ('reinstall', ['ri']), ('run_cmd', ['rcm']), ('run_target', ['rtg']), + ('run_test', ['rte']), ('show_vars', ['sv']), ('show_builds', ['sb']), ('show_build_names', ['sn']), @@ -235,7 +236,7 @@ def add_args(self, parser): parser.add_argument('-nb', '--no-build', action="store_true", help="""do not build the target even if it is out of date""") parser.add_argument('-ta', '--target-args', type=str, default="", - help="""command line arguments to pass to the target invokation""") + help="""arguments to pass to the target invokation""") parser.add_argument('-np', '--no-posix', action="store_true", help="""do not use posix mode when splitting the target arguments""") parser.add_argument('-wd', '--work-dir', type=str, default=None, @@ -248,6 +249,31 @@ def _exec(self, proj, args): proj.run_targets(args.target, target_args, args.work_dir) +class run_test(selectcmd): + """run tests in each build directory""" + def add_args(self, parser): + super().add_args(parser) + parser.add_argument('tests', default=[".*"], nargs='*', + help="""command to be run in each build directory""") + parser.add_argument('-ca', '--ctest-args', type=str, default=r"\-VV", + help="""arguments to pass to ctest""") + parser.add_argument('-np', '--no-posix', action="store_true", + help="""do not use posix mode if splitting the initial command string""") + parser.add_argument('-nc', '--no-check', action="store_true", + help="""do not check the error status of the command""") + parser.add_argument('-tg', '--target', nargs="+", default=[], + help="""build these targets before running the command""") + parser.add_argument('-wd', '--work-dir', type=str, default=".", + help="""the working directory. A relative path starting inside the build + directory. Defaults to ".", the build directory""") + def _exec(self, proj, args): + import shlex + if len(args.target) > 0: + proj.build() # targets are set + ctest_args = shlex.split(args.ctest_args, posix=not args.no_posix) + proj.run_tests(args.tests, ctest_args, args.work_dir, check=not args.no_check) + + class show_vars(selectcmd): """show the value of certain CMake cache vars""" def add_args(self, parser): diff --git a/src/c4/cmany/project.py b/src/c4/cmany/project.py index ea26be2..668d42a 100644 --- a/src/c4/cmany/project.py +++ b/src/c4/cmany/project.py @@ -363,10 +363,15 @@ def run_it(build): build.run_custom_cmd(cmd, **subprocess_args) self._execute(run_it, "Run cmd", silent=False) - def run_targets(self, targets, target_args, workdir=None): + def run_targets(self, targets, target_args, workdir): def run_it(build): build.run_targets(targets, target_args, workdir) - self._execute(run_it, "Run target", silent=False) + self._execute(run_it, "Run targets", silent=False) + + def run_tests(self, tests, ctest_args, workdir, check): + def run_it(build): + build.run_tests(tests, ctest_args, workdir, check) + self._execute(run_it, "Run tests", silent=False) def export_vs(self): confs = [] diff --git a/src/c4/cmany/util.py b/src/c4/cmany/util.py index 63f3e84..087b334 100644 --- a/src/c4/cmany/util.py +++ b/src/c4/cmany/util.py @@ -627,15 +627,21 @@ def runcmd_nocheck(cmd, *cmd_args, **run_args): # TODO: https://stackoverflow.com/questions/17742789/running-multiple-bash-commands-with-subprocess logdbg(f"running command: {cmd} '{cmd_args}'") logdbg(f" : run_args={run_args}") + posix_mode = in_unix() if run_args.get('posix_mode'): posix_mode = run_args.get('posix_mode') del run_args['posix_mode'] if isinstance(cmd, str): - cmd = shlex.split(cmd, posix_mode=posix_mode) + cmd = shlex.split(cmd, posix=posix_mode) logdbg(f" : split cmd={cmd}") elif isinstance(cmd, tuple): cmd = list(cmd) + elif isinstance(cmd, list): + pass + else: + raise Exception("could not understand command") cmd += list(cmd_args) + logdbg(f" : cmd={cmd}") scmd = shlex.join(cmd) cwd = os.path.realpath(run_args.get('cwd', os.getcwd())) logcmd(f'$ cd {cwd} && {scmd}') From cd4ee5763019f9972c8620f2e37f013997f775b7 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Mon, 23 Nov 2020 00:51:57 +0000 Subject: [PATCH 19/50] fix: usr: verbose flag was ignored because of caching --- src/c4/cmany/build.py | 18 ++++++++++++++---- src/c4/cmany/generator.py | 7 +++++-- src/c4/cmany/project.py | 1 + 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index 92b3f7a..62bd81e 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -87,6 +87,17 @@ def __init__(self, proj_root, build_root, install_root, if not self.deps_prefix: self.deps_prefix = self.builddir + def reset_kwargs(self, kwargs): + "the serialization/deserialization approach creates cache coherency problems" + util.logdbg(f"{self}: ressetting kwargs: {kwargs}") + self.kwargs = kwargs + # + # TODO complete this + + @property + def verbose(self): + return self.kwargs.get('verbose', False) + def _set_name_and_paths(self): self.tag = __class__.get_tag( self.system, self.architecture, @@ -107,19 +118,18 @@ def create_generator(self, num_jobs, fallback_generator="Unix Makefiles"): # toolchain_cache = cmake.get_toolchain_cache(self.toolchain_file) # print(toolchain_cache) # self.adjust(compiler=toolchain_cache['CMAKE_CXX_COMPILER']) - verbose = self.kwargs['verbose'] if self.compiler.is_msvc: vsi = vsinfo.VisualStudioInfo(self.compiler.name) - g = Generator(vsi.gen, self, num_jobs, verbose) + g = Generator(vsi.gen, self, num_jobs) arch = Architecture(vsi.architecture) self.adjust(architecture=arch) self.vsinfo = vsi return g else: if self.system.name == "windows": - return Generator(fallback_generator, self, num_jobs, verbose) + return Generator(fallback_generator, self, num_jobs) else: - return Generator(Generator.default_str(), self, num_jobs, verbose) + return Generator(Generator.default_str(), self, num_jobs) def adjust(self, **kwargs): for k, _ in kwargs.items(): diff --git a/src/c4/cmany/generator.py b/src/c4/cmany/generator.py index 71be427..de983c5 100644 --- a/src/c4/cmany/generator.py +++ b/src/c4/cmany/generator.py @@ -75,7 +75,7 @@ def default_str(): s = cmake.CMakeSysInfo.generator() return s - def __init__(self, name, build, num_jobs, verbose): + def __init__(self, name, build, num_jobs): if isinstance(name, list): #more_args = name[1:] name = name[0] @@ -88,7 +88,6 @@ def __init__(self, name, build, num_jobs, verbose): self.is_ninja = name.endswith("Ninja") self.is_msvc = name.startswith("Visual Studio") self.build = build - self.verbose = verbose # self.sysinfo_name = self.name if self.is_msvc: @@ -100,6 +99,10 @@ def __init__(self, name, build, num_jobs, verbose): # these vars would not change cmake --system-information # self.full_name += " ".join(self.build.flags.cmake_vars) + @property + def verbose(self): + return self.build.verbose + def configure_args(self, for_json=False, export_compile_commands=True): args = [] if self.name != "": diff --git a/src/c4/cmany/project.py b/src/c4/cmany/project.py index 668d42a..ac79173 100644 --- a/src/c4/cmany/project.py +++ b/src/c4/cmany/project.py @@ -139,6 +139,7 @@ def __init__(self, **kwargs): def _init_with_build_dir(self, pdir, **kwargs): build = Build.deserialize(pdir) + build.reset_kwargs(kwargs) self.builds = [build] def _init_with_glob(self, **kwargs): From 481567a7b81182ab2301e9269e84a643b88fcf3d Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Wed, 25 Nov 2020 13:34:27 +0000 Subject: [PATCH 20/50] fix: usr: run_cmd, run_target and run_test: do not build all targets before running --- src/c4/cmany/build.py | 26 ++++++++++++++++++-------- src/c4/cmany/main.py | 20 ++++++++------------ 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index 62bd81e..90ca880 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -94,6 +94,10 @@ def reset_kwargs(self, kwargs): # # TODO complete this + @property + def targets(self): + return self.kwargs.get('target') + @property def verbose(self): return self.kwargs.get('verbose', False) @@ -246,17 +250,11 @@ def export_compile_commands(self): copyfile(src, dst) util.loginfo("exported compile_commands.json:", dst) - def run_custom_cmd(self, cmd, **subprocess_args): - if self.needs_configure(): - self.configure() - try: - util.runcmd(cmd, **subprocess_args, cwd=self.builddir) - except subprocess.CalledProcessError as exc: - raise err.RunCmdFailed(self, cmd, exc) - def run_targets(self, targets, target_args, workdir=None): if self.needs_configure(): self.configure() + if not (self.kwargs.get('no_build') == True): + self.build(targets) try: for tgt_name in targets: t = self.get_target(tgt_name) @@ -269,6 +267,8 @@ def run_targets(self, targets, target_args, workdir=None): def run_tests(self, test_selection, ctest_args, workdir, check): if self.needs_configure(): self.configure() + if self.targets: + self.build(self.targets) try: for t in test_selection: cwd = workdir if workdir is not None else "." @@ -278,6 +278,16 @@ def run_tests(self, test_selection, ctest_args, workdir, check): except subprocess.CalledProcessError as exc: raise err.RunCmdFailed(self, cmd, exc) + def run_custom_cmd(self, cmd, **subprocess_args): + if self.needs_configure(): + self.configure() + if self.targets: + self.build(self.targets) + try: + util.runcmd(cmd, **subprocess_args, cwd=self.builddir) + except subprocess.CalledProcessError as exc: + raise err.RunCmdFailed(self, cmd, exc) + @property def cxx_compiler(self): return util.cacheattr(self, "_cxx_compiler", diff --git a/src/c4/cmany/main.py b/src/c4/cmany/main.py index 7562075..d704b39 100644 --- a/src/c4/cmany/main.py +++ b/src/c4/cmany/main.py @@ -72,7 +72,7 @@ def proj(self, args): """create a project given the configuration.""" return None def _exec(self, proj, args): - assert False, 'never call the base class method. Implement this in derived classes' + raise Exception('never call the base class method. Implement this in derived classes.') class help(cmdbase): @@ -210,7 +210,7 @@ def _exec(self, proj, args): class run_cmd(selectcmd): - """run a command in each build directory""" + """[EXPERIMENTAL] run a command in each build""" def add_args(self, parser): super().add_args(parser) parser.add_argument('command', nargs='+', @@ -222,13 +222,11 @@ def add_args(self, parser): parser.add_argument('-tg', '--target', nargs="+", default=[], help="""build these targets before running the command""") def _exec(self, proj, args): - if len(args.target) > 0: - proj.build() # targets are set - proj.run_cmd(args.command, posix_mode=not args.not_posix, check=not args.no_check) + proj.run_cmd(args.command, posix_mode=not args.no_posix, check=not args.no_check) class run_target(selectcmd): - """run targets in each build directory""" + """[EXPERIMENTAL] run targets in each build""" def add_args(self, parser): super().add_args(parser) parser.add_argument('target', default=[], nargs='*', @@ -236,21 +234,21 @@ def add_args(self, parser): parser.add_argument('-nb', '--no-build', action="store_true", help="""do not build the target even if it is out of date""") parser.add_argument('-ta', '--target-args', type=str, default="", - help="""arguments to pass to the target invokation""") + help="""arguments to pass to the target invokation. If multiple + targets are given, arguments are passed to every target. The + arguments need to be given as a single string.""") parser.add_argument('-np', '--no-posix', action="store_true", help="""do not use posix mode when splitting the target arguments""") parser.add_argument('-wd', '--work-dir', type=str, default=None, help="""the working directory. Defaults to each target file's directory""") def _exec(self, proj, args): import shlex - if not args.no_build: - proj.build() target_args = shlex.split(args.target_args, posix=not args.no_posix) proj.run_targets(args.target, target_args, args.work_dir) class run_test(selectcmd): - """run tests in each build directory""" + """[EXPERIMENTAL] run tests in each build directory""" def add_args(self, parser): super().add_args(parser) parser.add_argument('tests', default=[".*"], nargs='*', @@ -268,8 +266,6 @@ def add_args(self, parser): directory. Defaults to ".", the build directory""") def _exec(self, proj, args): import shlex - if len(args.target) > 0: - proj.build() # targets are set ctest_args = shlex.split(args.ctest_args, posix=not args.no_posix) proj.run_tests(args.tests, ctest_args, args.work_dir, check=not args.no_check) From ac9723ae03fd8235e7ebbe8432b50f3a39b356b2 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 26 Nov 2020 22:26:20 +0000 Subject: [PATCH 21/50] fix: import of InvalidGenerator --- src/c4/cmany/cmake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c4/cmany/cmake.py b/src/c4/cmany/cmake.py index 47909e0..0d69eff 100644 --- a/src/c4/cmany/cmake.py +++ b/src/c4/cmany/cmake.py @@ -341,7 +341,7 @@ def system_info(gen): logdbg("cmany: finished generating information for generator '{}'\n".format(gen), out, cmd) out = out.strip() if not out: - from err import InvalidGenerator + from .err import InvalidGenerator raise InvalidGenerator(gen, "for --system-information. cmd='{}'".format(cmd)) with open(p, "w") as f: f.write(out) From cb8f2464d1e7c64700ccb7dbcba591007804324f Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 7 Jan 2021 17:53:45 +0000 Subject: [PATCH 22/50] fix: shlex.join is only available in python 3.8+ --- src/c4/cmany/util.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/c4/cmany/util.py b/src/c4/cmany/util.py index 087b334..78ae72d 100644 --- a/src/c4/cmany/util.py +++ b/src/c4/cmany/util.py @@ -623,6 +623,13 @@ def get_output(cmd): return out +def shlex_join(cmd): + if sys.version_info >= (3, 8): + return shlex.join(cmd) + else: + return ' '.join(shlex.quote(x) for x in cmd) + + def runcmd_nocheck(cmd, *cmd_args, **run_args): # TODO: https://stackoverflow.com/questions/17742789/running-multiple-bash-commands-with-subprocess logdbg(f"running command: {cmd} '{cmd_args}'") @@ -642,7 +649,7 @@ def runcmd_nocheck(cmd, *cmd_args, **run_args): raise Exception("could not understand command") cmd += list(cmd_args) logdbg(f" : cmd={cmd}") - scmd = shlex.join(cmd) + scmd = shlex_join(cmd) cwd = os.path.realpath(run_args.get('cwd', os.getcwd())) logcmd(f'$ cd {cwd} && {scmd}') sp = subprocess.run(cmd, **run_args) From dd1c93d157914f02537272edfd6e623a97f8c07d Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 26 Jan 2021 23:00:27 +0000 Subject: [PATCH 23/50] [fix] fix missing arguments to exception --- src/c4/cmany/build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index 90ca880..fdcf713 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -253,7 +253,7 @@ def export_compile_commands(self): def run_targets(self, targets, target_args, workdir=None): if self.needs_configure(): self.configure() - if not (self.kwargs.get('no_build') == True): + if not (self.kwargs.get('no_build') is True): self.build(targets) try: for tgt_name in targets: @@ -276,6 +276,8 @@ def run_tests(self, test_selection, ctest_args, workdir, check): args = ctest_args + ["-R", t] util.runcmd("ctest", *args, cwd=cwd, check=check) except subprocess.CalledProcessError as exc: + cmd = ["ctest"] + args + cmd = util.shlex_join(cmd) raise err.RunCmdFailed(self, cmd, exc) def run_custom_cmd(self, cmd, **subprocess_args): From 3d593198438910c7e35dcd4cbf29da958a6d1a08 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 26 Jan 2021 23:21:24 +0000 Subject: [PATCH 24/50] [impl] run target: add options for calling the target from another command --- src/c4/cmany/build.py | 6 ++++-- src/c4/cmany/main.py | 6 +++++- src/c4/cmany/project.py | 4 ++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index fdcf713..e996c27 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -6,6 +6,7 @@ import glob from datetime import datetime from collections import OrderedDict as odict +import shlex from .generator import Generator from . import util, cmake, vsinfo @@ -250,15 +251,16 @@ def export_compile_commands(self): copyfile(src, dst) util.loginfo("exported compile_commands.json:", dst) - def run_targets(self, targets, target_args, workdir=None): + def run_targets(self, targets, target_args, cmd_wrap=None, workdir=None): if self.needs_configure(): self.configure() if not (self.kwargs.get('no_build') is True): self.build(targets) try: + cmd_wrap = [] if cmd_wrap is None else shlex.split(cmd_wrap) for tgt_name in targets: t = self.get_target(tgt_name) - cmd = [t.output_file] + target_args + cmd = cmd_wrap + [t.output_file] + target_args cwd = workdir if workdir is not None else t.subdir_abs util.runcmd(cmd, cwd=cwd) except subprocess.CalledProcessError as exc: diff --git a/src/c4/cmany/main.py b/src/c4/cmany/main.py index d704b39..a01f16d 100644 --- a/src/c4/cmany/main.py +++ b/src/c4/cmany/main.py @@ -241,10 +241,14 @@ def add_args(self, parser): help="""do not use posix mode when splitting the target arguments""") parser.add_argument('-wd', '--work-dir', type=str, default=None, help="""the working directory. Defaults to each target file's directory""") + parser.add_argument('-wp', '--wrap', type=str, default=None, + help="""wrap the target invokation with the given command, eg + valgrind or gdb. A string with spaces can be passed, in which + case it will be split before the command is formed.""") def _exec(self, proj, args): import shlex target_args = shlex.split(args.target_args, posix=not args.no_posix) - proj.run_targets(args.target, target_args, args.work_dir) + proj.run_targets(args.target, target_args, args.wrap, args.work_dir) class run_test(selectcmd): diff --git a/src/c4/cmany/project.py b/src/c4/cmany/project.py index ac79173..1a0d207 100644 --- a/src/c4/cmany/project.py +++ b/src/c4/cmany/project.py @@ -364,9 +364,9 @@ def run_it(build): build.run_custom_cmd(cmd, **subprocess_args) self._execute(run_it, "Run cmd", silent=False) - def run_targets(self, targets, target_args, workdir): + def run_targets(self, targets, target_args, cmd_wrap, workdir): def run_it(build): - build.run_targets(targets, target_args, workdir) + build.run_targets(targets, target_args, cmd_wrap, workdir) self._execute(run_it, "Run targets", silent=False) def run_tests(self, tests, ctest_args, workdir, check): From da13e9b376cd783c7e9b8afa42a774dccf138db8 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 10 Jul 2021 01:17:09 +0100 Subject: [PATCH 25/50] [fix] add intel 2021.1 --- src/c4/cmany/compiler.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/c4/cmany/compiler.py b/src/c4/cmany/compiler.py index fa21a4e..87ed54a 100644 --- a/src/c4/cmany/compiler.py +++ b/src/c4/cmany/compiler.py @@ -61,7 +61,7 @@ def __init__(self, spec): path = os.path.abspath(p) name, version, version_full = self.get_version(path) self.shortname = name - self.gcclike = self.shortname in ('gcc', 'clang', 'icc', 'g++', 'clang++', 'icpc') + self.gcclike = self.shortname in ('gcc', 'clang', 'icc', 'g++', 'clang++', 'icpc', 'icpx') self.is_msvc = self.shortname.startswith('vs') if not self.is_msvc: name += version @@ -93,6 +93,8 @@ def get_c_compiler(shortname, cxx_compiler): cc = cxx_compiler elif shortname == "icc" or shortname == "icpc": cc = re.sub(r'icpc', r'icc', cxx_compiler) + elif shortname == "icpx": + cc = re.sub(r'icpx', r'icpx', cxx_compiler) elif shortname == "gcc" or shortname == "g++": if re.search(r'g\+\+', cxx_compiler): cc = re.sub(r'g\+\+', r'gcc', cxx_compiler) @@ -126,7 +128,7 @@ def get_version(self, path): # print("cmp: version:", name, "---", version_full, "---") vregex = r'(\d+\.\d+)\.\d+' base = os.path.basename(path) - # print("cmp base:", base) + # print(name, "cmp base:", base, name) if base.startswith("c++") or base.startswith("cc"): try: # if this fails, just go on. It's not really needed. with tempfile.NamedTemporaryFile(suffix=".cpp", prefix="cmany.", delete=False) as f: @@ -165,6 +167,12 @@ def get_version(self, path): else: version = re.sub(r'icc \(ICC\) ' + vregex + '.*', r'\1', version_full) # print("icc version:", version, "---") + elif version_full.startswith("Intel(R) oneAPI"): + name = "intel" + rx = r'Intel\(R\) oneAPI .*? Compiler ([0-9.]*?) \(([0-9.]*?)\).*' + version = re.sub(rx, r'\1', version_full) + version_full = re.sub(rx, r'\2', version_full) + #print("intel version:", version, "---", version_full) else: version = slntout([path, '-dumpversion']) version = re.sub(vregex, r'\1', version) From 84ea77ddc394cd5d5dead0507e12502c04edce84 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 19 Aug 2021 18:22:56 +0200 Subject: [PATCH 26/50] [impl] add vs2022 --- doc/vs.rst | 174 ++++++++++++++++++++- src/c4/cmany/vsinfo.py | 35 ++++- test/run.bat | 4 +- test/test01vsinfo.py | 340 +++++++++++++++++++++++++++++++++-------- 4 files changed, 481 insertions(+), 72 deletions(-) diff --git a/doc/vs.rst b/doc/vs.rst index 69f4a76..dbf2a6a 100644 --- a/doc/vs.rst +++ b/doc/vs.rst @@ -11,7 +11,7 @@ is:: where: -* ```` is one of ``vs2019``, ``vs2017``, ``vs2015``, ``vs2013``, ``vs2012``, +* ```` is one of ``vs2022``, ``vs2019``, ``vs2017``, ``vs2015``, ``vs2013``, ``vs2012``, ``vs2010``, ``vs2008`` or ``vs2005`` * ```` is one of ``32``, ``64``, ``arm``, ``ia64``. Defaults to the native architecture when omitted. @@ -20,6 +20,7 @@ where: * when either ``clang`` or ``xp`` is given, uses the default toolset of the chosen VS version for either clang or xp * otherwise, one of: + * from ``vs2022``: ``v143``, ``v143_clang``, ``v143_clang_c2``, ``v143_xp`` * from ``vs2019``: ``v142``, ``v142_clang``, ``v142_clang_c2``, ``v142_xp`` * from ``vs2017``: ``v141``, ``v141_clang``, ``v141_clang_c2``, ``v141_xp`` * from ``vs2015``: ``v140``, ``v140_clang``, ``v140_clang_c2``, ``v140_xp`` @@ -239,6 +240,7 @@ Visual Studio toolset Here's the list of valid Visual Studio toolsets: +* ``vs2022`` compiler toolsets: ``v143``, ``v143_clang_c2``, ``v143_xp`` * ``vs2019`` compiler toolsets: ``v142``, ``v142_clang_c2``, ``v142_xp`` * ``vs2017`` compiler toolsets: ``v141``, ``v141_clang_c2``, ``v141_xp`` * ``vs2015`` compiler toolsets: ``v140``, ``v140_clang_c2``, ``v140_xp`` @@ -268,6 +270,175 @@ shows what each of them mean. (If you find any errors, please submit a bug or PR). +VS2022 +^^^^^^ + ++------------------------------+-----------------------------+--------------------+---------------------+ +| cmany compiler alias | project VS version | Target arch. | VS Toolset | ++==============================+=============================+====================+=====================+ +| ``vs2022`` | ``17 2022`` | ``(native)`` | ``v143`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_clang`` | ``17 2022`` | ``(native)`` | ``v143_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_xp`` | ``17 2022`` | ``(native)`` | ``v143_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v143`` | ``17 2022`` | ``(native)`` | ``v143`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v143_xp`` | ``17 2022`` | ``(native)`` | ``v143_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v143_clang`` | ``17 2022`` | ``(native)`` | ``v143_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v142`` | ``17 2022`` | ``(native)`` | ``v142`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v142_xp`` | ``17 2022`` | ``(native)`` | ``v142_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v142_clang`` | ``17 2022`` | ``(native)`` | ``v142_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v141`` | ``17 2022`` | ``(native)`` | ``v141`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v141_xp`` | ``17 2022`` | ``(native)`` | ``v141_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v141_clang`` | ``17 2022`` | ``(native)`` | ``v141_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v140`` | ``17 2022`` | ``(native)`` | ``v140`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v140_xp`` | ``17 2022`` | ``(native)`` | ``v140_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v140_clang`` | ``17 2022`` | ``(native)`` | ``v140_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v120`` | ``17 2022`` | ``(native)`` | ``v120`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v120_xp`` | ``17 2022`` | ``(native)`` | ``v120_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v110`` | ``17 2022`` | ``(native)`` | ``v110`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v110_xp`` | ``17 2022`` | ``(native)`` | ``v110_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v100`` | ``17 2022`` | ``(native)`` | ``v100`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v100_xp`` | ``17 2022`` | ``(native)`` | ``v100_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v90`` | ``17 2022`` | ``(native)`` | ``v90`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v90_xp`` | ``17 2022`` | ``(native)`` | ``v90_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_v80`` | ``17 2022`` | ``(native)`` | ``v80`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32`` | ``17 2022`` | ``x86`` | ``v143`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_clang`` | ``17 2022`` | ``x86`` | ``v143_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_xp`` | ``17 2022`` | ``x86`` | ``v143_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v143`` | ``17 2022`` | ``x86`` | ``v143`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v143_xp`` | ``17 2022`` | ``x86`` | ``v143_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v143_clang`` | ``17 2022`` | ``x86`` | ``v143_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v142`` | ``17 2022`` | ``x86`` | ``v142`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v142_xp`` | ``17 2022`` | ``x86`` | ``v142_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v142_clang`` | ``17 2022`` | ``x86`` | ``v142_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v141`` | ``17 2022`` | ``x86`` | ``v141`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v141_xp`` | ``17 2022`` | ``x86`` | ``v141_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v141_clang`` | ``17 2022`` | ``x86`` | ``v141_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v140`` | ``17 2022`` | ``x86`` | ``v140`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v140_xp`` | ``17 2022`` | ``x86`` | ``v140_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v140_clang`` | ``17 2022`` | ``x86`` | ``v140_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v120`` | ``17 2022`` | ``x86`` | ``v120`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v120_xp`` | ``17 2022`` | ``x86`` | ``v120_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v110`` | ``17 2022`` | ``x86`` | ``v110`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v110_xp`` | ``17 2022`` | ``x86`` | ``v110_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v100`` | ``17 2022`` | ``x86`` | ``v100`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v100_xp`` | ``17 2022`` | ``x86`` | ``v100_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v90`` | ``17 2022`` | ``x86`` | ``v90`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v90_xp`` | ``17 2022`` | ``x86`` | ``v90_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_32_v80`` | ``17 2022`` | ``x86`` | ``v80`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64`` | ``17 2022`` | ``x86_64`` | ``v142`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_clang`` | ``17 2022`` | ``x86_64`` | ``v142_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_xp`` | ``17 2022`` | ``x86_64`` | ``v142_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v142`` | ``17 2022`` | ``x86_64`` | ``v142`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v142_xp`` | ``17 2022`` | ``x86_64`` | ``v142_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v142_clang`` | ``17 2022`` | ``x86_64`` | ``v142_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v141`` | ``17 2022`` | ``x86_64`` | ``v141`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v141_xp`` | ``17 2022`` | ``x86_64`` | ``v141_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v141_clang`` | ``17 2022`` | ``x86_64`` | ``v141_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v140`` | ``17 2022`` | ``x86_64`` | ``v140`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v140_xp`` | ``17 2022`` | ``x86_64`` | ``v140_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v140_clang`` | ``17 2022`` | ``x86_64`` | ``v140_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v120`` | ``17 2022`` | ``x86_64`` | ``v120`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v120_xp`` | ``17 2022`` | ``x86_64`` | ``v120_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v110`` | ``17 2022`` | ``x86_64`` | ``v110`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v110_xp`` | ``17 2022`` | ``x86_64`` | ``v110_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v100`` | ``17 2022`` | ``x86_64`` | ``v100`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v100_xp`` | ``17 2022`` | ``x86_64`` | ``v100_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v90`` | ``17 2022`` | ``x86_64`` | ``v90`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v90_xp`` | ``17 2022`` | ``x86_64`` | ``v90_xp`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_64_v80`` | ``17 2022`` | ``x86_64`` | ``v80`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_arm`` | ``17 2022`` | ``arm`` | ``v142`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_arm_clang`` | ``17 2022`` | ``arm`` | ``v142_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_arm_v142`` | ``17 2022`` | ``arm`` | ``v142`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_arm_v142_clang`` | ``17 2022`` | ``arm`` | ``v142_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_arm_v141`` | ``17 2022`` | ``arm`` | ``v141`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_arm_v141_clang`` | ``17 2022`` | ``arm`` | ``v141_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_arm_v140`` | ``17 2022`` | ``arm`` | ``v140`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_arm_v140_clang`` | ``17 2022`` | ``arm`` | ``v140_clang_c2`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_arm_v120`` | ``17 2022`` | ``arm`` | ``v120`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_arm_v110`` | ``17 2022`` | ``arm`` | ``v110`` | ++------------------------------+-----------------------------+--------------------+---------------------+ +| ``vs2022_arm_v100`` | ``17 2022`` | ``arm`` | ``v100`` | ++------------------------------+-----------------------------+--------------------+---------------------+ + + + VS2019 ^^^^^^ @@ -927,4 +1098,3 @@ VS2005 +------------------------------+-----------------------------+--------------------+---------------------+ | ``vs2005_64_v80`` | ``8 2005`` | ``x86_64`` | ``v80`` | +------------------------------+-----------------------------+--------------------+---------------------+ - diff --git a/src/c4/cmany/vsinfo.py b/src/c4/cmany/vsinfo.py index 081cef9..a82d0ba 100644 --- a/src/c4/cmany/vsinfo.py +++ b/src/c4/cmany/vsinfo.py @@ -230,6 +230,7 @@ def find_any(): # ----------------------------------------------------------------------------- # a reversible dictionary for the VS version numbers _versions = { + 'vs2022':17, 17:'vs2022', 'vs2022_64':17, 'vs2022_32':17, 'vs2022_arm':17 , 'vs2022_arm32':17, 'vs2022_arm64':17, # nopep8 'vs2019':16, 16:'vs2019', 'vs2019_64':16, 'vs2019_32':16, 'vs2019_arm':16 , 'vs2019_arm32':16, 'vs2019_arm64':16, # nopep8 'vs2017':15, 15:'vs2017', 'vs2017_64':15, 'vs2017_32':15, 'vs2017_arm':15 , # nopep8 'vs2015':14, 14:'vs2015', 'vs2015_64':14, 'vs2015_32':14, 'vs2015_arm':14 , # nopep8 @@ -255,6 +256,12 @@ def find_any(): # a reversible dictionary for the names _names = { + 'vs2022' : ['Visual Studio 17 2022', '-A', _arc ], 'Visual Studio 17 2022' + _arc : 'vs2022' , # nopep8 + 'vs2022_32' : ['Visual Studio 17 2022', '-A', 'Win32'], 'Visual Studio 17 2022' : 'vs2022_32' , # nopep8 + 'vs2022_64' : ['Visual Studio 17 2022', '-A', 'x64' ], 'Visual Studio 17 2022 Win64' : 'vs2022_64' , # nopep8 + 'vs2022_arm' : ['Visual Studio 17 2022', '-A', 'ARM' ], 'Visual Studio 17 2022 ARM' : 'vs2022_arm' , # nopep8 + 'vs2022_arm32': ['Visual Studio 17 2022', '-A', 'ARM' ], 'Visual Studio 17 2022 ARM32' : 'vs2022_arm32', # nopep8 + 'vs2022_arm64': ['Visual Studio 17 2022', '-A', 'ARM64'], 'Visual Studio 17 2022 ARM64' : 'vs2022_arm64', # nopep8 'vs2019' : ['Visual Studio 16 2019', '-A', _arc ], 'Visual Studio 16 2019' + _arc : 'vs2019' , # nopep8 'vs2019_32' : ['Visual Studio 16 2019', '-A', 'Win32'], 'Visual Studio 16 2019' : 'vs2019_32' , # nopep8 'vs2019_64' : ['Visual Studio 16 2019', '-A', 'x64' ], 'Visual Studio 16 2019 Win64' : 'vs2019_64' , # nopep8 @@ -291,6 +298,22 @@ def find_any(): } _architectures = { + 'Visual Studio 17 2022' : 'x86' , + 'Visual Studio 17 2022 Win32' : 'x86' , + 'Visual Studio 17 2022 Win64' : 'x86_64' , + 'Visual Studio 17 2022 x86' : 'x86' , + 'Visual Studio 17 2022 x64' : 'x86_64' , + 'Visual Studio 17 2022 ARM' : 'arm' , + 'Visual Studio 17 2022 ARM32' : 'arm32' , + 'Visual Studio 17 2022 ARM64' : 'arm64' , + 'Visual Studio 17 2022 -A '+_arc: _arc2 , + 'Visual Studio 17 2022 -A Win32': 'x86' , + 'Visual Studio 17 2022 -A Win64': 'x86_64' , + 'Visual Studio 17 2022 -A x64' : 'x86_64' , + 'Visual Studio 17 2022 -A x86' : 'x86' , + 'Visual Studio 17 2022 -A ARM' : 'arm' , + 'Visual Studio 17 2022 -A ARM32': 'arm' , + 'Visual Studio 17 2022 -A ARM64': 'arm64' , 'Visual Studio 16 2019' : 'x86' , 'Visual Studio 16 2019 Win32' : 'x86' , 'Visual Studio 16 2019 Win64' : 'x86_64' , @@ -371,6 +394,8 @@ def to_gen(name_or_gen_or_ver): # ----------------------------------------------------------------------------- _toolsets = ( + # vs2022 compiler toolsets + 'v143_clang_c2', 'v143_clang', 'v143_xp', 'v143', # vs2019 compiler toolsets 'v142_clang_c2', 'v142_clang', 'v142_xp', 'v142', # vs2017 compiler toolsets @@ -412,7 +437,9 @@ def sep_name_toolset(name, canonize=True): if toolset in ('clang_c2', 'clang', 'xp'): assert re.match('vs....', name) year = int(re.sub(r'^vs(....).*', r'\1', name)) - if year == 2019: + if year == 2022: + vs_toolset = 'v143_' + toolset + elif year == 2019: vs_toolset = 'v142_' + toolset elif year == 2017: vs_toolset = 'v141_' + toolset @@ -484,6 +511,8 @@ def fn_201x(): d = cacheattr(sys.modules[__name__], '_vs2017dir', lambda: fn_201x()) elif ver == 16: d = cacheattr(sys.modules[__name__], '_vs2019dir', lambda: fn_201x()) + elif ver == 17: + d = cacheattr(sys.modules[__name__], '_vs2022dir', lambda: fn_201x()) else: raise Exception('VS Version not implemented: ' + str(ver)) return d @@ -508,7 +537,7 @@ def vcvarsall(name_or_gen_or_ver): d = vsdir(ver) if ver < 15: s = os.path.join(d, 'VC', 'vcvarsall.bat') - elif ver == 15 or ver == 16: + elif ver == 15 or ver == 16 or ver == 17: s = os.path.join(d, 'VC', 'Auxiliary', 'Build', 'vcvarsall.bat') else: raise Exception('VS Version not implemented: ' + str(ver)) @@ -545,7 +574,7 @@ def msbuild(name_or_gen_or_ver): root = vsdir(ver) val = '{}\\MSBuild\\{}.0\\bin\\{}MSBuild.exe' msbuild = val.format(root, ver, 'amd64\\' if util.in_64bit() else '') - elif ver == 16: + elif ver == 16 or ver == 17: # https://developercommunity.visualstudio.com/content/problem/400763/incorrect-path-to-msbuild-160-vs-2019-preview-1.html root = vsdir(ver) val = '{}\\MSBuild\\Current\\bin\\{}MSBuild.exe' diff --git a/test/run.bat b/test/run.bat index 97147f5..addc7d8 100755 --- a/test/run.bat +++ b/test/run.bat @@ -7,8 +7,8 @@ echo %PYTHON% echo %PIP% echo %PIPINSTALL% -if not defined PYTHON set PYTHON=python3 -if "%PYTHON%" == "" set PYTHON=python3 +if not defined PYTHON set PYTHON=python +if "%PYTHON%" == "" set PYTHON=python if not defined PIP set PIP=%PYTHON%\..\..\Scripts\pip if "%PIP%" == "" set PIP=%PYTHON%\..\..\Scripts\pip if not defined PIPINSTALL set "PIPINSTALL=%PIP% install" diff --git a/test/test01vsinfo.py b/test/test01vsinfo.py index 5c8a919..a214ad7 100644 --- a/test/test01vsinfo.py +++ b/test/test01vsinfo.py @@ -47,148 +47,344 @@ def test01_find_any(self): class Test00VisualStudioAliases(ut.TestCase): - def test01_name_to_gen(self): - def c(a, s): - sc = vsinfo.to_gen(a) - if sc != s: - self.fail(f"{a} should be '{s}' but is '{sc}'") + @staticmethod + def _test_name_to_gen(a, s): + sc = vsinfo.to_gen(a) + if sc != s: + self.fail(f"{a} should be '{s}' but is '{sc}'") + + def test01_name_to_gen_2022(self): + c = __class__._test_name_to_gen + c('vs2022' , ['Visual Studio 17 2022', '-A', _arc]) + c('vs2022_32' , ['Visual Studio 17 2022', '-A', 'Win32']) + c('vs2022_64' , ['Visual Studio 17 2022', '-A', 'x64']) + c('vs2022_arm' , ['Visual Studio 17 2022', '-A', 'ARM']) + c('vs2022_arm32', ['Visual Studio 17 2022', '-A', 'ARM']) + c('vs2022_arm64', ['Visual Studio 17 2022', '-A', 'ARM64']) + c('Visual Studio 17 2022' + _sfx , 'Visual Studio 17 2022' + _sfx ) + c('Visual Studio 17 2022' , 'Visual Studio 17 2022' ) + c('Visual Studio 17 2022 Win64' , 'Visual Studio 17 2022 Win64' ) + c('Visual Studio 17 2022 ARM' , 'Visual Studio 17 2022 ARM' ) + + def test01_name_to_gen_2019(self): + c = __class__._test_name_to_gen c('vs2019' , ['Visual Studio 16 2019', '-A', _arc]) c('vs2019_32' , ['Visual Studio 16 2019', '-A', 'Win32']) c('vs2019_64' , ['Visual Studio 16 2019', '-A', 'x64']) c('vs2019_arm' , ['Visual Studio 16 2019', '-A', 'ARM']) c('vs2019_arm32', ['Visual Studio 16 2019', '-A', 'ARM']) c('vs2019_arm64', ['Visual Studio 16 2019', '-A', 'ARM64']) - c('vs2017' , 'Visual Studio 15 2017' + _sfx ) - c('vs2017_32' , 'Visual Studio 15 2017' ) - c('vs2017_64' , 'Visual Studio 15 2017 Win64' ) - c('vs2017_arm' , 'Visual Studio 15 2017 ARM' ) - c('vs2015' , 'Visual Studio 14 2015' + _sfx ) - c('vs2015_32' , 'Visual Studio 14 2015' ) - c('vs2015_64' , 'Visual Studio 14 2015 Win64' ) - c('vs2015_arm' , 'Visual Studio 14 2015 ARM' ) - c('vs2013' , 'Visual Studio 12 2013' + _sfx ) - c('vs2013_32' , 'Visual Studio 12 2013' ) - c('vs2013_64' , 'Visual Studio 12 2013 Win64' ) - c('vs2013_arm' , 'Visual Studio 12 2013 ARM' ) - c('vs2012' , 'Visual Studio 11 2012' + _sfx ) - c('vs2012_32' , 'Visual Studio 11 2012' ) - c('vs2012_64' , 'Visual Studio 11 2012 Win64' ) - c('vs2012_arm' , 'Visual Studio 11 2012 ARM' ) - c('vs2010' , 'Visual Studio 10 2010' + _sfx ) - c('vs2010_32' , 'Visual Studio 10 2010' ) - c('vs2010_64' , 'Visual Studio 10 2010 Win64' ) - c('vs2010_ia64' , 'Visual Studio 10 2010 IA64' ) - c('vs2008' , 'Visual Studio 9 2008' + _sfx ) - c('vs2008_32' , 'Visual Studio 9 2008' ) - c('vs2008_64' , 'Visual Studio 9 2008 Win64' ) - c('vs2008_ia64' , 'Visual Studio 9 2008 IA64' ) - c('vs2005' , 'Visual Studio 8 2005' + _sfx ) - c('vs2005_32' , 'Visual Studio 8 2005' ) - c('vs2005_64' , 'Visual Studio 8 2005 Win64' ) c('Visual Studio 16 2019' + _sfx , 'Visual Studio 16 2019' + _sfx ) c('Visual Studio 16 2019' , 'Visual Studio 16 2019' ) c('Visual Studio 16 2019 Win64' , 'Visual Studio 16 2019 Win64' ) c('Visual Studio 16 2019 ARM' , 'Visual Studio 16 2019 ARM' ) + + def test01_name_to_gen_2017(self): + c = __class__._test_name_to_gen + c('vs2017' , 'Visual Studio 15 2017' + _sfx ) + c('vs2017_32' , 'Visual Studio 15 2017' ) + c('vs2017_64' , 'Visual Studio 15 2017 Win64' ) + c('vs2017_arm' , 'Visual Studio 15 2017 ARM' ) c('Visual Studio 15 2017' + _sfx , 'Visual Studio 15 2017' + _sfx ) c('Visual Studio 15 2017' , 'Visual Studio 15 2017' ) c('Visual Studio 15 2017 Win64' , 'Visual Studio 15 2017 Win64' ) c('Visual Studio 15 2017 ARM' , 'Visual Studio 15 2017 ARM' ) + + def test01_name_to_gen_2015(self): + c = __class__._test_name_to_gen + c('vs2015' , 'Visual Studio 14 2015' + _sfx ) + c('vs2015_32' , 'Visual Studio 14 2015' ) + c('vs2015_64' , 'Visual Studio 14 2015 Win64' ) + c('vs2015_arm' , 'Visual Studio 14 2015 ARM' ) c('Visual Studio 14 2015' + _sfx , 'Visual Studio 14 2015' + _sfx ) c('Visual Studio 14 2015' , 'Visual Studio 14 2015' ) c('Visual Studio 14 2015 Win64' , 'Visual Studio 14 2015 Win64' ) c('Visual Studio 14 2015 ARM' , 'Visual Studio 14 2015 ARM' ) + + def test01_name_to_gen_2013(self): + c = __class__._test_name_to_gen + c('vs2013' , 'Visual Studio 12 2013' + _sfx ) + c('vs2013_32' , 'Visual Studio 12 2013' ) + c('vs2013_64' , 'Visual Studio 12 2013 Win64' ) + c('vs2013_arm' , 'Visual Studio 12 2013 ARM' ) c('Visual Studio 12 2013' + _sfx , 'Visual Studio 12 2013' + _sfx ) c('Visual Studio 12 2013' , 'Visual Studio 12 2013' ) c('Visual Studio 12 2013 Win64' , 'Visual Studio 12 2013 Win64' ) c('Visual Studio 12 2013 ARM' , 'Visual Studio 12 2013 ARM' ) + + def test01_name_to_gen_2012(self): + c = __class__._test_name_to_gen + c('vs2012' , 'Visual Studio 11 2012' + _sfx ) + c('vs2012_32' , 'Visual Studio 11 2012' ) + c('vs2012_64' , 'Visual Studio 11 2012 Win64' ) + c('vs2012_arm' , 'Visual Studio 11 2012 ARM' ) c('Visual Studio 11 2012' + _sfx , 'Visual Studio 11 2012' + _sfx ) c('Visual Studio 11 2012' , 'Visual Studio 11 2012' ) c('Visual Studio 11 2012 Win64' , 'Visual Studio 11 2012 Win64' ) c('Visual Studio 11 2012 ARM' , 'Visual Studio 11 2012 ARM' ) + + def test01_name_to_gen_2010(self): + c = __class__._test_name_to_gen + c('vs2010' , 'Visual Studio 10 2010' + _sfx ) + c('vs2010_32' , 'Visual Studio 10 2010' ) + c('vs2010_64' , 'Visual Studio 10 2010 Win64' ) + c('vs2010_ia64' , 'Visual Studio 10 2010 IA64' ) c('Visual Studio 10 2010' + _sfx , 'Visual Studio 10 2010' + _sfx ) c('Visual Studio 10 2010' , 'Visual Studio 10 2010' ) c('Visual Studio 10 2010 Win64' , 'Visual Studio 10 2010 Win64' ) c('Visual Studio 10 2010 IA64' , 'Visual Studio 10 2010 IA64' ) + + def test01_name_to_gen_2008(self): + c = __class__._test_name_to_gen + c('vs2008' , 'Visual Studio 9 2008' + _sfx ) + c('vs2008_32' , 'Visual Studio 9 2008' ) + c('vs2008_64' , 'Visual Studio 9 2008 Win64' ) + c('vs2008_ia64' , 'Visual Studio 9 2008 IA64' ) c('Visual Studio 9 2008' + _sfx , 'Visual Studio 9 2008' + _sfx ) c('Visual Studio 9 2008' , 'Visual Studio 9 2008' ) c('Visual Studio 9 2008 Win64' , 'Visual Studio 9 2008 Win64' ) c('Visual Studio 9 2008 IA64' , 'Visual Studio 9 2008 IA64' ) + + def test01_name_to_gen_2005(self): + c = __class__._test_name_to_gen + c('vs2005' , 'Visual Studio 8 2005' + _sfx ) + c('vs2005_32' , 'Visual Studio 8 2005' ) + c('vs2005_64' , 'Visual Studio 8 2005 Win64' ) c('Visual Studio 8 2005' + _sfx , 'Visual Studio 8 2005' + _sfx ) c('Visual Studio 8 2005' , 'Visual Studio 8 2005' ) c('Visual Studio 8 2005 Win64' , 'Visual Studio 8 2005 Win64' ) - def test02_gen_to_name(self): - def c(a, s): - sc = vsinfo.to_name(a) - if sc != s: - self.fail("{} should be '{}' but is '{}'".format(a, s, sc)) + @staticmethod + def _test_gen_to_name(a, s): + sc = vsinfo.to_name(a) + if sc != s: + self.fail("{} should be '{}' but is '{}'".format(a, s, sc)) + + def test02_gen_to_name_2022(self): + c = __class__._test_gen_to_name + c('Visual Studio 17 2022' , 'vs2022_32' ) + c('Visual Studio 17 2022 Win64' , 'vs2022_64' ) + c('Visual Studio 17 2022 ARM' , 'vs2022_arm' ) + c('Visual Studio 17 2022 ARM32' , 'vs2022_arm32') + c('Visual Studio 17 2022 ARM64' , 'vs2022_arm64') + c('vs2022' , 'vs2022' ) + c('vs2022_32' , 'vs2022_32' ) + c('vs2022_64' , 'vs2022_64' ) + c('vs2022_arm' , 'vs2022_arm' ) + c('vs2022_arm32', 'vs2022_arm32') + c('vs2022_arm64', 'vs2022_arm64') + + def test02_gen_to_name_2019(self): + c = __class__._test_gen_to_name c('Visual Studio 16 2019' , 'vs2019_32' ) c('Visual Studio 16 2019 Win64' , 'vs2019_64' ) c('Visual Studio 16 2019 ARM' , 'vs2019_arm' ) c('Visual Studio 16 2019 ARM32' , 'vs2019_arm32') c('Visual Studio 16 2019 ARM64' , 'vs2019_arm64') - c('Visual Studio 15 2017' , 'vs2017_32' ) - c('Visual Studio 15 2017 Win64' , 'vs2017_64' ) - c('Visual Studio 15 2017 ARM' , 'vs2017_arm' ) - c('Visual Studio 14 2015' , 'vs2015_32' ) - c('Visual Studio 14 2015 Win64' , 'vs2015_64' ) - c('Visual Studio 14 2015 ARM' , 'vs2015_arm' ) - c('Visual Studio 12 2013' , 'vs2013_32' ) - c('Visual Studio 12 2013 Win64' , 'vs2013_64' ) - c('Visual Studio 12 2013 ARM' , 'vs2013_arm' ) - c('Visual Studio 11 2012' , 'vs2012_32' ) - c('Visual Studio 11 2012 Win64' , 'vs2012_64' ) - c('Visual Studio 11 2012 ARM' , 'vs2012_arm' ) - c('Visual Studio 10 2010' , 'vs2010_32' ) - c('Visual Studio 10 2010 Win64' , 'vs2010_64' ) - c('Visual Studio 10 2010 IA64' , 'vs2010_ia64' ) - c('Visual Studio 9 2008' , 'vs2008_32' ) - c('Visual Studio 9 2008 Win64' , 'vs2008_64' ) - c('Visual Studio 9 2008 IA64' , 'vs2008_ia64' ) - c('Visual Studio 8 2005' , 'vs2005_32' ) - c('Visual Studio 8 2005 Win64' , 'vs2005_64' ) c('vs2019' , 'vs2019' ) c('vs2019_32' , 'vs2019_32' ) c('vs2019_64' , 'vs2019_64' ) c('vs2019_arm' , 'vs2019_arm' ) c('vs2019_arm32', 'vs2019_arm32') c('vs2019_arm64', 'vs2019_arm64') + + def test02_gen_to_name_2017(self): + c = __class__._test_gen_to_name + c('Visual Studio 15 2017' , 'vs2017_32' ) + c('Visual Studio 15 2017 Win64' , 'vs2017_64' ) + c('Visual Studio 15 2017 ARM' , 'vs2017_arm' ) c('vs2017' , 'vs2017' ) c('vs2017_32' , 'vs2017_32' ) c('vs2017_64' , 'vs2017_64' ) c('vs2017_arm' , 'vs2017_arm' ) + + def test02_gen_to_name_2015(self): + c = __class__._test_gen_to_name + c('Visual Studio 14 2015' , 'vs2015_32' ) + c('Visual Studio 14 2015 Win64' , 'vs2015_64' ) + c('Visual Studio 14 2015 ARM' , 'vs2015_arm' ) c('vs2015' , 'vs2015' ) c('vs2015_32' , 'vs2015_32' ) c('vs2015_64' , 'vs2015_64' ) c('vs2015_arm' , 'vs2015_arm' ) + + def test02_gen_to_name_2013(self): + c = __class__._test_gen_to_name + c('Visual Studio 12 2013' , 'vs2013_32' ) + c('Visual Studio 12 2013 Win64' , 'vs2013_64' ) + c('Visual Studio 12 2013 ARM' , 'vs2013_arm' ) c('vs2013' , 'vs2013' ) c('vs2013_32' , 'vs2013_32' ) c('vs2013_64' , 'vs2013_64' ) c('vs2013_arm' , 'vs2013_arm' ) + + def test02_gen_to_name_2012(self): + c = __class__._test_gen_to_name + c('Visual Studio 11 2012' , 'vs2012_32' ) + c('Visual Studio 11 2012 Win64' , 'vs2012_64' ) + c('Visual Studio 11 2012 ARM' , 'vs2012_arm' ) c('vs2012' , 'vs2012' ) c('vs2012_32' , 'vs2012_32' ) c('vs2012_64' , 'vs2012_64' ) c('vs2012_arm' , 'vs2012_arm' ) + + def test02_gen_to_name_2010(self): + c = __class__._test_gen_to_name + c('Visual Studio 10 2010' , 'vs2010_32' ) + c('Visual Studio 10 2010 Win64' , 'vs2010_64' ) + c('Visual Studio 10 2010 IA64' , 'vs2010_ia64' ) c('vs2010' , 'vs2010' ) c('vs2010_32' , 'vs2010_32' ) c('vs2010_64' , 'vs2010_64' ) c('vs2010_ia64' , 'vs2010_ia64' ) + + def test02_gen_to_name_2008(self): + c = __class__._test_gen_to_name + c('Visual Studio 9 2008' , 'vs2008_32' ) + c('Visual Studio 9 2008 Win64' , 'vs2008_64' ) + c('Visual Studio 9 2008 IA64' , 'vs2008_ia64' ) c('vs2008' , 'vs2008' ) c('vs2008_32' , 'vs2008_32' ) c('vs2008_64' , 'vs2008_64' ) c('vs2008_ia64' , 'vs2008_ia64' ) + + def test02_gen_to_name_2005(self): + c = __class__._test_gen_to_name + c('Visual Studio 8 2005' , 'vs2005_32' ) + c('Visual Studio 8 2005 Win64' , 'vs2005_64' ) c('vs2005' , 'vs2005' ) c('vs2005_32' , 'vs2005_32' ) c('vs2005_64' , 'vs2005_64' ) - def test03_parse_toolset(self): - def t(spec, name_vs, ts_vs): - cname_vs,cts_vs = vsinfo.sep_name_toolset(spec) - if cname_vs != name_vs: - self.fail("{} should be '{}' but is '{}'".format(spec, name_vs, cname_vs)) - if cts_vs != ts_vs: - self.fail("{} should be '{}' but is '{}'".format(spec, ts_vs, cts_vs)) - + @staticmethod + def _test_parse_toolset(spec, name_vs, ts_vs): + cname_vs, cts_vs = vsinfo.sep_name_toolset(spec) + if cname_vs != name_vs: + self.fail("{} should be '{}' but is '{}'".format(spec, name_vs, cname_vs)) + if cts_vs != ts_vs: + self.fail("{} should be '{}' but is '{}'".format(spec, ts_vs, cts_vs)) + + def test03_parse_toolset_2022(self): + t = __class__._test_parse_toolset + t('vs2022' , 'vs2022' , None ) + t('vs2022_clang' , 'vs2022' , 'v143_clang_c2') + t('vs2022_xp' , 'vs2022' , 'v143_xp' ) + t('vs2022_v143' , 'vs2022' , 'v143' ) + t('vs2022_v143_xp' , 'vs2022' , 'v143_xp' ) + t('vs2022_v143_clang' , 'vs2022' , 'v143_clang_c2') + t('vs2022_v142' , 'vs2022' , 'v142' ) + t('vs2022_v142_xp' , 'vs2022' , 'v142_xp' ) + t('vs2022_v142_clang' , 'vs2022' , 'v142_clang_c2') + t('vs2022_v141' , 'vs2022' , 'v141' ) + t('vs2022_v141_xp' , 'vs2022' , 'v141_xp' ) + t('vs2022_v141_clang' , 'vs2022' , 'v141_clang_c2') + t('vs2022_v140' , 'vs2022' , 'v140' ) + t('vs2022_v140_xp' , 'vs2022' , 'v140_xp' ) + t('vs2022_v140_clang' , 'vs2022' , 'v140_clang_c2') + t('vs2022_v120' , 'vs2022' , 'v120' ) + t('vs2022_v120_xp' , 'vs2022' , 'v120_xp' ) + t('vs2022_v110' , 'vs2022' , 'v110' ) + t('vs2022_v110_xp' , 'vs2022' , 'v110_xp' ) + t('vs2022_v100' , 'vs2022' , 'v100' ) + t('vs2022_v100_xp' , 'vs2022' , 'v100_xp' ) + t('vs2022_v90' , 'vs2022' , 'v90' ) + t('vs2022_v90_xp' , 'vs2022' , 'v90_xp' ) + t('vs2022_v80' , 'vs2022' , 'v80' ) + + t('vs2022_32' , 'vs2022_32' , None ) + t('vs2022_32_clang' , 'vs2022_32' , 'v143_clang_c2') + t('vs2022_32_xp' , 'vs2022_32' , 'v143_xp' ) + t('vs2022_32_v143' , 'vs2022_32' , 'v143' ) + t('vs2022_32_v143_xp' , 'vs2022_32' , 'v143_xp' ) + t('vs2022_32_v143_clang' , 'vs2022_32' , 'v143_clang_c2') + t('vs2022_32_v142' , 'vs2022_32' , 'v142' ) + t('vs2022_32_v142_xp' , 'vs2022_32' , 'v142_xp' ) + t('vs2022_32_v142_clang' , 'vs2022_32' , 'v142_clang_c2') + t('vs2022_32_v141' , 'vs2022_32' , 'v141' ) + t('vs2022_32_v141_xp' , 'vs2022_32' , 'v141_xp' ) + t('vs2022_32_v141_clang' , 'vs2022_32' , 'v141_clang_c2') + t('vs2022_32_v140' , 'vs2022_32' , 'v140' ) + t('vs2022_32_v140_xp' , 'vs2022_32' , 'v140_xp' ) + t('vs2022_32_v140_clang' , 'vs2022_32' , 'v140_clang_c2') + t('vs2022_32_v120' , 'vs2022_32' , 'v120' ) + t('vs2022_32_v120_xp' , 'vs2022_32' , 'v120_xp' ) + t('vs2022_32_v110' , 'vs2022_32' , 'v110' ) + t('vs2022_32_v110_xp' , 'vs2022_32' , 'v110_xp' ) + t('vs2022_32_v100' , 'vs2022_32' , 'v100' ) + t('vs2022_32_v100_xp' , 'vs2022_32' , 'v100_xp' ) + t('vs2022_32_v90' , 'vs2022_32' , 'v90' ) + t('vs2022_32_v90_xp' , 'vs2022_32' , 'v90_xp' ) + t('vs2022_32_v80' , 'vs2022_32' , 'v80' ) + + t('vs2022_64' , 'vs2022_64' , None ) + t('vs2022_64_clang' , 'vs2022_64' , 'v143_clang_c2') + t('vs2022_64_xp' , 'vs2022_64' , 'v143_xp' ) + t('vs2022_64_v143' , 'vs2022_64' , 'v143' ) + t('vs2022_64_v143_xp' , 'vs2022_64' , 'v143_xp' ) + t('vs2022_64_v143_clang' , 'vs2022_64' , 'v143_clang_c2') + t('vs2022_64_v142' , 'vs2022_64' , 'v142' ) + t('vs2022_64_v142_xp' , 'vs2022_64' , 'v142_xp' ) + t('vs2022_64_v142_clang' , 'vs2022_64' , 'v142_clang_c2') + t('vs2022_64_v141' , 'vs2022_64' , 'v141' ) + t('vs2022_64_v141_xp' , 'vs2022_64' , 'v141_xp' ) + t('vs2022_64_v141_clang' , 'vs2022_64' , 'v141_clang_c2') + t('vs2022_64_v140' , 'vs2022_64' , 'v140' ) + t('vs2022_64_v140_xp' , 'vs2022_64' , 'v140_xp' ) + t('vs2022_64_v140_clang' , 'vs2022_64' , 'v140_clang_c2') + t('vs2022_64_v120' , 'vs2022_64' , 'v120' ) + t('vs2022_64_v120_xp' , 'vs2022_64' , 'v120_xp' ) + t('vs2022_64_v110' , 'vs2022_64' , 'v110' ) + t('vs2022_64_v110_xp' , 'vs2022_64' , 'v110_xp' ) + t('vs2022_64_v100' , 'vs2022_64' , 'v100' ) + t('vs2022_64_v100_xp' , 'vs2022_64' , 'v100_xp' ) + t('vs2022_64_v90' , 'vs2022_64' , 'v90' ) + t('vs2022_64_v90_xp' , 'vs2022_64' , 'v90_xp' ) + t('vs2022_64_v80' , 'vs2022_64' , 'v80' ) + + t('vs2022_arm' , 'vs2022_arm' , None ) + t('vs2022_arm_clang' , 'vs2022_arm' , 'v143_clang_c2') + t('vs2022_arm_v143' , 'vs2022_arm' , 'v143' ) + t('vs2022_arm_v143_clang' , 'vs2022_arm' , 'v143_clang_c2') + t('vs2022_arm_v142' , 'vs2022_arm' , 'v142' ) + t('vs2022_arm_v142_clang' , 'vs2022_arm' , 'v142_clang_c2') + t('vs2022_arm_v141' , 'vs2022_arm' , 'v141' ) + t('vs2022_arm_v141_clang' , 'vs2022_arm' , 'v141_clang_c2') + t('vs2022_arm_v140' , 'vs2022_arm' , 'v140' ) + t('vs2022_arm_v140_clang' , 'vs2022_arm' , 'v140_clang_c2') + t('vs2022_arm_v120' , 'vs2022_arm' , 'v120' ) + t('vs2022_arm_v110' , 'vs2022_arm' , 'v110' ) + t('vs2022_arm_v100' , 'vs2022_arm' , 'v100' ) + + t('vs2022_arm32' , 'vs2022_arm32' , None ) + t('vs2022_arm32_clang' , 'vs2022_arm32' , 'v143_clang_c2') + t('vs2022_arm32_v143' , 'vs2022_arm32' , 'v143' ) + t('vs2022_arm32_v143_clang' , 'vs2022_arm32' , 'v143_clang_c2') + t('vs2022_arm32_v142' , 'vs2022_arm32' , 'v142' ) + t('vs2022_arm32_v142_clang' , 'vs2022_arm32' , 'v142_clang_c2') + t('vs2022_arm32_v141' , 'vs2022_arm32' , 'v141' ) + t('vs2022_arm32_v141_clang' , 'vs2022_arm32' , 'v141_clang_c2') + t('vs2022_arm32_v140' , 'vs2022_arm32' , 'v140' ) + t('vs2022_arm32_v140_clang' , 'vs2022_arm32' , 'v140_clang_c2') + t('vs2022_arm32_v120' , 'vs2022_arm32' , 'v120' ) + t('vs2022_arm32_v110' , 'vs2022_arm32' , 'v110' ) + t('vs2022_arm32_v100' , 'vs2022_arm32' , 'v100' ) + + t('vs2022_arm64' , 'vs2022_arm64' , None ) + t('vs2022_arm64_clang' , 'vs2022_arm64' , 'v143_clang_c2') + t('vs2022_arm64_v143' , 'vs2022_arm64' , 'v143' ) + t('vs2022_arm64_v143_clang' , 'vs2022_arm64' , 'v143_clang_c2') + t('vs2022_arm64_v142' , 'vs2022_arm64' , 'v142' ) + t('vs2022_arm64_v142_clang' , 'vs2022_arm64' , 'v142_clang_c2') + t('vs2022_arm64_v141' , 'vs2022_arm64' , 'v141' ) + t('vs2022_arm64_v141_clang' , 'vs2022_arm64' , 'v141_clang_c2') + t('vs2022_arm64_v140' , 'vs2022_arm64' , 'v140' ) + t('vs2022_arm64_v140_clang' , 'vs2022_arm64' , 'v140_clang_c2') + t('vs2022_arm64_v120' , 'vs2022_arm64' , 'v120' ) + t('vs2022_arm64_v110' , 'vs2022_arm64' , 'v110' ) + t('vs2022_arm64_v100' , 'vs2022_arm64' , 'v100' ) + + def test03_parse_toolset_2019(self): + t = __class__._test_parse_toolset t('vs2019' , 'vs2019' , None ) t('vs2019_clang' , 'vs2019' , 'v142_clang_c2') t('vs2019_xp' , 'vs2019' , 'v142_xp' ) @@ -291,6 +487,8 @@ def t(spec, name_vs, ts_vs): t('vs2019_arm64_v110' , 'vs2019_arm64' , 'v110' ) t('vs2019_arm64_v100' , 'vs2019_arm64' , 'v100' ) + def test03_parse_toolset_2017(self): + t = __class__._test_parse_toolset t('vs2017' , 'vs2017' , None ) t('vs2017_clang' , 'vs2017' , 'v141_clang_c2') t('vs2017_xp' , 'vs2017' , 'v141_xp' ) @@ -358,6 +556,8 @@ def t(spec, name_vs, ts_vs): t('vs2017_arm_v110' , 'vs2017_arm' , 'v110' ) t('vs2017_arm_v100' , 'vs2017_arm' , 'v100' ) + def test03_parse_toolset_2015(self): + t = __class__._test_parse_toolset t('vs2015' , 'vs2015' , None ) t('vs2015_clang' , 'vs2015' , 'v140_clang_c2') t('vs2015_xp' , 'vs2015' , 'v140_xp' ) @@ -402,6 +602,8 @@ def t(spec, name_vs, ts_vs): t('vs2015_arm' , 'vs2015_arm' , None ) t('vs2015_arm_clang' , 'vs2015_arm' , 'v140_clang_c2') + def test03_parse_toolset_2013(self): + t = __class__._test_parse_toolset t('vs2013' , 'vs2013' , None ) t('vs2013_xp' , 'vs2013' , 'v120_xp' ) t('vs2013_v110' , 'vs2013' , 'v110' ) @@ -432,6 +634,8 @@ def t(spec, name_vs, ts_vs): t('vs2013_64_v90_xp' , 'vs2013_64' , 'v90_xp' ) t('vs2013_64_v80' , 'vs2013_64' , 'v80' ) + def test03_parse_toolset_2012(self): + t = __class__._test_parse_toolset t('vs2012' , 'vs2012' , None ) t('vs2012_xp' , 'vs2012' , 'v110_xp' ) t('vs2012_v110' , 'vs2012' , 'v110' ) @@ -462,6 +666,8 @@ def t(spec, name_vs, ts_vs): t('vs2012_64_v90_xp' , 'vs2012_64' , 'v90_xp' ) t('vs2012_64_v80' , 'vs2012_64' , 'v80' ) + def test03_parse_toolset_2010(self): + t = __class__._test_parse_toolset t('vs2010' , 'vs2010' , None ) t('vs2010_xp' , 'vs2010' , 'v100_xp' ) t('vs2010_v100' , 'vs2010' , 'v100' ) @@ -494,6 +700,8 @@ def t(spec, name_vs, ts_vs): t('vs2010_ia64_v90_xp' , 'vs2010_ia64' , 'v90_xp' ) t('vs2010_ia64_v80' , 'vs2010_ia64' , 'v80' ) + def test03_parse_toolset_2008(self): + t = __class__._test_parse_toolset t('vs2008' , 'vs2008' , None ) t('vs2008_xp' , 'vs2008' , 'v90_xp' ) t('vs2008_v90' , 'vs2008' , 'v90' ) @@ -518,6 +726,8 @@ def t(spec, name_vs, ts_vs): t('vs2008_ia64_v90_xp' , 'vs2008_ia64' , 'v90_xp' ) t('vs2008_ia64_v80' , 'vs2008_ia64' , 'v80' ) + def test03_parse_toolset_2005(self): + t = __class__._test_parse_toolset t('vs2005' , 'vs2005' , None ) t('vs2005_v80' , 'vs2005' , 'v80' ) From b6d7a550e50daed7de95e7ac2043f5d6e2526b13 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 12 Oct 2021 22:48:59 +0100 Subject: [PATCH 27/50] [fix] some targets have the executable created in a non-standard place --- src/c4/cmany/target.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/c4/cmany/target.py b/src/c4/cmany/target.py index e4ced81..45ba070 100644 --- a/src/c4/cmany/target.py +++ b/src/c4/cmany/target.py @@ -15,31 +15,36 @@ def __str__(self): class Target: + def _dbg(self, msg): + util.logdbg(f"[{self.subdir}] {self.name}: {msg}") + def __init__(self, name, build, subdir, projfile): assert os.path.exists(projfile) - util.logdbg(f"[{subdir}] {name}: creating target...") self.name = name self.build = build self.subdir = subdir self.subdir_abs = os.path.join(os.path.abspath(build.builddir), self.subdir) self.projfile = projfile # Makefile or {name}.vcxproj self.cmake_private_dir = util.chkf(self.subdir_abs, "CMakeFiles") + self._dbg("creating target...") # find the target type self._type = TargetType.custom gen = self.build.generator if gen.is_msvc: tgt_dir = os.path.join(self.subdir_abs, f"{name}.dir") if os.path.exists(tgt_dir): + self._dbg(f"target is binary: found {tgt_dir}") self._type |= TargetType.binary elif gen.is_makefile: tgt_dir = os.path.join(self.cmake_private_dir, f"{name}.dir") link_file = os.path.join(tgt_dir, "link.txt") if os.path.exists(link_file): - util.logdbg(f"{name}: found {link_file}") + self._dbg(f"target is binary: found {link_file}") self._type |= TargetType.binary else: raise Exception(f"unknown generator: {gen}") + @property def desc(self): return f"[{self.subdir}] {self.name} [{self._type}]" @@ -59,9 +64,26 @@ def output_file(self): # then this logic will fail gen = self.build.generator if gen.is_msvc: - return f"{self.subdir_abs}/{self.build.build_type}/{self.name}.exe" + f = f"{self.subdir_abs}/{self.build.build_type}/{self.name}.exe" + return f elif gen.is_makefile: - return f"{self.subdir_abs}/{self.name}" + f = f"{self.subdir_abs}/{self.name}" + if os.path.exists(f): + self._dbg(f"found target executable: {f}") + return f + else: + self._dbg(f"could not find standard target executable: {f}") + fallbacks = ( + f"{self.build.builddir}/bin/{self.name}", + ) + for f in fallbacks: + self._dbg(f"trying {f}") + if os.path.exists(f): + self._dbg(f"found target executable: {f}") + return f + else: + self._dbg(f"could not target executable: {f}") + raise Exception("could not find target executable") else: raise Exception(f"unknown generator: {gen}") From 7d70d1f8533a7b3edc501f8870e92ab1260405a1 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 7 Dec 2021 17:06:04 +0100 Subject: [PATCH 28/50] subprocess: PIPE --- src/c4/cmany/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c4/cmany/util.py b/src/c4/cmany/util.py index 78ae72d..a1cabf1 100644 --- a/src/c4/cmany/util.py +++ b/src/c4/cmany/util.py @@ -652,7 +652,7 @@ def runcmd_nocheck(cmd, *cmd_args, **run_args): scmd = shlex_join(cmd) cwd = os.path.realpath(run_args.get('cwd', os.getcwd())) logcmd(f'$ cd {cwd} && {scmd}') - sp = subprocess.run(cmd, **run_args) + sp = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **run_args) logdbg("finished running command") return sp From db387efeb531a82c931f85b8ca82eda3ff4e5b39 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 22 Jan 2022 23:32:33 +0000 Subject: [PATCH 29/50] [fix] target executables may be place in several places --- src/c4/cmany/target.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/c4/cmany/target.py b/src/c4/cmany/target.py index 45ba070..64e71b4 100644 --- a/src/c4/cmany/target.py +++ b/src/c4/cmany/target.py @@ -63,29 +63,30 @@ def output_file(self): # if the project's cmake scripts change the output path, # then this logic will fail gen = self.build.generator + da = self.subdir_abs + bt = self.build.build_type if gen.is_msvc: - f = f"{self.subdir_abs}/{self.build.build_type}/{self.name}.exe" - return f + return self._find_target_file_with_fallbacks( + f"{da}/{bt}/{self.name}.exe", + f"{self.build.builddir}/bin/{bt}/{self.name}.exe", + ) elif gen.is_makefile: - f = f"{self.subdir_abs}/{self.name}" + return self._find_target_file_with_fallbacks( + f"{da}/{self.name}", + f"{self.build.builddir}/bin/{self.name}" + ) + else: + raise Exception(f"unknown generator: {gen}") + + def _find_target_file_with_fallbacks(self, *paths): + for f in paths: + self._dbg(f"trying {f}") if os.path.exists(f): - self._dbg(f"found target executable: {f}") + self._dbg(f"... found it!") return f else: - self._dbg(f"could not find standard target executable: {f}") - fallbacks = ( - f"{self.build.builddir}/bin/{self.name}", - ) - for f in fallbacks: - self._dbg(f"trying {f}") - if os.path.exists(f): - self._dbg(f"found target executable: {f}") - return f - else: - self._dbg(f"could not target executable: {f}") - raise Exception("could not find target executable") - else: - raise Exception(f"unknown generator: {gen}") + self._dbg(f"... not found!") + raise Exception(f"could not find target executable: {paths}") @property def vcxproj(self): From c7c65429d60bbdfe2a8a55c013bb951256ef74ba Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Fri, 4 Mar 2022 11:24:29 +0000 Subject: [PATCH 30/50] [fix] improve loading of toolchains --- src/c4/cmany/cmake.py | 65 ++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/src/c4/cmany/cmake.py b/src/c4/cmany/cmake.py index 0d69eff..227a207 100644 --- a/src/c4/cmany/cmake.py +++ b/src/c4/cmany/cmake.py @@ -382,36 +382,37 @@ def _genid(gen): # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- -# def get_toolchain_cache(toolchain): -# d = os.path.join(USER_DIR, 'toolchains', re.sub(os.sep, '+', toolchain)) -# logdbg("toolchain cache: USER_DIR=", USER_DIR) -# logdbg("toolchain cache: d=", d) -# bd = os.path.join(d, 'build') -# logdbg("toolchain cache: bd=", bd) -# if not os.path.exists(d): -# os.makedirs(d) -# with setcwd(d): -# with open('main.cpp', 'w') as f: -# f.write("int main() {}") -# with open('CMakeLists.txt', 'w') as f: -# f.write(""" -# cmake_minimum_required(VERSION 2.6) -# project(toolchain_test) -# add_executable(main main.cpp) -# """) -# if not os.path.exists(bd): -# os.makedirs(bd) -# with setcwd(bd): -# cmd = ['cmake', '-DCMAKE_TOOLCHAIN_FILE='+toolchain, '..'] -# runsyscmd(cmd, echo_output=True) -# return loadvars(bd) +def get_toolchain_cache(toolchain): + d = os.path.join(USER_DIR, 'toolchains', re.sub(os.sep, '_', toolchain)) + logdbg("toolchain cache: USER_DIR=", USER_DIR) + logdbg("toolchain cache: d=", d) + bd = os.path.join(d, 'build') + logdbg("toolchain cache: bd=", bd) + if not os.path.exists(d): + os.makedirs(d) + with setcwd(d): + with open('main.cpp', 'w') as f: + f.write("int main() {}") + with open('CMakeLists.txt', 'w') as f: + f.write(""" +cmake_minimum_required(VERSION 2.6) +project(toolchain_test) +add_executable(main main.cpp) +""") + if not os.path.exists(bd): + os.makedirs(bd) + with setcwd(bd): + cmd = ['cmake', '-DCMAKE_TOOLCHAIN_FILE='+toolchain, '..'] + runsyscmd(cmd, echo_output=True) + return loadvars(bd) + + def extract_toolchain_compilers(toolchain): - with open(toolchain) as f: - lines = f.readlines() - out = odict() - for l in lines: - res = re.search(r'(set|SET)\ ?\(\ ?(CMAKE_.*?_COMPILER) (.*?)\ ?\)', l) - if res: - res = res.groups() - out[res[1]] = res[2] - return out + cache = get_toolchain_cache(toolchain) + out = odict() + for k, v in cache.items(): + logdbg(f"toolchain cache: {k}=={v}") + if k.startswith('CMAKE_') and k.endswith('_COMPILER'): + logdbg(f"toolchain cache: .................... {k}=={v}") + out[k] = v.val + return out From 85fdcebdfcceae76d88f1a1899e02bfb669df0bd Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Thu, 17 Mar 2022 12:02:50 +0000 Subject: [PATCH 31/50] [fix] fix runcmd_nocheck(): stdout was suppressed --- src/c4/cmany/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c4/cmany/util.py b/src/c4/cmany/util.py index a1cabf1..66466b3 100644 --- a/src/c4/cmany/util.py +++ b/src/c4/cmany/util.py @@ -652,7 +652,7 @@ def runcmd_nocheck(cmd, *cmd_args, **run_args): scmd = shlex_join(cmd) cwd = os.path.realpath(run_args.get('cwd', os.getcwd())) logcmd(f'$ cd {cwd} && {scmd}') - sp = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **run_args) + sp = subprocess.run(cmd, stderr=subprocess.STDOUT, **run_args) logdbg("finished running command") return sp From 9155833b32d2bfcf45ec9209a1305804d024f247 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Fri, 4 Mar 2022 17:04:11 +0000 Subject: [PATCH 32/50] [fix] runcmd: ensure stdout is printed --- src/c4/cmany/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/c4/cmany/util.py b/src/c4/cmany/util.py index 66466b3..50f5b25 100644 --- a/src/c4/cmany/util.py +++ b/src/c4/cmany/util.py @@ -649,6 +649,7 @@ def runcmd_nocheck(cmd, *cmd_args, **run_args): raise Exception("could not understand command") cmd += list(cmd_args) logdbg(f" : cmd={cmd}") + logdbg(f" : run_args={run_args}") scmd = shlex_join(cmd) cwd = os.path.realpath(run_args.get('cwd', os.getcwd())) logcmd(f'$ cd {cwd} && {scmd}') From 5254f14e3fa8ae2b00b882251a7c96d56e9a5e8a Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 17 May 2022 09:14:47 +0200 Subject: [PATCH 33/50] [fix] add missing argument when raising exception --- src/c4/cmany/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index e996c27..fd900cb 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -466,7 +466,7 @@ def _get_toolchain(self): tc = os.path.join(os.getcwd(), tc) tc = os.path.abspath(tc) if not os.path.exists(tc): - raise err.ToolchainFileNotFound(tc) + raise err.ToolchainFileNotFound(tc, self) return tc def _gather_flags(self, which, append_to_sysinfo_var=None, with_defines=False): From 3c306db64403163c2a829205db50e01fee821735 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 11 Jun 2022 02:10:40 +0200 Subject: [PATCH 34/50] [wip] improve behavior with toolchains --- src/c4/cmany/build.py | 6 +++++- src/c4/cmany/compiler.py | 41 +++++++++++++++++++++++++++------------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index fd900cb..516eec1 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -65,7 +65,11 @@ def __init__(self, proj_root, build_root, install_root, self.toolchain_file = self._get_toolchain() if self.toolchain_file: comps = cmake.extract_toolchain_compilers(self.toolchain_file) - c = Compiler(comps['CMAKE_CXX_COMPILER']) + if comps.get('CMAKE_CXX_COMPILER'): + c = Compiler(comps['CMAKE_CXX_COMPILER']) + else: + c = Compiler(os.environ.get('CXX')) + dbg(f"CMAKE_CXX_COMPILER not found, trying environment var CXX:", c) self.adjust(compiler=c) # # WATCHOUT: this may trigger a readjustment of this build's parameters diff --git a/src/c4/cmany/compiler.py b/src/c4/cmany/compiler.py index 87ed54a..af21624 100644 --- a/src/c4/cmany/compiler.py +++ b/src/c4/cmany/compiler.py @@ -1,6 +1,7 @@ import os import re import tempfile +import shlex from .build_item import BuildItem from .system import System @@ -55,10 +56,21 @@ def __init__(self, spec): spl = [path] p = util.which(path) if p is None: + util.logdbg("not found:", path) + shspl = shlex.split(path) + util.logdbg("trying split:", shspl) + if len(shspl) > 0: + p = util.which(shspl[0]) + util.logdbg("trying split:", p) + if p is None: + util.logdbg("no compiler found", path) raise err.CompilerNotFound(path) - # if p != path: - # print("compiler: selected {} for {}".format(p, path)) - path = os.path.abspath(p) + if p != path: + util.logdbg("compiler: selected {} for {}".format(p, path)) + if isinstance(path, str): + path = os.path.abspath(p) + else: + path[0] = os.path.abspath(p) name, version, version_full = self.get_version(path) self.shortname = name self.gcclike = self.shortname in ('gcc', 'clang', 'icc', 'g++', 'clang++', 'icpc', 'icpx') @@ -120,8 +132,11 @@ def get_version(self, path): if hasattr(self, "vs"): return self.vs.name, str(self.vs.year), self.vs.name # other compilers - # print("cmp: found compiler:", path) - out = slntout([path, '--version']) + util.logdbg("cmp: found compiler:", path) + if isinstance(path, str): + out = slntout([path, '--version']) + else: + out = slntout(path + ['--version']) version_full = out.split("\n")[0] splits = version_full.split(" ") name = splits[0].lower() @@ -144,14 +159,7 @@ def get_version(self, path): elif re.search("#define __GNUC__", m): name = "g++" if re.search(r"\+\+", path) else "gcc" break - if name.startswith("g++") or name.startswith("gcc"): - # print("g++: version:", name, name.find('++')) - name = "g++" if name.find('++') != -1 else 'gcc' - # print("g++: version:", name, name.find('++')) - version = slntout([path, '-dumpversion']) - version = re.sub(vregex, r'\1', version) - # print("gcc version:", version, "---") - elif name.startswith("clang"): + if name.startswith("clang++") or name.startswith("clang") or name.endswith("clang++") or name.endswith("clang"): name = "clang++" if path.find('clang++') != -1 else 'clang' if re.search('Apple LLVM', version_full): name = "apple_llvm" @@ -160,6 +168,13 @@ def get_version(self, path): else: version = re.sub(r'clang version ' + vregex + '.*', r'\1', version_full) # print("clang version:", version, "---") + elif name.startswith("g++") or name.startswith("gcc") or name.endswith("g++") or name.endswith("gcc"): + # print("g++: version:", name, name.find('++')) + name = "g++" if name.find('++') != -1 else 'gcc' + # print("g++: version:", name, name.find('++')) + version = slntout([path, '-dumpversion']) + version = re.sub(vregex, r'\1', version) + # print("gcc version:", version, "---") elif name.startswith("icpc") or name.startswith("icc"): name = "icc" if name.startswith("icc") else "icpc" if re.search(r'icpc \(ICC\) ' + vregex + '.*', version_full): From a7f2ef8a8635c66de9c6414d27fc0b807b42f3f5 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 27 Sep 2022 11:03:12 +0100 Subject: [PATCH 35/50] [wip] improve behavior with toolchains --- src/c4/cmany/architecture.py | 13 +++++------ src/c4/cmany/cmake.py | 43 ++++++++++++++++++------------------ src/c4/cmany/compiler.py | 34 ++++++++++++++-------------- 3 files changed, 45 insertions(+), 45 deletions(-) diff --git a/src/c4/cmany/architecture.py b/src/c4/cmany/architecture.py index 53b4203..004bfd0 100644 --- a/src/c4/cmany/architecture.py +++ b/src/c4/cmany/architecture.py @@ -2,6 +2,7 @@ from .build_item import BuildItem from . import util +from .cmake import CMakeSysInfo # ----------------------------------------------------------------------------- @@ -15,14 +16,10 @@ def default(): @staticmethod def default_str(): - # s = CMakeSysInfo.architecture() - # if s == "amd64": - # s = "x86_64" - # return s - if util.in_64bit(): - return "x86_64" - elif util.in_32bit(): - return "x86" + s = CMakeSysInfo.architecture() + if s == "amd64": + s = "x86_64" + return s @property def is64(self): diff --git a/src/c4/cmany/cmake.py b/src/c4/cmany/cmake.py index 227a207..7c0a26b 100644 --- a/src/c4/cmany/cmake.py +++ b/src/c4/cmany/cmake.py @@ -5,7 +5,8 @@ from collections import OrderedDict as odict from .conf import USER_DIR -from .util import cacheattr, setcwd, runsyscmd, logdbg, chkf +from .util import cacheattr, setcwd, runsyscmd, chkf +from .util import logdbg as dbg from . import util from . import err @@ -67,14 +68,14 @@ def loadvars(builddir): if os.path.exists(c): with open(c, 'r') as f: for line in f: - # logdbg("loadvars0", line.strip()) + # dbg("loadvars0", line.strip()) if not re.match(_cache_entry, line): continue ls = line.strip() name = re.sub(_cache_entry, r'\1', ls) vartype = re.sub(_cache_entry, r'\2', ls)[1:] value = re.sub(_cache_entry, r'\3', ls) - # logdbg("loadvars1", name, vartype, value) + # dbg("loadvars1", name, vartype, value) v[name] = CMakeCacheVar(name, value, vartype) return v @@ -86,7 +87,7 @@ def get_cxx_compiler(builddir): if len(files) != 1: raise Exception(f"could not find compiler settings: {expr}") cxxfile = files[0] - logdbg("") + dbg("") with open(cxxfile) as f: lines = f.readlines() lookup = "set(CMAKE_CXX_COMPILER " @@ -281,15 +282,15 @@ def info(which_generator="default"): def _getstr(var_name, which_generator): regex = r'^{} "(.*)"'.format(var_name) for l in __class__.info(which_generator): - #logdbg(l.strip("\n"), l.startswith(var_name), var_name) + #dbg(l.strip("\n"), l.startswith(var_name), var_name) if l.startswith(var_name): l = l.strip("\n").lstrip(" ").rstrip(" ") - #logdbg(var_name, "startswith :", l) + #dbg(var_name, "startswith :", l) if re.match(regex, l): s = re.sub(regex, r'\1', l) - #logdbg(var_name, "result: '" + s + "'") + #dbg(var_name, "result: '" + s + "'") return s - #logdbg("--------------------------------------\n", __class__.info(which_generator)) + #dbg("--------------------------------------\n", __class__.info(which_generator)) msg = "could not find variable {} in the output of `cmake --system-information -G '{}'`" raise err.Error(msg, var_name, which_generator) @@ -297,30 +298,30 @@ def _getstr(var_name, which_generator): def system_info(gen): """gen can be a string or a cmany.Generator object""" from .generator import Generator - logdbg("CMakeSystemInfo: asked info for", gen) + dbg("CMakeSystemInfo: asked info for", gen) p = _genid(gen) d = os.path.join(USER_DIR, 'cmake_info', p) p = os.path.join(d, 'info') - logdbg("CMakeSystemInfo: path=", p) + dbg("CMakeSystemInfo: path=", p) # https://stackoverflow.com/questions/7015587/python-difference-of-2-datetimes-in-months if os.path.exists(p) and util.time_since_modification(p).months < 1: - logdbg("CMakeSystemInfo: asked info for", gen, "... found", p) + dbg("CMakeSystemInfo: asked info for", gen, "... found", p) with open(p, "r") as f: i = f.readlines() if i: return i else: - logdbg("CMakeSystemInfo: info for gen", gen, "is empty...") + dbg("CMakeSystemInfo: info for gen", gen, "is empty...") # if isinstance(gen, Generator): cmd = ['cmake'] + gen.configure_args() + ['--system-information'] - logdbg("CMakeSystemInfo: from generator! '{}' ---> cmd={}".format(gen, cmd)) + dbg("CMakeSystemInfo: from generator! '{}' ---> cmd={}".format(gen, cmd)) else: if gen == "default" or gen == "": - logdbg("CMakeSystemInfo: default! '{}'".format(gen)) + dbg("CMakeSystemInfo: default! '{}'".format(gen)) cmd = ['cmake', '--system-information'] else: - logdbg("CMakeSystemInfo: assume vs! '{}'".format(gen)) + dbg("CMakeSystemInfo: assume vs! '{}'".format(gen)) from . import vsinfo gen = vsinfo.to_gen(gen) if isinstance(gen, list): @@ -338,7 +339,7 @@ def system_info(gen): os.makedirs(d) with setcwd(d): out = runsyscmd(cmd, echo_output=False, capture_output=True) - logdbg("cmany: finished generating information for generator '{}'\n".format(gen), out, cmd) + dbg("cmany: finished generating information for generator '{}'\n".format(gen), out, cmd) out = out.strip() if not out: from .err import InvalidGenerator @@ -384,10 +385,10 @@ def _genid(gen): # ----------------------------------------------------------------------------- def get_toolchain_cache(toolchain): d = os.path.join(USER_DIR, 'toolchains', re.sub(os.sep, '_', toolchain)) - logdbg("toolchain cache: USER_DIR=", USER_DIR) - logdbg("toolchain cache: d=", d) + dbg("toolchain cache: USER_DIR=", USER_DIR) + dbg("toolchain cache: d=", d) bd = os.path.join(d, 'build') - logdbg("toolchain cache: bd=", bd) + dbg("toolchain cache: bd=", bd) if not os.path.exists(d): os.makedirs(d) with setcwd(d): @@ -411,8 +412,8 @@ def extract_toolchain_compilers(toolchain): cache = get_toolchain_cache(toolchain) out = odict() for k, v in cache.items(): - logdbg(f"toolchain cache: {k}=={v}") + dbg(f"toolchain cache: {k}=={v}") if k.startswith('CMAKE_') and k.endswith('_COMPILER'): - logdbg(f"toolchain cache: .................... {k}=={v}") + dbg(f"toolchain cache: .................... {k}=={v}") out[k] = v.val return out diff --git a/src/c4/cmany/compiler.py b/src/c4/cmany/compiler.py index af21624..47da206 100644 --- a/src/c4/cmany/compiler.py +++ b/src/c4/cmany/compiler.py @@ -9,6 +9,7 @@ from . import util from . import vsinfo from . import err +from .util import logdbg as dbg # ----------------------------------------------------------------------------- @@ -56,17 +57,17 @@ def __init__(self, spec): spl = [path] p = util.which(path) if p is None: - util.logdbg("not found:", path) + dbg("not found:", path) shspl = shlex.split(path) - util.logdbg("trying split:", shspl) + dbg("trying split:", shspl) if len(shspl) > 0: p = util.which(shspl[0]) - util.logdbg("trying split:", p) + dbg("trying split:", p) if p is None: - util.logdbg("no compiler found", path) + dbg("no compiler found", path) raise err.CompilerNotFound(path) if p != path: - util.logdbg("compiler: selected {} for {}".format(p, path)) + dbg("compiler: selected {} for {}".format(p, path)) if isinstance(path, str): path = os.path.abspath(p) else: @@ -132,7 +133,7 @@ def get_version(self, path): if hasattr(self, "vs"): return self.vs.name, str(self.vs.year), self.vs.name # other compilers - util.logdbg("cmp: found compiler:", path) + dbg("cmp: found compiler:", path) if isinstance(path, str): out = slntout([path, '--version']) else: @@ -140,10 +141,10 @@ def get_version(self, path): version_full = out.split("\n")[0] splits = version_full.split(" ") name = splits[0].lower() - # print("cmp: version:", name, "---", version_full, "---") + dbg("cmp: version:", name, "---", version_full, "---") vregex = r'(\d+\.\d+)\.\d+' base = os.path.basename(path) - # print(name, "cmp base:", base, name) + dbg(name, "cmp base:", base, name) if base.startswith("c++") or base.startswith("cc"): try: # if this fails, just go on. It's not really needed. with tempfile.NamedTemporaryFile(suffix=".cpp", prefix="cmany.", delete=False) as f: @@ -164,30 +165,31 @@ def get_version(self, path): if re.search('Apple LLVM', version_full): name = "apple_llvm" version = re.sub(r'Apple LLVM version ' + vregex + '.*', r'\1', version_full) - print("apple_llvm version:", version, "---") + dbg("apple_llvm version:", version, "---") else: version = re.sub(r'clang version ' + vregex + '.*', r'\1', version_full) - # print("clang version:", version, "---") + dbg("clang version:", version, "---") elif name.startswith("g++") or name.startswith("gcc") or name.endswith("g++") or name.endswith("gcc"): - # print("g++: version:", name, name.find('++')) - name = "g++" if name.find('++') != -1 else 'gcc' - # print("g++: version:", name, name.find('++')) + dbg("g++: version:", name, name.find('++')) + #name = "g++" if name.find('++') != -1 else 'gcc' + #dbg("g++: version:", name, name.find('++')) version = slntout([path, '-dumpversion']) + dbg("g++: versiondump:", version) version = re.sub(vregex, r'\1', version) - # print("gcc version:", version, "---") + dbg("gcc version:", version, "---") elif name.startswith("icpc") or name.startswith("icc"): name = "icc" if name.startswith("icc") else "icpc" if re.search(r'icpc \(ICC\) ' + vregex + '.*', version_full): version = re.sub(r'icpc \(ICC\) ' + vregex + '.*', r'\1', version_full) else: version = re.sub(r'icc \(ICC\) ' + vregex + '.*', r'\1', version_full) - # print("icc version:", version, "---") + dbg("icc version:", version, "---") elif version_full.startswith("Intel(R) oneAPI"): name = "intel" rx = r'Intel\(R\) oneAPI .*? Compiler ([0-9.]*?) \(([0-9.]*?)\).*' version = re.sub(rx, r'\1', version_full) version_full = re.sub(rx, r'\2', version_full) - #print("intel version:", version, "---", version_full) + dbg("intel version:", version, "---", version_full) else: version = slntout([path, '-dumpversion']) version = re.sub(vregex, r'\1', version) From 384a055ea505174cc1c4b99ca0ad5f2632fd0c92 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 29 Oct 2022 12:26:39 +0100 Subject: [PATCH 36/50] [fix] improve caching of cmake sysinfo with toolchains --- src/c4/cmany/architecture.py | 8 +- src/c4/cmany/args.py | 36 ++----- src/c4/cmany/build.py | 17 +-- src/c4/cmany/build_flags.py | 5 +- src/c4/cmany/build_item.py | 33 ++++-- src/c4/cmany/cmake.py | 198 +++++++++++++++++++++-------------- src/c4/cmany/compiler.py | 12 +-- src/c4/cmany/generator.py | 6 +- src/c4/cmany/project.py | 20 ++-- src/c4/cmany/system.py | 8 +- src/c4/cmany/util.py | 1 + src/c4/cmany/variant.py | 4 +- test/test04combinations.py | 2 +- test/test06xcompile.py | 6 +- 14 files changed, 193 insertions(+), 163 deletions(-) diff --git a/src/c4/cmany/architecture.py b/src/c4/cmany/architecture.py index 004bfd0..cce6d8a 100644 --- a/src/c4/cmany/architecture.py +++ b/src/c4/cmany/architecture.py @@ -10,13 +10,13 @@ class Architecture(BuildItem): """Specifies a processor architecture""" @staticmethod - def default(): + def default(toolchain_file: str=None): """return the architecture of the current machine""" - return Architecture(__class__.default_str()) + return Architecture(__class__.default_str(toolchain_file)) @staticmethod - def default_str(): - s = CMakeSysInfo.architecture() + def default_str(toolchain_file: str=None): + s = CMakeSysInfo.architecture(toolchain=toolchain_file) if s == "amd64": s = "x86_64" return s diff --git a/src/c4/cmany/args.py b/src/c4/cmany/args.py index d4c7546..c0c8c12 100644 --- a/src/c4/cmany/args.py +++ b/src/c4/cmany/args.py @@ -254,47 +254,31 @@ def add_select(parser): given either as a comma-separated list or with repeated invokations of their arguments. Commas can be escaped by using a backslash, \\.""") - # - dft = [system.System.default_str()] g.add_argument("-s", "--systems", metavar="os1,os2,...", - default=dft, action=BuildItemArgument, + action=BuildItemArgument, help="""Specify a comma-separated list of operating systems - to combine. Defaults to the current system, """ + - _item_printer(dft) + """.""") - # - dft = [architecture.Architecture.default_str()] + to combine. Defaults to the current system.""") g.add_argument("-a", "--architectures", metavar="arch1,arch2,...", - default=dft, action=BuildItemArgument, + action=BuildItemArgument, help="""Specify a comma-separated list of processor architectures to combine. Defaults to CMake's default - architecture on this system, """ + - _item_printer(dft) + """.""") - # - dft = [compiler.Compiler.default_str()] + architecture on this system.""") g.add_argument("-c", "--compilers", metavar="compiler1,compiler2,...", - default=dft, action=BuildItemArgument, + action=BuildItemArgument, help="""Specify a comma-separated list of compilers to combine. Compilers can be given as an absolute path, or as a name, in which case that name will be searched for in the current shell's PATH. Defaults to CMake's default - compiler on this system, """ + - _item_printer(dft) + """.""") - # - # dft = [build_type.BuildType.default_str()] # avoid a circular dependency - dft = ["Release"] + compiler on this system.""") g.add_argument("-t", "--build-types", metavar="type1,type2,...", - default=dft, action=BuildItemArgument, + action=BuildItemArgument, help="""Specify a comma-separated list of build types - to combine. Defaults to """ + _item_printer(dft) + """.""") - # - # dft = [variant.Variant.default_str()] # avoid a circular dependency - dft = ["none"] + to combine.""") g.add_argument("-v", "--variants", metavar="variant1,variant2,...", - default=["none"], action=BuildItemArgument, + action=BuildItemArgument, help="""Specify a comma-separated list of variants to combine. The variant name 'none' is special and will be - omitted from the name of the resulting build. Defaults to - """ + _item_printer(dft) + """.""") + omitted from the name of the resulting build. Defaults to none.""") #add_combination_flags(parser) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index 516eec1..2c2e75e 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -63,16 +63,7 @@ def __init__(self, proj_root, build_root, install_root, super().__init__(tag) # self.toolchain_file = self._get_toolchain() - if self.toolchain_file: - comps = cmake.extract_toolchain_compilers(self.toolchain_file) - if comps.get('CMAKE_CXX_COMPILER'): - c = Compiler(comps['CMAKE_CXX_COMPILER']) - else: - c = Compiler(os.environ.get('CXX')) - dbg(f"CMAKE_CXX_COMPILER not found, trying environment var CXX:", c) - self.adjust(compiler=c) - # - # WATCHOUT: this may trigger a readjustment of this build's parameters + # WATCHOUT: this may trigger a readjustment of the build's parameters self.generator = self.create_generator(num_jobs) # # This will load the vars from the builddir cache, if it exists. @@ -117,7 +108,7 @@ def _set_name_and_paths(self): self.installdir = os.path.join(self.installroot, self.installtag) self.preload_file = os.path.join(self.builddir, Build.pfile) self.cachefile = os.path.join(self.builddir, 'CMakeCache.txt') - for prop in "projdir buildroot installroot buildtag installtag builddir installdir preload_file cachefile".split(" "): + for prop in "system architecture compiler build_type variant projdir buildroot installroot buildtag installtag builddir installdir preload_file cachefile".split(" "): dbg(" {}: {}={}".format(self.tag, prop, getattr(self, prop))) return self.tag @@ -138,7 +129,7 @@ def create_generator(self, num_jobs, fallback_generator="Unix Makefiles"): if self.system.name == "windows": return Generator(fallback_generator, self, num_jobs) else: - return Generator(Generator.default_str(), self, num_jobs) + return Generator(Generator.default_str(self.toolchain_file), self, num_jobs) def adjust(self, **kwargs): for k, _ in kwargs.items(): @@ -466,9 +457,11 @@ def _get_toolchain(self): tc = BuildFlags.merge_toolchains(tc, fs.toolchain) if not tc: return None + dbg("toolchain:", tc) if not os.path.isabs(tc): tc = os.path.join(os.getcwd(), tc) tc = os.path.abspath(tc) + dbg("toolchain:", tc) if not os.path.exists(tc): raise err.ToolchainFileNotFound(tc, self) return tc diff --git a/src/c4/cmany/build_flags.py b/src/c4/cmany/build_flags.py index f9cb695..5ba152c 100644 --- a/src/c4/cmany/build_flags.py +++ b/src/c4/cmany/build_flags.py @@ -1,6 +1,7 @@ from .named_item import NamedItem as NamedItem from . import util +from .util import logdbg as dbg from . import err @@ -59,9 +60,11 @@ def log(self, log_fn=print, msg=""): @staticmethod def merge_toolchains(tc1, tc2): - if ((tc1 != tc2) and (tc1 is not None and tc2 is not None)): + dbg("merge_toolchains:", tc1, tc2) + if ((tc1 != tc2) and ((tc1 is not None) and (tc2 is not None))): raise err.Error("conflicting toolchains: {} vs {}", tc1, tc2) if tc1 is None and tc2 is not None: + dbg("picking toolchain:", tc2) tc1 = tc2 return tc1 diff --git a/src/c4/cmany/build_item.py b/src/c4/cmany/build_item.py index 6c49c3d..c78cc85 100644 --- a/src/c4/cmany/build_item.py +++ b/src/c4/cmany/build_item.py @@ -8,25 +8,40 @@ from .combination_rules import CombinationRules -# some of parsing functions below are difficult; this is a debugging scaffold +# some of the parsing functions below are difficult; this is a +# debugging scaffold _dbg_parse = False def _dbg(fmt, *args): if not _dbg_parse: return print(fmt.format(*args)) +def dbg(fmt, *args, **kwargs): + util.logdbg(fmt.format(*args, **kwargs)) + + # ----------------------------------------------------------------------------- class BuildItem(NamedItem): """A base class for build items.""" @staticmethod - def create(map_of_class_name_to_tuple_of_class_and_specs): + def create_build_items(names_and_classes, **kwargs): items = BuildItemCollection() - for cls_name, (cls, spec_list) in map_of_class_name_to_tuple_of_class_and_specs.items(): - if isinstance(spec_list, str): - spec_list = util.splitesc_quoted(spec_list, ',') - for s in spec_list: - items.add_build_item(cls(s)) + for name, cls in names_and_classes.items(): + spec_list = kwargs.get(name) + if spec_list is None: + item = cls.default(kwargs.get('toolchain')) + dbg('{}: none given; picking default {}: {}', name, cls.__name__, item) + items.add_build_item(item) + else: + if isinstance(spec_list, str): + splitted = util.splitesc_quoted(spec_list, ',') + dbg('{}: str given; splitting: {} -> {}', name, spec_list, splitted) + spec_list = splitted + for s in spec_list: + dbg('{}: adding {}', name, s) + item = cls(s) + items.add_build_item(item) items.resolve_references() return items @@ -177,7 +192,7 @@ def parse_args(v_): if rest: vli.append(rest) _dbg("parse_args 3.2.3: rest={} vli={}", rest, vli) - if _dbg_parse: print("parse_args 4: vli=", vli) + _dbg("parse_args 4: vli=", vli) # unquote split elements vli = [util.unquote(v).strip(',') for v in vli] _dbg("parse_args 5: input=____{}____ output=__{}__", v_, vli) @@ -240,7 +255,7 @@ def __init__(self, *args, **kwargs): def __eq__(self, other): """code quality checkers complain that this class adds attributes without overriding __eq__. So just fool them!""" - return super().__init__(other) + return super().__eq__(other) def add_build_item(self, item): # convert the class name to snake case and append s for plural diff --git a/src/c4/cmany/cmake.py b/src/c4/cmany/cmake.py index 7c0a26b..1e38d35 100644 --- a/src/c4/cmany/cmake.py +++ b/src/c4/cmany/cmake.py @@ -111,7 +111,7 @@ def __init__(self, builddir=None): def __eq__(self, other): """code quality checkers complain that this class adds attributes without overriding __eq__. So just fool them!""" - return super().__init__(other) + return super().__eq__(other) def getvars(self, names): out = odict() @@ -247,41 +247,43 @@ class CMakeSysInfo: generator, etc.""" @staticmethod - def generator(): + def generator(toolchain=None): return cacheattr(__class__, '_generator_default', - lambda: __class__._getstr('CMAKE_GENERATOR', 'default')) + lambda: __class__._getstr('CMAKE_GENERATOR', 'default', toolchain)) @staticmethod - def system_name(which_generator="default"): - return __class__.var('CMAKE_SYSTEM_NAME', which_generator, lambda v: v.lower()) + def system_name(generator="default", toolchain=None): + return __class__.var('CMAKE_SYSTEM_NAME', generator, toolchain, + lambda v: v.lower()) @staticmethod - def architecture(which_generator="default"): - return __class__.var('CMAKE_SYSTEM_PROCESSOR', which_generator, lambda v: v.lower()) + def architecture(generator="default", toolchain=None): + return __class__.var('CMAKE_SYSTEM_PROCESSOR', generator, toolchain, + lambda v: v.lower()) @staticmethod - def cxx_compiler(which_generator="default"): - return __class__.var('CMAKE_CXX_COMPILER', which_generator) + def cxx_compiler(generator="default", toolchain=None): + return __class__.var('CMAKE_CXX_COMPILER', generator, toolchain) @staticmethod - def c_compiler(which_generator="default"): - return __class__.var('CMAKE_C_COMPILER', which_generator) + def c_compiler(generator="default", toolchain=None): + return __class__.var('CMAKE_C_COMPILER', generator, toolchain) @staticmethod - def var(var_name, which_generator="default", transform_fn=lambda x: x): - gs = __class__._getstr - return cacheattr(__class__, '_{}_{}'.format(var_name, _genid(which_generator)), - lambda: transform_fn(gs(var_name, which_generator))) + def var(var_name, generator="default", toolchain=None, transform_fn=lambda x: x): + attrname = '_{}_{}'.format(var_name, _gentc_id(generator, toolchain)) + return cacheattr(__class__, attrname, + lambda: transform_fn(__class__._getstr(var_name, generator, toolchain))) @staticmethod - def info(which_generator="default"): - return cacheattr(__class__, '_info_' + _genid(which_generator), - lambda: __class__.system_info(which_generator)) + def info(generator="default", toolchain=None): + return cacheattr(__class__, '_info_' + _gentc_id(generator, toolchain), + lambda: __class__.system_info(generator, toolchain)) @staticmethod - def _getstr(var_name, which_generator): + def _getstr(var_name, generator, toolchain): regex = r'^{} "(.*)"'.format(var_name) - for l in __class__.info(which_generator): + for l in __class__.info(generator, toolchain): #dbg(l.strip("\n"), l.startswith(var_name), var_name) if l.startswith(var_name): l = l.strip("\n").lstrip(" ").rstrip(" ") @@ -290,64 +292,78 @@ def _getstr(var_name, which_generator): s = re.sub(regex, r'\1', l) #dbg(var_name, "result: '" + s + "'") return s - #dbg("--------------------------------------\n", __class__.info(which_generator)) - msg = "could not find variable {} in the output of `cmake --system-information -G '{}'`" - raise err.Error(msg, var_name, which_generator) + #dbg("--------------------------------------\n", __class__.info(generator)) + raise err.Error("could not find variable {} in the output of `cmake --system-information -G '{}'`", + var_name, generator) @staticmethod - def system_info(gen): + def system_info(gen, toolchain_file=None): """gen can be a string or a cmany.Generator object""" - from .generator import Generator - dbg("CMakeSystemInfo: asked info for", gen) - p = _genid(gen) - d = os.path.join(USER_DIR, 'cmake_info', p) - p = os.path.join(d, 'info') + dbg("CMakeSystemInfo: asked info for", gen, + (("toolchain="+toolchain_file) if toolchain_file is not None else "")) + toolchain_args = [] + if toolchain_file is not None: + if not os.path.exists(toolchain_file): + raise err.ToolchainFileNotFound(toolchain_file) + toolchain_args = ['--toolchain', toolchain_file] + d = _geninfodir(gen, toolchain_file) + p = os.path.join(d, 'cmake_system_information') + dbg("CMakeSystemInfo: dir=", d) dbg("CMakeSystemInfo: path=", p) - # https://stackoverflow.com/questions/7015587/python-difference-of-2-datetimes-in-months - if os.path.exists(p) and util.time_since_modification(p).months < 1: - dbg("CMakeSystemInfo: asked info for", gen, "... found", p) - with open(p, "r") as f: - i = f.readlines() - if i: - return i - else: - dbg("CMakeSystemInfo: info for gen", gen, "is empty...") - # - if isinstance(gen, Generator): - cmd = ['cmake'] + gen.configure_args() + ['--system-information'] - dbg("CMakeSystemInfo: from generator! '{}' ---> cmd={}".format(gen, cmd)) - else: + if os.path.exists(p): + dbg('CMakeSystemInfo: found at', p) + if util.time_since_modification(p).months >= 1: + dbg("CMakeSystemInfo: older than 1 month. Refreshing", p) + else: + dbg("CMakeSystemInfo: less than 1 month. Choosing", p) + return _getnfo(p) + gen_args = [] + if isinstance(gen, str): if gen == "default" or gen == "": dbg("CMakeSystemInfo: default! '{}'".format(gen)) - cmd = ['cmake', '--system-information'] else: dbg("CMakeSystemInfo: assume vs! '{}'".format(gen)) from . import vsinfo gen = vsinfo.to_gen(gen) if isinstance(gen, list): - cmd = ['cmake', '-G'] + gen + ['--system-information'] + gen_args = ['-G'] + gen else: if not (gen.startswith('vs') or gen.startswith('Visual Studio')): raise Exception("unknown generator: {}".format(gen)) - cmd = ['cmake', '-G', gen, '--system-information'] + gen_args = ['-G', gen] + else: + gen_args = gen.configure_args() + dbg("CMakeSystemInfo: from generator! '{}' ---> {}".format(gen, gen_args)) + cmd = ['cmake'] + toolchain_args + gen_args + ['--system-information', os.path.basename(p)] + dbg("CMakeSystemInfo: cmd={}".format(cmd)) # remove export build commands as cmake reacts badly to it, # generating an empty info string _remove_invalid_args_from_sysinfo_cmd(cmd) + dbg("CMakeSystemInfo: filtered cmd={}".format(cmd)) print("\ncmany: CMake information for generator '{}' was not found. Creating and storing... cmd={}".format(gen, cmd)) # if not os.path.exists(d): os.makedirs(d) - with setcwd(d): - out = runsyscmd(cmd, echo_output=False, capture_output=True) - dbg("cmany: finished generating information for generator '{}'\n".format(gen), out, cmd) - out = out.strip() + with util.setcwd(d): + runsyscmd(cmd) + out = _getnfo(p) + dbg("cmany: finished generating information for generator '{}', cmd={}. Info=\n{}".format(gen, cmd, '\n'.join(out))) if not out: from .err import InvalidGenerator raise InvalidGenerator(gen, "for --system-information. cmd='{}'".format(cmd)) - with open(p, "w") as f: - f.write(out) - i = out.split("\n") - return i + return out + + +def _getnfo(p): + dbg("CMakeSystemInfo: loading=", p) + if not os.path.exists(p): + raise FileNotFoundError(p) + with open(p, "r") as f: + lines = f.readlines() + if not lines: + dbg("CMakeSystemInfo: info for gen", gen, "is empty...", p) + lines = [l.strip() for l in lines] + return lines def _remove_invalid_args_from_sysinfo_cmd(cmd): @@ -372,40 +388,62 @@ def _remove_invalid_args_from_sysinfo_cmd(cmd): # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- + + +def _geninfodir(gen, toolchain_file: str): + dbg('cmakeinfo USER_DIR=', USER_DIR) + if toolchain_file is None: + base = USER_DIR + else: + id = _toolchainid(toolchain_file) + base = os.path.join(USER_DIR, 'toolchains', id) + dbg('cmakeinfo toolchain=', toolchain_file) + dbg('cmakeinfo toolchain_id=', id) + dbg('cmakeinfo toolchain_base=', base) + id = _genid(gen) + d = os.path.join(base, 'cmake_info', id) + dbg('cmakeinfo base=', base) + dbg('cmakeinfo gen_id=', id) + dbg('cmakeinfo infodir=', d) + return d + + +def _toolchaindir(toolchain_file: str): + id = _toolchainid(toolchain_file) + base = os.path.join(USER_DIR, 'toolchains', id) + dbg('cmakeinfo USER_DIR=', USER_DIR) + return base + + +def _gentc_id(gen, toolchain): + if toolchain is None: + return _genid(gen) + else: + return _toolchainid(toolchain) + '_' + _genid(gen) + + def _genid(gen): - from .generator import Generator - p = gen.sysinfo_name if isinstance(gen, Generator) else gen - if isinstance(gen, list): p = " ".join(p) + if isinstance(gen, str): + p = gen + elif isinstance(gen, list): + p = " ".join(p) + else: + from .generator import Generator + p = gen.sysinfo_name p = re.sub(r'[() ]', '_', p) return p +def _toolchainid(toolchain: str): + id = re.sub(os.sep, '_', toolchain) + id = re.sub(r'[() +.]', '_', id) + return id + + + # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- -def get_toolchain_cache(toolchain): - d = os.path.join(USER_DIR, 'toolchains', re.sub(os.sep, '_', toolchain)) - dbg("toolchain cache: USER_DIR=", USER_DIR) - dbg("toolchain cache: d=", d) - bd = os.path.join(d, 'build') - dbg("toolchain cache: bd=", bd) - if not os.path.exists(d): - os.makedirs(d) - with setcwd(d): - with open('main.cpp', 'w') as f: - f.write("int main() {}") - with open('CMakeLists.txt', 'w') as f: - f.write(""" -cmake_minimum_required(VERSION 2.6) -project(toolchain_test) -add_executable(main main.cpp) -""") - if not os.path.exists(bd): - os.makedirs(bd) - with setcwd(bd): - cmd = ['cmake', '-DCMAKE_TOOLCHAIN_FILE='+toolchain, '..'] - runsyscmd(cmd, echo_output=True) - return loadvars(bd) def extract_toolchain_compilers(toolchain): diff --git a/src/c4/cmany/compiler.py b/src/c4/cmany/compiler.py index 47da206..ae31458 100644 --- a/src/c4/cmany/compiler.py +++ b/src/c4/cmany/compiler.py @@ -17,16 +17,16 @@ class Compiler(BuildItem): """Represents a compiler choice""" @staticmethod - def default(): - return Compiler(__class__.default_str()) + def default(toolchain_file: str=None): + return Compiler(__class__.default_str(toolchain_file)) @staticmethod - def default_str(): - if str(System.default()) != "windows": - cpp = CMakeSysInfo.cxx_compiler() + def default_str(toolchain_file: str=None): + if str(System.default(toolchain_file)) != "windows": + cpp = CMakeSysInfo.cxx_compiler(toolchain=toolchain_file) else: vs = vsinfo.find_any() - cpp = vs.name if vs is not None else CMakeSysInfo.cxx_compiler() + cpp = vs.name if vs is not None else CMakeSysInfo.cxx_compiler(toolchain_file) return cpp def is_trivial(self): diff --git a/src/c4/cmany/generator.py b/src/c4/cmany/generator.py index de983c5..06b7921 100644 --- a/src/c4/cmany/generator.py +++ b/src/c4/cmany/generator.py @@ -61,7 +61,7 @@ """ # ----------------------------------------------------------------------------- -class Generator(BuildItem): +class Generator(BuildItem): # NamedItem """Visual Studio aliases example: vs2013: use the bitness of the current system @@ -70,9 +70,9 @@ class Generator(BuildItem): """ @staticmethod - def default_str(): + def default_str(toolchain_file: str=None): """get the default generator from cmake""" - s = cmake.CMakeSysInfo.generator() + s = cmake.CMakeSysInfo.generator(toolchain_file) return s def __init__(self, name, build, num_jobs): diff --git a/src/c4/cmany/project.py b/src/c4/cmany/project.py index 1a0d207..63c9433 100644 --- a/src/c4/cmany/project.py +++ b/src/c4/cmany/project.py @@ -153,7 +153,7 @@ def _init_with_glob(self, **kwargs): self.builds.append(build) def _init_with_build_items(self, **kwargs): - s, a, c, t, v = __class__.get_build_items(**kwargs) + s, a, c, t, v = __class__.create_build_items(**kwargs) # cr = CombinationRules(kwargs.get('combination_rules', [])) combs = cr.valid_combinations(s, a, c, t, v) @@ -187,16 +187,14 @@ def _addnew(b, name): _addnew(b, 'variant') @staticmethod - def get_build_items(**kwargs): - d = odict() - for c, cls in ( - ('systems', System), - ('architectures', Architecture), - ('compilers', Compiler), - ('build_types', BuildType), - ('variants', Variant)): - d[c] = (cls, kwargs.get(c)) - coll = BuildItem.create(d) + def create_build_items(**kwargs): + coll = BuildItem.create_build_items({ + 'systems': System, + 'architectures': Architecture, + 'compilers': Compiler, + 'build_types': BuildType, + 'variants': Variant, + }, **kwargs) s = coll['systems'] a = coll['architectures'] c = coll['compilers'] diff --git a/src/c4/cmany/system.py b/src/c4/cmany/system.py index 9e5c0f9..9344309 100644 --- a/src/c4/cmany/system.py +++ b/src/c4/cmany/system.py @@ -7,13 +7,13 @@ class System(BuildItem): """Specifies an operating system""" @staticmethod - def default(): + def default(toolchain_file: str=None): """return the current operating system""" - return System(__class__.default_str()) + return System(__class__.default_str(toolchain_file)) @staticmethod - def default_str(): - s = CMakeSysInfo.system_name() + def default_str(toolchain_file: str=None): + s = CMakeSysInfo.system_name(toolchain=toolchain_file) if s == "mac os x" or s == "Darwin": s = "mac" return s diff --git a/src/c4/cmany/util.py b/src/c4/cmany/util.py index 50f5b25..982cb8f 100644 --- a/src/c4/cmany/util.py +++ b/src/c4/cmany/util.py @@ -497,6 +497,7 @@ def __exit__(self, exc_type, exc_value, traceback): def time_since_modification(path): """return the time elapsed since a path has been last modified, as a dateutil.relativedelta""" + # https://stackoverflow.com/questions/7015587/python-difference-of-2-datetimes-in-months mtime = os.path.getmtime(path) mtime = datetime.datetime.fromtimestamp(mtime) currt = datetime.datetime.now() diff --git a/src/c4/cmany/variant.py b/src/c4/cmany/variant.py index a4386ba..d955d18 100644 --- a/src/c4/cmany/variant.py +++ b/src/c4/cmany/variant.py @@ -8,11 +8,11 @@ class Variant(BuildItem): """for variations in build flags""" @staticmethod - def default_str(): + def default_str(toolchain_file: str=None): return 'none' @staticmethod - def default(): + def default(toolchain_file: str=None): return Variant(__class__.default_str()) @staticmethod diff --git a/test/test04combinations.py b/test/test04combinations.py index 5535ac8..d1148e3 100644 --- a/test/test04combinations.py +++ b/test/test04combinations.py @@ -54,7 +54,7 @@ def do_combination_t(test, input, expected_items, expected_combinations): ) for s, a, c, t, v in expected_combinations] # actual combinations - s, a, c, t, v = cmany.Project.get_build_items(**vars(args)) + s, a, c, t, v = cmany.Project.create_build_items(**vars(args)) cr = [] if hasattr(args, 'combination_rules'): cr = args.combination_rules diff --git a/test/test06xcompile.py b/test/test06xcompile.py index 7790298..c329047 100644 --- a/test/test06xcompile.py +++ b/test/test06xcompile.py @@ -21,8 +21,7 @@ def toolchain_compiler_exists(tc_file): - comps = cmake.extract_toolchain_compilers(tc_file) - return osp.exists(comps['CMAKE_CXX_COMPILER']) + return osp.exists(cmake.cxx_compiler(toolchain=tc_file)) def run_with_args(testdir, args_in): @@ -50,8 +49,7 @@ def do_toolchain_builds(toolchain, test, args, expected_builds): if not toolchain_compiler_exists(toolchain): return args = [a.format(toolchain=toolchain) for a in args] - comps = cmake.extract_toolchain_compilers(toolchain) - c = Compiler(comps['CMAKE_CXX_COMPILER']) + c = cmake.cxx_compiler(toolchain=toolchain) expected_builds = [b.format(compiler=Build.sanitize_compiler_name(c.name)) for b in expected_builds] for t in testdirs: From 8a41178c66a60a762b81b1fb55b8cea1a5cc9988 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Wed, 30 Nov 2022 16:06:15 +0000 Subject: [PATCH 37/50] [fix] allow args to BuildType ctor --- src/c4/cmany/build_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c4/cmany/build_type.py b/src/c4/cmany/build_type.py index cd9d86b..4591862 100644 --- a/src/c4/cmany/build_type.py +++ b/src/c4/cmany/build_type.py @@ -10,5 +10,5 @@ def default_str(): return "Release" @staticmethod - def default(): + def default(*kwargs): return BuildType(__class__.default_str()) From 600887f064e63d8f9ab72dc2128fbd164e384320 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Wed, 14 Dec 2022 10:42:44 +0000 Subject: [PATCH 38/50] [fix] fix VS generation --- src/c4/cmany/cmake.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/c4/cmany/cmake.py b/src/c4/cmany/cmake.py index 1e38d35..448bfb5 100644 --- a/src/c4/cmany/cmake.py +++ b/src/c4/cmany/cmake.py @@ -318,22 +318,22 @@ def system_info(gen, toolchain_file=None): dbg("CMakeSystemInfo: less than 1 month. Choosing", p) return _getnfo(p) gen_args = [] - if isinstance(gen, str): - if gen == "default" or gen == "": - dbg("CMakeSystemInfo: default! '{}'".format(gen)) - else: - dbg("CMakeSystemInfo: assume vs! '{}'".format(gen)) - from . import vsinfo - gen = vsinfo.to_gen(gen) - if isinstance(gen, list): - gen_args = ['-G'] + gen - else: - if not (gen.startswith('vs') or gen.startswith('Visual Studio')): - raise Exception("unknown generator: {}".format(gen)) - gen_args = ['-G', gen] - else: + from . import generator + if isinstance(gen, str) and (gen == "default" or gen == ""): + dbg("CMakeSystemInfo: default! '{}'".format(gen)) + elif isinstance(gen, generator.Generator): gen_args = gen.configure_args() dbg("CMakeSystemInfo: from generator! '{}' ---> {}".format(gen, gen_args)) + else: + dbg("CMakeSystemInfo: assume vs! '{}'".format(gen)) + from . import vsinfo + gen = vsinfo.to_gen(gen) + if isinstance(gen, list): + gen_args = ['-G'] + gen + else: + if not (gen.startswith('vs') or gen.startswith('Visual Studio')): + raise Exception("unknown generator: {}".format(gen)) + gen_args = ['-G', gen] cmd = ['cmake'] + toolchain_args + gen_args + ['--system-information', os.path.basename(p)] dbg("CMakeSystemInfo: cmd={}".format(cmd)) # remove export build commands as cmake reacts badly to it, @@ -426,7 +426,7 @@ def _genid(gen): if isinstance(gen, str): p = gen elif isinstance(gen, list): - p = " ".join(p) + p = " ".join(gen) else: from .generator import Generator p = gen.sysinfo_name From 91c9fc7b3d84aa95a952a397f3a91b56003ce592 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Fri, 10 Mar 2023 18:55:29 +0100 Subject: [PATCH 39/50] [fix] fix compiler heuristics for clang++ and g++ --- src/c4/cmany/compiler.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/c4/cmany/compiler.py b/src/c4/cmany/compiler.py index ae31458..51c6855 100644 --- a/src/c4/cmany/compiler.py +++ b/src/c4/cmany/compiler.py @@ -9,7 +9,11 @@ from . import util from . import vsinfo from . import err -from .util import logdbg as dbg +from .util import logdbg + + +def dbg(*args, **kwargs): + logdbg("compiler:", *args, **kwargs) # ----------------------------------------------------------------------------- @@ -67,7 +71,7 @@ def __init__(self, spec): dbg("no compiler found", path) raise err.CompilerNotFound(path) if p != path: - dbg("compiler: selected {} for {}".format(p, path)) + dbg("selected {} for {}".format(p, path)) if isinstance(path, str): path = os.path.abspath(p) else: @@ -108,12 +112,12 @@ def get_c_compiler(shortname, cxx_compiler): cc = re.sub(r'icpc', r'icc', cxx_compiler) elif shortname == "icpx": cc = re.sub(r'icpx', r'icpx', cxx_compiler) - elif shortname == "gcc" or shortname == "g++": + elif shortname == "gcc" or shortname == "g++" or shortname.startswith("g++"): if re.search(r'g\+\+', cxx_compiler): cc = re.sub(r'g\+\+', r'gcc', cxx_compiler) else: cc = re.sub(r'c\+\+', r'cc', cxx_compiler) - elif shortname == "clang" or shortname == "clang++": + elif shortname == "clang" or shortname == "clang++" or shortname.startswith("clang++"): if re.search(r'clang\+\+', cxx_compiler): cc = re.sub(r'clang\+\+', r'clang', cxx_compiler) else: @@ -133,7 +137,7 @@ def get_version(self, path): if hasattr(self, "vs"): return self.vs.name, str(self.vs.year), self.vs.name # other compilers - dbg("cmp: found compiler:", path) + dbg("found compiler:", path) if isinstance(path, str): out = slntout([path, '--version']) else: @@ -141,10 +145,11 @@ def get_version(self, path): version_full = out.split("\n")[0] splits = version_full.split(" ") name = splits[0].lower() - dbg("cmp: version:", name, "---", version_full, "---") + dbg("version:", name, "---", version_full, "---") vregex = r'(\d+\.\d+)\.\d+' base = os.path.basename(path) - dbg(name, "cmp base:", base, name) + dbg("base:", base) + dbg("name:", name) if base.startswith("c++") or base.startswith("cc"): try: # if this fails, just go on. It's not really needed. with tempfile.NamedTemporaryFile(suffix=".cpp", prefix="cmany.", delete=False) as f: @@ -160,21 +165,33 @@ def get_version(self, path): elif re.search("#define __GNUC__", m): name = "g++" if re.search(r"\+\+", path) else "gcc" break + elif version_full.startswith("Ubuntu clang version"): + name = "clang++" + # not elif! if name.startswith("clang++") or name.startswith("clang") or name.endswith("clang++") or name.endswith("clang"): name = "clang++" if path.find('clang++') != -1 else 'clang' if re.search('Apple LLVM', version_full): name = "apple_llvm" version = re.sub(r'Apple LLVM version ' + vregex + '.*', r'\1', version_full) dbg("apple_llvm version:", version, "---") + elif version_full.startswith("Ubuntu clang version"): + name = "clang++" + version = re.sub('Ubuntu clang version ' + vregex + '.*', r'\1', version_full) else: version = re.sub(r'clang version ' + vregex + '.*', r'\1', version_full) - dbg("clang version:", version, "---") + dbg("clang version:", version, "---") elif name.startswith("g++") or name.startswith("gcc") or name.endswith("g++") or name.endswith("gcc"): - dbg("g++: version:", name, name.find('++')) + dbg("g++: name=", name) + if (name.startswith("g++") and name != "g++"): + name = "g++" + if (name.startswith("gcc") and name != "gcc"): + name = "gcc" + dbg("g++: name=", name) #name = "g++" if name.find('++') != -1 else 'gcc' #dbg("g++: version:", name, name.find('++')) version = slntout([path, '-dumpversion']) dbg("g++: versiondump:", version) + dbg("g++: versionfull:", version_full) version = re.sub(vregex, r'\1', version) dbg("gcc version:", version, "---") elif name.startswith("icpc") or name.startswith("icc"): From b78e3e8cabe9d8637d994bdbccffa0a5945565d6 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Fri, 10 Mar 2023 19:21:58 +0100 Subject: [PATCH 40/50] [fix] detect when VS is not found --- src/c4/cmany/compiler.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/c4/cmany/compiler.py b/src/c4/cmany/compiler.py index 51c6855..a31b382 100644 --- a/src/c4/cmany/compiler.py +++ b/src/c4/cmany/compiler.py @@ -10,6 +10,7 @@ from . import vsinfo from . import err from .util import logdbg +from .util import logwarn def dbg(*args, **kwargs): @@ -30,7 +31,12 @@ def default_str(toolchain_file: str=None): cpp = CMakeSysInfo.cxx_compiler(toolchain=toolchain_file) else: vs = vsinfo.find_any() - cpp = vs.name if vs is not None else CMakeSysInfo.cxx_compiler(toolchain_file) + if vs is not None: + dbg("found visual studio:", vs.name) + cpp = vs.name + else: + logwarn("visual studio not found, using compiler from cmake --system-information, toolchain=", toolchain_file) + cpp = CMakeSysInfo.cxx_compiler(toolchain=toolchain_file) return cpp def is_trivial(self): @@ -59,6 +65,8 @@ def __init__(self, spec): if (util.in_windows() and len(path) == 1 and len(spl) > 1 and (spl[1][0] == '/' or spl[1][0] == '\\')): path = spec spl = [path] + if (util.in_windows() and path.endswith("cl.exe")): + raise err.VSNotFound(spec, msg="cannot use cl.exe directly") p = util.which(path) if p is None: dbg("not found:", path) From 64c28dd49ea3e30752a3be77ed391c12cc0b901b Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Fri, 10 Mar 2023 19:22:23 +0100 Subject: [PATCH 41/50] [chore] default to vs2022 --- src/c4/cmany/vsinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c4/cmany/vsinfo.py b/src/c4/cmany/vsinfo.py index a82d0ba..62b0dc5 100644 --- a/src/c4/cmany/vsinfo.py +++ b/src/c4/cmany/vsinfo.py @@ -212,7 +212,7 @@ def get_cmd_line(self, filename): # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # vs search order -order = ('vs2019', 'vs2017', 'vs2015', 'vs2013', 'vs2012', 'vs2010', +order = ('vs2022', 'vs2019', 'vs2017', 'vs2015', 'vs2013', 'vs2012', 'vs2010', #'vs2008', 'vs2005', ) From 44c8532a53f473023dde3313f08a241001caa1ec Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 14 Mar 2023 20:24:17 +0100 Subject: [PATCH 42/50] build type: be more specific with creating the type --- src/c4/cmany/build_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/c4/cmany/build_type.py b/src/c4/cmany/build_type.py index 4591862..a6b4bd1 100644 --- a/src/c4/cmany/build_type.py +++ b/src/c4/cmany/build_type.py @@ -10,5 +10,5 @@ def default_str(): return "Release" @staticmethod - def default(*kwargs): + def default(toolchain: str=None): return BuildType(__class__.default_str()) From 0bf8bece6e963399c3a0702f084063bd5d03d523 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Wed, 15 Mar 2023 00:23:42 +0100 Subject: [PATCH 43/50] add warning about flag arguments --- src/c4/cmany/args.py | 12 ++++++++++-- src/c4/cmany/main.py | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/c4/cmany/args.py b/src/c4/cmany/args.py index c0c8c12..4149140 100644 --- a/src/c4/cmany/args.py +++ b/src/c4/cmany/args.py @@ -372,6 +372,14 @@ def add_item_combination_flags(parser): following variants.""") +def dash_warning(flag): + return """To prevent flags/arguments starting with dash (-) or + double-dash (--) from being expanded by the shell and then + understood as a cmany flag, enclose them in quotes with leading + whitespace. Eg, instead of {} '--flag' use {} ' --flag', or + instead of {} '-f' use {} ' -f'.""".format(flag, flag, flag, flag) + + def add_cflags(parser): g = parser.add_argument_group( 'Flags: CMake cache variables, compiler flags and defines', @@ -405,7 +413,7 @@ def add_cflags(parser): to get help about this. Multiple invokations of -X are possible, in which case arguments will be appended and not overwritten. - To escape commas, use a backslash \\.""") + To escape commas, use a backslash \\. """ + dash_warning("-X")) g.add_argument("-C", "--cflags", default=[], action=FlagArgument, help="""Add C compiler flags. Accepts a comma-separated list of C compiler flags. @@ -416,7 +424,7 @@ def add_cflags(parser): to get help about this. Multiple invokations of -X are possible, in which case arguments will be appended and not overwritten. - To escape commas, use a backslash \\.""") + To escape commas, use a backslash \\. """ + dash_warning("-C")) # g.add_argument("-I", "--include-dirs", default=[], action=FlagArgument, # help="""add dirs to the include path of all builds # Multiple invokations of -I are possible, in which case arguments will be appended and not overwritten. diff --git a/src/c4/cmany/main.py b/src/c4/cmany/main.py index a01f16d..3d32207 100644 --- a/src/c4/cmany/main.py +++ b/src/c4/cmany/main.py @@ -236,7 +236,7 @@ def add_args(self, parser): parser.add_argument('-ta', '--target-args', type=str, default="", help="""arguments to pass to the target invokation. If multiple targets are given, arguments are passed to every target. The - arguments need to be given as a single string.""") + arguments need to be given as a single string. """ + c4args.dash_warning("-ta")) parser.add_argument('-np', '--no-posix', action="store_true", help="""do not use posix mode when splitting the target arguments""") parser.add_argument('-wd', '--work-dir', type=str, default=None, @@ -244,7 +244,7 @@ def add_args(self, parser): parser.add_argument('-wp', '--wrap', type=str, default=None, help="""wrap the target invokation with the given command, eg valgrind or gdb. A string with spaces can be passed, in which - case it will be split before the command is formed.""") + case it will be split to form the invokation command.""") def _exec(self, proj, args): import shlex target_args = shlex.split(args.target_args, posix=not args.no_posix) From 18b60c0bd8290750d953131b67d33e1bc5e8ce56 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Wed, 15 Mar 2023 00:23:34 +0100 Subject: [PATCH 44/50] fix tests --- src/c4/cmany/cmake.py | 25 +++------- src/c4/cmany/project.py | 4 +- src/c4/cmany/variant.py | 4 +- test/hello/CMakeLists.txt | 2 +- test/test00util.py | 12 +++-- test/test02flags.py | 28 ++++++------ test/test03variant.py | 17 +++---- test/test04combinations.py | 13 +++++- test/test05cmany.py | 18 ++++---- test/test06xcompile.py | 5 +- test/test07config.py | 94 +++++++++++++++++--------------------- 11 files changed, 107 insertions(+), 115 deletions(-) diff --git a/src/c4/cmany/cmake.py b/src/c4/cmany/cmake.py index 448bfb5..bcf615c 100644 --- a/src/c4/cmany/cmake.py +++ b/src/c4/cmany/cmake.py @@ -319,7 +319,9 @@ def system_info(gen, toolchain_file=None): return _getnfo(p) gen_args = [] from . import generator - if isinstance(gen, str) and (gen == "default" or gen == ""): + if gen is None: + dbg("CMakeSystemInfo: gen is None! picking default'") + elif isinstance(gen, str) and (gen == "default" or gen == ""): dbg("CMakeSystemInfo: default! '{}'".format(gen)) elif isinstance(gen, generator.Generator): gen_args = gen.configure_args() @@ -423,7 +425,9 @@ def _gentc_id(gen, toolchain): def _genid(gen): - if isinstance(gen, str): + if gen is None: + return "None" + elif isinstance(gen, str): p = gen elif isinstance(gen, list): p = " ".join(gen) @@ -438,20 +442,3 @@ def _toolchainid(toolchain: str): id = re.sub(os.sep, '_', toolchain) id = re.sub(r'[() +.]', '_', id) return id - - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- - - -def extract_toolchain_compilers(toolchain): - cache = get_toolchain_cache(toolchain) - out = odict() - for k, v in cache.items(): - dbg(f"toolchain cache: {k}=={v}") - if k.startswith('CMAKE_') and k.endswith('_COMPILER'): - dbg(f"toolchain cache: .................... {k}=={v}") - out[k] = v.val - return out diff --git a/src/c4/cmany/project.py b/src/c4/cmany/project.py index 63c9433..4df99cf 100644 --- a/src/c4/cmany/project.py +++ b/src/c4/cmany/project.py @@ -97,8 +97,8 @@ def __init__(self, **kwargs): cmakecache = None if _pexists(self.cmakelists): dbg("found CMakeLists.txt:", self.cmakelists) - self.build_dir = _getdir('build_dir', 'build', kwargs, cwd) - self.install_dir = _getdir('install_dir', 'install', kwargs, cwd) + self.build_dir = _getdir('build_root', 'build', kwargs, cwd) + self.install_dir = _getdir('install_root', 'install', kwargs, cwd) self.root_dir = pdir elif _pexists(pdir, "CMakeCache.txt"): cmakecache = os.path.join(pdir, "CMakeCache.txt") diff --git a/src/c4/cmany/variant.py b/src/c4/cmany/variant.py index d955d18..3719894 100644 --- a/src/c4/cmany/variant.py +++ b/src/c4/cmany/variant.py @@ -17,7 +17,7 @@ def default(toolchain_file: str=None): @staticmethod def create_variants(spec_list): - d = odict([('variants', (Variant, spec_list))]) - d = BuildItem.create(d) + name_and_class = odict([('variants', Variant)]) + d = BuildItem.create_build_items(name_and_class, variants=spec_list) assert d.collections == ['variants'] return d['variants'] diff --git a/test/hello/CMakeLists.txt b/test/hello/CMakeLists.txt index 8cb2ddb..6e3e6af 100644 --- a/test/hello/CMakeLists.txt +++ b/test/hello/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 2.12) project(hello) add_executable(hello main.cpp) diff --git a/test/test00util.py b/test/test00util.py index 5483b72..0cf33fa 100644 --- a/test/test00util.py +++ b/test/test00util.py @@ -15,8 +15,8 @@ class Test00splitesc(ut.TestCase): def test(self): self.assertEqual( - util.splitesc('hello\,world,yet\,another', ','), - ['hello\,world', 'yet\,another']) + util.splitesc(r'hello\,world,yet\,another', ','), + [r'hello\,world', r'yet\,another']) # ----------------------------------------------------------------------------- @@ -258,8 +258,12 @@ class Test09nested_merge(ut.TestCase): def test00reqs(self): import collections - self.assertTrue(isinstance(dict(), collections.Mapping)) - self.assertTrue(isinstance(collections.OrderedDict(), collections.Mapping)) + if sys.version_info.major == 3 and sys.version_info.minor >= 10: + from collections.abc import Mapping as CollectionsMapping + else: + from collections import Mapping as CollectionsMapping + self.assertTrue(isinstance(dict(), CollectionsMapping)) + self.assertTrue(isinstance(collections.OrderedDict(), CollectionsMapping)) def test01do_it(self): for fname in ('tc1', 'tc2', 'tc3'): diff --git a/test/test02flags.py b/test/test02flags.py index b48a74a..dd17a8f 100644 --- a/test/test02flags.py +++ b/test/test02flags.py @@ -177,18 +177,18 @@ def _c(a, r): self.check_one(n, o, a, r) _c('{} VAR2,VAR3', ['VAR2', 'VAR3']) _c('{} VAR4 {} VAR5', ['VAR4', 'VAR5']) _c('{} VAR6,VAR7 {} VAR8,VAR9', ['VAR6', 'VAR7', 'VAR8', 'VAR9']) - _c('{} VAR\,1', ['VAR,1']) - _c('{} VAR\,2,VAR\,3', ['VAR,2', 'VAR,3']) - _c('{} VAR\,4 {} VAR\,5', ['VAR,4', 'VAR,5']) - _c('{} VAR\,6,VAR\,7 {} VAR\,8,VAR\,9', ['VAR,6', 'VAR,7', 'VAR,8', 'VAR,9']) + _c(r'{} VAR\,1', ['VAR,1']) + _c(r'{} VAR\,2,VAR\,3', ['VAR,2', 'VAR,3']) + _c(r'{} VAR\,4 {} VAR\,5', ['VAR,4', 'VAR,5']) + _c(r'{} VAR\,6,VAR\,7 {} VAR\,8,VAR\,9', ['VAR,6', 'VAR,7', 'VAR,8', 'VAR,9']) _c('{} VAR1=1', ['VAR1=1']) _c('{} VAR2=2,VAR3=3', ['VAR2=2', 'VAR3=3']) _c('{} VAR4=4 {} VAR5=5', ['VAR4=4', 'VAR5=5']) _c('{} VAR6=6,VAR7=7 {} VAR8=8,VAR9=9', ['VAR6=6', 'VAR7=7', 'VAR8=8', 'VAR9=9']) - _c('{} VAR1=1\,a', ['VAR1=1,a']) - _c('{} VAR2=2\,a,VAR3=3\,a', ['VAR2=2,a', 'VAR3=3,a']) - _c('{} VAR4=4\,a {} VAR5=5\,a', ['VAR4=4,a', 'VAR5=5,a']) - _c('{} VAR6=6\,a,VAR7=7\,a {} VAR8=8\,a,VAR9=9\,a', + _c(r'{} VAR1=1\,a', ['VAR1=1,a']) + _c(r'{} VAR2=2\,a,VAR3=3\,a', ['VAR2=2,a', 'VAR3=3,a']) + _c(r'{} VAR4=4\,a {} VAR5=5\,a', ['VAR4=4,a', 'VAR5=5,a']) + _c(r'{} VAR6=6\,a,VAR7=7\,a {} VAR8=8\,a,VAR9=9\,a', ['VAR6=6,a', 'VAR7=7,a', 'VAR8=8,a', 'VAR9=9,a']) _c(['{}', 'VAR1="1 with spaces"'], ['VAR1="1 with spaces"']) _c(['{}', 'VAR2="2 with spaces",VAR3="3 with spaces"'], @@ -199,13 +199,13 @@ def _c(a, r): self.check_one(n, o, a, r) 'VAR8="8 with spaces",VAR9="9 with spaces"'], ['VAR6="6 with spaces"', 'VAR7="7 with spaces"', 'VAR8="8 with spaces"', 'VAR9="9 with spaces"']) - _c(['{}', 'VAR1="1\,a with spaces"'], ['VAR1="1,a with spaces"']) - _c(['{}', 'VAR2="2\,a with spaces",VAR3="3\,a with spaces"'], + _c(['{}', r'VAR1="1\,a with spaces"'], ['VAR1="1,a with spaces"']) + _c(['{}', r'VAR2="2\,a with spaces",VAR3="3\,a with spaces"'], ['VAR2="2,a with spaces"', 'VAR3="3,a with spaces"']) - _c(['{}', 'VAR4="4\,a with spaces"', '{}', 'VAR5="5\,a with spaces"'], + _c(['{}', r'VAR4="4\,a with spaces"', '{}', r'VAR5="5\,a with spaces"'], ['VAR4="4,a with spaces"', 'VAR5="5,a with spaces"']) - _c(['{}', 'VAR6="6\,a with spaces",VAR7="7\,a with spaces"', '{}', - 'VAR8="8\,a with spaces",VAR9="9\,a with spaces"'], + _c(['{}', r'VAR6="6\,a with spaces",VAR7="7\,a with spaces"', '{}', + r'VAR8="8\,a with spaces",VAR9="9\,a with spaces"'], ['VAR6="6,a with spaces"', 'VAR7="7,a with spaces"', 'VAR8="8,a with spaces"', 'VAR9="9,a with spaces"']) _c('{} "-fPIC,-Wall,-O3,-Os"', ['-fPIC', '-Wall', '-O3', '-Os']) @@ -247,7 +247,7 @@ def test10_mixed0(self): defines=['VARIANT2']) self.check_many('-X nortti,c++14 -D VARIANT3', cmake_vars=[], cxxflags=['nortti', 'c++14'], cflags=[], defines=['VARIANT3']) - self.check_many('-X "-fPIC,-Wl\,-rpath" -D VARIANT1', cmake_vars=[], cxxflags=['-fPIC', '-Wl,-rpath'], + self.check_many(r'-X "-fPIC,-Wl\,-rpath" -D VARIANT1', cmake_vars=[], cxxflags=['-fPIC', '-Wl,-rpath'], cflags=[], defines=['VARIANT1']) def test11_mixed1(self): diff --git a/test/test03variant.py b/test/test03variant.py index 1a25770..5f69c8f 100644 --- a/test/test03variant.py +++ b/test/test03variant.py @@ -30,7 +30,7 @@ def test01_nospecs(self): vars = ['var0','var1','var2'] for w in (vars, ','.join(vars)): out = variant.Variant.create_variants(w) - self.assertEquals(len(out), 3) + self.assertEqual(len(out), 3) self.c('var0', out[0], cmake_vars=[], defines=[], cflags=[], cxxflags=[]) self.c('var1', out[1], cmake_vars=[], defines=[], cflags=[], cxxflags=[]) self.c('var2', out[2], cmake_vars=[], defines=[], cflags=[], cxxflags=[]) @@ -42,7 +42,7 @@ def test02_norefs(self): ] for w in (vars, ','.join(vars)): out = variant.Variant.create_variants(w) - self.assertEquals(len(out), 3) + self.assertEqual(len(out), 3) self.c('var0', out[0], cmake_vars=[], defines=['VAR1', 'VAR_TYPE=1'], cflags=[], cxxflags=['-fPIC']) self.c('var1', out[1], cmake_vars=[], defines=['VAR2', 'VAR_TYPE=2'], cflags=[], cxxflags=['-Wall']) self.c('var2', out[2], cmake_vars=[], defines=['VAR3', 'VAR_TYPE=3'], cflags=[], cxxflags=['nortti', 'c++14']) @@ -55,7 +55,7 @@ def test10_refs_beginning(self): ] for w in (vars, ','.join(vars)): out = variant.Variant.create_variants(w) - self.assertEquals(len(out), 3) + self.assertEqual(len(out), 3) self.c('var0', out[0], cmake_vars=[], defines=['VAR1', 'VAR_TYPE=1'], cflags=[], cxxflags=['-fPIC']) self.c('var1', out[1], cmake_vars=[], defines=['VAR1', 'VAR_TYPE=1', 'VAR2', 'VAR_TYPE=2'], cflags=[], cxxflags=['-fPIC', '-Wall']) self.c('var2', out[2], cmake_vars=[], defines=['VAR1', 'VAR_TYPE=1', 'VAR2', 'VAR_TYPE=2', 'VAR3', 'VAR_TYPE=3'], cflags=[], cxxflags=['-fPIC', '-Wall', 'nortti', 'c++14']) @@ -68,7 +68,7 @@ def test11_refs_middle(self): ] for w in (vars, ','.join(vars)): out = variant.Variant.create_variants(w) - self.assertEquals(len(out), 3) + self.assertEqual(len(out), 3) self.c('var0', out[0], cmake_vars=[], defines=['VAR1', 'VAR_TYPE=1'], cflags=[], cxxflags=['-fPIC']) self.c('var1', out[1], cmake_vars=[], defines=['VAR1', 'VAR_TYPE=1', 'VAR2', 'VAR_TYPE=2'], cflags=[], cxxflags=['-Wall', '-fPIC']) self.c('var2', out[2], cmake_vars=[], defines=['VAR1', 'VAR_TYPE=1', 'VAR2', 'VAR_TYPE=2', 'VAR3', 'VAR_TYPE=3'], cflags=[], cxxflags=['nortti', 'c++14', '-Wall', '-fPIC']) @@ -81,7 +81,7 @@ def test12_refs_end(self): ] for w in (vars, ','.join(vars)): out = variant.Variant.create_variants(w) - self.assertEquals(len(out), 3) + self.assertEqual(len(out), 3) self.c('var0', out[0], cmake_vars=[], defines=['VAR1', 'VAR_TYPE=1'], cflags=[], cxxflags=['-fPIC']) self.c('var1', out[1], cmake_vars=[], defines=['VAR2', 'VAR_TYPE=2', 'VAR1', 'VAR_TYPE=1'], cflags=[], cxxflags=['-Wall', '-fPIC']) self.c('var2', out[2], cmake_vars=[], defines=['VAR3', 'VAR_TYPE=3', 'VAR2', 'VAR_TYPE=2', 'VAR1', 'VAR_TYPE=1'], cflags=[], cxxflags=['nortti', 'c++14', '-Wall', '-fPIC']) @@ -96,7 +96,7 @@ def test22_consecutive_refs_only(self): ] for w in (vars, ','.join(vars)): out = variant.Variant.create_variants(w) - self.assertEquals(len(out), 5) + self.assertEqual(len(out), 5) self.c('var0', out[0], cmake_vars=[], defines=['VAR1', 'VAR_TYPE=1'], cflags=[], cxxflags=['-fPIC']) self.c('var1', out[1], cmake_vars=[], defines=['VAR2', 'VAR_TYPE=2'], cflags=[], cxxflags=['-Wall']) self.c('var2', out[2], cmake_vars=[], defines=['VAR3', 'VAR_TYPE=3'], cflags=[], cxxflags=['-g3']) @@ -114,7 +114,7 @@ def t(self, input, expected): c4args.add_select(parser) args = parser.parse_args(input) with self.subTest(input=input, desc='argument parsing'): - self.assertEqual(args.variants, expected, msg=input) + self.assertEqual(args.variants, expected, msg="input="+str(input)) def tp(self, input, which_var, expected_name, **expected_props): parser = argparse.ArgumentParser() @@ -142,7 +142,7 @@ def rp(self, input, which_var, expected_name, **expected_props): self.tp(isl, which_var, expected_name, **expected_props) def test00(self): - self.t([], ['none']) + self.t([], None) self.t(["-v", "none"], ['none']) def test01(self): @@ -265,6 +265,7 @@ def test02(self): var2 = 'var2' self.t(var0 + ',' + var1 + ',' + var2, [var0, var1, var2]) + # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- diff --git a/test/test04combinations.py b/test/test04combinations.py index d1148e3..7748eca 100644 --- a/test/test04combinations.py +++ b/test/test04combinations.py @@ -32,10 +32,19 @@ def do_combination_t(test, input, expected_items, expected_combinations): args = _parser.parse_args(input) # util.logcmd(input, "\n", args) - for i in ('systems', 'architectures', 'compilers', 'build_types', 'variants'): + for i,default_str in ( + ('systems', ds), + ('architectures', da), + ('compilers', dc), + ('build_types', dt), + ('variants', dv) + ): + print("i=", i, "default_str=", ds) expected = expected_items[i] expected = [e.split(':')[0] for e in expected] - actual = getattr(args, i) + actual = getattr(args, i, None) + if actual is None: + actual = [default_str] actual = [a.split(':')[0] for a in actual] # print(i, "actual=", actual) # print(i, "expected=", expected) diff --git a/test/test05cmany.py b/test/test05cmany.py index ef1d90e..40fda2e 100644 --- a/test/test05cmany.py +++ b/test/test05cmany.py @@ -115,7 +115,7 @@ def run(self, args_, custom_root=None): if not os.path.exists(root): os.makedirs(root) projdir = os.path.abspath('.') - args.append(projdir) + args += ["--proj-dir", projdir] args = maincmd + args with util.setcwd(root): tmpfile, tmpname = tempfile.mkstemp(prefix="_cmany_tmp.out.") @@ -166,7 +166,7 @@ def run_projs(testobj, args, check_fn=None): with testobj.subTest(msg="default parameters", proj=p.proj): p.run(args + ['--build-root', bd, '--install-root', id]) if check_fn: - tb = TestBuild(proj=p, buildroot=bd, installroot=id, + tb = BuildVerifier(proj=p, buildroot=bd, installroot=id, compiler=cmany.Compiler.default(), build_type=cmany.BuildType.default(), variant=cmany.Variant.default(), @@ -179,7 +179,7 @@ def run_projs(testobj, args, check_fn=None): with testobj.subTest(msg="run in a non root dir", proj=p.proj): p.run(args, custom_root=rd) if check_fn: - tb = TestBuild(proj=p, buildroot=bd, installroot=id, + tb = BuildVerifier(proj=p, buildroot=bd, installroot=id, compiler=cmany.Compiler.default(), build_type=cmany.BuildType.default(), variant=cmany.Variant.default(), @@ -204,7 +204,7 @@ def run_projs(testobj, args, check_fn=None): for c in compiler_set: for t in build_types: for v in variant_set: - tb = TestBuild(proj=p, buildroot=bd, installroot=id, + tb = BuildVerifier(proj=p, buildroot=bd, installroot=id, compiler=c, build_type=t, variant=v, numbuilds=numbuilds) check_fn(tb) @@ -228,7 +228,7 @@ def run_projs(testobj, args, check_fn=None): for c in compiler_set: for t in build_types: for v in variant_set: - tb = TestBuild(proj=p, buildroot=bd, installroot=id, + tb = BuildVerifier(proj=p, buildroot=bd, installroot=id, compiler=c, build_type=t, variant=v, numbuilds=numbuilds) check_fn(tb) @@ -249,7 +249,7 @@ def run_projs(testobj, args, check_fn=None): '-v', v.full_specs, ]) if check_fn: - tb = TestBuild(proj=p, buildroot=bd, installroot=id, + tb = BuildVerifier(proj=p, buildroot=bd, installroot=id, compiler=c, build_type=t, variant=v, numbuilds=1) check_fn(tb) @@ -271,7 +271,7 @@ def run_projs(testobj, args, check_fn=None): p.run(args + ['--build-root', bd, '--install-root', id]) os.environ['CMANY_ARGS'] = '' if check_fn: - tb = TestBuild(proj=p, buildroot=bd, installroot=id, + tb = BuildVerifier(proj=p, buildroot=bd, installroot=id, compiler=c, build_type=t, variant=v, numbuilds=1) check_fn(tb) @@ -280,7 +280,7 @@ def run_projs(testobj, args, check_fn=None): # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- -class TestBuild: +class BuildVerifier: def __init__(self, proj, buildroot, installroot, compiler, build_type, variant, numbuilds): self.proj = proj @@ -323,7 +323,9 @@ def nsiblings(self, dir): def siblings(self, dir): res = os.path.join(self.proj.root, dir, '*') + print("wtf0", res) ch = glob.glob(res) + print("wtf1", ch) return ch diff --git a/test/test06xcompile.py b/test/test06xcompile.py index c329047..9b401a2 100644 --- a/test/test06xcompile.py +++ b/test/test06xcompile.py @@ -21,7 +21,7 @@ def toolchain_compiler_exists(tc_file): - return osp.exists(cmake.cxx_compiler(toolchain=tc_file)) + return cmake.CMakeSysInfo.cxx_compiler(toolchain=tc_file) def run_with_args(testdir, args_in): @@ -49,7 +49,8 @@ def do_toolchain_builds(toolchain, test, args, expected_builds): if not toolchain_compiler_exists(toolchain): return args = [a.format(toolchain=toolchain) for a in args] - c = cmake.cxx_compiler(toolchain=toolchain) + c = cmake.CMakeSysInfo.cxx_compiler(toolchain=toolchain) + c = Compiler(c) expected_builds = [b.format(compiler=Build.sanitize_compiler_name(c.name)) for b in expected_builds] for t in testdirs: diff --git a/test/test07config.py b/test/test07config.py index 80d6788..f2b2238 100644 --- a/test/test07config.py +++ b/test/test07config.py @@ -155,11 +155,36 @@ def do_roundtrip_load_save(self): # ----------------------------------------------------------------------------- # Test cases for build flags -# using the _ prefix prevents unittest from instantiating tests for this class -class _BFTCase(ConfFixtureBase): - """base for build flags test case""" - def _setUp(self, bf, yml): +from parameterized import parameterized +build_flags_parameters = [ + [ + "Empty", + BuildFlags("empty"), + "" + ], + [ + "CMakeVars", + BuildFlags("cmake_vars", + cmake_vars=["VARFOO=BAR", "CMAKE_VERBOSE_MAKEFILE=1"]), + """cmake_vars: VARFOO=BAR CMAKE_VERBOSE_MAKEFILE=1""" + ], + [ + "CMakeVarsAndCxxflags", + BuildFlags("cmake_vars_and_cxxflags", + cmake_vars=["VARFOO=BAR", "CMAKE_VERBOSE_MAKEFILE=1"], + cxxflags=["-std=c++11", "-Wall", ]), + """\ +cmake_vars: VARFOO=BAR CMAKE_VERBOSE_MAKEFILE=1 +cxxflags: -std=c++11 -Wall +""" + ], +] + +class Test02BuildFlags(ConfFixtureBase): + + def _prepare(self, name, bf, yml): + self.name = name self.bf = bf self.yml = yml self.name = self.__class__.__name__[7:].lower() @@ -191,68 +216,31 @@ def check_cm(self, cm): if a not in self.non_empty: self.assertTrue(not hasattr(cm, a)) - def test01Save(self): + @parameterized.expand(build_flags_parameters) + def test01Save(self, name, bf, yml): + self._prepare(name, bf, yml) cm = self.do_save() self.check_cm(cm) - def test02Load(self): + @parameterized.expand(build_flags_parameters) + def test02Load(self, name, bf, yml): + self._prepare(name, bf, yml) bf = self.do_load() self.check_bf(bf) - def test03RoundtripSaveLoad(self): + @parameterized.expand(build_flags_parameters) + def test03RoundtripSaveLoad(self, name, bf, yml): + self._prepare(name, bf, yml) bf = self.do_roundtrip_save_load() self.check_bf(bf) - def test04RoundtripLoadSave(self): + @parameterized.expand(build_flags_parameters) + def test04RoundtripLoadSave(self, name, bf, yml): + self._prepare(name, bf, yml) cm = self.do_roundtrip_load_save() self.check_cm(cm) -# ----------------------------------------------------------------------------- -class Test02BuildFlags01Empty(_BFTCase): - def setUp(self): - self._setUp(bf=BuildFlags("empty"), yml="") - - -class Test02BuildFlags02CMakeVars(_BFTCase): - def setUp(self): - super()._setUp( - bf=BuildFlags("cmake_vars", - cmake_vars=["VARFOO=BAR", "CMAKE_VERBOSE_MAKEFILE=1"]), - yml="""\ -cmake_vars: VARFOO=BAR CMAKE_VERBOSE_MAKEFILE=1 -""") - - -class Test02BuildFlags03CMakeVarsAndCxxflags(_BFTCase): - def setUp(self): - self._setUp( - bf=BuildFlags("cmake_vars_and_cxxflags", - cmake_vars=["VARFOO=BAR", "CMAKE_VERBOSE_MAKEFILE=1"], - cxxflags=["-std=c++11", "-Wall", ]), - yml="""\ -cmake_vars: VARFOO=BAR CMAKE_VERBOSE_MAKEFILE=1 -cxxflags: -std=c++11 -Wall -""") - - -class Test02BuildFlags04Full(_BFTCase): - def setUp(self): - self._setUp( - bf=BuildFlags("cmake_vars_and_cxxflags", - cmake_vars=["VARFOO=BAR", "CMAKE_VERBOSE_MAKEFILE=1"], - defines=["FOO", "BAR", "VFOO=baaaaaaaaaa"], - cflags=["-std=c++11", "-Wall", ], - cxxflags=["-std=c++11", "-Wall", ], - toolchain="somefile.cmake"), - yml="""\ -cmake_vars: VARFOO=BAR CMAKE_VERBOSE_MAKEFILE=1 -defines: FOO BAR VFOO=baaaaaaaaaa -cflags: -std=c++11 -Wall -cxxflags: -std=c++11 -Wall -toolchain: somefile.cmake -""") - # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- From 2b214020e11545a09c6f7b50ba5e3748bdeca284 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Wed, 15 Mar 2023 00:41:25 +0100 Subject: [PATCH 45/50] update test scripts --- requirements_test.txt | 4 +++- test/run.bat | 2 +- test/run.sh | 6 +----- test/{test00util.py => test_00util.py} | 0 test/{test01vsinfo.py => test_01vsinfo.py} | 0 test/{test02flags.py => test_02flags.py} | 0 test/{test03variant.py => test_03variant.py} | 0 test/{test04combinations.py => test_04combinations.py} | 0 test/{test05cmany.py => test_05cmany.py} | 0 test/{test06xcompile.py => test_06xcompile.py} | 0 test/{test07config.py => test_07config.py} | 0 11 files changed, 5 insertions(+), 7 deletions(-) rename test/{test00util.py => test_00util.py} (100%) rename test/{test01vsinfo.py => test_01vsinfo.py} (100%) rename test/{test02flags.py => test_02flags.py} (100%) rename test/{test03variant.py => test_03variant.py} (100%) rename test/{test04combinations.py => test_04combinations.py} (100%) rename test/{test05cmany.py => test_05cmany.py} (100%) rename test/{test06xcompile.py => test_06xcompile.py} (100%) rename test/{test07config.py => test_07config.py} (100%) diff --git a/requirements_test.txt b/requirements_test.txt index c11086f..3733776 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,5 +1,7 @@ wheel -nose +pytest +pytest-cov +parameterized twine coverage codecov diff --git a/test/run.bat b/test/run.bat index addc7d8..55fa98a 100755 --- a/test/run.bat +++ b/test/run.bat @@ -22,7 +22,7 @@ echo %PIPINSTALL% cd test if %ERRORLEVEL% neq 0 exit /b %ERRORLEVEL% set PYTHONPATH=%root%\src -%PYTHON% -m nose -d -v --with-id --nocapture --with-coverage --cover-erase --cover-package=..\src --cover-branches %* +%PYTHON% -m pytest --cov=..\src --cov-reset %* if %ERRORLEVEL% neq 0 exit /b %ERRORLEVEL% exit /b %ERRORLEVEL% diff --git a/test/run.sh b/test/run.sh index c20afe3..9fa8c31 100755 --- a/test/run.sh +++ b/test/run.sh @@ -5,14 +5,10 @@ set -e root=$(cd $(dirname $0)/.. ; pwd) PY=${PYTHON:-python3} -PIP=${PIP:-pip3} -PIP_INSTALL=${PIP_INSTALL:-pip install} # run the cmany unit tests cd $root/test export PYTHONPATH=$root/src -$PY -m nose -d -v --with-id --nocapture \ - --with-coverage --cover-erase --cover-package=../src --cover-branches \ - $* +$PY -m pytest --cov=../src --cov-reset $* exit $? diff --git a/test/test00util.py b/test/test_00util.py similarity index 100% rename from test/test00util.py rename to test/test_00util.py diff --git a/test/test01vsinfo.py b/test/test_01vsinfo.py similarity index 100% rename from test/test01vsinfo.py rename to test/test_01vsinfo.py diff --git a/test/test02flags.py b/test/test_02flags.py similarity index 100% rename from test/test02flags.py rename to test/test_02flags.py diff --git a/test/test03variant.py b/test/test_03variant.py similarity index 100% rename from test/test03variant.py rename to test/test_03variant.py diff --git a/test/test04combinations.py b/test/test_04combinations.py similarity index 100% rename from test/test04combinations.py rename to test/test_04combinations.py diff --git a/test/test05cmany.py b/test/test_05cmany.py similarity index 100% rename from test/test05cmany.py rename to test/test_05cmany.py diff --git a/test/test06xcompile.py b/test/test_06xcompile.py similarity index 100% rename from test/test06xcompile.py rename to test/test_06xcompile.py diff --git a/test/test07config.py b/test/test_07config.py similarity index 100% rename from test/test07config.py rename to test/test_07config.py From 314d1ef6471b27ff0ea9de669b23400401e085fb Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sun, 19 Mar 2023 03:49:35 +0100 Subject: [PATCH 46/50] [fix] fix dirty-state cache on every call, was triggering reconfigures --- src/c4/cmany/build.py | 95 +++++---- src/c4/cmany/cmake.py | 194 ++++++++++++++---- src/c4/cmany/err.py | 7 + test/CMakeCache.txt | 376 +++++++++++++++++++++++++++++++++++ test/run.bat | 2 +- test/run.sh | 2 +- test/test_01cmake.py | 443 ++++++++++++++++++++++++++++++++++++++++++ test/test_05cmany.py | 16 +- 8 files changed, 1052 insertions(+), 83 deletions(-) create mode 100644 test/CMakeCache.txt create mode 100644 test/test_01cmake.py diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index 2c2e75e..2edde15 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -16,13 +16,15 @@ from .compiler import Compiler from .architecture import Architecture from . import err -from .util import logdbg as dbg +from .util import logdbg from .target import Target # experimental. I don't think it will stay unless conan starts accepting args from .conan import Conan + + # ----------------------------------------------------------------------------- class Build(NamedItem): """Holds a build's settings""" @@ -30,6 +32,9 @@ class Build(NamedItem): pfile = "cmany_preload.cmake" sfile = "cmany_build.dill" + def dbg(self, *args, **kwargs): + logdbg(self.name + ":", *args, **kwargs) + def __init__(self, proj_root, build_root, install_root, system, arch, build_type, compiler, variant, flags, num_jobs, kwargs): @@ -52,11 +57,11 @@ def __init__(self, proj_root, build_root, install_root, # if util.in_64bit and self.architecture.is32: if self.compiler.gcclike: - dbg("making 32 bit") + logdbg("build: making 32 bit") self.compiler.make_32bit() elif util.in_32bit and self.architecture.is64: if self.compiler.gcclike: - dbg("making 64 bit") + logdbg("build: making 64 bit") self.compiler.make_64bit() # tag = self._set_name_and_paths() @@ -69,10 +74,12 @@ def __init__(self, proj_root, build_root, install_root, # This will load the vars from the builddir cache, if it exists. # It should be done only after creating the generator. self.varcache = cmake.CMakeCache(self.builddir) + self.dbg("cache is", "dirty" if self.varcache.dirty else "clean", "on load") # ... and this will overwrite (in memory) the vars with the input # arguments. This will make the cache dirty and so we know when it # needs to be committed back to CMakeCache.txt self.gather_input_cache_vars() + self.dbg("cache is", "dirty" if self.varcache.dirty else "clean", "after cmd vars") # self.deps = kwargs.get('deps', '') if self.deps and not os.path.isabs(self.deps): @@ -85,7 +92,7 @@ def __init__(self, proj_root, build_root, install_root, def reset_kwargs(self, kwargs): "the serialization/deserialization approach creates cache coherency problems" - util.logdbg(f"{self}: ressetting kwargs: {kwargs}") + self.dbg(f"{self}: ressetting kwargs: {kwargs}") self.kwargs = kwargs # # TODO complete this @@ -109,7 +116,8 @@ def _set_name_and_paths(self): self.preload_file = os.path.join(self.builddir, Build.pfile) self.cachefile = os.path.join(self.builddir, 'CMakeCache.txt') for prop in "system architecture compiler build_type variant projdir buildroot installroot buildtag installtag builddir installdir preload_file cachefile".split(" "): - dbg(" {}: {}={}".format(self.tag, prop, getattr(self, prop))) + # don't call self.dbg() yet + logdbg("build: {}: {}={}".format(self.tag, prop, getattr(self, prop))) return self.tag def create_generator(self, num_jobs, fallback_generator="Unix Makefiles"): @@ -119,6 +127,7 @@ def create_generator(self, num_jobs, fallback_generator="Unix Makefiles"): # print(toolchain_cache) # self.adjust(compiler=toolchain_cache['CMAKE_CXX_COMPILER']) if self.compiler.is_msvc: + self.dbg("compiler is msvc -- selecting appropriate Visual Studio") vsi = vsinfo.VisualStudioInfo(self.compiler.name) g = Generator(vsi.gen, self, num_jobs) arch = Architecture(vsi.architecture) @@ -138,12 +147,12 @@ def adjust(self, **kwargs): raise err.NoSupport(f"build adjustment for {k}. Must be one of {supported}") a = kwargs.get('architecture') if a and a != self.architecture: - dbg(self, "adjusting architecture:", self.architecture, "---->", a) + self.dbg(self, "adjusting architecture:", self.architecture, "---->", a) self.adjusted = True self.architecture = a c = kwargs.get('compiler') if c and c != self.compiler: - dbg(self, "adjusting compiler:", self.compiler, "---->", a) + self.dbg(self, "adjusting compiler:", self.compiler, "---->", a) self.adjusted = True self.compiler = c self._set_name_and_paths() @@ -209,7 +218,13 @@ def configure(self): cmd = self.configure_cmd() try: util.runsyscmd(cmd) - self.mark_configure_done(cmd) + first_time = self.mark_configure_done(cmd) + # workaround a problem where CMAKE_C_COMPILER set from + # either preload or command line will have its vartype + # set to STRING instead of FILEPATH. So override + # cmake's problem. + if first_time: + self.varcache.commit(self.builddir) except Exception as e: raise err.ConfigureFailed(self, cmd, e) if self.export_compile: @@ -295,8 +310,8 @@ def cxx_compiler(self): def vs_dev_cmd(self, target): cl_exe = self.vsinfo.cxx_compiler cl_version = self.vsinfo.cl_version - dbg("cl.exe:", cl_exe) - dbg("cl.exe version:", cl_version) + self.dbg("cl.exe:", cl_exe) + self.dbg("cl.exe version:", cl_version) return vsinfo.dev_env( vcvarsall=f'"{self.vsinfo.vcvarsall}"', arch=str(self.architecture.vs_dev_env_name), @@ -325,23 +340,39 @@ def _check_successful_configure(self, purpose): def mark_configure_done(self, cmd): self._serialize() with util.setcwd(self.builddir): - with open("cmany_configure.done", "w") as f: + first_time = False + name = "cmany_configure.done" + if not os.path.exists(name): + first_time = True + with open(name, "w") as f: f.write(" ".join(cmd) + "\n") + return first_time def needs_configure(self): + ret = False if not os.path.exists(self.builddir): - return True - with util.setcwd(self.builddir): - if not os.path.exists("cmany_configure.done"): - return True - if self.needs_cache_regeneration(): - return True - return False + self.dbg("builddir does not exist:", self.builddir) + ret = True + else: + with util.setcwd(self.builddir): + donefile = "cmany_configure.done" + if not os.path.exists(donefile): + self.dbg("not found:", os.path.join(self.builddir, donefile)) + ret = True + if self.needs_cache_regeneration(): + ret = True + self.dbg("needs configure" if ret else "does not need configure") + return ret def needs_cache_regeneration(self): - if os.path.exists(self.cachefile) and self.varcache.dirty: - return True - return False + ret = False + if os.path.exists(self.cachefile): + self.dbg("cache file found:", self.cachefile) + if self.varcache.dirty: + self.dbg("var cache is dirty") + ret = True + self.dbg("cache needs regeneration" if ret else "cache ok") + return ret def build(self, targets=[]): self.create_dir() @@ -373,11 +404,11 @@ def build_files(self, files, target): for f in files: try: cmd = self.generator.cmd_source_file(f, target) - dbg(f"building file {f}, target {target}. cmd={cmd}") + self.dbg(f"building file {f}, target {target}. cmd={cmd}") util.runsyscmd_str(cmd, cwd=self.builddir) - dbg(f"building file {f}, target {target}. success!") + self.dbg(f"building file {f}, target {target}. success!") except Exception as e: - dbg(f"building file {f}, target {target}. exception: {e}!") + self.dbg(f"building file {f}, target {target}. exception: {e}!") raise err.CompileFailed(self, cmd, e) def rebuild(self, targets=[]): @@ -457,11 +488,11 @@ def _get_toolchain(self): tc = BuildFlags.merge_toolchains(tc, fs.toolchain) if not tc: return None - dbg("toolchain:", tc) + self.dbg("toolchain:", tc) if not os.path.isabs(tc): tc = os.path.join(os.getcwd(), tc) tc = os.path.abspath(tc) - dbg("toolchain:", tc) + self.dbg("toolchain:", tc) if not os.path.exists(tc): raise err.ToolchainFileNotFound(tc, self) return tc @@ -635,9 +666,9 @@ def get_targets(self): ret = [] with util.setcwd(self.builddir): for sd in sorted(util.rglob(self.builddir, "CMakeFiles")): - util.logdbg(f"found {sd}...") + self.dbg(f"found {sd}...") sd = os.path.dirname(sd) - util.logdbg(f"descending into {sd}/ ...") + self.dbg(f"descending into {sd}/ ...") ret += self._dir_targets(sd) return ret @@ -700,14 +731,6 @@ def p(n, v): print("{}={}".format(n, v)) {vars} message(STATUS "cmany:preload----------------------") -# if(CMANY_INCLUDE_DIRECTORIES) -# include_directories(${{CMANY_INCLUDE_DIRECTORIES}}) -# endif() -# -# if(CMANY_LINK_DIRECTORIES) -# link_directories(${{CMANY_LINK_DIRECTORIES}}) -# endif() - # Do not edit. Will be overwritten. # Generated by cmany on {date} """) diff --git a/src/c4/cmany/cmake.py b/src/c4/cmany/cmake.py index bcf615c..ce9b3bc 100644 --- a/src/c4/cmany/cmake.py +++ b/src/c4/cmany/cmake.py @@ -10,7 +10,21 @@ from . import util from . import err + +def _can_change_cache_type(varname, prevtype, nexttype): + """when CMAKE_C_COMPILER et al are set in the preload file or from + the command line, cmake sets the vartype to STRING, so we work + around that by allowing a change to its proper type of FILEPATH, + which is the one that would prevail if nothing had been set.""" + if varname.startswith("CMAKE_") and varname.endswith("_COMPILER"): + if prevtype == "STRING" and nexttype == "FILEPATH": + return True + return False + + _cache_entry = r'^(.*?)(:.*?)=(.*)$' +def _named_cache_entry(name: str): + return r'^{}(:.*?)=(.*)$'.format(name) def hascache(builddir): @@ -20,34 +34,48 @@ def hascache(builddir): return None -def setcachevar(builddir, var, value): - setcachevars(builddir, odict([(var, value)])) - - -def getcachevar(builddir, var): - v = getcachevars(builddir, [var]) - return v[var] - - def setcachevars(builddir, varvalues): - with setcwd(builddir, silent=True): - with open('CMakeCache.txt', 'r') as f: - ilines = f.readlines() - olines = [] - for l in ilines: - for k, v in varvalues.items(): - if l.startswith(k + ':'): + assert os.path.isabs(builddir), builddir + cachefile = os.path.join(builddir, "CMakeCache.txt") + if not os.path.exists(cachefile): + raise err.CacheFileNotFound(cachefile, builddir, "setcachevars") + with open(cachefile, 'r') as f: + ilines = f.readlines() + olines = [] + rx = {} # save the splits and regular expressions + for k, v in varvalues.items(): + spl = k.split(':') + name = spl[0] + vartype = spl[1] if (len(spl) > 1) else None + rx[name] = (vartype, v, re.compile(_named_cache_entry(name))) + for l in ilines: + for k, (vartype, v, rxname) in rx.items(): + if rxname.match(l): + dbg("committing: before=", l.strip()) + if vartype is not None: + existing_type = re.sub(_cache_entry, r'\2', l) + assert existing_type.startswith(':') + existing_type = existing_type[1:] + existing_type = existing_type.strip("\r\n") + if vartype != existing_type: + if not _can_change_cache_type(k, existing_type, vartype): + raise err.CannotChangeCacheVarType(cachefile, k, existing_type, vartype) + n = "{}:{}={}\n".format(k, vartype, v) + else: n = re.sub(_cache_entry, r'\1\2=' + v, l) - l = n + dbg("committing: after=", n.strip()) + l = n olines.append(l) - with open('CMakeCache.txt', 'w') as f: - f.writelines(olines) + with open(cachefile, 'w') as f: + f.writelines(olines) def getcachevars(builddir, varlist): vlist = [v + ':' for v in varlist] values = odict() with setcwd(builddir, silent=True): + if not os.path.exists("CMakeCache.txt"): + raise err.CacheFileNotFound("CMakeCache.txt", builddir, "getcachevars") with open('CMakeCache.txt') as f: for line in f: for v in vlist: @@ -69,6 +97,8 @@ def loadvars(builddir): with open(c, 'r') as f: for line in f: # dbg("loadvars0", line.strip()) + if line.startswith('#'): + continue if not re.match(_cache_entry, line): continue ls = line.strip() @@ -76,7 +106,7 @@ def loadvars(builddir): vartype = re.sub(_cache_entry, r'\2', ls)[1:] value = re.sub(_cache_entry, r'\3', ls) # dbg("loadvars1", name, vartype, value) - v[name] = CMakeCacheVar(name, value, vartype) + v[name] = CMakeCacheVar(name, value, vartype, dirty=False, from_input=True) return v @@ -145,12 +175,20 @@ def i(self, name, val, **kwargs): return self.setvar(name, val, "INTERNAL", **kwargs) def setvar(self, name, val, vartype=None, **kwargs): + q = lambda v: "'" + str(v) + "'" + dbg("setting cache var:", name+':'+str(vartype), "=", q(val), "cache=dirty" if self.dirty else "cache=clean") v = self.get(name) if v is not None: - changed = v.reset(val, vartype, **kwargs) + dbg("setting cache var:", name, "was found with", q(v.val)) + if (vartype is not None) and (vartype != v.vartype): + if not _can_change_cache_type(name, v.vartype, vartype): + raise err.CannotChangeCacheVarType(None, v.name, v.vartype, vartype) + changed = v.reset(val, **kwargs) + dbg("setting cache var:", name, "changed" if changed else "same value") self.dirty |= changed return changed else: + dbg("setting cache var:", name, "was not found") v = CMakeCacheVar(name, val, vartype, dirty=True, **kwargs) self[name] = v self.dirty = True @@ -166,7 +204,12 @@ def commit(self, builddir): for _, v in self.items(): if not v.dirty: continue - tmp[v.name] = v.val + dbg("committing to cache: {}:{}={}".format(v.name, v.vartype, v.val)) + if v.vartype is not None: + name_and_type = "{}:{}".format(v.name, v.vartype) + else: + name_and_type = v.name + tmp[name_and_type] = v.val setcachevars(builddir, tmp) for _, v in self.items(): v.dirty = False @@ -174,56 +217,121 @@ def commit(self, builddir): return True +def _guess_var_type(name, val): + """make an informed guess of the var type""" + # first look at the name + uppername = name.upper() + if (("-ADVANCED" in uppername) or + (uppername.startswith("CMAKE_") and (uppername in ( + "CMAKE_CACHEFILE_DIR", + "CMAKE_CACHE_MAJOR_VERSION", + "CMAKE_CACHE_MINOR_VERSION", + "CMAKE_CACHE_PATCH_VERSION", + "CMAKE_CACHE_PATCH_VERSION", + "CMAKE_COMMAND", + "CMAKE_CPACK_COMMAND", + "CMAKE_CTEST_COMMAND", + "CMAKE_EDIT_COMMAND", + "CMAKE_EXECUTABLE_FORMAT", + "CMAKE_EXTRA_GENERATOR", + "CMAKE_GENERATOR", + "CMAKE_GENERATOR_INSTANCE", + "CMAKE_GENERATOR_PLATFORM", + "CMAKE_GENERATOR_TOOLSET", + "CMAKE_HOME_DIRECTORY", + "CMAKE_INSTALL_SO_NO_EXE", + "CMAKE_NUMBER_OF_MAKEFILES", + "CMAKE_PLATFORM_INFO_INITIALIZED", + "CMAKE_ROOT", + "CMAKE_UNAME", + ))) or + (uppername.startswith("_CMAKE"))): + return "INTERNAL" + elif uppername in ( + "CMAKE_EXPORT_COMPILE_COMMANDS", + "CMAKE_SKIP_RPATH", + "CMAKE_SKIP_INSTALL_RPATH", + ): + return "BOOL" + elif (uppername in ( + "CMAKE_FIND_PACKAGE_REDIRECTS_DIR", + "CMAKE_PROJECT_DESCRIPTION", + "CMAKE_PROJECT_HOMEPAGE_URL", + "CMAKE_PROJECT_NAME", + ) or ( + uppername.endswith("_BINARY_DIR") + or uppername.endswith("_IS_TOP_LEVEL") + or uppername.endswith("_SOURCE_DIR") + )): + return "STATIC" + elif ( + ("PATH" in uppername) or + uppername.endswith("COMPILER") or + uppername.endswith("COMPILER_AR") or + uppername.endswith("COMPILER_RANLIB") or + uppername.endswith("DLLTOOL") or + uppername.endswith("LINKER") + ): + return "FILEPATH" + # + # now look at the value + upperval = val.upper() + if upperval in ( + "ON", "OFF", "NO", "YES", "1", "0", + "TRUE", "FALSE", "T", "F", "N", "Y", + ): + # https://cmake.org/pipermail/cmake/2007-December/018548.html + return "BOOL" + elif os.path.isfile(val): + return "FILEPATH" + elif os.path.isdir(val) or "DIR" in name.upper() or os.path.isabs(val): + return "PATH" + else: + return "STRING" + + # ------------------------------------------------------------------------- class CMakeCacheVar: def __init__(self, name, val, vartype=None, dirty=False, from_input=False): self.name = name self.val = val - self.vartype = self._guess_var_type(name, val, vartype) + self.vartype = vartype if vartype else _guess_var_type(name, val) self.dirty = dirty self.from_input = from_input - def _guess_var_type(self, name, val, vartype): - """make an informed guess of the var type - @todo: add a test for this""" - if vartype is not None: - return vartype - elif val.upper() in ("ON", "OFF", "NO", "YES", "1", "0", "TRUE", "FALSE", "T", "F", "N", "Y"): - # https://cmake.org/pipermail/cmake/2007-December/018548.html - return "BOOL" - elif os.path.isfile(val) or "PATH" in name.upper(): - return "FILEPATH" - elif os.path.isdir(val) or "DIR" in name.upper() or os.path.isabs(val): - return "PATH" - else: - return "STRING" - - def reset(self, val, vartype='', **kwargs): + def reset(self, val, **kwargs): """ :param val: - :param vartype: :param kwargs: force_dirty, defaults to False from_input, defaults to None :return: """ + dbg("setting cache var:", self.name, "'{}' to '{}'".format(self.val, val), "(dirty={})".format(self.dirty)) + dbg("setting cache var:", self.name, "kwargs=", kwargs) force_dirty = kwargs.get('force_dirty', False) from_input = kwargs.get('from_input') + if kwargs.get("vartype") is not None: + if not _can_change_cache_type(self.name, self.vartype, kwargs.get("vartype")): + raise err.CannotChangeCacheVarType(None, self.name, self.vartype, kwargs.get("vartype")) if from_input is not None: self.from_input = from_input - if vartype == 'STRING' or (vartype is None and self.vartype == 'STRING'): + if self.vartype == 'STRING': + dbg("setting cache var:", self.name, "is string") candidates = (val, val.strip("'"), val.strip('"')) equal = False for c in candidates: if c == self.val: + dbg("setting cache var:", self.name, "is string equal to", c) equal = True break else: equal = (self.val == val) - if not equal or (vartype is not None and vartype != self.vartype): + dbg("setting cache var:", self.name, "is", "equal" if equal else "NOT equal") + if not equal: + dbg("setting cache var:", self.name, "CHANGE!") self.val = val - self.vartype = vartype if vartype is not None else self.vartype self.dirty = True return True if force_dirty: diff --git a/src/c4/cmany/err.py b/src/c4/cmany/err.py index 5d61d54..d9980fb 100644 --- a/src/c4/cmany/err.py +++ b/src/c4/cmany/err.py @@ -72,6 +72,13 @@ def __init__(self, cfile, bdir, purpose=None): super().__init__(msg, purpose, cfile, bdir, os.getcwd()) +class CannotChangeCacheVarType(Error): + def __init__(self, cachefile, var, oldtype, newtype): + msg = "{}cache variable '{}': cannot set type from '{}' to '{}')" + cachefile = "{}: ".format(cachefile) if cachefile else "" + super().__init__(msg, cachefile, var, oldtype, newtype) + + class BuildSerializationNotFound(Error): def __init__(self, sfile, bdir): msg = "build serialization does not exist: '{}' (build dir is '{}', curr dir is '{}')" diff --git a/test/CMakeCache.txt b/test/CMakeCache.txt new file mode 100644 index 0000000..ecd35ab --- /dev/null +++ b/test/CMakeCache.txt @@ -0,0 +1,376 @@ +# This is the CMakeCache file. +# For build in directory: /home/jpmag/proj/cmany/test/hello/build +# It was generated by CMake: /usr/bin/cmake +# You can edit this file to change values found and used by cmake. +# If you do not want to change any of the values, simply exit the editor. +# If you do want to change a value, simply edit, save, and exit the editor. +# The syntax for the file is as follows: +# KEY:TYPE=VALUE +# KEY is the name of a variable in the cache. +# TYPE is a hint to GUIs for the type of VALUE, DO NOT EDIT TYPE!. +# VALUE is the current value for the KEY. + +######################## +# EXTERNAL cache entries +######################## + +//Path to a program. +CMAKE_ADDR2LINE:FILEPATH=/usr/bin/addr2line + +//Path to a program. +CMAKE_AR:FILEPATH=/usr/bin/ar + +//Choose the type of build, options are: None Debug Release RelWithDebInfo +// MinSizeRel ... +CMAKE_BUILD_TYPE:STRING= + +//Enable/Disable color output during build. +CMAKE_COLOR_MAKEFILE:BOOL=ON + +//CXX compiler +CMAKE_CXX_COMPILER:FILEPATH=/usr/bin/c++ + +//A wrapper around 'ar' adding the appropriate '--plugin' option +// for the GCC compiler +CMAKE_CXX_COMPILER_AR:FILEPATH=/usr/bin/gcc-ar + +//A wrapper around 'ranlib' adding the appropriate '--plugin' option +// for the GCC compiler +CMAKE_CXX_COMPILER_RANLIB:FILEPATH=/usr/bin/gcc-ranlib + +//Flags used by the CXX compiler during all build types. +CMAKE_CXX_FLAGS:STRING= + +//Flags used by the CXX compiler during DEBUG builds. +CMAKE_CXX_FLAGS_DEBUG:STRING=-g + +//Flags used by the CXX compiler during MINSIZEREL builds. +CMAKE_CXX_FLAGS_MINSIZEREL:STRING=-Os -DNDEBUG + +//Flags used by the CXX compiler during RELEASE builds. +CMAKE_CXX_FLAGS_RELEASE:STRING=-O3 -DNDEBUG + +//Flags used by the CXX compiler during RELWITHDEBINFO builds. +CMAKE_CXX_FLAGS_RELWITHDEBINFO:STRING=-O2 -g -DNDEBUG + +//C compiler +CMAKE_C_COMPILER:FILEPATH=/usr/bin/cc + +//A wrapper around 'ar' adding the appropriate '--plugin' option +// for the GCC compiler +CMAKE_C_COMPILER_AR:FILEPATH=/usr/bin/gcc-ar + +//A wrapper around 'ranlib' adding the appropriate '--plugin' option +// for the GCC compiler +CMAKE_C_COMPILER_RANLIB:FILEPATH=/usr/bin/gcc-ranlib + +//Flags used by the C compiler during all build types. +CMAKE_C_FLAGS:STRING= + +//Flags used by the C compiler during DEBUG builds. +CMAKE_C_FLAGS_DEBUG:STRING=-g + +//Flags used by the C compiler during MINSIZEREL builds. +CMAKE_C_FLAGS_MINSIZEREL:STRING=-Os -DNDEBUG + +//Flags used by the C compiler during RELEASE builds. +CMAKE_C_FLAGS_RELEASE:STRING=-O3 -DNDEBUG + +//Flags used by the C compiler during RELWITHDEBINFO builds. +CMAKE_C_FLAGS_RELWITHDEBINFO:STRING=-O2 -g -DNDEBUG + +//Path to a program. +CMAKE_DLLTOOL:FILEPATH=CMAKE_DLLTOOL-NOTFOUND + +//Flags used by the linker during all build types. +CMAKE_EXE_LINKER_FLAGS:STRING= + +//Flags used by the linker during DEBUG builds. +CMAKE_EXE_LINKER_FLAGS_DEBUG:STRING= + +//Flags used by the linker during MINSIZEREL builds. +CMAKE_EXE_LINKER_FLAGS_MINSIZEREL:STRING= + +//Flags used by the linker during RELEASE builds. +CMAKE_EXE_LINKER_FLAGS_RELEASE:STRING= + +//Flags used by the linker during RELWITHDEBINFO builds. +CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO:STRING= + +//Enable/Disable output of compile commands during generation. +CMAKE_EXPORT_COMPILE_COMMANDS:BOOL= + +//Value Computed by CMake. +CMAKE_FIND_PACKAGE_REDIRECTS_DIR:STATIC=/home/jpmag/proj/cmany/test/hello/build/CMakeFiles/pkgRedirects + +//Install path prefix, prepended onto install directories. +CMAKE_INSTALL_PREFIX:PATH=/usr/local + +//Path to a program. +CMAKE_LINKER:FILEPATH=/usr/bin/ld + +//Path to a program. +CMAKE_MAKE_PROGRAM:FILEPATH=/usr/bin/make + +//Flags used by the linker during the creation of modules during +// all build types. +CMAKE_MODULE_LINKER_FLAGS:STRING= + +//Flags used by the linker during the creation of modules during +// DEBUG builds. +CMAKE_MODULE_LINKER_FLAGS_DEBUG:STRING= + +//Flags used by the linker during the creation of modules during +// MINSIZEREL builds. +CMAKE_MODULE_LINKER_FLAGS_MINSIZEREL:STRING= + +//Flags used by the linker during the creation of modules during +// RELEASE builds. +CMAKE_MODULE_LINKER_FLAGS_RELEASE:STRING= + +//Flags used by the linker during the creation of modules during +// RELWITHDEBINFO builds. +CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO:STRING= + +//Path to a program. +CMAKE_NM:FILEPATH=/usr/bin/nm + +//Path to a program. +CMAKE_OBJCOPY:FILEPATH=/usr/bin/objcopy + +//Path to a program. +CMAKE_OBJDUMP:FILEPATH=/usr/bin/objdump + +//Value Computed by CMake +CMAKE_PROJECT_DESCRIPTION:STATIC= + +//Value Computed by CMake +CMAKE_PROJECT_HOMEPAGE_URL:STATIC= + +//Value Computed by CMake +CMAKE_PROJECT_NAME:STATIC=hello + +//Path to a program. +CMAKE_RANLIB:FILEPATH=/usr/bin/ranlib + +//Path to a program. +CMAKE_READELF:FILEPATH=/usr/bin/readelf + +//Flags used by the linker during the creation of shared libraries +// during all build types. +CMAKE_SHARED_LINKER_FLAGS:STRING= + +//Flags used by the linker during the creation of shared libraries +// during DEBUG builds. +CMAKE_SHARED_LINKER_FLAGS_DEBUG:STRING= + +//Flags used by the linker during the creation of shared libraries +// during MINSIZEREL builds. +CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL:STRING= + +//Flags used by the linker during the creation of shared libraries +// during RELEASE builds. +CMAKE_SHARED_LINKER_FLAGS_RELEASE:STRING= + +//Flags used by the linker during the creation of shared libraries +// during RELWITHDEBINFO builds. +CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO:STRING= + +//If set, runtime paths are not added when installing shared libraries, +// but are added when building. +CMAKE_SKIP_INSTALL_RPATH:BOOL=NO + +//If set, runtime paths are not added when using shared libraries. +CMAKE_SKIP_RPATH:BOOL=NO + +//Flags used by the linker during the creation of static libraries +// during all build types. +CMAKE_STATIC_LINKER_FLAGS:STRING= + +//Flags used by the linker during the creation of static libraries +// during DEBUG builds. +CMAKE_STATIC_LINKER_FLAGS_DEBUG:STRING= + +//Flags used by the linker during the creation of static libraries +// during MINSIZEREL builds. +CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL:STRING= + +//Flags used by the linker during the creation of static libraries +// during RELEASE builds. +CMAKE_STATIC_LINKER_FLAGS_RELEASE:STRING= + +//Flags used by the linker during the creation of static libraries +// during RELWITHDEBINFO builds. +CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO:STRING= + +//Path to a program. +CMAKE_STRIP:FILEPATH=/usr/bin/strip + +//If this value is on, makefiles will be generated without the +// .SILENT directive, and all commands will be echoed to the console +// during the make. This is useful for debugging only. With Visual +// Studio IDE projects all commands are done without /nologo. +CMAKE_VERBOSE_MAKEFILE:BOOL=FALSE + +//Value Computed by CMake +hello_BINARY_DIR:STATIC=/home/jpmag/proj/cmany/test/hello/build + +//Value Computed by CMake +hello_IS_TOP_LEVEL:STATIC=ON + +//Value Computed by CMake +hello_SOURCE_DIR:STATIC=/home/jpmag/proj/cmany/test/hello + + +######################## +# INTERNAL cache entries +######################## + +//ADVANCED property for variable: CMAKE_ADDR2LINE +CMAKE_ADDR2LINE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_AR +CMAKE_AR-ADVANCED:INTERNAL=1 +//This is the directory where this CMakeCache.txt was created +CMAKE_CACHEFILE_DIR:INTERNAL=/home/jpmag/proj/cmany/test/hello/build +//Major version of cmake used to create the current loaded cache +CMAKE_CACHE_MAJOR_VERSION:INTERNAL=3 +//Minor version of cmake used to create the current loaded cache +CMAKE_CACHE_MINOR_VERSION:INTERNAL=25 +//Patch version of cmake used to create the current loaded cache +CMAKE_CACHE_PATCH_VERSION:INTERNAL=2 +//ADVANCED property for variable: CMAKE_COLOR_MAKEFILE +CMAKE_COLOR_MAKEFILE-ADVANCED:INTERNAL=1 +//Path to CMake executable. +CMAKE_COMMAND:INTERNAL=/usr/bin/cmake +//Path to cpack program executable. +CMAKE_CPACK_COMMAND:INTERNAL=/usr/bin/cpack +//Path to ctest program executable. +CMAKE_CTEST_COMMAND:INTERNAL=/usr/bin/ctest +//ADVANCED property for variable: CMAKE_CXX_COMPILER +CMAKE_CXX_COMPILER-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_COMPILER_AR +CMAKE_CXX_COMPILER_AR-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_COMPILER_RANLIB +CMAKE_CXX_COMPILER_RANLIB-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_FLAGS +CMAKE_CXX_FLAGS-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_FLAGS_DEBUG +CMAKE_CXX_FLAGS_DEBUG-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_FLAGS_MINSIZEREL +CMAKE_CXX_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_FLAGS_RELEASE +CMAKE_CXX_FLAGS_RELEASE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_CXX_FLAGS_RELWITHDEBINFO +CMAKE_CXX_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_COMPILER +CMAKE_C_COMPILER-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_COMPILER_AR +CMAKE_C_COMPILER_AR-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_COMPILER_RANLIB +CMAKE_C_COMPILER_RANLIB-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_FLAGS +CMAKE_C_FLAGS-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_FLAGS_DEBUG +CMAKE_C_FLAGS_DEBUG-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_FLAGS_MINSIZEREL +CMAKE_C_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_FLAGS_RELEASE +CMAKE_C_FLAGS_RELEASE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_C_FLAGS_RELWITHDEBINFO +CMAKE_C_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_DLLTOOL +CMAKE_DLLTOOL-ADVANCED:INTERNAL=1 +//Path to cache edit program executable. +CMAKE_EDIT_COMMAND:INTERNAL=/usr/bin/ccmake +//Executable file format +CMAKE_EXECUTABLE_FORMAT:INTERNAL=ELF +//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS +CMAKE_EXE_LINKER_FLAGS-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS_DEBUG +CMAKE_EXE_LINKER_FLAGS_DEBUG-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS_MINSIZEREL +CMAKE_EXE_LINKER_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS_RELEASE +CMAKE_EXE_LINKER_FLAGS_RELEASE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO +CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_EXPORT_COMPILE_COMMANDS +CMAKE_EXPORT_COMPILE_COMMANDS-ADVANCED:INTERNAL=1 +//Name of external makefile project generator. +CMAKE_EXTRA_GENERATOR:INTERNAL= +//Name of generator. +CMAKE_GENERATOR:INTERNAL=Unix Makefiles +//Generator instance identifier. +CMAKE_GENERATOR_INSTANCE:INTERNAL= +//Name of generator platform. +CMAKE_GENERATOR_PLATFORM:INTERNAL= +//Name of generator toolset. +CMAKE_GENERATOR_TOOLSET:INTERNAL= +//Source directory with the top level CMakeLists.txt file for this +// project +CMAKE_HOME_DIRECTORY:INTERNAL=/home/jpmag/proj/cmany/test/hello +//Install .so files without execute permission. +CMAKE_INSTALL_SO_NO_EXE:INTERNAL=0 +//ADVANCED property for variable: CMAKE_LINKER +CMAKE_LINKER-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_MAKE_PROGRAM +CMAKE_MAKE_PROGRAM-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS +CMAKE_MODULE_LINKER_FLAGS-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS_DEBUG +CMAKE_MODULE_LINKER_FLAGS_DEBUG-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS_MINSIZEREL +CMAKE_MODULE_LINKER_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS_RELEASE +CMAKE_MODULE_LINKER_FLAGS_RELEASE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO +CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_NM +CMAKE_NM-ADVANCED:INTERNAL=1 +//number of local generators +CMAKE_NUMBER_OF_MAKEFILES:INTERNAL=1 +//ADVANCED property for variable: CMAKE_OBJCOPY +CMAKE_OBJCOPY-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_OBJDUMP +CMAKE_OBJDUMP-ADVANCED:INTERNAL=1 +//Platform information initialized +CMAKE_PLATFORM_INFO_INITIALIZED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_RANLIB +CMAKE_RANLIB-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_READELF +CMAKE_READELF-ADVANCED:INTERNAL=1 +//Path to CMake installation. +CMAKE_ROOT:INTERNAL=/usr/share/cmake +//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS +CMAKE_SHARED_LINKER_FLAGS-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS_DEBUG +CMAKE_SHARED_LINKER_FLAGS_DEBUG-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL +CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS_RELEASE +CMAKE_SHARED_LINKER_FLAGS_RELEASE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO +CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_SKIP_INSTALL_RPATH +CMAKE_SKIP_INSTALL_RPATH-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_SKIP_RPATH +CMAKE_SKIP_RPATH-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS +CMAKE_STATIC_LINKER_FLAGS-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS_DEBUG +CMAKE_STATIC_LINKER_FLAGS_DEBUG-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL +CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS_RELEASE +CMAKE_STATIC_LINKER_FLAGS_RELEASE-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO +CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED:INTERNAL=1 +//ADVANCED property for variable: CMAKE_STRIP +CMAKE_STRIP-ADVANCED:INTERNAL=1 +//uname command +CMAKE_UNAME:INTERNAL=/usr/bin/uname +//ADVANCED property for variable: CMAKE_VERBOSE_MAKEFILE +CMAKE_VERBOSE_MAKEFILE-ADVANCED:INTERNAL=1 +//linker supports push/pop state +_CMAKE_LINKER_PUSHPOP_STATE_SUPPORTED:INTERNAL=TRUE + diff --git a/test/run.bat b/test/run.bat index 55fa98a..7669dc2 100755 --- a/test/run.bat +++ b/test/run.bat @@ -22,7 +22,7 @@ echo %PIPINSTALL% cd test if %ERRORLEVEL% neq 0 exit /b %ERRORLEVEL% set PYTHONPATH=%root%\src -%PYTHON% -m pytest --cov=..\src --cov-reset %* +%PYTHON% -m pytest --cov=..\src --cov-reset -s %* if %ERRORLEVEL% neq 0 exit /b %ERRORLEVEL% exit /b %ERRORLEVEL% diff --git a/test/run.sh b/test/run.sh index 9fa8c31..03dd241 100755 --- a/test/run.sh +++ b/test/run.sh @@ -9,6 +9,6 @@ PY=${PYTHON:-python3} # run the cmany unit tests cd $root/test export PYTHONPATH=$root/src -$PY -m pytest --cov=../src --cov-reset $* +$PY -m pytest --cov=../src --cov-reset -v $* exit $? diff --git a/test/test_01cmake.py b/test/test_01cmake.py new file mode 100644 index 0000000..3baeec2 --- /dev/null +++ b/test/test_01cmake.py @@ -0,0 +1,443 @@ +#!/usr/bin/env python3 + +import unittest as ut +from parameterized import parameterized + +import os.path as osp +import shutil +import os + +import c4.cmany.util as util +import c4.cmany.err as err +import c4.cmany.cmake as c4cmake + +from collections import OrderedDict as odict + + +srcdir = osp.abspath(osp.dirname(__file__)) +dstdir = osp.join(osp.join(srcdir, 'hello/.test/'), '0--cache') +src = osp.join(srcdir, 'CMakeCache.txt') +dst = osp.join(dstdir, 'CMakeCache.txt') +validtypes = ["STRING", "PATH", "FILEPATH", "BOOL", "STATIC", "INTERNAL"] + + +def _clear_cache(): + if osp.exists(dst): + os.remove(dst) + assert not osp.exists(dst), dst + return dstdir + + +def _setup_cache(): + _clear_cache() + os.makedirs(dstdir, exist_ok=True) + assert osp.exists(dstdir), dstdir + assert src != dst + shutil.copy(src, dst) + assert osp.exists(dst), dst + return dstdir + + +# ----------------------------------------------------------------------------- + +class Test00has_cache(ut.TestCase): + + def test_00detects_existing(self): + dir = _setup_cache() + self.assertTrue(c4cmake.hascache(dir), dir) + + def test_01detects_missing(self): + dir = _clear_cache() + self.assertFalse(c4cmake.hascache(dir), dir) + + +# ----------------------------------------------------------------------------- + +""" +1 CMAKE_AR FILEPATH /usr/bin/ar +2 CMAKE_BUILD_TYPE STRING +3 CMAKE_COLOR_MAKEFILE BOOL ON +4 CMAKE_CXX_COMPILER FILEPATH /usr/bin/c++ +5 CMAKE_CXX_COMPILER_AR FILEPATH /usr/bin/gcc-ar +6 CMAKE_CXX_COMPILER_RANLIB FILEPATH /usr/bin/gcc-ranlib +7 CMAKE_CXX_FLAGS STRING +8 CMAKE_CXX_FLAGS_DEBUG STRING -g +9 CMAKE_CXX_FLAGS_MINSIZEREL STRING -Os -DNDEBUG +10 CMAKE_CXX_FLAGS_RELEASE STRING -O3 -DNDEBUG +11 CMAKE_CXX_FLAGS_RELWITHDEBINFO STRING -O2 -g -DNDEBUG +12 CMAKE_C_COMPILER FILEPATH /usr/bin/cc +13 CMAKE_C_COMPILER_AR FILEPATH /usr/bin/gcc-ar +14 CMAKE_C_COMPILER_RANLIB FILEPATH /usr/bin/gcc-ranlib +15 CMAKE_C_FLAGS STRING +16 CMAKE_C_FLAGS_DEBUG STRING -g +17 CMAKE_C_FLAGS_MINSIZEREL STRING -Os -DNDEBUG +18 CMAKE_C_FLAGS_RELEASE STRING -O3 -DNDEBUG +19 CMAKE_C_FLAGS_RELWITHDEBINFO STRING -O2 -g -DNDEBUG +20 CMAKE_DLLTOOL FILEPATH CMAKE_DLLTOOL-NOTFOUND +21 CMAKE_EXE_LINKER_FLAGS STRING +22 CMAKE_EXE_LINKER_FLAGS_DEBUG STRING +23 CMAKE_EXE_LINKER_FLAGS_MINSIZEREL STRING +24 CMAKE_EXE_LINKER_FLAGS_RELEASE STRING +25 CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO STRING +26 CMAKE_EXPORT_COMPILE_COMMANDS BOOL +27 CMAKE_FIND_PACKAGE_REDIRECTS_DIR STATIC /home/usr/proj/cmany/test/hello/build/CMakeFiles/pkgRedirects +28 CMAKE_INSTALL_PREFIX PATH /usr/local +29 CMAKE_LINKER FILEPATH /usr/bin/ld +30 CMAKE_MAKE_PROGRAM FILEPATH /usr/bin/make +31 CMAKE_MODULE_LINKER_FLAGS STRING +32 CMAKE_MODULE_LINKER_FLAGS_DEBUG STRING +33 CMAKE_MODULE_LINKER_FLAGS_MINSIZEREL STRING +34 CMAKE_MODULE_LINKER_FLAGS_RELEASE STRING +35 CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO STRING +36 CMAKE_NM FILEPATH /usr/bin/nm +37 CMAKE_OBJCOPY FILEPATH /usr/bin/objcopy +38 CMAKE_OBJDUMP FILEPATH /usr/bin/objdump +39 CMAKE_PROJECT_DESCRIPTION STATIC +40 CMAKE_PROJECT_HOMEPAGE_URL STATIC +41 CMAKE_PROJECT_NAME STATIC hello +42 CMAKE_RANLIB FILEPATH /usr/bin/ranlib +43 CMAKE_READELF FILEPATH /usr/bin/readelf +44 CMAKE_SHARED_LINKER_FLAGS STRING +45 CMAKE_SHARED_LINKER_FLAGS_DEBUG STRING +46 CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL STRING +47 CMAKE_SHARED_LINKER_FLAGS_RELEASE STRING +48 CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO STRING +49 CMAKE_SKIP_INSTALL_RPATH BOOL NO +50 CMAKE_SKIP_RPATH BOOL NO +51 CMAKE_STATIC_LINKER_FLAGS STRING +52 CMAKE_STATIC_LINKER_FLAGS_DEBUG STRING +53 CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL STRING +54 CMAKE_STATIC_LINKER_FLAGS_RELEASE STRING +55 CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO STRING +56 CMAKE_STRIP FILEPATH /usr/bin/strip +57 CMAKE_VERBOSE_MAKEFILE BOOL FALSE +58 hello_BINARY_DIR STATIC /home/usr/proj/cmany/test/hello/build +59 hello_IS_TOP_LEVEL STATIC ON +60 hello_SOURCE_DIR STATIC /home/usr/proj/cmany/test/hello +61 CMAKE_ADDR2LINE-ADVANCED INTERNAL 1 +62 CMAKE_AR-ADVANCED INTERNAL 1 +63 CMAKE_CACHEFILE_DIR INTERNAL /home/usr/proj/cmany/test/hello/build +64 CMAKE_CACHE_MAJOR_VERSION INTERNAL 3 +65 CMAKE_CACHE_MINOR_VERSION INTERNAL 25 +66 CMAKE_CACHE_PATCH_VERSION INTERNAL 2 +67 CMAKE_COLOR_MAKEFILE-ADVANCED INTERNAL 1 +68 CMAKE_COMMAND INTERNAL /usr/bin/cmake +69 CMAKE_CPACK_COMMAND INTERNAL /usr/bin/cpack +70 CMAKE_CTEST_COMMAND INTERNAL /usr/bin/ctest +71 CMAKE_CXX_COMPILER-ADVANCED INTERNAL 1 +72 CMAKE_CXX_COMPILER_AR-ADVANCED INTERNAL 1 +73 CMAKE_CXX_COMPILER_RANLIB-ADVANCED INTERNAL 1 +74 CMAKE_CXX_FLAGS-ADVANCED INTERNAL 1 +75 CMAKE_CXX_FLAGS_DEBUG-ADVANCED INTERNAL 1 +76 CMAKE_CXX_FLAGS_MINSIZEREL-ADVANCED INTERNAL 1 +77 CMAKE_CXX_FLAGS_RELEASE-ADVANCED INTERNAL 1 +78 CMAKE_CXX_FLAGS_RELWITHDEBINFO-ADVANCED INTERNAL 1 +79 CMAKE_C_COMPILER-ADVANCED INTERNAL 1 +80 CMAKE_C_COMPILER_AR-ADVANCED INTERNAL 1 +81 CMAKE_C_COMPILER_RANLIB-ADVANCED INTERNAL 1 +82 CMAKE_C_FLAGS-ADVANCED INTERNAL 1 +83 CMAKE_C_FLAGS_DEBUG-ADVANCED INTERNAL 1 +84 CMAKE_C_FLAGS_MINSIZEREL-ADVANCED INTERNAL 1 +85 CMAKE_C_FLAGS_RELEASE-ADVANCED INTERNAL 1 +86 CMAKE_C_FLAGS_RELWITHDEBINFO-ADVANCED INTERNAL 1 +87 CMAKE_DLLTOOL-ADVANCED INTERNAL 1 +88 CMAKE_EDIT_COMMAND INTERNAL /usr/bin/ccmake +89 CMAKE_EXECUTABLE_FORMAT INTERNAL ELF +90 CMAKE_EXE_LINKER_FLAGS-ADVANCED INTERNAL 1 +91 CMAKE_EXE_LINKER_FLAGS_DEBUG-ADVANCED INTERNAL 1 +92 CMAKE_EXE_LINKER_FLAGS_MINSIZEREL-ADVANCED INTERNAL 1 +93 CMAKE_EXE_LINKER_FLAGS_RELEASE-ADVANCED INTERNAL 1 +94 CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED INTERNAL 1 +95 CMAKE_EXPORT_COMPILE_COMMANDS-ADVANCED INTERNAL 1 +96 CMAKE_EXTRA_GENERATOR INTERNAL +97 CMAKE_GENERATOR INTERNAL Unix Makefiles +98 CMAKE_GENERATOR_INSTANCE INTERNAL +99 CMAKE_GENERATOR_PLATFORM INTERNAL +100 CMAKE_GENERATOR_TOOLSET INTERNAL +101 CMAKE_HOME_DIRECTORY INTERNAL /home/usr/proj/cmany/test/hello +102 CMAKE_INSTALL_SO_NO_EXE INTERNAL 0 +103 CMAKE_LINKER-ADVANCED INTERNAL 1 +104 CMAKE_MAKE_PROGRAM-ADVANCED INTERNAL 1 +105 CMAKE_MODULE_LINKER_FLAGS-ADVANCED INTERNAL 1 +106 CMAKE_MODULE_LINKER_FLAGS_DEBUG-ADVANCED INTERNAL 1 +107 CMAKE_MODULE_LINKER_FLAGS_MINSIZEREL-ADVANCED INTERNAL 1 +108 CMAKE_MODULE_LINKER_FLAGS_RELEASE-ADVANCED INTERNAL 1 +109 CMAKE_MODULE_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED INTERNAL 1 +110 CMAKE_NM-ADVANCED INTERNAL 1 +111 CMAKE_NUMBER_OF_MAKEFILES INTERNAL 1 +112 CMAKE_OBJCOPY-ADVANCED INTERNAL 1 +113 CMAKE_OBJDUMP-ADVANCED INTERNAL 1 +114 CMAKE_PLATFORM_INFO_INITIALIZED INTERNAL 1 +115 CMAKE_RANLIB-ADVANCED INTERNAL 1 +116 CMAKE_READELF-ADVANCED INTERNAL 1 +117 CMAKE_ROOT INTERNAL /usr/share/cmake +118 CMAKE_SHARED_LINKER_FLAGS-ADVANCED INTERNAL 1 +119 CMAKE_SHARED_LINKER_FLAGS_DEBUG-ADVANCED INTERNAL 1 +120 CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL-ADVANCED INTERNAL 1 +121 CMAKE_SHARED_LINKER_FLAGS_RELEASE-ADVANCED INTERNAL 1 +122 CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED INTERNAL 1 +123 CMAKE_SKIP_INSTALL_RPATH-ADVANCED INTERNAL 1 +124 CMAKE_SKIP_RPATH-ADVANCED INTERNAL 1 +125 CMAKE_STATIC_LINKER_FLAGS-ADVANCED INTERNAL 1 +126 CMAKE_STATIC_LINKER_FLAGS_DEBUG-ADVANCED INTERNAL 1 +127 CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL-ADVANCED INTERNAL 1 +128 CMAKE_STATIC_LINKER_FLAGS_RELEASE-ADVANCED INTERNAL 1 +129 CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO-ADVANCED INTERNAL 1 +130 CMAKE_STRIP-ADVANCED INTERNAL 1 +131 CMAKE_UNAME INTERNAL /usr/bin/uname +132 CMAKE_VERBOSE_MAKEFILE-ADVANCED INTERNAL 1 +133 _CMAKE_LINKER_PUSHPOP_STATE_SUPPORTED INTERNAL TRUE +""" + +class Test01guess_var_type(ut.TestCase): + + def test_00internal(self): + self.assertEqual(c4cmake._guess_var_type("CMAKE_NM-ADVANCED", "1"), "INTERNAL") + self.assertEqual(c4cmake._guess_var_type("CMAKE_SHARED_LINKER_FLAGS-ADVANCED", "1"), "INTERNAL") + + def test_01bool(self): + self.assertEqual(c4cmake._guess_var_type("foo", "ON"), "BOOL") + self.assertEqual(c4cmake._guess_var_type("foo", "OFF"), "BOOL") + self.assertEqual(c4cmake._guess_var_type("foo", "YES"), "BOOL") + self.assertEqual(c4cmake._guess_var_type("foo", "NO"), "BOOL") + self.assertEqual(c4cmake._guess_var_type("foo", "1"), "BOOL") + self.assertEqual(c4cmake._guess_var_type("foo", "0"), "BOOL") + self.assertEqual(c4cmake._guess_var_type("foo", "TRUE"), "BOOL") + self.assertEqual(c4cmake._guess_var_type("foo", "FALSE"), "BOOL") + self.assertEqual(c4cmake._guess_var_type("foo", "T"), "BOOL") + self.assertEqual(c4cmake._guess_var_type("foo", "F"), "BOOL") + self.assertEqual(c4cmake._guess_var_type("foo", "N"), "BOOL") + self.assertEqual(c4cmake._guess_var_type("foo", "Y"), "BOOL") + self.assertEqual(c4cmake._guess_var_type("CMAKE_EXPORT_COMPILE_COMMANDS", None), "BOOL") + self.assertEqual(c4cmake._guess_var_type("CMAKE_EXPORT_COMPILE_COMMANDS", "ON"), "BOOL") + self.assertEqual(c4cmake._guess_var_type("CMAKE_EXPORT_COMPILE_COMMANDS", "1"), "BOOL") + + def test_10full(self): + cachedict = c4cmake.loadvars(_setup_cache()) + self.assertGreater(len(cachedict), 0) + for i, (k, v) in enumerate(cachedict.items()): + #print(i, k, v.vartype, v.val) + self.assertEqual(k, v.name) + self.assertEqual(c4cmake._guess_var_type(v.name, v.val), v.vartype, v.name) + + +# ----------------------------------------------------------------------------- +class Test02CMakeCacheVar(ut.TestCase): + + def test_00default_args(self): + v = c4cmake.CMakeCacheVar("foo", "bar") + self.assertEqual(v.name, "foo") + self.assertEqual(v.val, "bar") + self.assertEqual(v.vartype, "STRING") + self.assertEqual(v.dirty, False) + self.assertEqual(v.from_input, False) + + def test_01vartype_is_correct(self): + """set the variable without the vartype, then check it is correctly guessed""" + cachedict = c4cmake.loadvars(_setup_cache()) + self.assertGreater(len(cachedict), 0) + for i, (k, v) in enumerate(cachedict.items()): + vnew = c4cmake.CMakeCacheVar(v.name, v.val) + self.assertEqual(vnew.vartype, v.vartype, v.name) + + def test_10reset_val_taints(self): + v = c4cmake.CMakeCacheVar("foo", "bar", "STRING") + self.assertEqual(v.name, "foo") + self.assertEqual(v.val, "bar") + self.assertEqual(v.vartype, "STRING") + self.assertFalse(v.dirty) + self.assertFalse(v.from_input) + v.reset(val="newval") + self.assertTrue(v.dirty) + + def test_11reset_same_val_preserves(self): + v = c4cmake.CMakeCacheVar("foo", "bar", "STRING") + self.assertEqual(v.name, "foo") + self.assertEqual(v.val, "bar") + self.assertEqual(v.vartype, "STRING") + self.assertFalse(v.dirty) + self.assertFalse(v.from_input) + v.reset(val=v.val) + self.assertFalse(v.dirty) + + def test_20reset_type_fails(self): + v = c4cmake.CMakeCacheVar("foo", "bar", "STRING") + self.assertEqual(v.name, "foo") + self.assertEqual(v.val, "bar") + self.assertEqual(v.vartype, "STRING") + self.assertFalse(v.dirty) + with self.assertRaises(err.CannotChangeCacheVarType) as cm: + v.reset(val="foo", vartype="FILEPATH") + + +# ----------------------------------------------------------------------------- + +class Test03loadvars(ut.TestCase): + + def test_00empty_when_cache_missing(self): + v = c4cmake.loadvars(_clear_cache()) + self.assertEqual(len(v), 0) + + def test_01non_empty_when_cache_exists(self): + v = c4cmake.loadvars(_setup_cache()) + self.assertGreater(len(v), 0) + + def test_02all_members_are_CMakeVar(self): + cachedict = c4cmake.loadvars(_setup_cache()) + self.assertGreater(len(cachedict), 0) + for k, v in cachedict.items(): + self.assertTrue(isinstance(v, c4cmake.CMakeCacheVar)) + + def test_03all_vars_are_set(self): + cachedict = c4cmake.loadvars(_setup_cache()) + self.assertGreater(len(cachedict), 0) + for i, (k, v) in enumerate(cachedict.items()): + #print(i, k, v.vartype, v.val) + self.assertTrue(isinstance(v, c4cmake.CMakeCacheVar)) + self.assertEqual(k, v.name) + self.assertFalse(v.dirty) + self.assertTrue(v.from_input) + self.assertIn(v.vartype, validtypes, v.name) + + +# ----------------------------------------------------------------------------- + +class Test04setcachevars(ut.TestCase): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.dir = _setup_cache() + self.original = c4cmake.loadvars(self.dir) + + def _set_and_load(self, varvalues): + c4cmake.setcachevars(self.dir, varvalues) + return c4cmake.loadvars(self.dir) + + def test_00_mutate_existing_val_succeeds(self): + name = "CMAKE_C_FLAGS" # use this because there are several other vars with the same prefix + self.assertIn(name, self.original.keys()) + result = self._set_and_load({ + name: self.original[name].val + ".changed", + }) + self.assertNotEqual(result[name].val, self.original[name].val) + self.assertEqual(result[name].vartype, self.original[name].vartype) + # check no entries were added/removed + self.assertEqual(len(result.keys()), len(self.original.keys())) + self.assertEqual(list(sorted(result.keys())), list(sorted(self.original.keys()))) + for k, v in result.items(): + if k == name: + self.assertNotEqual(v.val, self.original[k].val) + self.assertEqual(v.vartype, self.original[k].vartype) + else: + self.assertEqual(v.val, self.original[k].val) + self.assertEqual(v.vartype, self.original[k].vartype) + + def test_01_same_existing_val_succeeds(self): + name = "CMAKE_C_FLAGS" # use this because there are several other vars with the same prefix + self.assertIn(name, self.original.keys()) + result = self._set_and_load({ + name: self.original[name].val, + }) + self.assertEqual(result[name].val, self.original[name].val) + self.assertEqual(result[name].vartype, self.original[name].vartype) + for k, v in result.items(): + self.assertEqual(v.val, self.original[k].val) + self.assertEqual(v.vartype, self.original[k].vartype) + + def test_10_set_multiple_val_succeeds(self): + names = [ + "CMAKE_C_FLAGS", + "CMAKE_C_COMPILER", + "CMAKE_CXX_FLAGS", + "CMAKE_CXX_COMPILER", + ] + varvalues = odict() + for name in names: + self.assertIn(name, self.original.keys()) + varvalues[name] = self.original[name].val + "asdads" + result = self._set_and_load(varvalues) + for name in names: + self.assertNotEqual(result[name].val, self.original[name].val) + self.assertEqual(result[name].vartype, self.original[name].vartype) + # check no entries were added/removed + self.assertEqual(len(result.keys()), len(self.original.keys())) + self.assertEqual(list(sorted(result.keys())), list(sorted(self.original.keys()))) + for k, v in result.items(): + if k in names: + self.assertNotEqual(v.val, self.original[k].val) + self.assertEqual(v.vartype, self.original[k].vartype) + else: + self.assertEqual(v.val, self.original[k].val) + self.assertEqual(v.vartype, self.original[k].vartype) + + def test_20_set_same_type_succeeds(self): + names = [ + "CMAKE_C_FLAGS", + "CMAKE_C_COMPILER", + "CMAKE_CXX_FLAGS", + "CMAKE_CXX_COMPILER", + ] + varvalues = odict() + for name in names: + self.assertIn(name, self.original.keys()) + var = self.original[name] + varvalues[name + ":" + var.vartype] = var.val + result = self._set_and_load(varvalues) + for name in names: + self.assertEqual(result[name].val, self.original[name].val) + self.assertEqual(result[name].vartype, self.original[name].vartype) + # check no entries were added/removed + self.assertEqual(len(result.keys()), len(self.original.keys())) + self.assertEqual(list(sorted(result.keys())), list(sorted(self.original.keys()))) + for k, v in result.items(): + self.assertEqual(v.val, self.original[k].val) + self.assertEqual(v.vartype, self.original[k].vartype) + + @parameterized.expand([ + [ + "all_wrong", [ + ("CMAKE_C_FLAGS","FILEPATH"), + ("CMAKE_C_COMPILER","STRING"), + ("CMAKE_CXX_FLAGS","FILEPATH"), + ("CMAKE_CXX_COMPILER","STRING"), + ], + ], + [ + "one_wrong", [ + ("CMAKE_C_FLAGS","STRING"), + ("CMAKE_C_COMPILER","STRING"), + ("CMAKE_CXX_FLAGS","STRING"), + ("CMAKE_CXX_COMPILER","FILEPATH"), + ], + ], + ]) + def test_21_change_type_fails(self, casename, names): + varvalues = odict() + for name, vartype in names: + self.assertIn(name, self.original.keys()) + varvalues[name+":"+vartype] = self.original[name].val + with self.assertRaises(err.CannotChangeCacheVarType) as cm: + c4cmake.setcachevars(self.dir, varvalues) + + +# ----------------------------------------------------------------------------- + +class Test05CMakeCache(ut.TestCase): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + _setup_cache() + + def test_00basic(self): + c = c4cmake.CMakeCache(dstdir) + self.assertGreater(len(c), 0) + + +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +if __name__ == '__main__': + if len(sys.argv) > 1 and sys.argv[1] == 'echo': + print(sys.argv[2:]) # for test_invoke() + else: + ut.main() diff --git a/test/test_05cmany.py b/test/test_05cmany.py index 40fda2e..d74e8cc 100644 --- a/test/test_05cmany.py +++ b/test/test_05cmany.py @@ -303,11 +303,20 @@ def __init__(self, proj, buildroot, installroot, compiler, build_type, variant, num_jobs=cpu_count(), kwargs={} ) + assert self.build_obj.varcache["CMAKE_C_COMPILER"].vartype == "FILEPATH" + assert self.build_obj.varcache["CMAKE_CXX_COMPILER"].vartype == "FILEPATH" def checkc(self, tester): tester.assertEqual(self.nsiblings(self.buildroot), self.numbuilds, msg=self.buildroot + str(self.siblings(self.buildroot))) - build_type = cmake.getcachevar(self.build_obj.builddir, 'CMAKE_BUILD_TYPE') - tester.assertEqual(build_type, str(self.build_type)) + cache = cmake.CMakeCache(self.build_obj.builddir) + tester.assertEqual(cache["CMAKE_BUILD_TYPE"].val, str(self.build_type)) + tester.assertEqual(cache["CMAKE_BUILD_TYPE"].vartype, "STRING") + tester.assertEqual(self.build_obj.varcache["CMAKE_C_COMPILER"].vartype, "FILEPATH") + tester.assertEqual(self.build_obj.varcache["CMAKE_CXX_COMPILER"].vartype, "FILEPATH") + tester.assertEqual(cache["CMAKE_C_COMPILER"].vartype, self.build_obj.varcache["CMAKE_C_COMPILER"].vartype) + tester.assertEqual(cache["CMAKE_CXX_COMPILER"].vartype, self.build_obj.varcache["CMAKE_CXX_COMPILER"].vartype) + tester.assertEqual(cache["CMAKE_C_COMPILER"].vartype, "FILEPATH") + tester.assertEqual(cache["CMAKE_CXX_COMPILER"].vartype, "FILEPATH") def checkv(self, tester): pass @@ -459,6 +468,8 @@ def test4x_subcommand_help_long_compare(self): # ----------------------------------------------------------------------------- class Test01Configure(ut.TestCase): + + def test00_default(self): run_projs(self, ['c'], lambda tb: tb.checkc(self)) @@ -474,6 +485,7 @@ class Test02Build(ut.TestCase): def test00_default(self): run_projs(self, ['b'], lambda tb: tb.checkb(self)) + # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- From 9f671038723b8eae797b2682309106668d0ec9d1 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sun, 19 Mar 2023 15:29:30 +0100 Subject: [PATCH 47/50] [fix] loaded cache vars must not be from input --- src/c4/cmany/build.py | 2 -- src/c4/cmany/cmake.py | 2 +- test/test_01cmake.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index 2edde15..47f09d7 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -23,8 +23,6 @@ from .conan import Conan - - # ----------------------------------------------------------------------------- class Build(NamedItem): """Holds a build's settings""" diff --git a/src/c4/cmany/cmake.py b/src/c4/cmany/cmake.py index ce9b3bc..e413819 100644 --- a/src/c4/cmany/cmake.py +++ b/src/c4/cmany/cmake.py @@ -106,7 +106,7 @@ def loadvars(builddir): vartype = re.sub(_cache_entry, r'\2', ls)[1:] value = re.sub(_cache_entry, r'\3', ls) # dbg("loadvars1", name, vartype, value) - v[name] = CMakeCacheVar(name, value, vartype, dirty=False, from_input=True) + v[name] = CMakeCacheVar(name, value, vartype, dirty=False, from_input=False) return v diff --git a/test/test_01cmake.py b/test/test_01cmake.py index 3baeec2..fca5e34 100644 --- a/test/test_01cmake.py +++ b/test/test_01cmake.py @@ -296,7 +296,7 @@ def test_03all_vars_are_set(self): self.assertTrue(isinstance(v, c4cmake.CMakeCacheVar)) self.assertEqual(k, v.name) self.assertFalse(v.dirty) - self.assertTrue(v.from_input) + self.assertFalse(v.from_input) self.assertIn(v.vartype, validtypes, v.name) From 28e88364a7a88798b77ea303f8b03f76f9c7a5b2 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Mon, 20 Mar 2023 18:35:13 +0000 Subject: [PATCH 48/50] [fix] icc --- src/c4/cmany/compiler.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/c4/cmany/compiler.py b/src/c4/cmany/compiler.py index a31b382..31d133d 100644 --- a/src/c4/cmany/compiler.py +++ b/src/c4/cmany/compiler.py @@ -49,11 +49,13 @@ def is_trivial(self): return True def __init__(self, spec): + dbg("initing from '{}'".format(spec)) if util.is_quoted(spec): spec = util.unquote(spec) spl = spec.split(':') path = spl[0] if path.startswith("vs") or path.startswith("Visual Studio"): + dbg("vs-like") if not util.in_windows(): raise err.CompilerNotFound(spec, "visual studio is only available in windows platforms") vs = vsinfo.VisualStudioInfo(path) @@ -86,7 +88,7 @@ def __init__(self, spec): path[0] = os.path.abspath(p) name, version, version_full = self.get_version(path) self.shortname = name - self.gcclike = self.shortname in ('gcc', 'clang', 'icc', 'g++', 'clang++', 'icpc', 'icpx') + self.gcclike = self.shortname in ('gcc', 'clang', 'icc', 'icx', 'g++', 'clang++', 'icpc', 'icpx') self.is_msvc = self.shortname.startswith('vs') if not self.is_msvc: name += version @@ -112,14 +114,17 @@ def __init__(self, spec): @staticmethod def get_c_compiler(shortname, cxx_compiler): + exename = os.path.basename(cxx_compiler) + dbg("getting c compiler. shortname:", shortname) + dbg("getting c compiler. cxx_compiler:", cxx_compiler) # if cxx_compiler.endswith("c++") or cxx_compiler.endswith('c++.exe'): # cc = re.sub(r'c\+\+', r'cc', cxx_compiler) if shortname.startswith('vs') or re.search(r'sual Studio', cxx_compiler): cc = cxx_compiler - elif shortname == "icc" or shortname == "icpc": + elif exename == "icc" or exename == "icpc": cc = re.sub(r'icpc', r'icc', cxx_compiler) - elif shortname == "icpx": - cc = re.sub(r'icpx', r'icpx', cxx_compiler) + elif exename == "icpx": + cc = re.sub(r'icpx', r'icx', cxx_compiler) elif shortname == "gcc" or shortname == "g++" or shortname.startswith("g++"): if re.search(r'g\+\+', cxx_compiler): cc = re.sub(r'g\+\+', r'gcc', cxx_compiler) @@ -136,6 +141,7 @@ def get_c_compiler(shortname, cxx_compiler): cc = re.sub(r"c\+\+", "cc", cxx_compiler) else: cc = "cc" + dbg("c compiler is:", cc) return cc def get_version(self, path): @@ -158,6 +164,7 @@ def get_version(self, path): base = os.path.basename(path) dbg("base:", base) dbg("name:", name) + dbg(name, "cmp base:", base, name) if base.startswith("c++") or base.startswith("cc"): try: # if this fails, just go on. It's not really needed. with tempfile.NamedTemporaryFile(suffix=".cpp", prefix="cmany.", delete=False) as f: @@ -203,6 +210,7 @@ def get_version(self, path): version = re.sub(vregex, r'\1', version) dbg("gcc version:", version, "---") elif name.startswith("icpc") or name.startswith("icc"): + dbg("intel:", name, name.find('++')) name = "icc" if name.startswith("icc") else "icpc" if re.search(r'icpc \(ICC\) ' + vregex + '.*', version_full): version = re.sub(r'icpc \(ICC\) ' + vregex + '.*', r'\1', version_full) From 742b2bac5b2eea55dc586d9ea4f043c14ace74ae Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 4 May 2024 15:03:57 +0200 Subject: [PATCH 49/50] run_target: wrap arguments for gdb, ddd, kdbg --- src/c4/cmany/build.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/c4/cmany/build.py b/src/c4/cmany/build.py index 47f09d7..e8208fe 100644 --- a/src/c4/cmany/build.py +++ b/src/c4/cmany/build.py @@ -268,6 +268,14 @@ def run_targets(self, targets, target_args, cmd_wrap=None, workdir=None): cmd_wrap = [] if cmd_wrap is None else shlex.split(cmd_wrap) for tgt_name in targets: t = self.get_target(tgt_name) + if cmd_wrap: + w = cmd_wrap[0] + if w.startswith("gdb"): + cmd_wrap += ["--args"] + elif w.startswith("ddd"): + cmd_wrap += ["--args"] + elif w.startswith("kdbg"): + target_args = ["-a", util.shlex_join(target_args)] cmd = cmd_wrap + [t.output_file] + target_args cwd = workdir if workdir is not None else t.subdir_abs util.runcmd(cmd, cwd=cwd) From f24acfb4616c930947b52aaa72d44fa8badf4267 Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Tue, 14 May 2024 21:00:02 +0200 Subject: [PATCH 50/50] add kflags for C and C++ --- src/c4/cmany/args.py | 2 ++ src/c4/cmany/build_flags.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/c4/cmany/args.py b/src/c4/cmany/args.py index 4149140..c75d087 100644 --- a/src/c4/cmany/args.py +++ b/src/c4/cmany/args.py @@ -403,6 +403,8 @@ def add_cflags(parser): Multiple invokations of -D are possible, in which case arguments will be appended and not overwritten. To escape commas, use a backslash \\.""") + g.add_argument("-K", "--kflags", default=[], action=FlagArgument, + help="""Add C and C++ compiler flags. See -C and -X.""") g.add_argument("-X", "--cxxflags", default=[], action=FlagArgument, help="""Add C++ compiler flags. Accepts a comma-separated list of C++ compiler flags. diff --git a/src/c4/cmany/build_flags.py b/src/c4/cmany/build_flags.py index 5ba152c..3032260 100644 --- a/src/c4/cmany/build_flags.py +++ b/src/c4/cmany/build_flags.py @@ -17,6 +17,10 @@ def __init__(self, name, **kwargs): self.cflags = kwargs.get('cflags', []) self.cxxflags = kwargs.get('cxxflags', []) self.toolchain = kwargs.get('toolchain') + kflags = kwargs.get('kflags') + if kflags is not None: + self.cflags += kflags + self.cxxflags += kflags # self.include_dirs = kwargs['include_dirs'] # self.link_dirs = kwargs['link_dirs']