Skip to content

Commit

Permalink
Merge pull request #938 from centricular/fix-unity-builds
Browse files Browse the repository at this point in the history 
Several commits that fix unity builds
  • Loading branch information
@jpakkane
jpakkane committed Oct 22, 2016
2 parents 8b27a48 + 7e9203f commit 1e3e22c
Show file tree
Hide file tree
Showing 14 changed files with 202 additions and 99 deletions.
83 changes: 38 additions & 45 deletions mesonbuild/backend/backends.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from .. import compilers
import json
import subprocess
from ..mesonlib import MesonException
from ..mesonlib import MesonException, get_compiler_for_source, classify_unity_sources

class InstallData():
def __init__(self, source_dir, build_dir, prefix):
Expand Down Expand Up @@ -78,21 +78,6 @@ def __init__(self, build):
priv_dirname = self.get_target_private_dir_abs(t)
os.makedirs(priv_dirname, exist_ok=True)

def get_compiler_for_lang(self, lang):
for i in self.build.compilers:
if i.language == lang:
return i
raise RuntimeError('No compiler for language ' + lang)

def get_compiler_for_source(self, src, is_cross):
comp = self.build.cross_compilers if is_cross else self.build.compilers
for i in comp:
if i.can_compile(src):
return i
if isinstance(src, mesonlib.File):
src = src.fname
raise RuntimeError('No specified compiler can handle file ' + src)

def get_target_filename(self, t):
if isinstance(t, build.CustomTarget):
if len(t.get_outputs()) != 1:
Expand Down Expand Up @@ -153,14 +138,17 @@ def get_target_generated_dir(self, target, gensrc, src):
# target that the GeneratedList is used in
return os.path.join(self.get_target_private_dir(target), src)

def get_unity_source_filename(self, target, suffix):
return target.name + '-unity.' + suffix

def generate_unity_files(self, target, unity_src):
langlist = {}
abs_files = []
result = []
compsrcs = classify_unity_sources(target.compilers.values(), unity_src)

def init_language_file(language, suffix):
def init_language_file(suffix):
outfilename = os.path.join(self.get_target_private_dir_abs(target),
target.name + '-unity' + suffix)
self.get_unity_source_filename(target, suffix))
outfileabs = os.path.join(self.environment.get_build_dir(),
outfilename)
outfileabs_tmp = outfileabs + '.tmp'
Expand All @@ -171,20 +159,12 @@ def init_language_file(language, suffix):
result.append(outfilename)
return open(outfileabs_tmp, 'w')

try:
for src in unity_src:
comp = self.get_compiler_for_source(src, target.is_cross)
language = comp.get_language()
try:
ofile = langlist[language]
except KeyError:
suffix = '.' + comp.get_default_suffix()
ofile = langlist[language] = init_language_file(language,
suffix)
ofile.write('#include<%s>\n' % src)
finally:
for x in langlist.values():
x.close()
# For each language, generate a unity source file and return the list
for comp, srcs in compsrcs.items():
lang = comp.get_language()
with init_language_file(comp.get_default_suffix()) as ofile:
for src in srcs:
ofile.write('#include<%s>\n' % src)
[mesonlib.replace_if_different(x, x + '.tmp') for x in abs_files]
return result

Expand Down Expand Up @@ -278,24 +258,37 @@ def determine_linker(self, target, src):
for s in src:
if c.can_compile(s):
return c
raise RuntimeError('Unreachable code')
raise AssertionError("BUG: Couldn't determine linker for sources {!r}".format(src))

def object_filename_from_source(self, target, source):
return source.fname.replace('/', '_').replace('\\', '_') + '.' + self.environment.get_object_suffix()
if isinstance(source, mesonlib.File):
source = source.fname
return source.replace('/', '_').replace('\\', '_') + '.' + self.environment.get_object_suffix()

def determine_ext_objs(self, extobj, proj_dir_to_build_root=''):
def determine_ext_objs(self, extobj, proj_dir_to_build_root):
result = []
targetdir = self.get_target_private_dir(extobj.target)
# With unity builds, there's just one object that contains all the
# sources, so if we want all the objects, just return that.
if self.environment.coredata.get_builtin_option('unity'):
if not extobj.unity_compatible:
# This should never happen
msg = 'BUG: Meson must not allow extracting single objects ' \
'in Unity builds'
raise AssertionError(msg)
comp = get_compiler_for_source(extobj.target.compilers.values(),
extobj.srclist[0])
# The unity object name uses the full absolute path of the source file
osrc = os.path.join(self.get_target_private_dir_abs(extobj.target),
self.get_unity_source_filename(extobj.target,
comp.get_default_suffix()))
objname = self.object_filename_from_source(extobj.target, osrc)
objpath = os.path.join(proj_dir_to_build_root, targetdir, objname)
return [objpath]
for osrc in extobj.srclist:
# If extracting in a subproject, the subproject
# name gets duplicated in the file name.
pathsegs = osrc.subdir.split(os.sep)
if pathsegs[0] == 'subprojects':
pathsegs = pathsegs[2:]
fixedpath = os.sep.join(pathsegs)
objname = os.path.join(proj_dir_to_build_root, targetdir,
self.object_filename_from_source(extobj.target, osrc))
result.append(objname)
objname = self.object_filename_from_source(extobj.target, osrc)
objpath = os.path.join(proj_dir_to_build_root, targetdir, objname)
result.append(objpath)
return result

def get_pch_include_args(self, compiler, target):
Expand Down
84 changes: 60 additions & 24 deletions mesonbuild/backend/ninjabackend.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from .. import mlog
from .. import dependencies
from .. import compilers
from ..mesonlib import File, MesonException
from ..mesonlib import File, MesonException, get_compiler_for_source
from .backends import InstallData
from ..build import InvalidArguments
import os, sys, pickle, re
Expand Down Expand Up @@ -258,6 +258,20 @@ def get_target_sources(self, target):
srcs[f] = s
return srcs

# Languages that can mix with C or C++ but don't support unity builds yet
# because the syntax we use for unity builds is specific to C/++/ObjC/++.
langs_cant_unity = ('d', 'fortran')
def get_target_source_can_unity(self, target, source):
if isinstance(source, File):
source = source.fname
suffix = os.path.splitext(source)[1][1:]
for lang in self.langs_cant_unity:
if not lang in target.compilers:
continue
if suffix in target.compilers[lang].file_suffixes:
return False
return True

def generate_target(self, target, outfile):
if isinstance(target, build.CustomTarget):
self.generate_custom_target(target, outfile)
Expand Down Expand Up @@ -319,6 +333,18 @@ def generate_target(self, target, outfile):
header_deps += self.get_generated_headers(target)
src_list = []

if is_unity:
# Warn about incompatible sources if a unity build is enabled
langs = set(target.compilers.keys())
langs_cant = langs.intersection(self.langs_cant_unity)
if langs_cant:
langs_are = langs = ', '.join(langs_cant).upper()
langs_are += ' are' if len(langs_cant) > 1 else ' is'
msg = '{} not supported in Unity builds yet, so {} ' \
'sources in the {!r} target will be compiled normally' \
''.format(langs_are, langs, target.name)
mlog.log(mlog.red('FIXME'), msg)

# Get a list of all generated *sources* (sources files, headers,
# objects, etc). Needed to determine the linker.
generated_output_sources = []
Expand All @@ -329,13 +355,14 @@ def generate_target(self, target, outfile):
generated_source_files = []
for rel_src, gensrc in generated_sources.items():
generated_output_sources.append(rel_src)
raw_src = RawFilename(rel_src)
if self.environment.is_source(rel_src) and not self.environment.is_header(rel_src):
if is_unity:
unity_deps.append(rel_src)
if is_unity and self.get_target_source_can_unity(target, rel_src):
unity_deps.append(raw_src)
abs_src = os.path.join(self.environment.get_build_dir(), rel_src)
unity_src.append(abs_src)
else:
generated_source_files.append(RawFilename(rel_src))
generated_source_files.append(raw_src)
elif self.environment.is_object(rel_src):
obj_list.append(rel_src)
elif self.environment.is_library(rel_src):
Expand All @@ -344,40 +371,48 @@ def generate_target(self, target, outfile):
# Assume anything not specifically a source file is a header. This is because
# people generate files with weird suffixes (.inc, .fh) that they then include
# in their source files.
header_deps.append(RawFilename(rel_src))
header_deps.append(raw_src)
# These are the generated source files that need to be built for use by
# this target. We create the Ninja build file elements for this here
# because we need `header_deps` to be fully generated in the above loop.
for src in generated_source_files:
src_list.append(src)
obj_list.append(self.generate_single_compile(target, outfile, src, True,
header_deps=header_deps))

# Generate compilation targets for C sources generated from Vala
# sources. This can be extended to other $LANG->C compilers later if
# necessary. This needs to be separate for at least Vala
vala_generated_source_files = []
for src in vala_generated_sources:
raw_src = RawFilename(src)
src_list.append(src)
if is_unity:
unity_src.append(os.path.join(self.environment.get_build_dir(), src))
header_deps.append(src)
header_deps.append(raw_src)
else:
# Generated targets are ordered deps because the must exist
# before the sources compiling them are used. After the first
# compile we get precise dependency info from dep files.
# This should work in all cases. If it does not, then just
# move them from orderdeps to proper deps.
if self.environment.is_header(src):
header_deps.append(src)
header_deps.append(raw_src)
else:
# Passing 'vala' here signifies that we want the compile
# arguments to be specialized for C code generated by
# valac. For instance, no warnings should be emitted.
obj_list.append(self.generate_single_compile(target, outfile, src, 'vala', [], header_deps))
# We gather all these and generate compile rules below
# after `header_deps` (above) is fully generated
vala_generated_source_files.append(raw_src)
for src in vala_generated_source_files:
# Passing 'vala' here signifies that we want the compile
# arguments to be specialized for C code generated by
# valac. For instance, no warnings should be emitted.
obj_list.append(self.generate_single_compile(target, outfile, src, 'vala', [], header_deps))

# Generate compile targets for all the pre-existing sources for this target
for f, src in target_sources.items():
if not self.environment.is_header(src):
src_list.append(src)
if is_unity:
if is_unity and self.get_target_source_can_unity(target, src):
abs_src = os.path.join(self.environment.get_build_dir(),
src.rel_to_builddir(self.build_to_src))
unity_src.append(abs_src)
Expand All @@ -386,7 +421,7 @@ def generate_target(self, target, outfile):
obj_list += self.flatten_object_list(target)
if is_unity:
for src in self.generate_unity_files(target, unity_src):
obj_list.append(self.generate_single_compile(target, outfile, src, True, unity_deps + header_deps))
obj_list.append(self.generate_single_compile(target, outfile, RawFilename(src), True, unity_deps + header_deps))
linker = self.determine_linker(target, src_list + generated_output_sources)
elem = self.generate_link(target, outfile, outname, obj_list, linker, pch_objects)
self.generate_shlib_aliases(target, self.get_target_dir(target))
Expand Down Expand Up @@ -1668,14 +1703,14 @@ def get_link_debugfile_args(self, linker, target, outname):

def generate_single_compile(self, target, outfile, src, is_generated=False, header_deps=[], order_deps=[]):
"""
Compiles only C/C++ and ObjC/ObjC++ sources
Compiles C/C++, ObjC/ObjC++, and D sources
"""
if(isinstance(src, str) and src.endswith('.h')):
raise RuntimeError('Fug')
if isinstance(src, str) and src.endswith('.h'):
raise AssertionError('BUG: sources should not contain headers')
if isinstance(src, RawFilename) and src.fname.endswith('.h'):
raise RuntimeError('Fug')
raise AssertionError('BUG: sources should not contain headers')
extra_orderdeps = []
compiler = self.get_compiler_for_source(src, target.is_cross)
compiler = get_compiler_for_source(target.compilers.values(), src)
commands = []
# The first thing is implicit include directories: source, build and private.
commands += compiler.get_include_args(self.get_target_private_dir(target), False)
Expand Down Expand Up @@ -1716,12 +1751,12 @@ def generate_single_compile(self, target, outfile, src, is_generated=False, head
break
if isinstance(src, RawFilename):
rel_src = src.fname
elif is_generated:
if self.has_dir_part(src):
rel_src = src
if os.path.isabs(src.fname):
abs_src = src.fname
else:
rel_src = os.path.join(self.get_target_private_dir(target), src)
abs_src = os.path.join(self.environment.get_source_dir(), rel_src)
abs_src = os.path.join(self.environment.get_build_dir(), src.fname)
elif is_generated:
raise AssertionError('BUG: broken generated source file handling for {!r}'.format(src))
else:
if isinstance(src, File):
rel_src = src.rel_to_builddir(self.build_to_src)
Expand Down Expand Up @@ -1805,6 +1840,7 @@ def generate_single_compile(self, target, outfile, src, is_generated=False, head
return rel_obj

def has_dir_part(self, fname):
# FIXME FIXME: The usage of this is a terrible and unreliable hack
return '/' in fname or '\\' in fname

# Fortran is a bit weird (again). When you link against a library, just compiling a source file
Expand Down Expand Up @@ -1857,7 +1893,7 @@ def generate_pch(self, target, outfile):
'directory as source, please put it in a subdirectory.' \
''.format(target.get_basename())
raise InvalidArguments(msg)
compiler = self.get_compiler_for_lang(lang)
compiler = target.compilers[lang]
if compiler.id == 'msvc':
src = os.path.join(self.build_to_src, target.get_source_subdir(), pch[-1])
(commands, dep, dst, objs) = self.generate_msvc_pch_command(target, compiler, pch)
Expand Down

0 comments on commit 1e3e22c

Please sign in to comment.