Implement EMCC_FORCE_STDLIBS using --whole-archive (#7274)

Previously when using EMCC_FORCE_STDLIBS we were building a second
`.bc` copy of any `.a` libraries.  Not that we have
--whole-archive/--no-whole-archive we no longer need to do this.

This change also refactors a bunch of code in system_libs that
includes a couple of fixes:

- Libraries can now depend in other libraries that appear earlier in
  the list.
- Libraries that are depended on by forced libraries are not themselves
  treated as forced.

embuilder.py

@@ -108,30 +108,6 @@
 logger = logging.getLogger(__file__)
 
 def build(src, result_libs, args=[]):
-  # if a library is a .a, also build the .bc, as we need it when forcing a
-  # a system library - in that case, we always want all the code linked in
-  if result_libs:
-    need_forced = []
-    with_forced = []
-    for result_lib in result_libs:
-      if result_lib.endswith('.a'):
-        short = result_lib[:-2]
-        if short in SYSTEM_TASKS:
-          need_forced.append(short.replace('_noexcept', ''))
-          with_forced.append(short + '.bc')
-          continue
-      with_forced.append(result_lib)
-
-    if need_forced:
-      if os.environ.get('EMCC_FORCE_STDLIBS'):
-        print('skipping forced (.bc) versions of .a libraries, since EMCC_FORCE_STDLIBS already set')
-      else:
-        os.environ['EMCC_FORCE_STDLIBS'] = ','.join(need_forced)
-        try:
-          build(src, with_forced, args)
-        finally:
-          del os.environ['EMCC_FORCE_STDLIBS']
-
   # build in order to generate the libraries
   # do it all in a temp dir where everything will be cleaned up
   temp_dir = temp_files.get_dir()

tests/test_other.py

@@ -8229,7 +8229,7 @@ def test(filename, expectations):
                  0, [],                         ['tempDoublePtr', 'waka'],     8,   0,    0), # noqa; totally empty!
       # but we don't metadce with linkable code! other modules may want it
       (['-O3', '-s', 'MAIN_MODULE=1'],
-              1529, ['invoke_i'],               ['waka'],                 469663, 149, 1439),
+              1529, ['invoke_i'],               ['waka'],                 469663, 149, 1443),
     ]) # noqa
 
     print('test on a minimal pure computational thing')

tests/test_sanity.py

@@ -749,8 +749,8 @@ def test_embuilder(self):
       ([PYTHON, EMBUILDER, 'build', 'emmalloc'], ['building and verifying emmalloc', 'success'], True, ['emmalloc.bc']),
       ([PYTHON, EMBUILDER, 'build', 'emmalloc_debug'], ['building and verifying emmalloc', 'success'], True, ['emmalloc_debug.bc']),
       ([PYTHON, EMBUILDER, 'build', 'pthreads'], ['building and verifying pthreads', 'success'], True, ['pthreads.bc']),
-      ([PYTHON, EMBUILDER, 'build', 'libcxx'], ['success'], True, ['libcxx.a', 'libcxx.bc']),
-      ([PYTHON, EMBUILDER, 'build', 'libcxx_noexcept'], ['success'], True, ['libcxx_noexcept.a', 'libcxx_noexcept.bc']),
+      ([PYTHON, EMBUILDER, 'build', 'libcxx'], ['success'], True, ['libcxx.a']),
+      ([PYTHON, EMBUILDER, 'build', 'libcxx_noexcept'], ['success'], True, ['libcxx_noexcept.a']),
       ([PYTHON, EMBUILDER, 'build', 'libcxxabi'], ['success'], True, ['libcxxabi.bc']),
       ([PYTHON, EMBUILDER, 'build', 'gl'], ['success'], True, ['gl.bc']),
       ([PYTHON, EMBUILDER, 'build', 'native_optimizer'], ['success'], True, ['optimizer.2.exe']),

tools/system_libs.py

@@ -12,6 +12,7 @@
 import sys
 import tarfile
 import zipfile
+from collections import namedtuple
 
 from . import ports
 from . import shared
@@ -502,9 +503,13 @@ def create_wasm_libc_rt(libname):
   # You can provide 1 to include everything, or a comma-separated list with the ones you want
   force = os.environ.get('EMCC_FORCE_STDLIBS')
   force_all = force == '1'
-  force = set((force.split(',') if force else []) + forced)
-  if force:
-    logging.debug('forcing stdlibs: ' + str(force))
+  force_include = set((force.split(',') if force else []) + forced)
+  if force_include:
+    logging.debug('forcing stdlibs: ' + str(force_include))
+
+  # Set of libraries to include on the link line, as opposed to `force` which
+  # is the set of libraries to force include (with --whole-archive).
+  always_include = set()
 
   # Setting this will only use the forced libs in EMCC_FORCE_STDLIBS. This avoids spending time checking
   # for unresolved symbols in your project files, which can speed up linking, but if you do not have
@@ -538,7 +543,7 @@ def add_back_deps(need):
       add_back_deps(need) # recurse to get deps of deps
 
   # Scan symbols
-  symbolses = shared.Building.parallel_llvm_nm(list(map(os.path.abspath, temp_files)))
+  symbolses = shared.Building.parallel_llvm_nm([os.path.abspath(t) for t in temp_files])
 
   if len(symbolses) == 0:
     class Dummy(object):
@@ -574,97 +579,124 @@ class Dummy(object):
     libc_deps += ['wasm-libc']
   if shared.Settings.USE_PTHREADS:
     libc_name = 'libc-mt'
-    force.add('pthreads')
-    force.add('pthreads_asmjs')
+    always_include.add('pthreads')
+    always_include.add('pthreads_asmjs')
+  always_include.add(malloc_name())
+  if shared.Settings.WASM_BACKEND:
+    always_include.add('compiler-rt')
+
+  Library = namedtuple('Library', ['shortname', 'suffix', 'create', 'symbols', 'deps', 'can_noexcept'])
 
-  system_libs = [('libcxx',        'a', create_libcxx,      libcxx_symbols,      ['libcxxabi'], True), # noqa
-                 ('libcxxabi',     ext, create_libcxxabi,   libcxxabi_symbols,   [libc_name],   False), # noqa
-                 ('gl',            ext, create_gl,          gl_symbols,          [libc_name],   False), # noqa
-                 ('al',            ext, create_al,          al_symbols,          [libc_name],   False), # noqa
-                 ('html5',         ext, create_html5,       html5_symbols,       [],            False), # noqa
-                 ('compiler-rt',   'a', create_compiler_rt, compiler_rt_symbols, [libc_name],   False), # noqa
-                 (malloc_name(),   ext, create_malloc,      [],                  [],            False)] # noqa
+  system_libs = [Library('libcxx',        'a', create_libcxx,      libcxx_symbols,      ['libcxxabi'], True), # noqa
+                 Library('libcxxabi',     ext, create_libcxxabi,   libcxxabi_symbols,   [libc_name],   False), # noqa
+                 Library('gl',            ext, create_gl,          gl_symbols,          [libc_name],   False), # noqa
+                 Library('al',            ext, create_al,          al_symbols,          [libc_name],   False), # noqa
+                 Library('html5',         ext, create_html5,       html5_symbols,       [],            False), # noqa
+                 Library('compiler-rt',   'a', create_compiler_rt, compiler_rt_symbols, [libc_name],   False), # noqa
+                 Library(malloc_name(),   ext, create_malloc,      [],                  [],            False)] # noqa
 
   if shared.Settings.USE_PTHREADS:
-    system_libs += [('pthreads',       ext, create_pthreads,       pthreads_symbols,       [libc_name],  False), # noqa
-                    ('pthreads_asmjs', ext, create_pthreads_asmjs, asmjs_pthreads_symbols, [libc_name],  False)] # noqa
+    system_libs += [Library('pthreads',       ext, create_pthreads,       pthreads_symbols,       [libc_name],  False), # noqa
+                    Library('pthreads_asmjs', ext, create_pthreads_asmjs, asmjs_pthreads_symbols, [libc_name],  False)] # noqa
 
-  system_libs += [(libc_name, ext, create_libc, libc_symbols, libc_deps, False)]
+  system_libs.append(Library(libc_name, ext, create_libc, libc_symbols, libc_deps, False))
 
   # if building to wasm, we need more math code, since we have less builtins
   if shared.Settings.WASM:
-    system_libs += [('wasm-libc', ext, create_wasm_libc, wasm_libc_symbols, [], False)]
+    system_libs.append(Library('wasm-libc', ext, create_wasm_libc, wasm_libc_symbols, [], False))
 
   # Add libc-extras at the end, as libc may end up requiring them, and they depend on nothing.
-  system_libs += [('libc-extras', ext, create_libc_extras, libc_extras_symbols, [], False)]
+  system_libs.append(Library('libc-extras', ext, create_libc_extras, libc_extras_symbols, [], False))
 
-  force.add(malloc_name())
-  if shared.Settings.WASM_BACKEND:
-    force.add('compiler-rt')
+  libs_to_link = []
+  already_included = set()
 
-  # Go over libraries to figure out which we must include
   def maybe_noexcept(name):
     if shared.Settings.DISABLE_EXCEPTION_CATCHING:
       name += '_noexcept'
     return name
-  ret = []
-  has = need = None
 
-  all_names = [s[0] for s in system_libs]
+  system_libs_map = {l.shortname: l for l in system_libs}
+
+  def add_library(lib):
+    if lib.shortname in already_included:
+      return
+    already_included.add(lib.shortname)
 
-  for shortname, suffix, create, library_symbols, deps, can_noexcept in system_libs:
-    assert all(d in all_names for d in deps)
-    force_this = force_all or shortname in force
-    if can_noexcept:
+    shortname = lib.shortname
+    if lib.can_noexcept:
       shortname = maybe_noexcept(shortname)
-    if force_this and not shared.Settings.WASM_OBJECT_FILES:
-      # .a files do not always link in all their parts; don't use them when forced
-      # When using wasm object files there are only .a archives but they get
-      # included via --whole-archive.
-      suffix = 'bc'
-    name = shortname + '.' + suffix
-
-    if not force_this:
-      need = set()
-      has = set()
+    name = shortname + '.' + lib.suffix
+
+    logging.debug('including %s' % name)
+
+    def do_create():
+      return lib.create(name)
+
+    libfile = shared.Cache.get(name, do_create, extension=lib.suffix)
+    need_whole_archive = lib.shortname in force_include and lib.suffix != 'bc'
+    libs_to_link.append((libfile, need_whole_archive))
+
+    # Recursively add dependencies
+    for d in lib.deps:
+      add_library(system_libs_map[d])
+
+  # Go over libraries to figure out which we must include
+  for lib in system_libs:
+    if lib.shortname in already_included:
+      continue
+    force_this = lib.shortname in force_include
+    if not force_this and only_forced:
+      continue
+    include_this = force_this or lib.shortname in always_include
+
+    if not include_this:
+      need_syms = set()
+      has_syms = set()
       for symbols in symbolses:
         if shared.Settings.VERBOSE:
           logging.debug('undefs: ' + str(symbols.undefs))
-        for library_symbol in library_symbols:
+        for library_symbol in lib.symbols:
           if library_symbol in symbols.undefs:
-            need.add(library_symbol)
+            need_syms.add(library_symbol)
           if library_symbol in symbols.defs:
-            has.add(library_symbol)
-      for haz in has: # remove symbols that are supplied by another of the inputs
-        if haz in need:
-          need.remove(haz)
+            has_syms.add(library_symbol)
+      for haz in has_syms:
+        if haz in need_syms:
+          # remove symbols that are supplied by another of the inputs
+          need_syms.remove(haz)
       if shared.Settings.VERBOSE:
-        logging.debug('considering %s: we need %s and have %s' % (name, str(need), str(has)))
-    if force_this or (len(need) and not only_forced):
-      # We need to build and link the library in
-      logging.debug('including %s' % name)
-
-      def do_create():
-        return create(name)
+        logging.debug('considering %s: we need %s and have %s' % (lib.shortname, str(need_syms), str(has_syms)))
+      if not len(need_syms):
+        continue
 
-      libfile = shared.Cache.get(name, do_create, extension=suffix)
-      ret.append(libfile)
-      force = force.union(deps)
+    # We need to build and link the library in
+    add_library(lib)
 
   if shared.Settings.WASM_BACKEND:
-    ret.append(shared.Cache.get('wasm_compiler_rt.a', lambda: create_wasm_compiler_rt('wasm_compiler_rt.a'), extension='a'))
-    ret.append(shared.Cache.get('wasm_libc_rt.a', lambda: create_wasm_libc_rt('wasm_libc_rt.a'), extension='a'))
-
-  ret.sort(key=lambda x: x.endswith('.a')) # make sure to put .a files at the end.
-
-  for actual in ret:
-    if os.path.basename(actual) == 'libcxxabi.bc':
-      # libcxxabi and libcxx *static* linking is tricky. e.g. cxa_demangle.cpp disables c++
-      # exceptions, but since the string methods in the headers are *weakly* linked, then
-      # we might have exception-supporting versions of them from elsewhere, and if libcxxabi
-      # is first then it would "win", breaking exception throwing from those string
-      # header methods. To avoid that, we link libcxxabi last.
-      ret = [f for f in ret if f != actual] + [actual]
+    libs_to_link.append((shared.Cache.get('wasm_compiler_rt.a', lambda: create_wasm_compiler_rt('wasm_compiler_rt.a'), extension='a'), False))
+    libs_to_link.append((shared.Cache.get('wasm_libc_rt.a', lambda: create_wasm_libc_rt('wasm_libc_rt.a'), extension='a'), False))
+
+  libs_to_link.sort(key=lambda x: x[0].endswith('.a')) # make sure to put .a files at the end.
+
+  # libcxxabi and libcxx *static* linking is tricky. e.g. cxa_demangle.cpp disables c++
+  # exceptions, but since the string methods in the headers are *weakly* linked, then
+  # we might have exception-supporting versions of them from elsewhere, and if libcxxabi
+  # is first then it would "win", breaking exception throwing from those string
+  # header methods. To avoid that, we link libcxxabi last.
+  libs_to_link.sort(key=lambda x: x[0].endswith('libcxxabi.bc'))
+
+  # Wrap libraries in --whole-archive, as needed.  We need to do this last
+  # since otherwise the abort sorting won't make sense.
+  if force_all:
+    ret = ['--whole-archive'] + [r[0] for r in libs_to_link] + ['--no-whole-archive']
+  else:
+    ret = []
+    for name, need_whole_archive in libs_to_link:
+      if need_whole_archive:
+        ret += ['--whole-archive', name, '--no-whole-archive']
+      else:
+        ret.append(name)
 
   return ret