[lldb] Add windows support for LLDB_EXPORT_ALL_SYMBOLS

[River707]

LLDB_EXPORT_ALL_SYMBOLS is useful when building out-of-tree plugins and extensions that rely on LLDB's internal symbols. For example, this is how the Mojo language provides its REPL and debugger support.

Supporting this on windows is kind of tricky because this is normally expected to be done using dllexport/dllimport, but lldb uses these with the public api. This PR takes an approach similar to what LLVM does with LLVM_EXPORT_SYMBOLS_FOR_PLUGINS, and what chromium does for abseil, and uses a python script to extract the necessary symbols by looking at the symbol table for the various lldb libraries.

[llvmbot]

@llvm/pr-subscribers-lldb

Changes

LLDB_EXPORT_ALL_SYMBOLS is useful when building out-of-tree plugins and extensions that rely on LLDB's internal symbols. For example, this is how the Mojo language provides its REPL and debugger support.

Supporting this on windows is kind of tricky because this is normally expected to be done using dllexport/dllimport, but lldb uses these with the public api. This PR takes an approach similar to what LLVM does with LLVM_EXPORT_SYMBOLS_FOR_PLUGINS, and what chromium does for abseil, and uses a python script to extract the necessary symbols by looking at the symbol table for the various lldb libraries.

---

Full diff: https://github.com/llvm/llvm-project/pull/67628.diff

3 Files Affected:

• (modified) lldb/cmake/modules/LLDBConfig.cmake (+2-8)
• (added) lldb/scripts/msvc_extract_private_symbols.py (+100)
• (modified) lldb/source/API/CMakeLists.txt (+37)

diff --git a/lldb/cmake/modules/LLDBConfig.cmake b/lldb/cmake/modules/LLDBConfig.cmake
index 19283b3cbb0194f..380016ce48015fa 100644
--- a/lldb/cmake/modules/LLDBConfig.cmake
+++ b/lldb/cmake/modules/LLDBConfig.cmake
@@ -122,14 +122,8 @@ if(APPLE AND CMAKE_GENERATOR STREQUAL Xcode)
   endif()
 endif()
 
-if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
-  set(LLDB_EXPORT_ALL_SYMBOLS 0 CACHE BOOL
-    "Causes lldb to export all symbols when building liblldb.")
-else()
-  # Windows doesn't support toggling this, so don't bother making it a
-  # cache variable.
-  set(LLDB_EXPORT_ALL_SYMBOLS 0)
-endif()
+set(LLDB_EXPORT_ALL_SYMBOLS 0 CACHE BOOL
+  "Causes lldb to export all symbols when building liblldb.")
 
 if ((NOT MSVC) OR MSVC12)
   add_definitions( -DHAVE_ROUND )
diff --git a/lldb/scripts/msvc_extract_private_symbols.py b/lldb/scripts/msvc_extract_private_symbols.py
new file mode 100644
index 000000000000000..91ed02ffa8665f0
--- /dev/null
+++ b/lldb/scripts/msvc_extract_private_symbols.py
@@ -0,0 +1,100 @@
+"""A tool for extracting a list of private lldb symbols to export for MSVC.
+
+When exporting symbols from a dll or exe we either need to mark the symbols in
+the source code as __declspec(dllexport) or supply a list of symbols to the
+linker. Private symbols in LLDB don't explicitly specific dllexport, so we
+automate that by examining the symbol table.
+"""
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+
+
+def extract_symbols(nm_path: str, lib: str):
+    """Extract all of the private lldb symbols from the given path to llvm-nm and
+    library to extract from."""
+
+    # Matches mangled symbols containing 'lldb_private'.
+    lldb_sym_re = r"0* [BT] (?P<symbol>[?]+[^?].*lldb_private.*)"
+
+    # '-g' means we only get global symbols.
+    # '-p' do not waste time sorting the symbols.
+    process = subprocess.Popen(
+        [nm_path, "-g", "-p", lib],
+        bufsize=1,
+        stdout=subprocess.PIPE,
+        stdin=subprocess.PIPE,
+        universal_newlines=True,
+    )
+    process.stdin.close()
+
+    lldb_symbols = set()
+    for line in process.stdout:
+        match = re.match(lldb_sym_re, line)
+        if match:
+            symbol = match.group("symbol")
+            assert symbol.count(" ") == 0, (
+                "Regex matched too much, probably got undecorated name as well"
+            )
+            # Deleting destructors start with ?_G or ?_E and can be discarded
+            # because link.exe gives you a warning telling you they can't be
+            # exported if you don't.
+            if symbol.startswith("??_G") or symbol.startswith("??_E"):
+                continue
+            lldb_symbols.add(symbol)
+
+    return lldb_symbols
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate LLDB dll exports")
+    parser.add_argument(
+        "-o", metavar="file", type=str, help="The name of the resultant export file."
+    )
+    parser.add_argument("--nm", help="Path to the llvm-nm executable.")
+    parser.add_argument(
+        "libs",
+        metavar="lib",
+        type=str,
+        nargs="+",
+        help="The libraries to extract symbols from.",
+    )
+    args = parser.parse_args()
+
+    # Get the list of libraries to extract symbols from
+    libs = list()
+    for lib in args.libs:
+        # When invoked by cmake the arguments are the cmake target names of the
+        # libraries, so we need to add .lib/.a to the end and maybe lib to the
+        # start to get the filename. Also allow objects.
+        suffixes = [".lib", ".a", ".obj", ".o"]
+        if not any([lib.endswith(s) for s in suffixes]):
+            for s in suffixes:
+                if os.path.exists(lib + s):
+                    lib = lib + s
+                    break
+                if os.path.exists("lib" + lib + s):
+                    lib = "lib" + lib + s
+                    break
+        if not any([lib.endswith(s) for s in suffixes]):
+            print("Don't know what to do with argument " + lib, file=sys.stderr)
+            exit(1)
+        libs.append(lib)
+
+    # Extract symbols from the input libraries.
+    symbols = set()
+    for lib in libs:
+        for sym in list(extract_symbols(args.nm, lib)):
+            symbols.add(sym)
+
+    # Write out the symbols to the output file.
+    with open(args.o, "w", newline="") as f:
+        for s in sorted(symbols):
+            f.write(f"{s}\n")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/lldb/source/API/CMakeLists.txt b/lldb/source/API/CMakeLists.txt
index 910cb667e16b708..7cfa3aaafdae188 100644
--- a/lldb/source/API/CMakeLists.txt
+++ b/lldb/source/API/CMakeLists.txt
@@ -184,6 +184,43 @@ if (NOT CMAKE_SYSTEM_NAME MATCHES "Windows")
     add_llvm_symbol_exports(liblldb ${CMAKE_CURRENT_SOURCE_DIR}/liblldb-private.exports)
   endif()
   set_target_properties(liblldb_exports PROPERTIES FOLDER "lldb misc")
+elseif (LLDB_EXPORT_ALL_SYMBOLS)
+  MESSAGE("-- Symbols (liblldb): exporting all symbols from the lldb and lldb_private namespaces")
+
+  # Pull out the various lldb libraries linked into liblldb, these will be used
+  # when looking for symbols to extract. We ignore plugin libraries here,
+  # because these symbols aren't publicly exposed.
+  get_target_property(all_liblldb_libs liblldb LINK_LIBRARIES)
+  set(lldb_libs "")
+  foreach(lib ${all_liblldb_libs})
+    if(TARGET ${lib} AND ${lib} MATCHES "^lldb" AND
+       NOT ${lib} MATCHES "^lldbPlugin")
+      get_target_property(lib_type ${lib} TYPE)
+      if("${lib_type}" STREQUAL "STATIC_LIBRARY")
+        list(APPEND lldb_libs ${lib})
+      endif()
+    endif()
+  endforeach(lib)
+  list(REMOVE_DUPLICATES lldb_libs)
+
+  # Extract all of the private symbols and produce a single file we can use for
+  # the exports.
+  set(exported_symbol_file ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}/liblldb_private.symbols)
+  get_host_tool_path(llvm-nm LLVM_NM llvm_nm_exe llvm_nm_target)
+  add_custom_command(
+    OUTPUT ${exported_symbol_file}
+    COMMAND "${Python3_EXECUTABLE}"
+      ${LLDB_SOURCE_DIR}/scripts/msvc_extract_private_symbols.py
+      ${lldb_libs} -o ${exported_symbol_file} --nm=${llvm_nm_exe}
+    WORKING_DIRECTORY ${LLVM_LIBRARY_OUTPUT_INTDIR}
+    DEPENDS ${LLDB_SOURCE_DIR}/scripts/msvc_extract_private_symbols.py
+      ${lldb_libs} ${llvm_nm_target}
+    VERBATIM
+    COMMENT "Generating liblldb private export list"
+  )
+
+  add_llvm_symbol_exports(liblldb ${exported_symbol_file})
+  set_target_properties(liblldb_exports PROPERTIES FOLDER "lldb misc")
 endif()
 
 if (NOT MSVC)

[walter-erquinigo]

Other than a few nits, this is awesome!

[walter-erquinigo]

I'd rewrite this like

Suggested change

	for s in suffixes:
	if os.path.exists(lib + s):
	lib = lib + s
	break
	if os.path.exists("lib" + lib + s):
	lib = "lib" + lib + s
	break
	for suffix in suffixes:
	for path in [lib + suffix, "lib" + lib + suffix]:
	if os.path.exists(path):
	lib = path
	break

[River707]

Tried this, but it actually ends up being a bit more convoluted because there are two loops to break from instead of just one (either need to recheck the suffix, or use the weird else after for loop + continue/break)