Reland "Make it possible for lit.site.cfg to contain relative paths, and use it for llvm and clang"

The problem on Windows was that the \b in "..\bin" was interpreted
as an escape sequence. Use r"" strings to prevent that.

This reverts commit ab11b9e,
with raw strings in the lit.site.cfg.py.in files.

Differential Revision: https://reviews.llvm.org/D77184

Author: nico

clang/test/CMakeLists.txt

@@ -35,13 +35,30 @@ configure_lit_site_cfg(
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+  PATHS
+  "LLVM_SOURCE_DIR"
+  "LLVM_BINARY_DIR"
+  "LLVM_TOOLS_DIR"
+  "LLVM_LIBS_DIR"
+  "SHLIBDIR"
+  "LLVM_LIT_TOOLS_DIR"
+  "CLANG_BINARY_DIR"
+  "CLANG_SOURCE_DIR"
+  "CLANG_TOOLS_DIR"
   )
 
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.cfg.py
+  PATHS
+  "LLVM_SOURCE_DIR"
+  "LLVM_BINARY_DIR"
+  "LLVM_TOOLS_DIR"
+  "LLVM_LIBS_DIR"
+  "CLANG_BINARY_DIR"
+  "SHLIBDIR"
   )
 
 option(CLANG_TEST_USE_VG "Run Clang tests under Valgrind" OFF)

clang/test/Unit/lit.site.cfg.py.in

@@ -2,14 +2,14 @@
 
 import sys
 
-config.llvm_src_root = "@LLVM_SOURCE_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
-config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
+config.llvm_src_root = path(r"@LLVM_SOURCE_DIR@")
+config.llvm_obj_root = path(r"@LLVM_BINARY_DIR@")
+config.llvm_tools_dir = path(r"@LLVM_TOOLS_DIR@")
+config.llvm_libs_dir = path(r"@LLVM_LIBS_DIR@")
 config.llvm_build_mode = "@LLVM_BUILD_MODE@"
-config.clang_obj_root = "@CLANG_BINARY_DIR@"
+config.clang_obj_root = path(r"@CLANG_BINARY_DIR@")
 config.enable_shared = @ENABLE_SHARED@
-config.shlibdir = "@SHLIBDIR@"
+config.shlibdir = path(r"@SHLIBDIR@")
 config.target_triple = "@TARGET_TRIPLE@"
 
 # Support substitution of the tools_dir, libs_dirs, and build_mode with user
@@ -25,4 +25,5 @@ except KeyError:
     lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
 
 # Let the main config do the real work.
-lit_config.load_config(config, "@CLANG_SOURCE_DIR@/test/Unit/lit.cfg.py")
+lit_config.load_config(
+    config, os.path.join(path(r"@CLANG_SOURCE_DIR@"), "test/Unit/lit.cfg.py"))

clang/test/lit.site.cfg.py.in

@@ -2,16 +2,16 @@
 
 import sys
 
-config.llvm_src_root = "@LLVM_SOURCE_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
-config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
-config.llvm_shlib_dir = "@SHLIBDIR@"
+config.llvm_src_root = path(r"@LLVM_SOURCE_DIR@")
+config.llvm_obj_root = path(r"@LLVM_BINARY_DIR@")
+config.llvm_tools_dir = path(r"@LLVM_TOOLS_DIR@")
+config.llvm_libs_dir = path(r"@LLVM_LIBS_DIR@")
+config.llvm_shlib_dir = path(r"@SHLIBDIR@")
 config.llvm_plugin_ext = "@LLVM_PLUGIN_EXT@"
-config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
-config.clang_obj_root = "@CLANG_BINARY_DIR@"
-config.clang_src_dir = "@CLANG_SOURCE_DIR@"
-config.clang_tools_dir = "@CLANG_TOOLS_DIR@"
+config.lit_tools_dir = path(r"@LLVM_LIT_TOOLS_DIR@")
+config.clang_obj_root = path(r"@CLANG_BINARY_DIR@")
+config.clang_src_dir = path(r"@CLANG_SOURCE_DIR@")
+config.clang_tools_dir = path(r"@CLANG_TOOLS_DIR@")
 config.host_triple = "@LLVM_HOST_TRIPLE@"
 config.target_triple = "@TARGET_TRIPLE@"
 config.host_cxx = "@CMAKE_CXX_COMPILER@"
@@ -50,4 +50,5 @@ if not "@CLANG_DEFAULT_LINKER@":
     config.available_features.add('platform-linker')
 
 # Let the main config do the real work.
-lit_config.load_config(config, "@CLANG_SOURCE_DIR@/test/lit.cfg.py")
+lit_config.load_config(
+    config, os.path.join(config.clang_src_dir, "test/lit.cfg.py"))

llvm/cmake/modules/AddLLVM.cmake

@@ -1396,8 +1396,14 @@ endmacro()
 # variables needed for the 'lit.site.cfg' files. This function bundles the
 # common variables that any Lit instance is likely to need, and custom
 # variables can be passed in.
+# The keyword PATHS is followed by a list of cmake variable names that are
+# mentioned as `path("@varname@")` in the lit.cfg.py.in file. Variables in that
+# list are treated as paths that are relative to the directory the generated
+# lit.cfg.py file is in, and the `path()` function converts the relative
+# path back to absolute form. This makes it possible to move a build directory
+# containing lit.cfg.py files from one machine to another.
 function(configure_lit_site_cfg site_in site_out)
-  cmake_parse_arguments(ARG "" "" "MAIN_CONFIG;OUTPUT_MAPPING" ${ARGN})
+  cmake_parse_arguments(ARG "" "" "MAIN_CONFIG;OUTPUT_MAPPING;PATHS" ${ARGN})
 
   if ("${ARG_MAIN_CONFIG}" STREQUAL "")
     get_filename_component(INPUT_DIR ${site_in} DIRECTORY)
@@ -1447,12 +1453,17 @@ function(configure_lit_site_cfg site_in site_out)
   set(HOST_CXX "${CMAKE_CXX_COMPILER} ${CMAKE_CXX_COMPILER_ARG1}")
   set(HOST_LDFLAGS "${CMAKE_EXE_LINKER_FLAGS}")
 
-  set(LIT_SITE_CFG_IN_HEADER  "## Autogenerated from ${site_in}\n## Do not edit!")
+  set(LIT_SITE_CFG_IN_HEADER  "# Autogenerated from ${site_in}\n# Do not edit!")
+
+  string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}\n\n"
+    "# Allow generated lit.site.cfg.py to be relocatable.\n"
+    "def path(p): return os.path.join(os.path.dirname(__file__), p) if p else ''\n"
+    )
 
   # Override config_target_triple (and the env)
   if(LLVM_TARGET_TRIPLE_ENV)
     # This is expanded into the heading.
-    string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}\n\n"
+    string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}"
       "import os\n"
       "target_env = \"${LLVM_TARGET_TRIPLE_ENV}\"\n"
       "config.target_triple = config.environment[target_env] = os.environ.get(target_env, \"${TARGET_TRIPLE}\")\n"
@@ -1462,7 +1473,45 @@ function(configure_lit_site_cfg site_in site_out)
     set(TARGET_TRIPLE "\"+config.target_triple+\"")
   endif()
 
+  if (ARG_PATHS)
+    # Walk ARG_PATHS and collect the current value of the variables in there.
+    foreach(path ${ARG_PATHS})
+      list(APPEND ARG_PATH_VALUES "${${path}}")
+    endforeach()
+
+    # Compute paths relative to the directory containing output lit.site.cfg.py.
+    # Passing ARG_PATH_VALUES as-is to execute_process() makes cmake strip
+    # empty list entries. So escape the ;s in the list and do the splitting
+    # outselves. cmake has no relpath function, so use Python for that.
+    string(REPLACE ";" "\\;" ARG_PATH_VALUES_ESCAPED "${ARG_PATH_VALUES}")
+    get_filename_component(OUTPUT_DIR ${site_out} DIRECTORY)
+    execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c"
+      "import os, sys; sys.stdout.write(';'.join(os.path.relpath(p, sys.argv[1]) if p else '' for p in sys.argv[2].split(';')))"
+      ${OUTPUT_DIR}
+      ${ARG_PATH_VALUES_ESCAPED}
+      OUTPUT_VARIABLE ARG_PATH_VALUES_RELATIVE)
+
+    list(LENGTH ARG_PATHS len_paths)
+    list(LENGTH ARG_PATH_VALUES len_path_values)
+    list(LENGTH ARG_PATH_VALUES_RELATIVE len_path_value_rels)
+    if ((NOT ${len_paths} EQUAL ${len_path_values}) OR
+        (NOT ${len_paths} EQUAL ${len_path_value_rels}))
+      message(SEND_ERROR "PATHS lengths got confused")
+    endif()
+
+    # Transform variables mentioned in ARG_PATHS to relative paths for
+    # the configure_file() call. Variables are copied to subscopeds by cmake,
+    # so this only modifies the local copy of the variables.
+    math(EXPR arg_path_limit "${len_paths} - 1")
+    foreach(i RANGE ${arg_path_limit})
+      list(GET ARG_PATHS ${i} val1)
+      list(GET ARG_PATH_VALUES_RELATIVE ${i} val2)
+      set(${val1} ${val2})
+    endforeach()
+  endif()
+
   configure_file(${site_in} ${site_out} @ONLY)
+
   if (EXISTS "${ARG_MAIN_CONFIG}")
     set(PYTHON_STATEMENT "map_config('${ARG_MAIN_CONFIG}', '${site_out}')")
     get_property(LLVM_LIT_CONFIG_MAP GLOBAL PROPERTY LLVM_LIT_CONFIG_MAP)

llvm/test/CMakeLists.txt

@@ -22,12 +22,23 @@ configure_lit_site_cfg(
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+  PATHS
+  "LLVM_SOURCE_DIR"
+  "LLVM_BINARY_DIR"
+  "LLVM_TOOLS_DIR"
+  "LLVM_LIBRARY_DIR"
+  "SHLIBDIR"
   )
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.cfg.py
+  PATHS
+  "LLVM_SOURCE_DIR"
+  "LLVM_BINARY_DIR"
+  "LLVM_TOOLS_DIR"
+  "SHLIBDIR"
   )
 
 # Set the depends list as a variable so that it can grow conditionally.

llvm/test/Unit/lit.site.cfg.py.in

@@ -2,12 +2,12 @@
 
 import sys
 
-config.llvm_src_root = "@LLVM_SOURCE_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
+config.llvm_src_root = path(r"@LLVM_SOURCE_DIR@")
+config.llvm_obj_root = path(r"@LLVM_BINARY_DIR@")
+config.llvm_tools_dir = path(r"@LLVM_TOOLS_DIR@")
 config.llvm_build_mode = "@LLVM_BUILD_MODE@"
 config.enable_shared = @ENABLE_SHARED@
-config.shlibdir = "@SHLIBDIR@"
+config.shlibdir = path(r"@SHLIBDIR@")
 
 # Support substitution of the tools_dir and build_mode with user parameters.
 # This is used when we can't determine the tool dir at configuration time.
@@ -20,4 +20,5 @@ except KeyError:
     lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
 
 # Let the main config do the real work.
-lit_config.load_config(config, "@LLVM_SOURCE_DIR@/test/Unit/lit.cfg.py")
+lit_config.load_config(
+    config, os.path.join(config.llvm_src_root, "test/Unit/lit.cfg.py"))

llvm/test/lit.site.cfg.py.in

@@ -4,14 +4,14 @@ import sys
 
 config.host_triple = "@LLVM_HOST_TRIPLE@"
 config.target_triple = "@TARGET_TRIPLE@"
-config.llvm_src_root = "@LLVM_SOURCE_DIR@"
-config.llvm_obj_root = "@LLVM_BINARY_DIR@"
-config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
-config.llvm_lib_dir = "@LLVM_LIBRARY_DIR@"
-config.llvm_shlib_dir = "@SHLIBDIR@"
+config.llvm_src_root = path(r"@LLVM_SOURCE_DIR@")
+config.llvm_obj_root = path(r"@LLVM_BINARY_DIR@")
+config.llvm_tools_dir = path(r"@LLVM_TOOLS_DIR@")
+config.llvm_lib_dir = path(r"@LLVM_LIBRARY_DIR@")
+config.llvm_shlib_dir = path(r"@SHLIBDIR@")
 config.llvm_shlib_ext = "@SHLIBEXT@"
 config.llvm_exe_ext = "@EXEEXT@"
-config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
+config.lit_tools_dir = path(r"@LLVM_LIT_TOOLS_DIR@")
 config.python_executable = "@PYTHON_EXECUTABLE@"
 config.gold_executable = "@GOLD_EXECUTABLE@"
 config.ld64_executable = "@LD64_EXECUTABLE@"
@@ -63,4 +63,5 @@ import lit.llvm
 lit.llvm.initialize(lit_config, config)
 
 # Let the main config do the real work.
-lit_config.load_config(config, "@LLVM_SOURCE_DIR@/test/lit.cfg.py")
+lit_config.load_config(
+    config, os.path.join(config.llvm_src_root, "test/lit.cfg.py"))

llvm/utils/gn/secondary/clang/test/BUILD.gn

@@ -12,7 +12,10 @@ template("write_lit_config") {
     input = invoker.input
     output = invoker.output
     values = [
-      "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
+      # FIXME: Write relative paths for path()s.
+      "LIT_SITE_CFG_IN_HEADER=" +
+          "## Autogenerated from $input, do not edit\n\n" +
+          "def path(p): return p if p else \"\"",
       "CLANG_BINARY_DIR=" +
           rebase_path(get_label_info("//clang", "target_out_dir")),
       "CLANG_SOURCE_DIR=" + rebase_path("//clang"),

llvm/utils/gn/secondary/llvm/test/BUILD.gn

@@ -14,7 +14,10 @@ template("write_lit_config") {
     input = invoker.input
     output = invoker.output
     values = [
-      "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
+      # FIXME: Write relative paths for path()s.
+      "LIT_SITE_CFG_IN_HEADER=" +
+          "## Autogenerated from $input, do not edit\n\n" +
+          "def path(p): return p if p else \"\"",
       "ENABLE_SHARED=0",
       "LLVM_BINARY_DIR=" +
           rebase_path(get_label_info("//llvm", "target_out_dir")),