build: Add SPIRV-Tools to known_good.json

Previously we relied on glslang to fetch its known-good SPIRV-Tools, but
this doesn't work if we need a newer version than they provide. This
change adds an entry for SPIRV-Tools to the repo known_good.json so we
can manage the dependency ourselves.

Change-Id: I61e2d83eb3698b2564a9f1c935f140f0b918134d

BUILD.md

@@ -116,28 +116,27 @@ the validation layers. You must also take note of the headers' install
 directory and pass it on the CMake command line for building this repository,
 as described below.
 
-#### glslang
+#### SPIRV-Headers
 
 This repository has a required dependency on the
-[glslang repository](https://github.com/KhronosGroup/glslang).
-The glslang repository is required because it contains components that are
-required to build the validation layers. You must clone the glslang repository
-and build its `install` target. Follow the build instructions in the glslang
-[README.md](https://github.com/KhronosGroup/glslang/blob/master/README.md)
-file. Ensure that the `update_glslang_sources.py` script has been run as part
-of building glslang. You must also take note of the glslang install directory
+[SPIRV-Headers repository](https://github.com/KhronosGroup/SPIRV-Headers).
+The SPIRV-Headers repository is required because it supports components that are
+required to build the validation layers. You must clone the SPIRV-Headers repository
+and build its `install` target. Follow the build instructions in the SPIRV-Headers
+[README.md](https://github.com/KhronosGroup/SPIRV-Headers/blob/master/README.md)
+file. You must also take note of the SPIRV-headers install directory
 and pass it on the CMake command line for building this repository, as
 described below.
 
-#### SPIRV-Headers
+#### SPIRV-Tools
 
 This repository has a required dependency on the
-[SPIRV-headers repository](https://github.com/KhronosGroup/SPIRV-Headers).
-The SPIRV-headers repository is required because it supports components that are
-required to build the validation layers. You must clone the SPIRV-headers repository
-and build its `install` target. Follow the build instructions in the SPIRV-headers
-[README.md](https://github.com/KhronosGroup/SPIRV-Headers/blob/master/README.md)
-file. You must also take note of the SPIRV-headers install directory
+[SPIRV-Tools repository](https://github.com/KhronosGroup/SPIRV-Tools).
+The SPIRV-Tools repository is required because it contains components that are
+required to build the validation layers. You must clone the SPIRV-Tools repository
+and build its `install` target. Follow the build instructions in the SPIRV-Tools
+[README.md](https://github.com/KhronosGroup/SPIRV-Tools/blob/master/README.md)
+file. You must also take note of the SPIRV-Tools install directory
 and pass it on the CMake command line for building this repository, as
 described below.
 
@@ -149,6 +148,17 @@ which provides substantial performance improvements on all platforms.
 Since there is nothing to build, the install directory for this repository is the
 directory where the the repository is cloned.
 
+#### glslang
+
+The validation layer tests depend on the
+[glslang repository](https://github.com/KhronosGroup/glslang).
+You must clone the glslang repository
+and build its `install` target. Follow the build instructions in the glslang
+[README.md](https://github.com/KhronosGroup/glslang/blob/master/README.md)
+file. You must also take note of the glslang install directory
+and pass it on the CMake command line for building this repository, as
+described below.
+
 #### Google Test
 
 The validation layer tests depend on the
@@ -451,6 +461,7 @@ location of the loader's install directory:
     cmake -A x64 -DVULKAN_HEADERS_INSTALL_DIR=absolute_path_to_install_dir \
                  -DVULKAN_LOADER_INSTALL_DIR=absolute_path_to_install_dir \
                  -DGLSLANG_INSTALL_DIR=absolute_path_to_install_dir \
+                 -DSPIRV_HEADERS_INSTALL_DIR=absolute_path_to_install_dir \
                  -DSPIRV_TOOLS_INSTALL_DIR=absolute_path_to_install_dir \
                  -DROBIN_HOOD_HASHING_INSTALL_DIR=absolute_path_to_install_dir \
                  ..

CMakeLists.txt

@@ -245,8 +245,7 @@ if (USE_ROBIN_HOOD_HASHING)
     set(ROBIN_HOOD_HASHING_INCLUDE_DIR "${ROBIN_HOOD_HASHING_INSTALL_DIR}/src/include" PATH "Path to robin-hood-hashing/src/include")
 endif()
 
-if(BUILD_TESTS OR BUILD_LAYERS)
-
+if(BUILD_TESTS)
     set(GLSLANG_INSTALL_DIR "GLSLANG-NOTFOUND" CACHE PATH "Absolute path to a glslang install directory")
     if(NOT GLSLANG_INSTALL_DIR AND NOT DEFINED ENV{GLSLANG_INSTALL_DIR} AND NOT TARGET glslang)
         message(FATAL_ERROR "Must define location of glslang binaries -- see BUILD.md")
@@ -274,8 +273,7 @@ if(BUILD_TESTS OR BUILD_LAYERS)
         message(STATUS "Using glslang install located at ${GLSLANG_INSTALL_DIR}")
         set(GLSLANG_SEARCH_PATH "${GLSLANG_INSTALL_DIR}/lib")
         set(GLSLANG_DEBUG_SEARCH_PATH "${GLSLANG_INSTALL_DIR}/lib")
-        set(GLSLANG_SPIRV_INCLUDE_DIR "${GLSLANG_INSTALL_DIR}/include/glslang" CACHE PATH "Path to glslang spirv headers")
-        set(SPIRV_HEADERS_INCLUDE_DIR "${SPIRV_HEADERS_INSTALL_DIR}/include" CACHE PATH "Path to spirv-headers")
+        set(GLSLANG_INCLUDE_DIR "${GLSLANG_INSTALL_DIR}/include" CACHE PATH "Path to glslang headers")
 
         set(GLSLANG_LIBRARIES glslang OGLCompiler OSDependent MachineIndependent GenericCodeGen HLSL SPIRV SPVRemapper)
 
@@ -301,16 +299,19 @@ if(BUILD_TESTS OR BUILD_LAYERS)
                                              "${GLSLANG_LIBRARIES}")
         endforeach()
     else()
-        set(GLSLANG_SPIRV_INCLUDE_DIR "${glslang_SOURCE_DIR}" CACHE PATH "Path to glslang spirv headers")
+        set(GLSLANG_INCLUDE_DIR "${glslang_SOURCE_DIR}" CACHE PATH "Path to glslang headers")
         set(GLSLANG_LIBRARIES glslang SPIRV SPVRemapper)
     endif()
+endif()
 
+if(BUILD_TESTS OR BUILD_LAYERS)
     # spirv-tools
     if (NOT TARGET SPIRV-Tools)
         if(NOT SPIRV_TOOLS_INSTALL_DIR)
             set(SPIRV_TOOLS_INSTALL_DIR "${GLSLANG_INSTALL_DIR}")
         endif()
 
+        set(SPIRV_HEADERS_INCLUDE_DIR "${SPIRV_HEADERS_INSTALL_DIR}/include" CACHE PATH "Path to spirv-headers")
         set(SPIRV_TOOLS_BINARY_ROOT "${SPIRV_TOOLS_INSTALL_DIR}/lib"
             CACHE PATH "User defined path to the SPIRV-Tools binaries for this project")
         set(SPIRV_TOOLS_OPT_BINARY_ROOT "${SPIRV_TOOLS_INSTALL_DIR}/lib"
@@ -324,34 +325,11 @@ if(BUILD_TESTS OR BUILD_LAYERS)
         find_library(SPIRV_TOOLS_LIB NAMES SPIRV-Tools HINTS ${SPIRV_TOOLS_SEARCH_PATH})
         find_library(SPIRV_TOOLS_OPT_LIB NAMES SPIRV-Tools-opt HINTS ${SPIRV_TOOLS_OPT_SEARCH_PATH})
 
-        if(WIN32)
-            add_library(SPIRV-Tools-opt STATIC IMPORTED)
-            add_library(SPIRV-Tools STATIC IMPORTED)
-
-            find_library(SPIRV_TOOLS_DLIB NAMES SPIRV-Toolsd HINTS ${SPIRV_TOOLS_DEBUG_SEARCH_PATH})
-            find_library(SPIRV_TOOLS_OPT_DLIB NAMES SPIRV-Tools-optd HINTS ${SPIRV_TOOLS_OPT_DEBUG_SEARCH_PATH})
-
-            set_target_properties(SPIRV-Tools
-                                  PROPERTIES IMPORTED_LOCATION
-                                             "${SPIRV_TOOLS_LIB}"
-                                             IMPORTED_LOCATION_DEBUG
-                                             "${SPIRV_TOOLS_DLIB}")
-            set_target_properties(SPIRV-Tools-opt
-                                  PROPERTIES IMPORTED_LOCATION
-                                             "${SPIRV_TOOLS_OPT_LIB}"
-                                             IMPORTED_LOCATION_DEBUG
-                                             "${SPIRV_TOOLS_OPT_DLIB}")
-
-            set(SPIRV_TOOLS_LIBRARIES SPIRV-Tools-opt SPIRV-Tools)
-        else()
-            set(SPIRV_TOOLS_LIBRARIES ${SPIRV_TOOLS_OPT_LIB} ${SPIRV_TOOLS_LIB})
-        endif()
+        set(SPIRV_TOOLS_LIBRARIES ${SPIRV_TOOLS_OPT_LIB} ${SPIRV_TOOLS_LIB})
     else()
         set(SPIRV_TOOLS_LIBRARIES SPIRV-Tools SPIRV-Tools-opt)
         set(SPIRV_TOOLS_INCLUDE_DIR "${spirv-tools_SOURCE_DIR}/include" CACHE PATH "Path to spirv tools headers")
     endif()
-
-    set(GLSLANG_LIBRARIES ${GLSLANG_LIBRARIES} ${SPIRV_TOOLS_LIBRARIES})
 endif()
 
 # Generate dependent helper files ------------------------------------------------------------------------------------------------

build-gn/secondary/build_overrides/spirv_tools.gni

@@ -1,4 +1,4 @@
-# Copyright (c) 2019 LunarG, Inc.
+# Copyright (c) 2019,2021 LunarG, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,4 +17,4 @@ spirv_tools_standalone = false
 
 # Paths to SPIRV-Tools dependencies
 spirv_tools_googletest_dir = "//external/googletest"
-spirv_tools_spirv_headers_dir = "//external/glslang/External/spirv-tools/external/spirv-headers"
+spirv_tools_spirv_headers_dir = "//external/SPIRV-Headers"

build-gn/secondary/build_overrides/vulkan_validation_layers.gni

@@ -14,7 +14,7 @@
 
 # Paths to validation layer dependencies
 vulkan_headers_dir = "//external/Vulkan-Headers"
-vvl_spirv_tools_dir = "//external/glslang/External/spirv-tools"
+vvl_spirv_tools_dir = "//external/SPIRV-Tools"
 vvl_glslang_dir = "//external/glslang/"
 robin_hood_headers_dir = "//external/robin-hood-hashing/src/include"
 

build-gn/update_deps.sh

@@ -19,7 +19,6 @@ cd "$(dirname $0)/.."
 
 # Use update_deps.py to update source dependencies from /scripts/known_good.json
 scripts/update_deps.py --dir="external" --no-build
-(cd external/glslang; ./update_glslang_sources.py)
 
 # Use gclient to update toolchain dependencies from /build-gn/DEPS (from chromium)
 cat << EOF > .gclient

layers/CMakeLists.txt

@@ -291,7 +291,7 @@ if(BUILD_LAYERS)
     endif()
 
     # Khronos validation additional dependencies
-    target_include_directories(VkLayer_khronos_validation PRIVATE ${GLSLANG_SPIRV_INCLUDE_DIR})
+    target_include_directories(VkLayer_khronos_validation PRIVATE ${GLSLANG_INCLUDE_DIR})
     target_include_directories(VkLayer_khronos_validation PRIVATE ${SPIRV_TOOLS_INCLUDE_DIR})
     target_include_directories(VkLayer_khronos_validation PRIVATE ${SPIRV_HEADERS_INCLUDE_DIR})
     if(INSTRUMENT_OPTICK)

scripts/known_good.json

@@ -7,9 +7,6 @@
       "build_dir" : "glslang/build",
       "install_dir" : "glslang/build/install",
       "commit" : "ae2a562936cc8504c9ef2757cceaff163147834f",
-      "prebuild" : [
-        "python update_glslang_sources.py"
-      ],
       "cmake_options" : [
         "-DUSE_CCACHE=ON"
       ]
@@ -30,6 +27,17 @@
       "install_dir": "SPIRV-Headers/build/install",
       "commit": "07f259e68af3a540038fa32df522554e74f53ed5"
     },
+    {
+      "name": "SPIRV-Tools",
+      "url": "https://github.com/KhronosGroup/SPIRV-Tools.git",
+      "sub_dir": "SPIRV-Tools",
+      "build_dir": "SPIRV-Tools/build",
+      "install_dir": "SPIRV-Tools/build/install",
+      "cmake_options": [
+        "-DSPIRV-Headers_SOURCE_DIR={repo_dir}/../SPIRV-Headers"
+      ],
+      "commit": "5dd2f76918bb2d0d67628e338f60f724f3e02e13"
+    },
     {
       "name": "robin-hood-hashing",
       "url": "https://github.com/martinus/robin-hood-hashing.git",
@@ -44,6 +52,7 @@
       "glslang" : "GLSLANG_INSTALL_DIR",
       "Vulkan-Headers" : "VULKAN_HEADERS_INSTALL_DIR",
       "SPIRV-Headers" : "SPIRV_HEADERS_INSTALL_DIR",
+      "SPIRV-Tools" : "SPIRV_TOOLS_INSTALL_DIR",
       "robin-hood-hashing" : "ROBIN_HOOD_HASHING_INSTALL_DIR"
   }
 }

scripts/update_deps.py

@@ -300,6 +300,9 @@ def command_output(cmd, directory, fail_ok=False):
         print(stdout)
     return stdout
 
+def escape(path):
+    return path.replace('\\', '\\\\')
+
 class GoodRepo(object):
     """Represents a repository at a known-good commit."""
 
@@ -451,7 +454,7 @@ def CMakeConfig(self, repos):
 
         # Add any CMake options
         for option in self.cmake_options:
-            cmake_cmd.append(option)
+            cmake_cmd.append(escape(option.format(**self.__dict__)))
 
         # Set build config for single-configuration generators
         if platform.system() == 'Linux' or platform.system() == 'Darwin':
@@ -586,8 +589,6 @@ def CreateHelper(args, repos, filename):
     This information is baked into the CMake files of the home repo and so
     this dictionary is kept with the repo via the json file.
     """
-    def escape(path):
-        return path.replace('\\', '\\\\')
     install_names = GetInstallNames(args)
     with open(filename, 'w') as helper_file:
         for repo in repos:

tests/CMakeLists.txt

@@ -114,7 +114,7 @@ target_include_directories(vk_layer_validation_tests
                            PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
                                   ${PROJECT_SOURCE_DIR}/layers
                                   ${PROJECT_SOURCE_DIR}/layers/generated
-                                  ${GLSLANG_SPIRV_INCLUDE_DIR}
+                                  ${GLSLANG_INCLUDE_DIR}
                                   ${SPIRV_TOOLS_INCLUDE_DIR}
                                   ${SPIRV_HEADERS_INCLUDE_DIR}
                                   ${CMAKE_CURRENT_BINARY_DIR}
@@ -149,7 +149,8 @@ if(WIN32)
     target_link_libraries(vk_layer_validation_tests
                           PRIVATE gtest
                                   gtest_main
-                                  ${GLSLANG_LIBRARIES})
+                                  ${GLSLANG_LIBRARIES}
+                                  ${SPIRV_TOOLS_LIBRARIES})
 else()
     target_compile_options(vk_layer_validation_tests PRIVATE "-Wno-sign-compare")
     if(BUILD_WSI_XCB_SUPPORT OR BUILD_WSI_XLIB_SUPPORT)
@@ -159,13 +160,15 @@ else()
                                       gtest
                                       gtest_main
                                       dl
-                                      ${GLSLANG_LIBRARIES})
+                                      ${GLSLANG_LIBRARIES}
+                                      ${SPIRV_TOOLS_LIBRARIES})
     else()
         target_link_libraries(vk_layer_validation_tests
                               PRIVATE gtest
                                       gtest_main
                                       dl
-                                      ${GLSLANG_LIBRARIES})
+                                      ${GLSLANG_LIBRARIES}
+                                      ${SPIRV_TOOLS_LIBRARIES})
     endif()
 endif()
 

tests/vktestframework.cpp

@@ -32,8 +32,8 @@
 #endif
 // TODO FIXME remove this once glslang doesn't define this
 #undef BadValue
-#include "SPIRV/GlslangToSpv.h"
-#include "SPIRV/SPVRemapper.h"
+#include "glslang/SPIRV/GlslangToSpv.h"
+#include "glslang/SPIRV/SPVRemapper.h"
 #if (defined(_MSC_VER) && _MSC_VER < 1900 /*vs2015*/)
 #pragma warning(pop)
 #endif

tests/vktestframework.h

@@ -22,7 +22,7 @@
 #ifndef VKTESTFRAMEWORK_H
 #define VKTESTFRAMEWORK_H
 
-#include "SPIRV/GLSL.std.450.h"
+#include "glslang/SPIRV/GLSL.std.450.h"
 #include "spirv-tools/libspirv.h"
 #include "glslang/Public/ShaderLang.h"
 #include "icd-spv.h"