From a22cc256e17b5f6156b6896ae020b99c0594dfb1 Mon Sep 17 00:00:00 2001 From: Bryan B Date: Wed, 19 Oct 2022 20:18:53 -0700 Subject: [PATCH] Revert "Remove third_party/webnn from build. (#675)" This reverts commit d5506fd019c7d30307379be5950031462d1314fd. --- .github/workflows/win_webnn_rel.yaml | 124 +++++++++++++++ .gitignore | 1 + build_overrides/gpgmm_features.gni | 7 + build_overrides/webnn.gni | 23 +++ scripts/regression_check.py | 1 + scripts/webnn.deps | 229 +++++++++++++++++++++++++++ scripts/webnn.gclient | 10 ++ src/BUILD.gn | 20 +++ third_party/BUILD.gn | 4 + 9 files changed, 419 insertions(+) create mode 100644 .github/workflows/win_webnn_rel.yaml create mode 100644 build_overrides/webnn.gni create mode 100644 scripts/webnn.deps create mode 100644 scripts/webnn.gclient create mode 100644 src/BUILD.gn diff --git a/.github/workflows/win_webnn_rel.yaml b/.github/workflows/win_webnn_rel.yaml new file mode 100644 index 000000000..893c439ed --- /dev/null +++ b/.github/workflows/win_webnn_rel.yaml @@ -0,0 +1,124 @@ +name: Windows WebNN (Release) + +on: + push: + branches: main + paths: + - 'src/**' + - 'scripts/webnn.deps' + + pull_request: + branches: main + paths: + - 'src/**' + - 'scripts/webnn.deps' + +jobs: + + win_webnn_rel: + + runs-on: windows-2019 + + steps: + - name: Git config + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - name: Install depot_tools + shell: cmd + run: | + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git ..\depot_tools + set "PATH=%CD%\..\depot_tools;%PATH%" + gclient + + - name: Set up Python 3.x + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - uses: actions/checkout@v2 + with: + ref: main + path: base + fetch-depth: 0 + + - name: Sync code for main branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd base + copy scripts\webnn.deps DEPS + copy scripts\webnn.gclient .gclient + gclient sync + + - name: Generate project for main branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd base + gn gen out\Release --args="webnn_enable_dmlx=true is_debug=false gpgmm_enable_assert_on_warning=true" + + - name: Build for main branch + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd base + ninja -C out\Release + + - name: Run webnn_end2end_tests + shell: cmd + continue-on-error: true + run: | + cd base + out\Release\webnn_end2end_tests.exe --gtest_output=json:${{ github.workspace }}\..\baseline_end2end_tests.json + + - name: Clean baseline directory + shell: cmd + run: | + rmdir /s /q base + + - uses: actions/checkout@v2 + with: + path: test + fetch-depth: 0 + + - name: Sync code for main branch (with patch) + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd test + copy scripts\webnn.deps DEPS + copy scripts\webnn.gclient .gclient + gclient sync + + - name: Generate project for main branch (with patch) + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd test + gn gen out\Release --args="webnn_enable_dmlx=true is_debug=false gpgmm_enable_assert_on_warning=true" + + - name: Build for main branch (with patch) + shell: cmd + run: | + set "PATH=%CD%\..\depot_tools;%PATH%" + set "DEPOT_TOOLS_WIN_TOOLCHAIN=0" + cd test + ninja -C out\Release + + - name: Run webnn_end2end_tests (with patch) + shell: cmd + continue-on-error: true + run: | + cd test + out\Release\webnn_end2end_tests.exe --gtest_output=json:${{ github.workspace }}\..\test_end2end_tests.json || true + + - name: Regression check end2end tests + run: | + python test\scripts\regression_check.py ${{ github.workspace }}\..\baseline_end2end_tests.json ${{ github.workspace }}\..\test_end2end_tests.json diff --git a/.gitignore b/.gitignore index d36f9d132..619868270 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ /third_party/jsoncpp* /third_party/llvm-build /third_party/vulkan* +/third_party/webnn_native /third_party/libFuzzer /third_party/*benchmark* /tools diff --git a/build_overrides/gpgmm_features.gni b/build_overrides/gpgmm_features.gni index 20af31c48..1caaebd46 100644 --- a/build_overrides/gpgmm_features.gni +++ b/build_overrides/gpgmm_features.gni @@ -21,6 +21,10 @@ if (!defined(checkout_dawn)) { checkout_dawn = false } +if (!defined(checkout_webnn)) { + checkout_webnn = false +} + declare_args() { # Enable GPGMM's ASSERTs even in release builds # Sets -dGPGMM_ENABLE_ASSERTS @@ -37,6 +41,9 @@ declare_args() { # Enables compilation of Dawn's end2end tests. gpgmm_enable_dawn = checkout_dawn + # Enables compilation of WebNN's end2end tests. + gpgmm_enable_webnn = checkout_webnn + # Build GPGMM as a shared library. # Sets -dGPGMM_SHARED_LIBRARY gpgmm_shared_library = false diff --git a/build_overrides/webnn.gni b/build_overrides/webnn.gni new file mode 100644 index 000000000..fca6028ec --- /dev/null +++ b/build_overrides/webnn.gni @@ -0,0 +1,23 @@ +# Copyright 2022 The GPGMM Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The paths to WebNN's dependencies +webnn_jinja2_dir = "//third_party/webnn_native/third_party/jinja2" + +# TODO(webnn): WebNN hard codes //third_party/dawn paths within BUILD.gn +# files so we must use GPGMM-Dawn version instead of WebNN-Dawn version to +# workaround. +webnn_dawn_root = "//third_party/dawn" + +webnn_gpgmm_dir = "//" diff --git a/scripts/regression_check.py b/scripts/regression_check.py index 624c59b09..c610b4ef9 100644 --- a/scripts/regression_check.py +++ b/scripts/regression_check.py @@ -1,3 +1,4 @@ +# Copyright 2021 The WebNN Authors # Copyright 2022 The GPGMM Authors # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/scripts/webnn.deps b/scripts/webnn.deps new file mode 100644 index 000000000..ac370e080 --- /dev/null +++ b/scripts/webnn.deps @@ -0,0 +1,229 @@ +use_relative_paths = True + +gclient_gn_args_file = 'build/config/gclient_args.gni' +gclient_gn_args = [ + 'checkout_dawn', + 'checkout_webnn', +] + +vars = { + 'chromium_git': 'https://chromium.googlesource.com', + 'dawn_git': 'https://dawn.googlesource.com', + 'github_git': 'https://github.com', + + 'gpgmm_standalone': True, + + # Checkout and download Dawn by default. This can be disabled with custom_vars. + 'checkout_dawn': False, + + # Checkout and download WebNN by default. This can be disabled with custom_vars. + 'checkout_webnn': False, + + 'gn_version': 'git_revision:fc295f3ac7ca4fe7acc6cb5fb052d22909ef3a8f', +} + +deps = { + # Dependencies required to test integrations + # Note: rolling Dawn also may require vulkan-deps to be rolled below. + # TODO(gpgmm): Consider linking vulkan-deps to Dawn like Tint. + # TODO(gpgmm): WebNN hard codes builds to third_party/dawn and should be fixed if the + # build errors are related to Dawn version mismatches. + 'third_party/dawn': { + 'url': '{dawn_git}/dawn.git@bf1c0cf52377b4db2bf3a433dc5056620aad7cdd', + 'condition': 'checkout_dawn or checkout_webnn', + }, + + 'third_party/webnn_native': { + 'url': '{github_git}/webmachinelearning/webnn-native.git@294de6bca89094ede3f7693bb7d0c7edf44e5141', + 'condition': 'checkout_webnn', + }, + + # Dependencies required to use GN/Clang in standalone + 'build': { + 'url': '{chromium_git}/chromium/src/build@555c8b467c21e2c4b22d00e87e3faa0431df9ac2', + 'condition': 'gpgmm_standalone', + }, + 'buildtools': { + 'url': '{chromium_git}/chromium/src/buildtools@f78b4b9f33bd8ef9944d5ce643daff1c31880189', + 'condition': 'gpgmm_standalone', + }, + 'buildtools/linux64': { + 'packages': [{ + 'package': 'gn/gn/linux-amd64', + 'version': Var('gn_version'), + }], + 'dep_type': 'cipd', + 'condition': 'gpgmm_standalone and host_os == "linux"', + }, + 'buildtools/win': { + 'packages': [{ + 'package': 'gn/gn/windows-amd64', + 'version': Var('gn_version'), + }], + 'dep_type': 'cipd', + 'condition': 'gpgmm_standalone and host_os == "win"', + }, + 'buildtools/third_party/libc++/trunk': { + 'url': '{chromium_git}/external/github.com/llvm/llvm-project/libcxx.git@79a2e924d96e2fc1e4b937c42efd08898fa472d7', + 'condition': 'gpgmm_standalone', + }, + 'buildtools/third_party/libc++abi/trunk': { + 'url': '{chromium_git}/external/github.com/llvm/llvm-project/libcxxabi.git@2715a6c0de8dac4c7674934a6b3d30ba0c685271', + 'condition': 'gpgmm_standalone', + }, + 'tools/clang': { + 'url': '{chromium_git}/chromium/src/tools/clang@8b7330592cb85ba09505a6be7bacabd0ad6160a3', + 'condition': 'gpgmm_standalone', + }, + 'tools/clang/dsymutil': { + 'packages': [ + { + 'package': 'chromium/llvm-build-tools/dsymutil', + 'version': 'M56jPzDv1620Rnm__jTMYS62Zi8rxHVq7yw0qeBFEgkC', + } + ], + 'condition': 'checkout_mac or checkout_ios', + 'dep_type': 'cipd', + }, + + # Testing, GTest and GMock + 'testing': { + 'url': '{chromium_git}/chromium/src/testing@3e2640a325dc34ec3d9cb2802b8da874aecaf52d', + 'condition': 'gpgmm_standalone', + }, + 'third_party/googletest': { + 'url': '{chromium_git}/external/github.com/google/googletest@2828773179fa425ee406df61890a150577178ea2', + 'condition': 'gpgmm_standalone', + }, + 'third_party/vulkan-deps': { + 'url': '{chromium_git}/vulkan-deps@7e9ab0686bf4d4fa9c52eeb8def33b2057624987', + 'condition': 'gpgmm_standalone', + }, + 'third_party/jsoncpp/source': { + 'url': '{chromium_git}/external/github.com/open-source-parsers/jsoncpp@9059f5cad030ba11d37818847443a53918c327b1', + 'condition': 'gpgmm_standalone', + }, + 'third_party/google_benchmark/src': { + 'url': '{chromium_git}/external/github.com/google/benchmark.git@f730846b0a3c0dc0699978846fb14ffb2fad0bdc', + 'condition': 'gpgmm_standalone', + }, +} + +hooks = [ + # Pull the compilers and system libraries for hermetic builds + { + 'name': 'sysroot_x86', + 'pattern': '.', + 'condition': 'checkout_linux and ((checkout_x86 or checkout_x64) and gpgmm_standalone)', + 'action': ['python3', 'build/linux/sysroot_scripts/install-sysroot.py', + '--arch=x86'], + }, + { + 'name': 'sysroot_x64', + 'pattern': '.', + 'condition': 'checkout_linux and (checkout_x64 and gpgmm_standalone)', + 'action': ['python3', 'build/linux/sysroot_scripts/install-sysroot.py', + '--arch=x64'], + }, + { + # Update the Mac toolchain if possible, this makes builders use "hermetic XCode" which is + # is more consistent (only changes when rolling build/) and is cached. + 'name': 'mac_toolchain', + 'pattern': '.', + 'condition': 'checkout_mac', + 'action': ['python3', 'build/mac_toolchain.py'], + }, + { + # Update the Windows toolchain if necessary. Must run before 'clang' below. + 'name': 'win_toolchain', + 'pattern': '.', + 'condition': 'checkout_win and gpgmm_standalone', + 'action': ['python3', 'build/vs_toolchain.py', 'update', '--force'], + }, + { + # Note: On Win, this should run after win_toolchain, as it may use it. + 'name': 'clang', + 'pattern': '.', + 'action': ['python3', 'tools/clang/scripts/update.py'], + 'condition': 'gpgmm_standalone', + }, + { + # Pull rc binaries using checked-in hashes. + 'name': 'rc_win', + 'pattern': '.', + 'condition': 'checkout_win and (host_os == "win" and gpgmm_standalone)', + 'action': [ 'download_from_google_storage', + '--no_resume', + '--no_auth', + '--bucket', 'chromium-browser-clang/rc', + '-s', 'build/toolchain/win/rc/win/rc.exe.sha1', + ], + }, + # Pull clang-format binaries using checked-in hashes. + { + 'name': 'clang_format_win', + 'pattern': '.', + 'condition': 'host_os == "win"', + 'action': [ 'download_from_google_storage', + '--no_resume', + '--no_auth', + '--bucket', 'chromium-clang-format', + '-s', 'buildtools/win/clang-format.exe.sha1', + ], + }, + { + 'name': 'clang_format_mac', + 'pattern': '.', + 'condition': 'host_os == "mac"', + 'action': [ 'download_from_google_storage', + '--no_resume', + '--no_auth', + '--bucket', 'chromium-clang-format', + '-s', 'buildtools/mac/clang-format.sha1', + ], + }, + { + 'name': 'clang_format_linux', + 'pattern': '.', + 'condition': 'host_os == "linux"', + 'action': [ 'download_from_google_storage', + '--no_resume', + '--no_auth', + '--bucket', 'chromium-clang-format', + '-s', 'buildtools/linux64/clang-format.sha1', + ], + }, + # Update build/util/LASTCHANGE. + { + 'name': 'lastchange', + 'pattern': '.', + 'condition': 'gpgmm_standalone', + 'action': ['python3', 'build/util/lastchange.py', + '-o', 'build/util/LASTCHANGE'], + }, + # Apply Dawn-GPGMM integration patch. + # Patch can be removed should GPGMM be merged into upstream. + { + 'name': 'apply_dawn_integration_patch', + 'pattern': '.', + 'condition': 'checkout_dawn', + 'action': [ 'git', '-C', './third_party/dawn/', + 'apply', '--ignore-space-change', '--ignore-whitespace', + '../../.github/workflows/.patches/dawn.diff', + ], + }, +] + +recursedeps = [ + # buildtools provides clang_format, libc++, and libc++abi + 'buildtools', + + # vulkan-deps provides vulkan-headers, spirv-tools, and gslang + 'third_party/vulkan-deps', + + # Dawn and Tint's revision are linked + 'third_party/dawn', + + # WebNN and DirectML revision are linked + 'third_party/webnn_native', +] diff --git a/scripts/webnn.gclient b/scripts/webnn.gclient new file mode 100644 index 000000000..07dae7ae5 --- /dev/null +++ b/scripts/webnn.gclient @@ -0,0 +1,10 @@ +solutions = [ + { "name" : ".", + "url" : "https://github.com/intel/gpgmm.git", + "deps_file" : "DEPS", + "managed" : False, + "custom_vars": { + "checkout_webnn": True, + }, + }, +] diff --git a/src/BUILD.gn b/src/BUILD.gn new file mode 100644 index 000000000..e58827c65 --- /dev/null +++ b/src/BUILD.gn @@ -0,0 +1,20 @@ +# Copyright 2022 The GPGMM Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import("../build_overrides/gpgmm_features.gni") +import("../build_overrides/gpgmm_overrides_with_defaults.gni") + +# TODO: Remove once WebNN no longer depends on src:gpgmm. +group("gpgmm") { + deps = [ "gpgmm:gpgmm" ] +} diff --git a/third_party/BUILD.gn b/third_party/BUILD.gn index f3101cb41..fa963e242 100644 --- a/third_party/BUILD.gn +++ b/third_party/BUILD.gn @@ -20,4 +20,8 @@ group("external_tests") { if (gpgmm_enable_dawn) { deps += [ "dawn/src/dawn/tests:dawn_end2end_tests" ] } + + if (gpgmm_enable_webnn) { + deps += [ "webnn_native/src/webnn/tests:webnn_end2end_tests" ] + } }