From 53ca3fe60235c9d25b8374752b689fd135df7831 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Thu, 25 Feb 2021 11:05:45 -0800 Subject: [PATCH 01/24] Compile a simple hello world example using emsdk's bazel toolchain --- .bazelrc | 5 + WORKSPACE | 20 + emscripten_toolchain/BUILD.bazel | 80 ++ emscripten_toolchain/crosstool.bzl | 1090 +++++++++++++++++++++++ emscripten_toolchain/emar.sh | 5 + emscripten_toolchain/emcc.sh | 5 + emscripten_toolchain/emcc_link.sh | 5 + emscripten_toolchain/emscripten.BUILD | 6 + emscripten_toolchain/emscripten_config | 12 + emscripten_toolchain/env.sh | 5 + emscripten_toolchain/link_wrapper.py | 146 +++ emscripten_toolchain/wasm_binary.py | 84 ++ emscripten_toolchain/wasm_cc_binary.bzl | 150 ++++ emscripten_toolchain/wasm_rules.bzl | 6 + tfjs-backend-wasm/src/cc/BUILD | 18 + tfjs-backend-wasm/src/cc/hello_world.cc | 6 + tfjs-backend-wasm/yarn.lock | 16 +- 17 files changed, 1645 insertions(+), 14 deletions(-) create mode 100644 emscripten_toolchain/BUILD.bazel create mode 100644 emscripten_toolchain/crosstool.bzl create mode 100755 emscripten_toolchain/emar.sh create mode 100755 emscripten_toolchain/emcc.sh create mode 100755 emscripten_toolchain/emcc_link.sh create mode 100644 emscripten_toolchain/emscripten.BUILD create mode 100644 emscripten_toolchain/emscripten_config create mode 100755 emscripten_toolchain/env.sh create mode 100644 emscripten_toolchain/link_wrapper.py create mode 100644 emscripten_toolchain/wasm_binary.py create mode 100644 emscripten_toolchain/wasm_cc_binary.bzl create mode 100644 emscripten_toolchain/wasm_rules.bzl create mode 100644 tfjs-backend-wasm/src/cc/hello_world.cc diff --git a/.bazelrc b/.bazelrc index e571676ad6..6c9d929621 100644 --- a/.bazelrc +++ b/.bazelrc @@ -25,3 +25,8 @@ build:wasm --cxxopt="-fomit-frame-pointer" # Disable sandbox environment because emsdk caches files by writing to # home directory. build:wasm --spawn_strategy=local + + +build:wasm_new --crosstool_top=//emscripten_toolchain:everything +build:wasm_new --cpu=wasm +build:wasm_new --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ No newline at end of file diff --git a/WORKSPACE b/WORKSPACE index c2c22aaf04..1bb9b36dcb 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -17,6 +17,26 @@ yarn_install( load("//toolchain:cc_toolchain_config.bzl", "emsdk_configure") emsdk_configure(name = "emsdk") +# npm_install used for emscripten dependencies +load("@build_bazel_rules_nodejs//:index.bzl", "npm_install") + +# emscripten 2.0.14 +http_archive( + name = "emscripten", + sha256 = "e466cd47ddd4bf0acd645412fdf08eda6d232484e48e5a2643e08062a7a4cf56", + strip_prefix = "install", + url = "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/fc5562126762ab26c4757147a3b4c24e85a7289e/wasm-binaries.tbz2", + build_file = "//emscripten_toolchain:emscripten.BUILD", + type = "tar.bz2", +) + +# Install emscripten dependencies +npm_install( + name = "emsdk_npm", + package_json = "@emscripten//:emscripten/package.json", + package_lock_json = "@emscripten//:emscripten/package-lock.json", +) + load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") # xnnpack used for fast vectorized wasm operations git_repository( diff --git a/emscripten_toolchain/BUILD.bazel b/emscripten_toolchain/BUILD.bazel new file mode 100644 index 0000000000..a676ad7cd6 --- /dev/null +++ b/emscripten_toolchain/BUILD.bazel @@ -0,0 +1,80 @@ +load(":crosstool.bzl", "emscripten_cc_toolchain_config_rule") + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "common-script-includes", + srcs = [ + "emar.sh", + "emcc.sh", + "emscripten_config", + "env.sh", + "@emscripten//:all", + "@nodejs//:node_files", + "@emsdk_npm//:node_modules", + ], +) + +filegroup( + name = "compile-emscripten", + srcs = [":common-script-includes"], +) + +filegroup( + name = "link-emscripten", + srcs = [ + "emcc_link.sh", + "link_wrapper.py", + ":common-script-includes", + "@emscripten//:all", + "@nodejs//:node_files", + ], +) + +filegroup( + name = "every-file", + srcs = [ + ":compile-emscripten", + ":link-emscripten", + "@emscripten//:all", + "@nodejs//:node_files", + ], +) + +filegroup(name = "empty") + +# dlmalloc.bc is implictly added by the emscripten toolchain +cc_library(name = "malloc") + +emscripten_cc_toolchain_config_rule( + name = "wasm", + cpu = "wasm", + emscripten_version = "emscripten", +) + +cc_toolchain( + name = "cc-compiler-wasm", + all_files = ":every-file", + ar_files = ":common-script-includes", + as_files = ":empty", + compiler_files = ":compile-emscripten", + dwp_files = ":empty", + linker_files = ":link-emscripten", + objcopy_files = ":empty", + strip_files = ":empty", + toolchain_config = "wasm", + toolchain_identifier = "emscripten-wasm", +) + +cc_toolchain_suite( + name = "everything", + toolchains = { + "wasm": ":cc-compiler-wasm", + "wasm|emscripten": ":cc-compiler-wasm", + }, +) + +py_binary( + name = "wasm_binary", + srcs = ["wasm_binary.py"], +) diff --git a/emscripten_toolchain/crosstool.bzl b/emscripten_toolchain/crosstool.bzl new file mode 100644 index 0000000000..0da1cb0175 --- /dev/null +++ b/emscripten_toolchain/crosstool.bzl @@ -0,0 +1,1090 @@ +"""This module encapsulates logic to create emscripten_cc_toolchain_config rule.""" + +load( + "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", + "action_config", + "env_entry", + "env_set", + "feature", + "feature_set", + "flag_group", + "tool", + "tool_path", + "variable_with_value", + "with_feature_set", + _flag_set = "flag_set", +) +load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") + +def flag_set(flags = None, features = None, not_features = None, **kwargs): + """Extension to flag_set which allows for a "simple" form. + + The simple form allows specifying flags as a simple list instead of a flag_group + if enable_if or expand_if semantics are not required. + + Similarly, the simple form allows passing features/not_features if they are a simple + list of semantically "and" features. + (i.e. "asan" and "dbg", rather than "asan" or "dbg") + + Args: + flags: list, set of flags + features: list, set of features required to be enabled. + not_features: list, set of features required to not be enabled. + **kwargs: The rest of the args for flag_set. + + Returns: + flag_set + """ + if flags: + if kwargs.get("flag_groups"): + fail("Cannot set flags and flag_groups") + else: + kwargs["flag_groups"] = [flag_group(flags = flags)] + + if features or not_features: + if kwargs.get("with_features"): + fail("Cannot set features/not_feature and with_features") + kwargs["with_features"] = [with_feature_set( + features = features or [], + not_features = not_features or [], + )] + return _flag_set(**kwargs) + +CROSSTOOL_DEFAULT_WARNINGS = [ + "-Wall", +] + +def _impl(ctx): + target_cpu = ctx.attr.cpu + toolchain_identifier = "emscripten-" + target_cpu + target_system_name = target_cpu + "-unknown-emscripten" + + host_system_name = "i686-unknown-linux-gnu" + + target_libc = "musl/js" + + abi_version = "emscripten_syscalls" + + compiler = "emscripten" + abi_libc_version = "default" + + cc_target_os = "emscripten" + emscripten_version = ctx.attr.emscripten_version + + builtin_sysroot = "external/emscripten/emscripten/cache/sysroot" + + ################################################################ + # Tools + ################################################################ + clang_tool = tool(path = "emcc.sh") + clif_match_tool = tool(path = "dummy_clif_matcher") + link_tool = tool(path = "emcc_link.sh") + archive_tool = tool(path = "emar.sh") + strip_tool = tool(path = "NOT_USED_STRIP_TOOL") + + #### Legacy tool paths (much of this is redundant with action_configs, but + #### these are still used for some things) + tool_paths = [ + tool_path(name = "ar", path = "emar.sh"), + tool_path(name = "cpp", path = "/bin/false"), + tool_path(name = "gcc", path = "emcc.sh"), + tool_path(name = "gcov", path = "/bin/false"), + tool_path(name = "ld", path = "emcc_link.sh"), + tool_path(name = "nm", path = "NOT_USED"), + tool_path(name = "objdump", path = "/bin/false"), + tool_path(name = "strip", path = "NOT_USED"), + ] + + ################################################################ + # Action Configs + ################################################################ + + cpp_compile_action = action_config( + action_name = ACTION_NAMES.cpp_compile, + tools = [clang_tool], + ) + + cpp_module_compile_action = action_config( + action_name = ACTION_NAMES.cpp_module_compile, + tools = [clang_tool], + ) + + cpp_module_codegen_action = action_config( + action_name = ACTION_NAMES.cpp_module_codegen, + tools = [clang_tool], + ) + + clif_match_action = action_config( + action_name = ACTION_NAMES.clif_match, + tools = [clif_match_tool], + ) + + cpp_link_dynamic_library_action = action_config( + action_name = ACTION_NAMES.cpp_link_dynamic_library, + tools = [link_tool], + ) + + strip_action = action_config( + action_name = ACTION_NAMES.strip, + tools = [strip_tool], + ) + + preprocess_assemble_action = action_config( + action_name = ACTION_NAMES.preprocess_assemble, + tools = [clang_tool], + ) + + cpp_header_parsing_action = action_config( + action_name = ACTION_NAMES.cpp_header_parsing, + tools = [clang_tool], + ) + + cpp_link_static_library_action = action_config( + action_name = ACTION_NAMES.cpp_link_static_library, + enabled = True, + flag_sets = [ + flag_set( + flag_groups = [ + flag_group( + flags = ["rcsD", "%{output_execpath}"], + expand_if_available = "output_execpath", + ), + ], + ), + flag_set( + flag_groups = [ + flag_group( + iterate_over = "libraries_to_link", + flag_groups = [ + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file", + ), + ), + flag_group( + flags = ["%{libraries_to_link.object_files}"], + iterate_over = "libraries_to_link.object_files", + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + ], + expand_if_available = "libraries_to_link", + ), + ], + ), + flag_set( + flag_groups = [ + flag_group( + flags = ["@%{linker_param_file}"], + expand_if_available = "linker_param_file", + ), + ], + ), + ], + tools = [archive_tool], + ) + + c_compile_action = action_config( + action_name = ACTION_NAMES.c_compile, + tools = [clang_tool], + ) + + linkstamp_compile_action = action_config( + action_name = ACTION_NAMES.linkstamp_compile, + tools = [clang_tool], + ) + + assemble_action = action_config( + action_name = ACTION_NAMES.assemble, + tools = [clang_tool], + ) + + cpp_link_executable_action = action_config( + action_name = ACTION_NAMES.cpp_link_executable, + tools = [link_tool], + ) + + cpp_link_nodeps_dynamic_library_action = action_config( + action_name = ACTION_NAMES.cpp_link_nodeps_dynamic_library, + tools = [link_tool], + ) + + action_configs = [ + strip_action, + c_compile_action, + cpp_compile_action, + linkstamp_compile_action, + assemble_action, + preprocess_assemble_action, + cpp_header_parsing_action, + cpp_module_compile_action, + cpp_module_codegen_action, + cpp_link_executable_action, + cpp_link_dynamic_library_action, + cpp_link_nodeps_dynamic_library_action, + cpp_link_static_library_action, + clif_match_action, + ] + + all_compile_actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ACTION_NAMES.lto_backend, + ] + + all_cpp_compile_actions = [ + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ] + + preprocessor_compile_actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, + ] + + all_link_actions = [ + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ] + + ################################################################ + # Features + ################################################################ + + features = [ + # This set of magic "feature"s are important configuration information for blaze. + feature(name = "no_legacy_features", enabled = True), + feature( + name = "has_configured_linker_path", + enabled = True, + ), + + # Blaze requests this feature by default, but we don't care. + feature(name = "dependency_file"), + + # Blaze requests this feature by default, but we don't care. + feature(name = "random_seed"), + + # Formerly "needsPic" attribute + feature(name = "supports_pic", enabled = False), + + # Blaze requests this feature by default. + # Blaze also tests if this feature is supported, before setting the "pic" build-variable. + feature(name = "pic"), + + # Blaze requests this feature if fission is requested + # Blaze also tests if it's supported to see if we support fission. + feature(name = "per_object_debug_info"), + + # Blaze requests this feature by default. + # Blaze also tests if this feature is supported before setting preprocessor_defines + # (...but why?) + feature(name = "preprocessor_defines"), + + # Blaze requests this feature by default. + # Blaze also tests if this feature is supported before setting includes. (...but why?) + feature(name = "include_paths"), + + # Blaze tests if this feature is enabled in order to create implicit + # "nodeps" .so outputs from cc_library rules. + feature(name = "supports_dynamic_linker", enabled = False), + + # Blaze requests this feature when linking a cc_binary which is + # "dynamic" aka linked against nodeps-dynamic-library cc_library + # outputs. + feature(name = "dynamic_linking_mode"), + + #### Configuration features + feature( + name = "crosstool_cpu", + enabled = True, + implies = ["crosstool_cpu_" + target_cpu], + ), + feature( + name = "crosstool_cpu_asmjs", + provides = ["variant:crosstool_cpu"], + ), + feature( + name = "crosstool_cpu_wasm", + provides = ["variant:crosstool_cpu"], + ), + + # These 3 features will be automatically enabled by blaze in the + # corresponding build mode. + feature( + name = "opt", + provides = ["variant:crosstool_build_mode"], + ), + feature( + name = "dbg", + provides = ["variant:crosstool_build_mode"], + ), + feature( + name = "fastbuild", + provides = ["variant:crosstool_build_mode"], + ), + + #### User-settable features + + # Set if enabling exceptions. + feature(name = "exceptions"), + + # This feature overrides the default optimization to prefer execution speed + # over binary size (like clang -O3). + feature( + name = "optimized_for_speed", + provides = ["variant:crosstool_optimization_mode"], + ), + + # This feature overrides the default optimization to prefer binary size over + # execution speed (like clang -Oz). + feature( + name = "optimized_for_size", + provides = ["variant:crosstool_optimization_mode"], + ), + + # Convenience aliases / alt-spellings. + feature( + name = "optimize_for_speed", + implies = ["optimized_for_speed"], + ), + feature( + name = "optimize_for_size", + implies = ["optimized_for_size"], + ), + + # This feature allows easier use of profiling tools by preserving mangled + # C++ names. This does everything profiling_funcs does and more. + feature(name = "profiling"), + + # This feature emits only enough debug info for function names to appear + # in profiles. + feature(name = "profiling_funcs"), + + # This feature allows source maps to be generated. + feature( + name = "source_maps", + implies = ["full_debug_info"], + ), + feature( + name = "dwarf_debug_info", + implies = ["profiling"], + ), + + # Turns on full debug info (-g4). + feature(name = "full_debug_info"), + + # Enables the use of "Emscripten" Pthread implementation. + # https://kripken.github.io/emscripten-site/docs/porting/pthreads.html + # https://github.com/kripken/emscripten/wiki/Pthreads-with-WebAssembly + feature(name = "use_pthreads"), + + # If enabled, the runtime will exit when main() completes. + feature(name = "exit_runtime"), + + # Primarily for toolchain maintainers: + feature(name = "emcc_debug"), + feature(name = "emcc_debug_link"), + feature( + name = "llvm_backend", + requires = [feature_set(features = ["crosstool_cpu_wasm"])], + enabled = True, + ), + + # Remove once flag is flipped. + # See https://github.com/bazelbuild/bazel/issues/7687 + feature( + name = "do_not_split_linking_cmdline", + ), + + # Adds simd support, only available with the llvm backend. + feature( + name = "wasm_simd", + requires = [feature_set(features = ["llvm_backend"])], + ), + feature( + name = "precise_long_double_printf", + enabled = True, + ), + feature( + name = "wasm_warnings_as_errors", + enabled = True, + ), + + # ASan and UBSan. See also: + # https://emscripten.org/docs/debugging/Sanitizers.html + feature(name = "wasm_asan"), + feature(name = "wasm_ubsan"), + ] + + crosstool_default_flag_sets = [ + # Compile, Link, and CC_FLAGS make variable + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = ["--sysroot=%{sysroot}"], + expand_if_available = "sysroot", + ), + ], + ), + # Compile + Link + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + # This forces color diagnostics even on Forge (where we don't have an + # attached terminal). + flags = [ + "-fdiagnostics-color", + ], + ), + # C++ compiles (and implicitly link) + flag_set( + actions = all_cpp_compile_actions, + flags = [ + "-fno-exceptions", + ], + not_features = ["exceptions"], + ), + flag_set( + actions = all_cpp_compile_actions, + flags = [ + "-fexceptions", + ], + features = ["exceptions"], + ), + # All compiles (and implicitly link) + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = [ + "-fno-strict-aliasing", + "-funsigned-char", + "-no-canonical-prefixes", + ], + ), + # Language Features + flag_set( + actions = all_cpp_compile_actions, + flags = ["-std=gnu++17", "-nostdinc", "-nostdinc++",], + ), + + # Emscripten-specific settings: + flag_set( + actions = all_compile_actions + all_link_actions, + flags = ["-s", "WASM=0"], + features = ["crosstool_cpu_asmjs"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-s", "USE_PTHREADS=1"], + features = ["use_pthreads"], + ), + flag_set( + actions = all_link_actions, + flags = ["-s", "EXIT_RUNTIME=1"], + features = ["exit_runtime"], + ), + flag_set( + actions = all_compile_actions + all_link_actions, + flags = ["-pthread"], + features = ["llvm_backend", "use_pthreads"], + ), + flag_set( + actions = all_compile_actions + all_link_actions, + flags = ["-msimd128"], + features = ["wasm_simd"], + ), + flag_set( + actions = all_link_actions, + flags = ["-s", "PRINTF_LONG_DOUBLE=1"], + features = ["precise_long_double_printf"], + ), + + # Opt + flag_set( + actions = preprocessor_compile_actions, + flags = ["-DNDEBUG"], + features = ["opt"], + ), + flag_set( + actions = all_compile_actions, + flags = ["-fomit-frame-pointer"], + features = ["opt"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-O3"], + features = ["opt"], + ), + # Users can override opt-level with semantic names... + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-Oz"], + features = ["optimized_for_size", "opt"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-O3"], + features = ["optimized_for_speed", "opt"], + ), + + # Fastbuild + flag_set( + actions = all_compile_actions, + flags = ["-fomit-frame-pointer"], + features = ["fastbuild"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-O2"], + features = ["fastbuild"], + ), + + # Dbg + flag_set( + actions = all_compile_actions, + flags = ["-fno-omit-frame-pointer"], + features = ["dbg"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-g", "-O0"], + features = ["dbg"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = [ + "-g4", + "-fsanitize=address", + "-O1", + "-DADDRESS_SANITIZER=1", + "-fno-omit-frame-pointer", + ], + features = ["wasm_asan"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = [ + "-g4", + "-fsanitize=undefined", + "-O1", + "-DUNDEFINED_BEHAVIOR_SANITIZER=1", + "-fno-omit-frame-pointer", + "-fno-sanitize=vptr", + ], + features = ["wasm_ubsan"], + ), + + # Profiling provides full debug info and a special --profiling flag + # to control name mangling + flag_set( + actions = all_link_actions, + flags = ["--profiling"], + features = ["profiling"], + ), + flag_set( + actions = all_link_actions, + flags = ["--profiling_funcs"], + features = ["profiling_funcs"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-g4"], + features = ["full_debug_info"], + ), + flag_set( + actions = all_link_actions, + flags = ["-gseparate-dwarf"], + features = ["dwarf_debug_info"], + ), + flag_set( + actions = all_compile_actions + + all_link_actions, + flags = ["-fdebug-compilation-dir=."], + features = ["dwarf_debug_info"], + ), + # Generic warning flag list + flag_set( + actions = all_compile_actions, + flags = CROSSTOOL_DEFAULT_WARNINGS, + ), + + # Defines and Includes and Paths and such + flag_set( + actions = all_compile_actions, + flag_groups = [ + flag_group(flags = ["-fPIC"], expand_if_available = "pic"), + ], + ), + flag_set( + actions = preprocessor_compile_actions, + flag_groups = [ + flag_group( + flags = ["-D%{preprocessor_defines}"], + iterate_over = "preprocessor_defines", + ), + ], + ), + flag_set( + actions = preprocessor_compile_actions, + flag_groups = [ + flag_group( + flags = ["-include", "%{includes}"], + iterate_over = "includes", + expand_if_available = "includes", + ), + ], + ), + flag_set( + actions = preprocessor_compile_actions, + flag_groups = [ + flag_group( + flags = ["-iquote", "%{quote_include_paths}"], + iterate_over = "quote_include_paths", + ), + flag_group( + flags = ["-I%{include_paths}"], + iterate_over = "include_paths", + ), + flag_group( + flags = ["-isystem", "%{system_include_paths}"], + iterate_over = "system_include_paths", + ), + ], + ), + + ## Linking options (not libs -- those go last) + + # Generic link options + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flags = ["-shared"], + ), + + # Linker search paths and objects: + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + iterate_over = "runtime_library_search_directories", + flag_groups = [ + flag_group( + flags = [ + "-Wl,-rpath,$EXEC_ORIGIN/%{runtime_library_search_directories}", + ], + expand_if_true = "is_cc_test", + ), + flag_group( + flags = [ + "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", + ], + expand_if_false = "is_cc_test", + ), + ], + expand_if_available = "runtime_library_search_directories", + ), + ], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-L%{library_search_directories}"], + iterate_over = "library_search_directories", + expand_if_available = "library_search_directories", + ), + ], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + # This is actually a list of object files from the linkstamp steps + flags = ["%{linkstamp_paths}"], + iterate_over = "linkstamp_paths", + expand_if_available = "linkstamp_paths", + ), + ], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["@%{thinlto_param_file}"], + expand_if_available = "libraries_to_link", + expand_if_true = "thinlto_param_file", + ), + flag_group( + iterate_over = "libraries_to_link", + flag_groups = [ + flag_group( + flags = ["-Wl,--start-lib"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + flag_group( + flags = ["-Wl,-whole-archive"], + expand_if_true = "libraries_to_link.is_whole_archive", + ), + flag_group( + flags = ["%{libraries_to_link.object_files}"], + iterate_over = "libraries_to_link.object_files", + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "interface_library", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "static_library", + ), + ), + flag_group( + flags = ["-l%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "dynamic_library", + ), + ), + flag_group( + flags = ["-l:%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "versioned_dynamic_library", + ), + ), + flag_group( + flags = ["-Wl,-no-whole-archive"], + expand_if_true = "libraries_to_link.is_whole_archive", + ), + flag_group( + flags = ["-Wl,--end-lib"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + ], + expand_if_available = "libraries_to_link", + ), + ], + ), + + # Configure the header parsing and preprocessing. + flag_set( + actions = [ACTION_NAMES.cpp_header_parsing], + flags = ["-xc++-header", "-fsyntax-only"], + features = ["parse_headers"], + ), + + # Note: user compile flags should be nearly last -- you probably + # don't want to put any more features after this! + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["%{user_compile_flags}"], + iterate_over = "user_compile_flags", + expand_if_available = "user_compile_flags", + ), + ], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["%{user_link_flags}"], + iterate_over = "user_link_flags", + expand_if_available = "user_link_flags", + ), + ], + ), + ## Options which need to go late -- after all the user options -- go here. + flag_set( + # One might hope that these options would only be needed for C++ + # compiles. But, sadly, users compile ".c" files with custom + # copts=["-x", "c++"], and expect that to be able to find C++ stdlib + # headers. It might be worth pondering how blaze could support this sort + # of use-case better. + actions = preprocessor_compile_actions + + [ACTION_NAMES.cc_flags_make_variable], + flags = [ + "-iwithsysroot" + "/include/c++/v1", + "-iwithsysroot" + "/include/compat", + "-iwithsysroot" + "/include", + "-isystem", "external/emscripten/lib/clang/13.0.0/include", + ], + ), + # Inputs and outputs + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["-MD", "-MF", "%{dependency_file}"], + expand_if_available = "dependency_file", + ), + ], + ), + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["-c", "%{source_file}"], + expand_if_available = "source_file", + ), + ], + ), + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["-S"], + expand_if_available = "output_assembly_file", + ), + flag_group( + flags = ["-E"], + expand_if_available = "output_preprocess_file", + ), + flag_group( + flags = ["-o", "%{output_file}"], + expand_if_available = "output_file", + ), + ], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-o", "%{output_execpath}"], + expand_if_available = "output_execpath", + ), + ], + ), + # And finally, the params file! + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["@%{linker_param_file}"], + expand_if_available = "linker_param_file", + ), + ], + ), + flag_set( + actions = all_compile_actions, + flags = [ + "-Wno-builtin-macro-redefined", + # Genrules may not escape quotes enough for these, so + # don't put them into $(CC_FLAGS): + '-D__DATE__="redacted"', + '-D__TIMESTAMP__="redacted"', + '-D__TIME__="redacted"', + ], + ), + flag_set( + actions = all_compile_actions, + flags = ["-Werror"], + features = ["wasm_warnings_as_errors"], + ), + ] + + crosstool_default_env_sets = [ + # Use llvm backend. Off by default, enabled via --features=llvm_backend + env_set( + actions = all_compile_actions + + all_link_actions + + [ACTION_NAMES.cpp_link_static_library], + env_entries = [env_entry(key = "EMCC_WASM_BACKEND", value = "1")], + with_features = [with_feature_set(features = ["llvm_backend"])], + ), + # Debug compile and link. Off by default, enabled via --features=emcc_debug + env_set( + actions = all_compile_actions, + env_entries = [env_entry(key = "EMCC_DEBUG", value = "1")], + with_features = [with_feature_set(features = ["emcc_debug"])], + ), + + # Debug only link step. Off by default, enabled via --features=emcc_debug_link + env_set( + actions = all_link_actions, + env_entries = [env_entry(key = "EMCC_DEBUG", value = "1")], + with_features = [ + with_feature_set(features = ["emcc_debug"]), + with_feature_set(features = ["emcc_debug_link"]), + ], + ), + ] + + crosstool_default_flags_feature = feature( + name = "crosstool_default_flags", + enabled = True, + flag_sets = crosstool_default_flag_sets, + env_sets = crosstool_default_env_sets, + ) + + features.append(crosstool_default_flags_feature) + + cxx_builtin_include_directories = [ + "external/emscripten/emscripten/cache/sysroot/include/c++/v1", + "external/emscripten/emscripten/cache/sysroot/include/compat", + "external/emscripten/emscripten/cache/sysroot/include", + "external/emscripten/lib/clang/13.0.0/include", + ] + + artifact_name_patterns = [] + + make_variables = [] + + out = ctx.actions.declare_file(ctx.label.name) + ctx.actions.write(out, "Fake executable") + return [ + cc_common.create_cc_toolchain_config_info( + ctx = ctx, + features = features, + action_configs = action_configs, + artifact_name_patterns = artifact_name_patterns, + cxx_builtin_include_directories = cxx_builtin_include_directories, + toolchain_identifier = toolchain_identifier, + host_system_name = host_system_name, + target_system_name = target_system_name, + target_cpu = target_cpu, + target_libc = target_libc, + compiler = compiler, + abi_version = abi_version, + abi_libc_version = abi_libc_version, + tool_paths = tool_paths, + make_variables = make_variables, + builtin_sysroot = builtin_sysroot, + cc_target_os = cc_target_os, + ), + DefaultInfo( + executable = out, + ), + ] + +emscripten_cc_toolchain_config_rule = rule( + implementation = _impl, + attrs = { + "cpu": attr.string(mandatory = True, values = ["asmjs", "wasm"]), + "emscripten_version": attr.string(mandatory = True), + }, + provides = [CcToolchainConfigInfo], + executable = True, +) diff --git a/emscripten_toolchain/emar.sh b/emscripten_toolchain/emar.sh new file mode 100755 index 0000000000..965442e34c --- /dev/null +++ b/emscripten_toolchain/emar.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source emscripten_toolchain/env.sh + +exec python3 $EMSCRIPTEN/emar.py "$@" diff --git a/emscripten_toolchain/emcc.sh b/emscripten_toolchain/emcc.sh new file mode 100755 index 0000000000..66d00b6c9a --- /dev/null +++ b/emscripten_toolchain/emcc.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source emscripten_toolchain/env.sh + +exec python3 external/emscripten/emscripten/emcc.py "$@" diff --git a/emscripten_toolchain/emcc_link.sh b/emscripten_toolchain/emcc_link.sh new file mode 100755 index 0000000000..9d0f8e208b --- /dev/null +++ b/emscripten_toolchain/emcc_link.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source emscripten_toolchain/env.sh + +exec python3 emscripten_toolchain/link_wrapper.py "$@" diff --git a/emscripten_toolchain/emscripten.BUILD b/emscripten_toolchain/emscripten.BUILD new file mode 100644 index 0000000000..6f11852874 --- /dev/null +++ b/emscripten_toolchain/emscripten.BUILD @@ -0,0 +1,6 @@ +package(default_visibility = ['//visibility:public']) + +filegroup( + name = "all", + srcs = glob(["**"]), +) diff --git a/emscripten_toolchain/emscripten_config b/emscripten_toolchain/emscripten_config new file mode 100644 index 0000000000..b7948aca41 --- /dev/null +++ b/emscripten_toolchain/emscripten_config @@ -0,0 +1,12 @@ +import os +import platform + +ROOT_DIR = os.environ["ROOT_DIR"] +EMSCRIPTEN_ROOT = os.environ["EMSCRIPTEN"] +LLVM_ROOT = ROOT_DIR + "/external/emscripten/bin" +BINARYEN_ROOT = ROOT_DIR + "/external/emscripten" +FROZEN_CACHE = True + +system = platform.system() +nodejs_binary = "node.exe" if(system =="Windows") else "bin/node" +NODE_JS = ROOT_DIR + "/external/nodejs_{}_amd64/{}".format(system.lower(), nodejs_binary) diff --git a/emscripten_toolchain/env.sh b/emscripten_toolchain/env.sh new file mode 100755 index 0000000000..dfb4ddc29d --- /dev/null +++ b/emscripten_toolchain/env.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +export ROOT_DIR=`(pwd -P)` +export EMSCRIPTEN=${ROOT_DIR}/external/emscripten/emscripten +export EM_CONFIG=${ROOT_DIR}/emscripten_toolchain/emscripten_config diff --git a/emscripten_toolchain/link_wrapper.py b/emscripten_toolchain/link_wrapper.py new file mode 100644 index 0000000000..a746ae8fd3 --- /dev/null +++ b/emscripten_toolchain/link_wrapper.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +"""wrapper around emcc link step. + +This wrapper currently serves the following purposes. + +1. Ensures we always link to file with .js extension. The upstream default + it to link to an llvm bitcode file which is never (AFAICT) want to do that. + +2. When building with --config=wasm the final output is multiple files, usually + at least one .js and one .wasm file. Since the cc_binary link step only + allows a single output, we must tar up the outputs into a single file. + +3. Add quotes around arguments that need them in the response file to work + around a bazel quirk. +""" + +from __future__ import print_function + +import os +import subprocess +import sys + +# Only argument should be @path/to/parameter/file +assert sys.argv[1][0] == '@' +param_filename = sys.argv[1][1:] +param_file_args = [l.strip() for l in open(param_filename, 'r').readlines()] + +output_index = param_file_args.index('-o') + 1 +orig_output = js_output = param_file_args[output_index] +outdir = os.path.dirname(orig_output) + +# google3-only(TODO(b/139440956): Default to False once the bug is fixed) +replace_response_file = any(' ' in a for a in param_file_args) + +if not os.path.splitext(orig_output)[1]: + js_output = orig_output + '.js' + param_file_args[output_index] = js_output + replace_response_file = True + +# Re-write response file if needed. +if replace_response_file: + new_param_filename = param_filename + '.modified' + with open(new_param_filename, 'w') as f: + for param in param_file_args: + if ' ' in param: + f.write('"%s"' % param) + else: + f.write(param) + f.write('\n') + sys.argv[1] = '@' + new_param_filename + +emcc_py = os.path.join(os.environ['EMSCRIPTEN'], 'emcc.py') +rtn = subprocess.call(['python3', emcc_py] + sys.argv[1:]) +if rtn != 0: + sys.exit(1) + +js_name = os.path.basename(js_output) +base_name = os.path.splitext(js_name)[0] + +files = [] +extensions = [ + '.js', + '.wasm', + '.wasm.map', + '.js.mem', + '.fetch.js', + '.worker.js', + '.data', + '.js.symbols', + '.wasm.debug.wasm' +] + +for ext in extensions: + filename = base_name + ext + if os.path.exists(os.path.join(outdir, filename)): + files.append(filename) + +wasm_base = os.path.join(outdir, base_name + '.wasm') +if os.path.exists(wasm_base + '.debug.wasm') and os.path.exists(wasm_base): + # If we have a .wasm.debug.wasm file and a .wasm file, we need to rewrite the + # section in the .wasm file that refers to it. The path that's in there + # is the blaze output path; we want it to be just the filename. + + llvm_objcopy = os.path.join( + os.environ['EMSCRIPTEN'], 'llvm-bin/llvm-objcopy') + # First, check to make sure the .wasm file has the header that needs to be + # rewritten. + rtn = subprocess.call([ + llvm_objcopy, + '--dump-section=external_debug_info=/dev/null', + wasm_base], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if rtn == 0: + # If llvm-objcopy did not return an error, the external_debug_info section + # must exist, so we're good to continue. + + # Next we need to convert length of the filename to LEB128. + # Start by converting the length of the filename to a bit string. + bit_string = '{0:b}'.format(len(base_name + '.wasm.debug.wasm')) + + # Pad the bit string with 0s so that its length is a multiple of 7. + while len(bit_string) % 7 != 0: + bit_string = '0' + bit_string + + # Break up our bit string into chunks of 7. + # We do this backwards because the final format is little-endian. + final_bytes = bytearray() + for i in reversed(range(0, len(bit_string), 7)): + binary_part = bit_string[i:i + 7] + if i != 0: + # Every chunk except the last one needs to be prepended with '1'. + # The length of each chunk is 7, so that one has an implicit '0'. + binary_part = '1' + binary_part + final_bytes.append(int(binary_part, 2)) + # Finally, add the actual filename. + final_bytes.extend(base_name + '.wasm.debug.wasm') + + # Write our length + filename bytes to a temp file. + with open('debugsection.tmp', 'wb+') as f: + f.write(final_bytes) + f.close() + + # First delete the old section. + subprocess.check_call([ + llvm_objcopy, + wasm_base, + '--remove-section=external_debug_info']) + # Rewrite section with the new size and filename from the temp file. + subprocess.check_call([ + llvm_objcopy, + wasm_base, + '--add-section=external_debug_info=debugsection.tmp']) + +# If we have more than one output file then create tarball +if len(files) > 1: + cmd = ['tar', 'cf', 'tmp.tar'] + files + subprocess.check_call(cmd, cwd=outdir) + os.rename(os.path.join(outdir, 'tmp.tar'), orig_output) +elif len(files) == 1: + # Otherwise, if only have a single output than move it to the expected name + if files[0] != os.path.basename(orig_output): + os.rename(os.path.join(outdir, files[0]), orig_output) +else: + print('emcc.py did not appear to output any known files!') + sys.exit(1) + +sys.exit(0) diff --git a/emscripten_toolchain/wasm_binary.py b/emscripten_toolchain/wasm_binary.py new file mode 100644 index 0000000000..466e789e61 --- /dev/null +++ b/emscripten_toolchain/wasm_binary.py @@ -0,0 +1,84 @@ +"""Unpackages a bazel emscripten archive for use in a bazel BUILD rule. + +This script will take a tar archive containing the output of the emscripten +toolchain. This file contains any output files produced by a wasm_cc_binary or a +cc_binary built with --config=wasm. The files are extracted into the given +output path. + +The name of archive is expected to be of the format `foo` or `foo.XXX` and +the contents are expected to be foo.js and foo.wasm. + +Several optional files may also be in the archive, including but not limited to +foo.js.mem, pthread-main.js, and foo.wasm.map. + +If the file is not a tar archive, the passed file will simply be copied to its +destination. + +This script and its accompanying Bazel rule should allow you to extract a +WebAssembly binary into a larger web application. +""" + +import os +import subprocess +import sys + +from absl import app +from absl import flags + +FLAGS = flags.FLAGS +flags.DEFINE_string('archive', None, 'The the archive to extract from.') +flags.DEFINE_string('output_path', None, 'The path to extract into.') + + +def ensure(f): + if not os.path.exists(f): + with open(f, 'w'): + pass + + +def check(f): + if not os.path.exists(f): + raise Exception('Expected file in archive: %s' % f) + + +def main(argv): + basename = os.path.basename(FLAGS.archive) + stem = basename.split('.')[0] + + # Check the type of the input file + mimetype_bytes = subprocess.check_output(['file', '-Lb', '--mime-type', '--mime-encoding', FLAGS.archive]) + mimetype = mimetype_bytes.decode(sys.stdout.encoding) + + # If we have a tar, extract all files. If we have just a single file, copy it. + if 'tar' in mimetype: + subprocess.check_call( + ['tar', 'xf', FLAGS.archive, '-C', FLAGS.output_path]) + elif 'binary' in mimetype: + subprocess.check_call([ + 'cp', + FLAGS.archive, + os.path.join(FLAGS.output_path, stem + '.wasm')]) + elif 'text' in mimetype: + subprocess.check_call([ + 'cp', + FLAGS.archive, + os.path.join(FLAGS.output_path, stem + '.js')]) + else: + subprocess.check_call(['cp', FLAGS.archive, FLAGS.output_path]) + + # At least one of these two files should exist at this point. + ensure(os.path.join(FLAGS.output_path, stem + '.js')) + ensure(os.path.join(FLAGS.output_path, stem + '.wasm')) + + # And can optionally contain these extra files. + ensure(os.path.join(FLAGS.output_path, stem + '.wasm.map')) + ensure(os.path.join(FLAGS.output_path, stem + '.worker.js')) + ensure(os.path.join(FLAGS.output_path, stem + '.js.mem')) + ensure(os.path.join(FLAGS.output_path, stem + '.data')) + ensure(os.path.join(FLAGS.output_path, stem + '.fetch.js')) + ensure(os.path.join(FLAGS.output_path, stem + '.js.symbols')) + ensure(os.path.join(FLAGS.output_path, stem + '.wasm.debug.wasm')) + + +if __name__ == '__main__': + app.run(main) diff --git a/emscripten_toolchain/wasm_cc_binary.bzl b/emscripten_toolchain/wasm_cc_binary.bzl new file mode 100644 index 0000000000..01780e2cd4 --- /dev/null +++ b/emscripten_toolchain/wasm_cc_binary.bzl @@ -0,0 +1,150 @@ +"""wasm_cc_binary rule for compiling C++ targets to WebAssembly. +""" + +def _wasm_transition_impl(settings, attr): + _ignore = (settings, attr) + + features = list(settings["//command_line_option:features"]) + linkopts = list(settings["//command_line_option:linkopt"]) + + if attr.threads == "emscripten": + # threads enabled + features.append("use_pthreads") + elif attr.threads == "off": + # threads disabled + features.append("-use_pthreads") + + if attr.exit_runtime == True: + features.append("exit_runtime") + + if attr.backend == "llvm": + features.append("llvm_backend") + elif attr.backend == "emscripten": + features.append("-llvm_backend") + + if attr.simd: + features.append("wasm_simd") + + return { + "//command_line_option:compiler": "emscripten", + "//command_line_option:crosstool_top": "//emscripten_toolchain:everything", + "//command_line_option:cpu": "wasm", + "//command_line_option:features": features, + "//command_line_option:dynamic_mode": "off", + "//command_line_option:linkopt": linkopts, + "//command_line_option:platforms": [], + "//command_line_option:custom_malloc": "//emscripten_toolchain:malloc", + } + +_wasm_transition = transition( + implementation = _wasm_transition_impl, + inputs = [ + "//command_line_option:features", + "//command_line_option:linkopt", + ], + outputs = [ + "//command_line_option:compiler", + "//command_line_option:cpu", + "//command_line_option:crosstool_top", + "//command_line_option:features", + "//command_line_option:dynamic_mode", + "//command_line_option:linkopt", + "//command_line_option:platforms", + "//command_line_option:custom_malloc", + ], +) + +def _wasm_binary_impl(ctx): + cc_target = ctx.attr.cc_target[0] + + args = [ + "--output_path={}".format(ctx.outputs.loader.dirname), + ] + [ + ctx.expand_location("--archive=$(location {})".format( + cc_target.label, + ), [cc_target]), + ] + outputs = [ + ctx.outputs.loader, + ctx.outputs.wasm, + ctx.outputs.map, + ctx.outputs.mem, + ctx.outputs.fetch, + ctx.outputs.worker, + ctx.outputs.data, + ctx.outputs.symbols, + ctx.outputs.dwarf, + ] + + ctx.actions.run( + inputs = ctx.files.cc_target, + outputs = outputs, + arguments = args, + executable = ctx.executable._wasm_binary_extractor, + ) + + return DefaultInfo( + files = depset(outputs), + # This is needed since rules like web_test usually have a data + # dependency on this target. + data_runfiles = ctx.runfiles(transitive_files = depset(outputs)), + ) + +def _wasm_binary_outputs(name, cc_target): + basename = cc_target.name + basename = basename.split(".")[0] + outputs = { + "loader": "{}/{}.js".format(name, basename), + "wasm": "{}/{}.wasm".format(name, basename), + "map": "{}/{}.wasm.map".format(name, basename), + "mem": "{}/{}.js.mem".format(name, basename), + "fetch": "{}/{}.fetch.js".format(name, basename), + "worker": "{}/{}.worker.js".format(name, basename), + "data": "{}/{}.data".format(name, basename), + "symbols": "{}/{}.js.symbols".format(name, basename), + "dwarf": "{}/{}.wasm.debug.wasm".format(name, basename), + } + + return outputs + +# Wraps a C++ Blaze target, extracting the appropriate files. +# +# This rule will transition to the emscripten toolchain in order +# to build the the cc_target as a WebAssembly binary. +# +# Args: +# name: The name of the rule. +# cc_target: The cc_binary or cc_library to extract files from. +wasm_cc_binary = rule( + implementation = _wasm_binary_impl, + attrs = { + "backend": attr.string( + default = "_default", + values = ["_default", "emscripten", "llvm"], + ), + "cc_target": attr.label( + cfg = _wasm_transition, + mandatory = True, + ), + "exit_runtime": attr.bool( + default = False, + ), + "threads": attr.string( + default = "_default", + values = ["_default", "emscripten", "off"], + ), + "simd": attr.bool( + default = False, + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + "_wasm_binary_extractor": attr.label( + executable = True, + allow_files = True, + cfg = "exec", + default = Label("//emscripten_toolchain:wasm_binary"), + ), + }, + outputs = _wasm_binary_outputs, +) diff --git a/emscripten_toolchain/wasm_rules.bzl b/emscripten_toolchain/wasm_rules.bzl new file mode 100644 index 0000000000..1c1c409ed5 --- /dev/null +++ b/emscripten_toolchain/wasm_rules.bzl @@ -0,0 +1,6 @@ +"""Rules related to C++ and WebAssembly. +""" + +load("//emscripten_toolchain:wasm_cc_binary.bzl", _wasm_cc_binary = "wasm_cc_binary") + +wasm_cc_binary = _wasm_cc_binary diff --git a/tfjs-backend-wasm/src/cc/BUILD b/tfjs-backend-wasm/src/cc/BUILD index 245be751a8..fecf70a189 100644 --- a/tfjs-backend-wasm/src/cc/BUILD +++ b/tfjs-backend-wasm/src/cc/BUILD @@ -1,4 +1,5 @@ load(":build_defs.bzl", "tfjs_cc_library", "tfjs_unit_test") +load("//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") # Emcripten produces a much larger wasm bundle unless the cc_binary has srcs # explicitly pointing to files with exported methods (EMSCRIPTEN_KEEPALIVE). @@ -36,6 +37,23 @@ cc_binary( ], ) +wasm_cc_binary( + name = "tfjs-backend-wasm-new", + cc_target = ":tfjs-backend-wasm.js", +) + + +# REMOVE THIS +cc_binary( + name = "hello_world", + srcs = ["hello_world.cc"], +) + +wasm_cc_binary( + name = "hello_world_wasm", + cc_target = ":hello_world", +) + # This build rule generates tfjs-backend-wasm-simd.{js,wasm}. # # We only need the .wasm file, not the .js file, because it will be loadsed by diff --git a/tfjs-backend-wasm/src/cc/hello_world.cc b/tfjs-backend-wasm/src/cc/hello_world.cc new file mode 100644 index 0000000000..ee72c53171 --- /dev/null +++ b/tfjs-backend-wasm/src/cc/hello_world.cc @@ -0,0 +1,6 @@ +#include + +int main(int argc, char** argv) { + std::cout << "hello world!" << std::endl; + return 0; +} diff --git a/tfjs-backend-wasm/yarn.lock b/tfjs-backend-wasm/yarn.lock index 18aa43888d..7881565c49 100644 --- a/tfjs-backend-wasm/yarn.lock +++ b/tfjs-backend-wasm/yarn.lock @@ -936,11 +936,6 @@ resolved "https://registry.yarnpkg.com/@bazel/buildifier/-/buildifier-4.0.0.tgz#c99fb21295c7e2858fa176c2b950c589217ae8ac" integrity sha512-e242AK+shp+5bpAc9l93pmG5YRYRfM0INV3gfbdxNeBxccx1MJUagaiwxqAV+Mw55zk92gsb99f2TAc9tt6C7w== -"@bazel/hide-bazel-files@latest": - version "1.7.0" - resolved "https://registry.yarnpkg.com/@bazel/hide-bazel-files/-/hide-bazel-files-1.7.0.tgz#7cb140c23c4269d6464c24be0a2acf0241d2a31d" - integrity sha512-pvdyRX/EsU8n+oElFb+OZ9i5M48HNFR+Z4D3vc0qDGiJ8oly9fZcUb2gbw4CzyeovJz0IzjSxjqMS6cp5gKoeg== - "@istanbuljs/schema@^0.1.2": version "0.1.2" resolved "https://registry.yarnpkg.com/@istanbuljs/schema/-/schema-0.1.2.tgz#26520bf09abe4a5644cd5414e37125a8954241dd" @@ -989,18 +984,11 @@ "@tensorflow/tfjs-backend-cpu@link:../tfjs-backend-cpu": version "0.0.0" - dependencies: - "@types/seedrandom" "2.4.27" - seedrandom "2.4.3" + uid "" "@tensorflow/tfjs-core@link:../tfjs-core": version "0.0.0" - dependencies: - "@types/offscreencanvas" "~2019.3.0" - "@types/seedrandom" "2.4.27" - "@types/webgl-ext" "0.0.30" - node-fetch "~2.6.1" - seedrandom "2.4.3" + uid "" "@types/emscripten@~0.0.34": version "0.0.34" From 35761df35ca5f028cda49cdb35ce21c90a26bf85 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Thu, 25 Feb 2021 14:41:27 -0800 Subject: [PATCH 02/24] xnnpack hello world not working with emsdk toolchain --- tfjs-backend-wasm/src/cc/BUILD | 12 +++++++ .../src/cc/hello_world_xnnpack.cc | 33 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 tfjs-backend-wasm/src/cc/hello_world_xnnpack.cc diff --git a/tfjs-backend-wasm/src/cc/BUILD b/tfjs-backend-wasm/src/cc/BUILD index fecf70a189..31c673ee98 100644 --- a/tfjs-backend-wasm/src/cc/BUILD +++ b/tfjs-backend-wasm/src/cc/BUILD @@ -54,6 +54,18 @@ wasm_cc_binary( cc_target = ":hello_world", ) +cc_binary( + name = "hello_world_xnnpack", + srcs = ["hello_world_xnnpack.cc"], + deps = [":backend"] +) + +wasm_cc_binary( + name = "hello_world_xnnpack_wasm", + cc_target = ":hello_world_xnnpack", +) + + # This build rule generates tfjs-backend-wasm-simd.{js,wasm}. # # We only need the .wasm file, not the .js file, because it will be loadsed by diff --git a/tfjs-backend-wasm/src/cc/hello_world_xnnpack.cc b/tfjs-backend-wasm/src/cc/hello_world_xnnpack.cc new file mode 100644 index 0000000000..3cc268cfa2 --- /dev/null +++ b/tfjs-backend-wasm/src/cc/hello_world_xnnpack.cc @@ -0,0 +1,33 @@ +#include +#include + + +int main(int argc, char** argv) { + const size_t channels = 1, input_stride = 1, output_stride = 1; + const uint32_t flags = 1; + xnn_operator_t unary_op = nullptr; + + xnn_status create_status = + xnn_create_bankers_rounding_nc_f32( + channels, input_stride, output_stride, flags, &unary_op); + if (create_status != xnn_status_success) { + std::cout << "Failed to create op" << std::endl; + return 1; + } + + const float* in_buf = new float[1]; + float* out_buf = new float[1]; + + xnn_status setup_status = + xnn_setup_bankers_rounding_nc_f32( + unary_op, 1, in_buf, out_buf, nullptr /* no thread pool */); + + if (setup_status != xnn_status_success) { + std::cout << "Failed to setup op" << std::endl; + return 1; + } + + xnn_run_operator(unary_op, nullptr /* no threadpool */); + std::cout << "created xnnpack op" << std::endl; + return 0; +} From 2e719ca19ecaf19ca541b7bb25bac9a472eb4996 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Fri, 26 Feb 2021 14:32:24 -0800 Subject: [PATCH 03/24] Remove unused variables --- tfjs-backend-wasm/src/cc/binary.cc | 1 - tfjs-backend-wasm/src/cc/kernels/CropAndResize.cc | 4 ---- tfjs-backend-wasm/src/cc/kernels/FloorDiv.cc | 1 - tfjs-backend-wasm/src/cc/kernels/Max.cc | 1 - tfjs-backend-wasm/src/cc/kernels/Mean.cc | 2 -- tfjs-backend-wasm/src/cc/kernels/Min.cc | 1 - tfjs-backend-wasm/src/cc/kernels/Pow.cc | 7 ++++++- tfjs-backend-wasm/src/cc/kernels/Prod.cc | 2 -- tfjs-backend-wasm/src/cc/kernels/Sum.cc | 2 -- tfjs-backend-wasm/src/cc/non_max_suppression_impl.cc | 2 +- 10 files changed, 7 insertions(+), 16 deletions(-) diff --git a/tfjs-backend-wasm/src/cc/binary.cc b/tfjs-backend-wasm/src/cc/binary.cc index 538d1c5f65..2d76ae3c79 100644 --- a/tfjs-backend-wasm/src/cc/binary.cc +++ b/tfjs-backend-wasm/src/cc/binary.cc @@ -61,7 +61,6 @@ void binary_xnn_f32(const size_t a_id, const size_t* a_shape_ptr, } else { binary_op = cache_result->second; } - const size_t batch_size = out_info.size; xnn_status status = setup_op(binary_op, a_shape_len, a_shape_ptr, b_shape_len, b_shape_ptr, a_buf, b_buf, out_buf, tfjs::backend::threadpool); diff --git a/tfjs-backend-wasm/src/cc/kernels/CropAndResize.cc b/tfjs-backend-wasm/src/cc/kernels/CropAndResize.cc index dd2d4a0e11..0bb95b43cb 100644 --- a/tfjs-backend-wasm/src/cc/kernels/CropAndResize.cc +++ b/tfjs-backend-wasm/src/cc/kernels/CropAndResize.cc @@ -91,16 +91,12 @@ void CropAndResize(size_t images_id, size_t boxes_id, size_t box_ind_id, auto& out_info = backend::get_tensor_info_out(out_id); const float* images_buf = images_info.f32(); - const size_t images_size = images_info.size; const float* boxes_buf = boxes_info.f32(); - const size_t boxes_size = boxes_info.size; const int* box_ind_buf = box_ind_info.i32(); - const size_t box_ind_size = box_ind_info.size; float* out_buf = out_info.f32_write(); - const size_t out_size = out_info.size; const size_t batch = images_shape[0]; const size_t image_height = images_shape[1]; diff --git a/tfjs-backend-wasm/src/cc/kernels/FloorDiv.cc b/tfjs-backend-wasm/src/cc/kernels/FloorDiv.cc index ba4cf9c800..2c1fbd0b38 100644 --- a/tfjs-backend-wasm/src/cc/kernels/FloorDiv.cc +++ b/tfjs-backend-wasm/src/cc/kernels/FloorDiv.cc @@ -35,7 +35,6 @@ void FloorDiv(const size_t a_id, const size_t* a_shape_ptr, const size_t a_shape_len, const size_t b_id, const size_t* b_shape_ptr, const size_t b_shape_len, const DType dtype, const size_t out_id) { - auto& a_info = backend::get_tensor_info(a_id); switch (dtype) { case DType::float32: binary_f32(a_id, b_id, out_id, diff --git a/tfjs-backend-wasm/src/cc/kernels/Max.cc b/tfjs-backend-wasm/src/cc/kernels/Max.cc index 7d91daaa2b..4d791e593b 100644 --- a/tfjs-backend-wasm/src/cc/kernels/Max.cc +++ b/tfjs-backend-wasm/src/cc/kernels/Max.cc @@ -33,7 +33,6 @@ void Max(const size_t x_id, const size_t reduce_size, const size_t out_id) { auto& out_info = backend::get_tensor_info_out(out_id); const float* x_buf = x_info.f32(); - const size_t x_size = x_info.size; float* out_buf = out_info.f32_write(); const size_t out_size = out_info.size; diff --git a/tfjs-backend-wasm/src/cc/kernels/Mean.cc b/tfjs-backend-wasm/src/cc/kernels/Mean.cc index 3230b2c291..f1328c410a 100644 --- a/tfjs-backend-wasm/src/cc/kernels/Mean.cc +++ b/tfjs-backend-wasm/src/cc/kernels/Mean.cc @@ -34,7 +34,6 @@ void Mean(const size_t x_id, const size_t reduce_size, const size_t out_id) { auto& out_info = backend::get_tensor_info_out(out_id); const float* x_buf = x_info.f32(); - const size_t x_size = x_info.size; float* out_buf = out_info.f32_write(); const size_t out_size = out_info.size; @@ -42,7 +41,6 @@ void Mean(const size_t x_id, const size_t reduce_size, const size_t out_id) { const float* x_offset = x_buf; for (size_t i = 0; i < out_size; ++i) { - const size_t offset = i * reduce_size; float sum = 0; const float* x_iter_end = x_offset + reduce_size; diff --git a/tfjs-backend-wasm/src/cc/kernels/Min.cc b/tfjs-backend-wasm/src/cc/kernels/Min.cc index bffa69cac7..d0e05748ba 100644 --- a/tfjs-backend-wasm/src/cc/kernels/Min.cc +++ b/tfjs-backend-wasm/src/cc/kernels/Min.cc @@ -33,7 +33,6 @@ void Min(const size_t x_id, const size_t reduce_size, const size_t out_id) { auto& out_info = backend::get_tensor_info_out(out_id); const float* x_buf = x_info.f32(); - const size_t x_size = x_info.size; float* out_buf = out_info.f32_write(); const size_t out_size = out_info.size; diff --git a/tfjs-backend-wasm/src/cc/kernels/Pow.cc b/tfjs-backend-wasm/src/cc/kernels/Pow.cc index f778382cc1..1436d4ce23 100644 --- a/tfjs-backend-wasm/src/cc/kernels/Pow.cc +++ b/tfjs-backend-wasm/src/cc/kernels/Pow.cc @@ -27,8 +27,13 @@ template inline T power(T a, T b) { return pow(a, b); } + +inline bool power_bool(bool a, bool b) { + return static_cast(pow(static_cast(a), static_cast(b))); +} } // namespace + namespace tfjs { namespace wasm { // We use C-style API to interface with Javascript. @@ -48,7 +53,7 @@ void Pow(const size_t a_id, const size_t* a_shape_ptr, const size_t a_shape_len, binary_i32(a_id, b_id, out_id, power); break; case DType::boolean: - binary_bool(a_id, b_id, out_id, power); + binary_bool(a_id, b_id, out_id, power_bool); break; default: util::warn("Pow for tensor ids %d and %d failed. Unknown dtype %d", a_id, diff --git a/tfjs-backend-wasm/src/cc/kernels/Prod.cc b/tfjs-backend-wasm/src/cc/kernels/Prod.cc index 28016385c4..e4effe5313 100644 --- a/tfjs-backend-wasm/src/cc/kernels/Prod.cc +++ b/tfjs-backend-wasm/src/cc/kernels/Prod.cc @@ -29,7 +29,6 @@ void prod(const size_t x_id, const size_t reduce_size, const size_t out_id) { auto& out_info = tfjs::backend::get_tensor_info_out(out_id); const T* x_buf = reinterpret_cast(x_info.memory_offset); - const size_t x_size = x_info.size; T* out_buf = reinterpret_cast(out_info.memory_offset); const size_t out_size = out_info.size; @@ -37,7 +36,6 @@ void prod(const size_t x_id, const size_t reduce_size, const size_t out_id) { const T* x_offset = x_buf; for (size_t i = 0; i < out_size; ++i) { - const size_t offset = i * reduce_size; T product = 1; const T* x_iter_end = x_offset + reduce_size; diff --git a/tfjs-backend-wasm/src/cc/kernels/Sum.cc b/tfjs-backend-wasm/src/cc/kernels/Sum.cc index 83a26d1063..4a21e9c9f1 100644 --- a/tfjs-backend-wasm/src/cc/kernels/Sum.cc +++ b/tfjs-backend-wasm/src/cc/kernels/Sum.cc @@ -34,7 +34,6 @@ void Sum(const size_t x_id, const size_t reduce_size, const size_t out_id) { auto& out_info = backend::get_tensor_info_out(out_id); const float* x_buf = x_info.f32(); - const size_t x_size = x_info.size; float* out_buf = out_info.f32_write(); const size_t out_size = out_info.size; @@ -42,7 +41,6 @@ void Sum(const size_t x_id, const size_t reduce_size, const size_t out_id) { const float* x_offset = x_buf; for (size_t i = 0; i < out_size; ++i) { - const size_t offset = i * reduce_size; float sum = 0; const float* x_iter_end = x_offset + reduce_size; diff --git a/tfjs-backend-wasm/src/cc/non_max_suppression_impl.cc b/tfjs-backend-wasm/src/cc/non_max_suppression_impl.cc index a23f6c4b29..2aeda8d276 100644 --- a/tfjs-backend-wasm/src/cc/non_max_suppression_impl.cc +++ b/tfjs-backend-wasm/src/cc/non_max_suppression_impl.cc @@ -116,7 +116,7 @@ const NonMaxSuppressionResult* non_max_suppression_impl( std::vector selected_indices; std::vector selected_scores; Candidate candidate; - float iou, original_score; + float original_score; while (selected_indices.size() < max_out_size && !candidate_priority_queue.empty()) { From 97859822ee008929f85a2d13bca5994d973a716e Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Mon, 1 Mar 2021 11:46:25 -0800 Subject: [PATCH 04/24] WASM simd compiling locally with -copt='-msimd128' --- WORKSPACE | 12 ++++++++---- tfjs-backend-wasm/src/cc/BUILD | 17 +++++++++++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 1bb9b36dcb..d0e99caed8 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -39,11 +39,15 @@ npm_install( load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") # xnnpack used for fast vectorized wasm operations -git_repository( +# git_repository( +# name = "xnnpack", +# commit = "55d53a4e7079d38e90acd75dd9e4f9e781d2da35", +# remote = "https://github.com/google/XNNPACK.git", +# shallow_since = "1614036677 -0800", +# ) +local_repository( name = "xnnpack", - commit = "55d53a4e7079d38e90acd75dd9e4f9e781d2da35", - remote = "https://github.com/google/XNNPACK.git", - shallow_since = "1614036677 -0800", + path = "/home/msoulanille/XNNPACK", ) # The libraries below are transitive dependencies of XNNPACK that we need to diff --git a/tfjs-backend-wasm/src/cc/BUILD b/tfjs-backend-wasm/src/cc/BUILD index 31c673ee98..80f45851cd 100644 --- a/tfjs-backend-wasm/src/cc/BUILD +++ b/tfjs-backend-wasm/src/cc/BUILD @@ -27,8 +27,8 @@ cc_binary( "-s EXIT_RUNTIME=0", "-s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]'", "-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"cwrap\"]'", - "-s MODULARIZE=1", - "-s EXPORT_NAME=WasmBackendModule", + #"-s MODULARIZE=1", + #"-s EXPORT_NAME=WasmBackendModule", "-s MALLOC=emmalloc", ], deps = [ @@ -42,6 +42,19 @@ wasm_cc_binary( cc_target = ":tfjs-backend-wasm.js", ) +wasm_cc_binary( + name = "tfjs-backend-wasm-simd-new", + cc_target = ":tfjs-backend-wasm.js", + simd = True, +) + +wasm_cc_binary( + name = "tfjs-backend-wasm-threaded-simd-new", + cc_target = ":tfjs-backend-wasm.js", + simd = True, + threads = "emscripten", +) + # REMOVE THIS cc_binary( From e9530cf79b0c6df79d80b4c5896ac79a74e5d711 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Mon, 1 Mar 2021 13:15:16 -0800 Subject: [PATCH 05/24] Add msimd128 copt when simd is true --- emscripten_toolchain/wasm_cc_binary.bzl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/emscripten_toolchain/wasm_cc_binary.bzl b/emscripten_toolchain/wasm_cc_binary.bzl index 01780e2cd4..ec8d8a7765 100644 --- a/emscripten_toolchain/wasm_cc_binary.bzl +++ b/emscripten_toolchain/wasm_cc_binary.bzl @@ -6,6 +6,7 @@ def _wasm_transition_impl(settings, attr): features = list(settings["//command_line_option:features"]) linkopts = list(settings["//command_line_option:linkopt"]) + copts = list(settings["//command_line_option:copt"]) if attr.threads == "emscripten": # threads enabled @@ -24,6 +25,7 @@ def _wasm_transition_impl(settings, attr): if attr.simd: features.append("wasm_simd") + copts.append("-msimd128") return { "//command_line_option:compiler": "emscripten", @@ -32,6 +34,7 @@ def _wasm_transition_impl(settings, attr): "//command_line_option:features": features, "//command_line_option:dynamic_mode": "off", "//command_line_option:linkopt": linkopts, + "//command_line_option:copt": copts, "//command_line_option:platforms": [], "//command_line_option:custom_malloc": "//emscripten_toolchain:malloc", } @@ -41,6 +44,7 @@ _wasm_transition = transition( inputs = [ "//command_line_option:features", "//command_line_option:linkopt", + "//command_line_option:copt", ], outputs = [ "//command_line_option:compiler", @@ -49,6 +53,7 @@ _wasm_transition = transition( "//command_line_option:features", "//command_line_option:dynamic_mode", "//command_line_option:linkopt", + "//command_line_option:copt", "//command_line_option:platforms", "//command_line_option:custom_malloc", ], From 6438cd4d75133d477e3f913a2219b396f099cf15 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Mon, 1 Mar 2021 15:01:41 -0800 Subject: [PATCH 06/24] Compile wasm bundles from the same cc target --- .bazelrc | 28 +++++--- tfjs-backend-wasm/scripts/build-wasm.sh | 39 +++++++---- tfjs-backend-wasm/src/cc/BUILD | 93 +++++++++++++------------ 3 files changed, 90 insertions(+), 70 deletions(-) diff --git a/.bazelrc b/.bazelrc index 6c9d929621..a2ae67c057 100644 --- a/.bazelrc +++ b/.bazelrc @@ -7,26 +7,32 @@ build --symlink_prefix=dist/ # Use our custom-configured c++ toolchain. -build:wasm --crosstool_top=//toolchain:emscripten +#build:wasm --crosstool_top=//toolchain:emscripten # Use --cpu as a differentiator. -build:wasm --cpu=wasm +#build:wasm --cpu=wasm # Use the default C++ toolchain to build the tools used during the build. -build:wasm --host_crosstool_top=@bazel_tools//tools/cpp:toolchain +#build:wasm --host_crosstool_top=@bazel_tools//tools/cpp:toolchain # These compile flags are active no matter which build mode we are in # (dbg vs opt). For flags specific to build mode, see cc_toolchain_config.bzl. -build:wasm --cxxopt="-std=c++11" -build:wasm --cxxopt="-fno-rtti" -build:wasm --cxxopt="-fno-exceptions" -build:wasm --cxxopt="-fomit-frame-pointer" +build --cxxopt="-std=c++11" +build --cxxopt="-fno-rtti" +build --cxxopt="-fno-exceptions" +build --cxxopt="-fomit-frame-pointer" # Disable sandbox environment because emsdk caches files by writing to # home directory. -build:wasm --spawn_strategy=local +#build:wasm --spawn_strategy=local -build:wasm_new --crosstool_top=//emscripten_toolchain:everything -build:wasm_new --cpu=wasm -build:wasm_new --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ No newline at end of file +#build:wasm_new --crosstool_top=//emscripten_toolchain:everything +#build:wasm_new --cpu=wasm +#build:wasm_new --host_crosstool_top=@bazel_tools//tools/cpp:toolchain + +# This flag is needed to so that the bazel cache is not invalidated +# when running bazel via `yarn bazel`. +# See https://github.com/angular/angular/issues/27514. +#build --incompatible_strict_action_env +#run --incompatible_strict_action_env diff --git a/tfjs-backend-wasm/scripts/build-wasm.sh b/tfjs-backend-wasm/scripts/build-wasm.sh index 6262b93696..383085302a 100755 --- a/tfjs-backend-wasm/scripts/build-wasm.sh +++ b/tfjs-backend-wasm/scripts/build-wasm.sh @@ -24,25 +24,36 @@ set -e set -x # Default build. -yarn bazel build -c opt //tfjs-backend-wasm/src/cc:tfjs-backend-wasm.js --config=wasm +yarn bazel build -c opt //tfjs-backend-wasm/src/cc:tfjs-backend-wasm # The typescript code and karma config expect the output of emscripten to be in # wasm-out/ so we copy the bazel output there. -cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm.js \ - ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm.wasm \ +cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm/tfjs-backend-wasm.js \ + ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm/tfjs-backend-wasm.wasm \ ../wasm-out/ if [[ "$1" != "--dev" ]]; then - # SIMD build. - yarn bazel build -c opt //tfjs-backend-wasm/src/cc:tfjs-backend-wasm-simd.js --config=wasm --copt="-msimd128" - cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-simd.wasm \ - ../wasm-out/ - - # Threaded + SIMD build. - yarn bazel build -c opt //tfjs-backend-wasm/src/cc:tfjs-backend-wasm-threaded-simd.js --config=wasm --copt="-pthread" --copt="-msimd128" - cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd.js \ - ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd.worker.js \ - ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd.wasm \ - ../wasm-out/ + # SIMD and threaded + SIMD builds. + yarn bazel build -c opt //tfjs-backend-wasm/src/cc:tfjs-backend-wasm-simd \ + //tfjs-backend-wasm/src/cc:tfjs-backend-wasm-threaded-simd + # Copy SIMD + cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-simd/tfjs-backend-wasm.wasm \ + ../wasm-out/tfjs-backend-wasm-simd.wasm + + # cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-simd.wasm \ + # ../wasm-out/ + + # Copy threaded + cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd/tfjs-backend-wasm.js \ + ../wasm-out/tfjs-backend-wasm-threaded-simd.js + cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd/tfjs-backend-wasm.worker.js \ + ../wasm-out/tfjs-backend-wasm-threaded-simd.worker.js + cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd/tfjs-backend-wasm.wasm \ + ../wasm-out/tfjs-backend-wasm-threaded-simd.wasm + + # cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd.js \ + # ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd.worker.js \ + # ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd.wasm \ + # ../wasm-out/ node ./create-worker-module.js node ./patch-threaded-simd-module.js diff --git a/tfjs-backend-wasm/src/cc/BUILD b/tfjs-backend-wasm/src/cc/BUILD index 80f45851cd..1e1d7b2208 100644 --- a/tfjs-backend-wasm/src/cc/BUILD +++ b/tfjs-backend-wasm/src/cc/BUILD @@ -27,8 +27,8 @@ cc_binary( "-s EXIT_RUNTIME=0", "-s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]'", "-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"cwrap\"]'", - #"-s MODULARIZE=1", - #"-s EXPORT_NAME=WasmBackendModule", + "-s MODULARIZE=1", + "-s EXPORT_NAME=WasmBackendModule", "-s MALLOC=emmalloc", ], deps = [ @@ -38,18 +38,18 @@ cc_binary( ) wasm_cc_binary( - name = "tfjs-backend-wasm-new", + name = "tfjs-backend-wasm", cc_target = ":tfjs-backend-wasm.js", ) wasm_cc_binary( - name = "tfjs-backend-wasm-simd-new", + name = "tfjs-backend-wasm-simd", cc_target = ":tfjs-backend-wasm.js", simd = True, ) wasm_cc_binary( - name = "tfjs-backend-wasm-threaded-simd-new", + name = "tfjs-backend-wasm-threaded-simd", cc_target = ":tfjs-backend-wasm.js", simd = True, threads = "emscripten", @@ -85,49 +85,52 @@ wasm_cc_binary( # the tfjs-backend-wasm.js file (generated from the previous build rule). # # See scripts/build-wasm.sh where we only copy the .wasm file to wasm-out. -cc_binary( - name = "tfjs-backend-wasm-simd.js", - srcs = ["backend.cc"] + KERNELS_WITH_KEEPALIVE, - linkopts = [ - "-s ALLOW_MEMORY_GROWTH=1", - "-s DEFAULT_LIBRARY_FUNCS_TO_INCLUDE=[]", - "-s DISABLE_EXCEPTION_CATCHING=1", - "-s FILESYSTEM=0", - "-s EXIT_RUNTIME=0", - "-s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]'", - "-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"cwrap\"]'", - "-s MALLOC=emmalloc", - ], - deps = [ - ":all_kernels", - ":backend", - ], -) +# cc_binary( +# name = "tfjs-backend-wasm-simd.js", +# srcs = ["backend.cc"] + KERNELS_WITH_KEEPALIVE, +# copts = [ +# "-msimd128", +# ], +# linkopts = [ +# "-s ALLOW_MEMORY_GROWTH=1", +# "-s DEFAULT_LIBRARY_FUNCS_TO_INCLUDE=[]", +# "-s DISABLE_EXCEPTION_CATCHING=1", +# "-s FILESYSTEM=0", +# "-s EXIT_RUNTIME=0", +# "-s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]'", +# "-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"cwrap\"]'", +# "-s MALLOC=emmalloc", +# ], +# deps = [ +# ":all_kernels", +# ":backend", +# ], +# ) # This build rule generates tfjs-backend-wasm-threaded-simd.{js,wasm} and # tfjs-backend-wasm-threaded-simd.worker.js. -cc_binary( - name = "tfjs-backend-wasm-threaded-simd.js", - srcs = ["backend.cc"] + KERNELS_WITH_KEEPALIVE, - linkopts = [ - "-s ALLOW_MEMORY_GROWTH=1", - "-s DEFAULT_LIBRARY_FUNCS_TO_INCLUDE=[]", - "-s DISABLE_EXCEPTION_CATCHING=1", - "-s FILESYSTEM=0", - "-s EXIT_RUNTIME=0", - "-s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]'", - "-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"cwrap\"]'", - "-s MODULARIZE=1", - "-s EXPORT_NAME=WasmBackendModuleThreadedSimd", - "-s MALLOC=emmalloc", - "-s USE_PTHREADS=1", - "-s PROXY_TO_PTHREAD=1", - ], - deps = [ - ":all_kernels", - ":backend", - ], -) +# cc_binary( +# name = "tfjs-backend-wasm-threaded-simd.js", +# srcs = ["backend.cc"] + KERNELS_WITH_KEEPALIVE, +# linkopts = [ +# "-s ALLOW_MEMORY_GROWTH=1", +# "-s DEFAULT_LIBRARY_FUNCS_TO_INCLUDE=[]", +# "-s DISABLE_EXCEPTION_CATCHING=1", +# "-s FILESYSTEM=0", +# "-s EXIT_RUNTIME=0", +# "-s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]'", +# "-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"cwrap\"]'", +# "-s MODULARIZE=1", +# "-s EXPORT_NAME=WasmBackendModuleThreadedSimd", +# "-s MALLOC=emmalloc", +# "-s USE_PTHREADS=1", +# "-s PROXY_TO_PTHREAD=1", +# ], +# deps = [ +# ":all_kernels", +# ":backend", +# ], +# ) test_suite( name = "cc_tests", From 602e2c5442a32b8b35a1b8a9eae43e9a7ed0870c Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Mon, 1 Mar 2021 15:20:57 -0800 Subject: [PATCH 07/24] Use a remote repo instead of local for patched XNNPACK --- WORKSPACE | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index d0e99caed8..9a3bcce317 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -39,15 +39,10 @@ npm_install( load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") # xnnpack used for fast vectorized wasm operations -# git_repository( -# name = "xnnpack", -# commit = "55d53a4e7079d38e90acd75dd9e4f9e781d2da35", -# remote = "https://github.com/google/XNNPACK.git", -# shallow_since = "1614036677 -0800", -# ) -local_repository( +git_repository( name = "xnnpack", - path = "/home/msoulanille/XNNPACK", + branch = "emsdk_fix", + remote = "https://github.com/mattsoulanille/XNNPACK.git", ) # The libraries below are transitive dependencies of XNNPACK that we need to From b434002af126f5227b7312c3c50a22da039638de Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Tue, 2 Mar 2021 08:15:51 -0800 Subject: [PATCH 08/24] Enable incompatible_strict_action_env --- .bazelrc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index a2ae67c057..6f6eea3aa1 100644 --- a/.bazelrc +++ b/.bazelrc @@ -34,5 +34,5 @@ build --cxxopt="-fomit-frame-pointer" # This flag is needed to so that the bazel cache is not invalidated # when running bazel via `yarn bazel`. # See https://github.com/angular/angular/issues/27514. -#build --incompatible_strict_action_env -#run --incompatible_strict_action_env +build --incompatible_strict_action_env +run --incompatible_strict_action_env From 3eb59d451cad0202e94e5710ffc84954cc639341 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Tue, 2 Mar 2021 08:44:27 -0800 Subject: [PATCH 09/24] Enable remote cache in CI --- .bazelrc | 27 ++++++++++++------------- .gitignore | 5 ++++- tfjs-backend-wasm/scripts/build-ci.sh | 3 +-- tfjs-backend-wasm/scripts/build-wasm.sh | 12 ++--------- 4 files changed, 20 insertions(+), 27 deletions(-) diff --git a/.bazelrc b/.bazelrc index 6f6eea3aa1..b673c6679d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -6,15 +6,6 @@ # editor's search path. build --symlink_prefix=dist/ -# Use our custom-configured c++ toolchain. -#build:wasm --crosstool_top=//toolchain:emscripten - -# Use --cpu as a differentiator. -#build:wasm --cpu=wasm - -# Use the default C++ toolchain to build the tools used during the build. -#build:wasm --host_crosstool_top=@bazel_tools//tools/cpp:toolchain - # These compile flags are active no matter which build mode we are in # (dbg vs opt). For flags specific to build mode, see cc_toolchain_config.bzl. build --cxxopt="-std=c++11" @@ -22,17 +13,25 @@ build --cxxopt="-fno-rtti" build --cxxopt="-fno-exceptions" build --cxxopt="-fomit-frame-pointer" -# Disable sandbox environment because emsdk caches files by writing to -# home directory. -#build:wasm --spawn_strategy=local - - #build:wasm_new --crosstool_top=//emscripten_toolchain:everything #build:wasm_new --cpu=wasm #build:wasm_new --host_crosstool_top=@bazel_tools//tools/cpp:toolchain +# Remote cache config. Users should override this in .bazelrc.user +build:remote --remote_http_cache=https://storage.googleapis.com/bazel-remote-cache-tfjs +build:remote --google_default_credentials + # This flag is needed to so that the bazel cache is not invalidated # when running bazel via `yarn bazel`. # See https://github.com/angular/angular/issues/27514. build --incompatible_strict_action_env run --incompatible_strict_action_env + +# Load any settings specific to the current user. +# .bazelrc.user should appear in .gitignore so that settings are not shared with team members +# This needs to be last statement in this +# config, as the user configuration should be able to overwrite flags from this file. +# See https://docs.bazel.build/versions/master/best-practices.html#bazelrc +# (Note that we use .bazelrc.user so the file appears next to .bazelrc in directory listing, +# rather than user.bazelrc as suggested in the Bazel docs) +try-import %workspace%/.bazelrc.user diff --git a/.gitignore b/.gitignore index ac42a7a5a6..20e852c787 100644 --- a/.gitignore +++ b/.gitignore @@ -59,4 +59,7 @@ tfjs-backend-wasm/wasm-out/*.js tfjs-backend-wasm/wasm-out/*.wasm yalc.lock yarn-error.log -cloudbuild_generated.yml \ No newline at end of file +cloudbuild_generated.yml + +# User-specific .bazelrc +.bazelrc.user \ No newline at end of file diff --git a/tfjs-backend-wasm/scripts/build-ci.sh b/tfjs-backend-wasm/scripts/build-ci.sh index c154f902d7..d8e1e867fa 100755 --- a/tfjs-backend-wasm/scripts/build-ci.sh +++ b/tfjs-backend-wasm/scripts/build-ci.sh @@ -17,5 +17,4 @@ set -e yarn tsc - -./scripts/build-wasm.sh +BAZEL_REMOTE="--config=remote" ./scripts/build-wasm.sh diff --git a/tfjs-backend-wasm/scripts/build-wasm.sh b/tfjs-backend-wasm/scripts/build-wasm.sh index 383085302a..617975f38f 100755 --- a/tfjs-backend-wasm/scripts/build-wasm.sh +++ b/tfjs-backend-wasm/scripts/build-wasm.sh @@ -24,7 +24,7 @@ set -e set -x # Default build. -yarn bazel build -c opt //tfjs-backend-wasm/src/cc:tfjs-backend-wasm +yarn bazel build $BAZEL_REMOTE -c opt //tfjs-backend-wasm/src/cc:tfjs-backend-wasm # The typescript code and karma config expect the output of emscripten to be in # wasm-out/ so we copy the bazel output there. cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm/tfjs-backend-wasm.js \ @@ -33,15 +33,12 @@ cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm/tfjs-backend-was if [[ "$1" != "--dev" ]]; then # SIMD and threaded + SIMD builds. - yarn bazel build -c opt //tfjs-backend-wasm/src/cc:tfjs-backend-wasm-simd \ + yarn bazel build $BAZEL_REMOTE -c opt //tfjs-backend-wasm/src/cc:tfjs-backend-wasm-simd \ //tfjs-backend-wasm/src/cc:tfjs-backend-wasm-threaded-simd # Copy SIMD cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-simd/tfjs-backend-wasm.wasm \ ../wasm-out/tfjs-backend-wasm-simd.wasm - # cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-simd.wasm \ - # ../wasm-out/ - # Copy threaded cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd/tfjs-backend-wasm.js \ ../wasm-out/tfjs-backend-wasm-threaded-simd.js @@ -50,11 +47,6 @@ if [[ "$1" != "--dev" ]]; then cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd/tfjs-backend-wasm.wasm \ ../wasm-out/tfjs-backend-wasm-threaded-simd.wasm - # cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd.js \ - # ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd.worker.js \ - # ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-threaded-simd.wasm \ - # ../wasm-out/ - node ./create-worker-module.js node ./patch-threaded-simd-module.js fi From ee64690a281569cd19de91bc5ba248ea37e269d8 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Tue, 2 Mar 2021 08:55:28 -0800 Subject: [PATCH 10/24] Cleanup and format --- .bazelrc | 4 -- tfjs-backend-wasm/src/cc/BUILD | 77 ---------------------------------- 2 files changed, 81 deletions(-) diff --git a/.bazelrc b/.bazelrc index b673c6679d..06a7b5989e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -13,10 +13,6 @@ build --cxxopt="-fno-rtti" build --cxxopt="-fno-exceptions" build --cxxopt="-fomit-frame-pointer" -#build:wasm_new --crosstool_top=//emscripten_toolchain:everything -#build:wasm_new --cpu=wasm -#build:wasm_new --host_crosstool_top=@bazel_tools//tools/cpp:toolchain - # Remote cache config. Users should override this in .bazelrc.user build:remote --remote_http_cache=https://storage.googleapis.com/bazel-remote-cache-tfjs build:remote --google_default_credentials diff --git a/tfjs-backend-wasm/src/cc/BUILD b/tfjs-backend-wasm/src/cc/BUILD index 1e1d7b2208..18cfa9502f 100644 --- a/tfjs-backend-wasm/src/cc/BUILD +++ b/tfjs-backend-wasm/src/cc/BUILD @@ -55,83 +55,6 @@ wasm_cc_binary( threads = "emscripten", ) - -# REMOVE THIS -cc_binary( - name = "hello_world", - srcs = ["hello_world.cc"], -) - -wasm_cc_binary( - name = "hello_world_wasm", - cc_target = ":hello_world", -) - -cc_binary( - name = "hello_world_xnnpack", - srcs = ["hello_world_xnnpack.cc"], - deps = [":backend"] -) - -wasm_cc_binary( - name = "hello_world_xnnpack_wasm", - cc_target = ":hello_world_xnnpack", -) - - -# This build rule generates tfjs-backend-wasm-simd.{js,wasm}. -# -# We only need the .wasm file, not the .js file, because it will be loadsed by -# the tfjs-backend-wasm.js file (generated from the previous build rule). -# -# See scripts/build-wasm.sh where we only copy the .wasm file to wasm-out. -# cc_binary( -# name = "tfjs-backend-wasm-simd.js", -# srcs = ["backend.cc"] + KERNELS_WITH_KEEPALIVE, -# copts = [ -# "-msimd128", -# ], -# linkopts = [ -# "-s ALLOW_MEMORY_GROWTH=1", -# "-s DEFAULT_LIBRARY_FUNCS_TO_INCLUDE=[]", -# "-s DISABLE_EXCEPTION_CATCHING=1", -# "-s FILESYSTEM=0", -# "-s EXIT_RUNTIME=0", -# "-s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]'", -# "-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"cwrap\"]'", -# "-s MALLOC=emmalloc", -# ], -# deps = [ -# ":all_kernels", -# ":backend", -# ], -# ) - -# This build rule generates tfjs-backend-wasm-threaded-simd.{js,wasm} and -# tfjs-backend-wasm-threaded-simd.worker.js. -# cc_binary( -# name = "tfjs-backend-wasm-threaded-simd.js", -# srcs = ["backend.cc"] + KERNELS_WITH_KEEPALIVE, -# linkopts = [ -# "-s ALLOW_MEMORY_GROWTH=1", -# "-s DEFAULT_LIBRARY_FUNCS_TO_INCLUDE=[]", -# "-s DISABLE_EXCEPTION_CATCHING=1", -# "-s FILESYSTEM=0", -# "-s EXIT_RUNTIME=0", -# "-s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]'", -# "-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"cwrap\"]'", -# "-s MODULARIZE=1", -# "-s EXPORT_NAME=WasmBackendModuleThreadedSimd", -# "-s MALLOC=emmalloc", -# "-s USE_PTHREADS=1", -# "-s PROXY_TO_PTHREAD=1", -# ], -# deps = [ -# ":all_kernels", -# ":backend", -# ], -# ) - test_suite( name = "cc_tests", ) From 927ec38295047440d80373657a904ce579a834ea Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Tue, 2 Mar 2021 09:03:35 -0800 Subject: [PATCH 11/24] Apply buildifier lints --- tfjs-backend-wasm/src/cc/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfjs-backend-wasm/src/cc/BUILD b/tfjs-backend-wasm/src/cc/BUILD index 18cfa9502f..509045b585 100644 --- a/tfjs-backend-wasm/src/cc/BUILD +++ b/tfjs-backend-wasm/src/cc/BUILD @@ -1,5 +1,5 @@ -load(":build_defs.bzl", "tfjs_cc_library", "tfjs_unit_test") load("//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") +load(":build_defs.bzl", "tfjs_cc_library", "tfjs_unit_test") # Emcripten produces a much larger wasm bundle unless the cc_binary has srcs # explicitly pointing to files with exported methods (EMSCRIPTEN_KEEPALIVE). From 30b0ac93fd730950177623b6fe03f1b937a76503 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Tue, 2 Mar 2021 09:13:24 -0800 Subject: [PATCH 12/24] Remove hello world test cc files --- tfjs-backend-wasm/src/cc/hello_world.cc | 6 ---- .../src/cc/hello_world_xnnpack.cc | 33 ------------------- 2 files changed, 39 deletions(-) delete mode 100644 tfjs-backend-wasm/src/cc/hello_world.cc delete mode 100644 tfjs-backend-wasm/src/cc/hello_world_xnnpack.cc diff --git a/tfjs-backend-wasm/src/cc/hello_world.cc b/tfjs-backend-wasm/src/cc/hello_world.cc deleted file mode 100644 index ee72c53171..0000000000 --- a/tfjs-backend-wasm/src/cc/hello_world.cc +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main(int argc, char** argv) { - std::cout << "hello world!" << std::endl; - return 0; -} diff --git a/tfjs-backend-wasm/src/cc/hello_world_xnnpack.cc b/tfjs-backend-wasm/src/cc/hello_world_xnnpack.cc deleted file mode 100644 index 3cc268cfa2..0000000000 --- a/tfjs-backend-wasm/src/cc/hello_world_xnnpack.cc +++ /dev/null @@ -1,33 +0,0 @@ -#include -#include - - -int main(int argc, char** argv) { - const size_t channels = 1, input_stride = 1, output_stride = 1; - const uint32_t flags = 1; - xnn_operator_t unary_op = nullptr; - - xnn_status create_status = - xnn_create_bankers_rounding_nc_f32( - channels, input_stride, output_stride, flags, &unary_op); - if (create_status != xnn_status_success) { - std::cout << "Failed to create op" << std::endl; - return 1; - } - - const float* in_buf = new float[1]; - float* out_buf = new float[1]; - - xnn_status setup_status = - xnn_setup_bankers_rounding_nc_f32( - unary_op, 1, in_buf, out_buf, nullptr /* no thread pool */); - - if (setup_status != xnn_status_success) { - std::cout << "Failed to setup op" << std::endl; - return 1; - } - - xnn_run_operator(unary_op, nullptr /* no threadpool */); - std::cout << "created xnnpack op" << std::endl; - return 0; -} From f9dc556ecd9e26c85a4fb6f908d0d18e28d66b6f Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Tue, 2 Mar 2021 09:15:49 -0800 Subject: [PATCH 13/24] Remove old emscripten toolchain --- WORKSPACE | 5 - toolchain/BUILD | 35 ----- toolchain/cc_toolchain_config.bzl | 212 ------------------------------ toolchain/emar.sh | 19 --- toolchain/emcc.sh | 19 --- toolchain/false.sh | 17 --- 6 files changed, 307 deletions(-) delete mode 100644 toolchain/BUILD delete mode 100644 toolchain/cc_toolchain_config.bzl delete mode 100755 toolchain/emar.sh delete mode 100755 toolchain/emcc.sh delete mode 100644 toolchain/false.sh diff --git a/WORKSPACE b/WORKSPACE index 9a3bcce317..d8aeef6fc7 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -12,11 +12,6 @@ yarn_install( yarn_lock = "//:yarn.lock", ) -# Make all files under $HOME/emsdk/* visible to the toolchain. The files are -# available as external/emsdk/emsdk/* -load("//toolchain:cc_toolchain_config.bzl", "emsdk_configure") -emsdk_configure(name = "emsdk") - # npm_install used for emscripten dependencies load("@build_bazel_rules_nodejs//:index.bzl", "npm_install") diff --git a/toolchain/BUILD b/toolchain/BUILD deleted file mode 100644 index c92e688e8e..0000000000 --- a/toolchain/BUILD +++ /dev/null @@ -1,35 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load(":cc_toolchain_config.bzl", "cc_toolchain_config") - -cc_toolchain_config(name = "wasm_toolchain_config") - -cc_toolchain_suite( - name = "emscripten", - toolchains = { - "wasm": ":wasm_toolchain", - }, -) - -filegroup( - name = "all", - srcs = [ - "emar.sh", - "emcc.sh", - "false.sh", - "@emsdk//:all", - ], -) - -cc_toolchain( - name = "wasm_toolchain", - all_files = ":all", - compiler_files = ":all", - dwp_files = ":empty", - linker_files = ":all", - objcopy_files = ":empty", - strip_files = ":empty", - supports_param_files = 0, - toolchain_config = ":wasm_toolchain_config", - toolchain_identifier = "wasm-toolchain", -) diff --git a/toolchain/cc_toolchain_config.bzl b/toolchain/cc_toolchain_config.bzl deleted file mode 100644 index 4c12bd1ff7..0000000000 --- a/toolchain/cc_toolchain_config.bzl +++ /dev/null @@ -1,212 +0,0 @@ -load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") -load( - "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", - "feature", - "flag_group", - "flag_set", - "tool_path", - "with_feature_set", -) - -def _impl(ctx): - tool_paths = [ - tool_path( - name = "gcc", - path = "emcc.sh", - ), - tool_path( - name = "ld", - path = "emcc.sh", - ), - tool_path( - name = "ar", - path = "emar.sh", - ), - tool_path( - name = "cpp", - path = "false.sh", - ), - tool_path( - name = "gcov", - path = "false.sh", - ), - tool_path( - name = "nm", - path = "NOT_USED", - ), - tool_path( - name = "objdump", - path = "false.sh", - ), - tool_path( - name = "strip", - path = "NOT_USED", - ), - ] - preprocessor_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, - ] - - all_link_actions = [ - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ] - - all_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ACTION_NAMES.lto_backend, - ] - toolchain_include_directories_feature = feature( - name = "toolchain_include_directories", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = [ - flag_group( - flags = [ - # The clang compiler comes with a definition of - # max_align_t struct in $emsdk/upstream/lib/clang/13.0.0/include/__stddef_max_align_t.h. - # It conflicts with the one defined in - # $emsdk/upstream/emscripten/cache/sysroot/include/bits/alltypes.h. - # We need both include paths to make things work. - # - # To workaround this, we are defining the following - # symbol through compiler flag so that the max_align_t - # defined in clang's header file will be skipped. - "-D", - "__CLANG_MAX_ALIGN_T_DEFINED", - # We are using emscripten 2.0.14 for this build. It - # comes with clang 13.0.0. Future emscripten release - # might change the clang version number below. - # - # Also need to change the version number in - # cxx_cxx_builtin_include_directories below. - "-isystem", - "external/emsdk/emsdk/upstream/lib/clang/13.0.0/include", - ], - ), - ], - ), - ], - ) - - crosstool_default_flag_sets = [ - # Opt. - flag_set( - actions = preprocessor_compile_actions, - flag_groups = [flag_group(flags = ["-DNDEBUG"])], - with_features = [with_feature_set(features = ["opt"])], - ), - flag_set( - actions = all_compile_actions + all_link_actions, - flag_groups = [flag_group(flags = ["-g0", "-O3"])], - with_features = [with_feature_set(features = ["opt"])], - ), - # Fastbuild. - flag_set( - actions = all_compile_actions + all_link_actions, - flag_groups = [flag_group(flags = ["-O2"])], - with_features = [with_feature_set(features = ["fastbuild"])], - ), - # Dbg. - flag_set( - actions = all_compile_actions + all_link_actions, - flag_groups = [flag_group(flags = ["-g2", "-O0"])], - with_features = [with_feature_set(features = ["dbg"])], - ), - ] - - features = [ - toolchain_include_directories_feature, - # These 3 features will be automatically enabled by blaze in the - # corresponding build mode. - feature( - name = "opt", - provides = ["variant:crosstool_build_mode"], - ), - feature( - name = "dbg", - provides = ["variant:crosstool_build_mode"], - ), - feature( - name = "fastbuild", - provides = ["variant:crosstool_build_mode"], - ), - feature( - name = "crosstool_default_flags", - enabled = True, - flag_sets = crosstool_default_flag_sets, - ), - ] - - cxx_builtin_include_directories = [ - "external/emsdk/emsdk/upstream/emscripten/cache/sysroot/include/c++/v1", - "external/emsdk/emsdk/upstream/emscripten/cache/sysroot/include/compat", - "external/emsdk/emsdk/upstream/emscripten/cache/sysroot/include", - # We are using emscripten 2.0.14 for this build. It comes with clang - # 13.0.0. Future emscripten release might change the clang version - # number below. - # - # Also need to change the version number in - # toolchain_include_directories_feature above. - "external/emsdk/emsdk/upstream/lib/clang/13.0.0/include", - ] - - builtin_sysroot = "external/emsdk/emsdk/upstream/emscripten/cache/sysroot" - - return cc_common.create_cc_toolchain_config_info( - ctx = ctx, - toolchain_identifier = "wasm-toolchain", - host_system_name = "i686-unknown-linux-gnu", - target_system_name = "wasm-unknown-emscripten", - target_cpu = "wasm", - target_libc = "musl/js", - compiler = "emscripten", - abi_version = "emscripten_syscalls", - abi_libc_version = "default", - tool_paths = tool_paths, - features = features, - builtin_sysroot = builtin_sysroot, - cxx_builtin_include_directories = cxx_builtin_include_directories, - ) - -cc_toolchain_config = rule( - implementation = _impl, - attrs = {}, - provides = [CcToolchainConfigInfo], -) - -def _emsdk_impl(ctx): - if "EMSDK" not in ctx.os.environ or ctx.os.environ["EMSDK"].strip() == "": - fail("The environment variable EMSDK is not found. " + - "Did you run source ./emsdk_env.sh ?") - path = ctx.os.environ["EMSDK"] - ctx.symlink(path, "emsdk") - ctx.file("BUILD", """ -filegroup( - name = "all", - srcs = glob(["emsdk/**"]), - visibility = ["//visibility:public"], -) -""") - -emsdk_configure = repository_rule( - implementation = _emsdk_impl, - local = True, -) diff --git a/toolchain/emar.sh b/toolchain/emar.sh deleted file mode 100755 index 65eed16b42..0000000000 --- a/toolchain/emar.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2019 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -set -euo pipefail - -external/emsdk/emsdk/upstream/emscripten/emar "$@" diff --git a/toolchain/emcc.sh b/toolchain/emcc.sh deleted file mode 100755 index 85044e097e..0000000000 --- a/toolchain/emcc.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2019 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -set -euo pipefail - -external/emsdk/emsdk/upstream/emscripten/emcc "$@" diff --git a/toolchain/false.sh b/toolchain/false.sh deleted file mode 100644 index b6ce9a0456..0000000000 --- a/toolchain/false.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2019 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================= - -exit 1 From d2ae7b6b42e3ec82622cbbde57b7b11dc2fe29e5 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Tue, 2 Mar 2021 12:46:14 -0800 Subject: [PATCH 14/24] Update dockerfiles --- dockers/release-docker/Dockerfile | 6 +++++- dockers/wasm-docker/Dockerfile | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/dockers/release-docker/Dockerfile b/dockers/release-docker/Dockerfile index 122d52c950..03688a51cb 100644 --- a/dockers/release-docker/Dockerfile +++ b/dockers/release-docker/Dockerfile @@ -16,7 +16,8 @@ RUN apt-get update -qqy && apt-get install -qqy \ lsb-release \ openssh-client \ git \ - gnupg && \ + gnupg + file && \ pip3 install -U crcmod && \ export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \ @@ -36,3 +37,6 @@ RUN npm install -g yarn # Install virtualenv RUN pip3 install virtualenv + +# Install absl +RUN pip3 install absl-py diff --git a/dockers/wasm-docker/Dockerfile b/dockers/wasm-docker/Dockerfile index a94b771c3b..ef5047ec97 100644 --- a/dockers/wasm-docker/Dockerfile +++ b/dockers/wasm-docker/Dockerfile @@ -3,3 +3,19 @@ # Install yarn RUN npm install -g yarn + +RUN apt-get update -qqy && apt-get install -qqy \ + curl \ + gcc \ + python \ + python3 \ + python3-setuptools \ + python3-dev \ + python3-pip \ + apt-transport-https \ + lsb-release \ + openssh-client \ + git \ + gnupg \ + file && \ + pip3 install -U absl-py From a81863f5f7bf548465bf4264f081699e59e2f588 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Tue, 2 Mar 2021 13:06:24 -0800 Subject: [PATCH 15/24] Add trailing slash to shell command --- dockers/release-docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockers/release-docker/Dockerfile b/dockers/release-docker/Dockerfile index 03688a51cb..2fad7f3cd0 100644 --- a/dockers/release-docker/Dockerfile +++ b/dockers/release-docker/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update -qqy && apt-get install -qqy \ lsb-release \ openssh-client \ git \ - gnupg + gnupg \ file && \ pip3 install -U crcmod && \ export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ From f643328185ea765e9e8d3c7f3839a5d024d0f85f Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Tue, 2 Mar 2021 13:59:53 -0800 Subject: [PATCH 16/24] Revert "Add trailing slash to shell command" This reverts commit a81863f5f7bf548465bf4264f081699e59e2f588. --- dockers/release-docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockers/release-docker/Dockerfile b/dockers/release-docker/Dockerfile index 2fad7f3cd0..03688a51cb 100644 --- a/dockers/release-docker/Dockerfile +++ b/dockers/release-docker/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update -qqy && apt-get install -qqy \ lsb-release \ openssh-client \ git \ - gnupg \ + gnupg file && \ pip3 install -U crcmod && \ export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ From 7dfdb991578b3870cc895c44a6185cec367049b7 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Tue, 2 Mar 2021 14:00:04 -0800 Subject: [PATCH 17/24] Revert "Update dockerfiles" This reverts commit d2ae7b6b42e3ec82622cbbde57b7b11dc2fe29e5. --- dockers/release-docker/Dockerfile | 6 +----- dockers/wasm-docker/Dockerfile | 16 ---------------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/dockers/release-docker/Dockerfile b/dockers/release-docker/Dockerfile index 03688a51cb..122d52c950 100644 --- a/dockers/release-docker/Dockerfile +++ b/dockers/release-docker/Dockerfile @@ -16,8 +16,7 @@ RUN apt-get update -qqy && apt-get install -qqy \ lsb-release \ openssh-client \ git \ - gnupg - file && \ + gnupg && \ pip3 install -U crcmod && \ export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \ @@ -37,6 +36,3 @@ RUN npm install -g yarn # Install virtualenv RUN pip3 install virtualenv - -# Install absl -RUN pip3 install absl-py diff --git a/dockers/wasm-docker/Dockerfile b/dockers/wasm-docker/Dockerfile index ef5047ec97..a94b771c3b 100644 --- a/dockers/wasm-docker/Dockerfile +++ b/dockers/wasm-docker/Dockerfile @@ -3,19 +3,3 @@ # Install yarn RUN npm install -g yarn - -RUN apt-get update -qqy && apt-get install -qqy \ - curl \ - gcc \ - python \ - python3 \ - python3-setuptools \ - python3-dev \ - python3-pip \ - apt-transport-https \ - lsb-release \ - openssh-client \ - git \ - gnupg \ - file && \ - pip3 install -U absl-py From e09b52397a8899185118117a7c1a732f2c3d4aa9 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Wed, 3 Mar 2021 13:03:30 -0800 Subject: [PATCH 18/24] Use local spawn strategy for bazel build performance --- .bazelrc | 21 ++++++++++++++------- tfjs-backend-wasm/scripts/build-ci.sh | 3 ++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.bazelrc b/.bazelrc index 06a7b5989e..3e035f9ecc 100644 --- a/.bazelrc +++ b/.bazelrc @@ -13,9 +13,12 @@ build --cxxopt="-fno-rtti" build --cxxopt="-fno-exceptions" build --cxxopt="-fomit-frame-pointer" -# Remote cache config. Users should override this in .bazelrc.user +# Remote cache config. Users can add credentials in their .bazelrc.user files. build:remote --remote_http_cache=https://storage.googleapis.com/bazel-remote-cache-tfjs -build:remote --google_default_credentials + +# Config for Google Cloud continuous integration that uses default credentials. +build:ci --config=remote +build:ci --google_default_credentials # This flag is needed to so that the bazel cache is not invalidated # when running bazel via `yarn bazel`. @@ -23,11 +26,15 @@ build:remote --google_default_credentials build --incompatible_strict_action_env run --incompatible_strict_action_env +# Don't use a sandboxed build since it hurts performance. +# When sandboxed, the wasm backend takes hours to build on a 32 core machine. +build --spawn_strategy=worker,local + # Load any settings specific to the current user. -# .bazelrc.user should appear in .gitignore so that settings are not shared with team members -# This needs to be last statement in this -# config, as the user configuration should be able to overwrite flags from this file. +# .bazelrc.user should appear in .gitignore so that settings are not shared with +# team members. This needs to be last statement in this config, as the user +# configuration should be able to overwrite flags from this file. # See https://docs.bazel.build/versions/master/best-practices.html#bazelrc -# (Note that we use .bazelrc.user so the file appears next to .bazelrc in directory listing, -# rather than user.bazelrc as suggested in the Bazel docs) +# (Note that we use .bazelrc.user so the file appears next to .bazelrc in +# directory listing, rather than user.bazelrc as suggested in the Bazel docs). try-import %workspace%/.bazelrc.user diff --git a/tfjs-backend-wasm/scripts/build-ci.sh b/tfjs-backend-wasm/scripts/build-ci.sh index d8e1e867fa..d4c6976be4 100755 --- a/tfjs-backend-wasm/scripts/build-ci.sh +++ b/tfjs-backend-wasm/scripts/build-ci.sh @@ -17,4 +17,5 @@ set -e yarn tsc -BAZEL_REMOTE="--config=remote" ./scripts/build-wasm.sh +BAZEL_REMOTE="--config=ci" ./scripts/build-wasm.sh +./scripts/build-wasm.sh From a01ee3c1cd039b4f1ddf00f79a8fad6922d48acc Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Wed, 3 Mar 2021 17:04:47 -0800 Subject: [PATCH 19/24] Update XNNPACK --- WORKSPACE | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index d8aeef6fc7..6ad88f1eb5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -36,8 +36,9 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") # xnnpack used for fast vectorized wasm operations git_repository( name = "xnnpack", - branch = "emsdk_fix", - remote = "https://github.com/mattsoulanille/XNNPACK.git", + commit = "f69ac5d218a4f2a68bc2981d61603d43966e8068", + remote = "https://github.com/google/XNNPACK.git", + shallow_since = "1614818900 -0800", ) # The libraries below are transitive dependencies of XNNPACK that we need to From 95e0a46e26dd9eda5d50cc11f35b858e897b73d9 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Thu, 4 Mar 2021 08:21:54 -0800 Subject: [PATCH 20/24] Stop accidentally building twice --- tfjs-backend-wasm/scripts/build-ci.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tfjs-backend-wasm/scripts/build-ci.sh b/tfjs-backend-wasm/scripts/build-ci.sh index d4c6976be4..a503705f4c 100755 --- a/tfjs-backend-wasm/scripts/build-ci.sh +++ b/tfjs-backend-wasm/scripts/build-ci.sh @@ -18,4 +18,3 @@ set -e yarn tsc BAZEL_REMOTE="--config=ci" ./scripts/build-wasm.sh -./scripts/build-wasm.sh From ca4f269d90fce89d1f3f86a92d342ea0f9c9e33e Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Tue, 9 Mar 2021 14:11:58 -0800 Subject: [PATCH 21/24] 'remote_http_cache' -> 'remote_cache' --- .bazelrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 3e035f9ecc..1266ee0c27 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,7 +14,7 @@ build --cxxopt="-fno-exceptions" build --cxxopt="-fomit-frame-pointer" # Remote cache config. Users can add credentials in their .bazelrc.user files. -build:remote --remote_http_cache=https://storage.googleapis.com/bazel-remote-cache-tfjs +build:remote --remote_cache=https://storage.googleapis.com/bazel-remote-cache-tfjs # Config for Google Cloud continuous integration that uses default credentials. build:ci --config=remote From 5a4f3bc7fd7769f5b6d6f4a2b5a1dba2abc880ad Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Wed, 10 Mar 2021 07:28:38 -0800 Subject: [PATCH 22/24] Fix typo --- .bazelrc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index 1266ee0c27..15327ed801 100644 --- a/.bazelrc +++ b/.bazelrc @@ -20,8 +20,8 @@ build:remote --remote_cache=https://storage.googleapis.com/bazel-remote-cache-tf build:ci --config=remote build:ci --google_default_credentials -# This flag is needed to so that the bazel cache is not invalidated -# when running bazel via `yarn bazel`. +# This flag is needed to prevent the bazel cache from being invalidated when +# running bazel via `yarn bazel`. # See https://github.com/angular/angular/issues/27514. build --incompatible_strict_action_env run --incompatible_strict_action_env From 4c199571602590a9f4d56251b05ef2d317645d97 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Mon, 29 Mar 2021 15:31:09 -0700 Subject: [PATCH 23/24] Update emscripten toolchain --- WORKSPACE | 31 +- emscripten_toolchain/BUILD.bazel | 80 -- emscripten_toolchain/crosstool.bzl | 1090 ----------------- emscripten_toolchain/emar.sh | 5 - emscripten_toolchain/emcc.sh | 5 - emscripten_toolchain/emcc_link.sh | 5 - emscripten_toolchain/emscripten.BUILD | 6 - emscripten_toolchain/emscripten_config | 12 - emscripten_toolchain/env.sh | 5 - emscripten_toolchain/link_wrapper.py | 146 --- emscripten_toolchain/wasm_binary.py | 84 -- emscripten_toolchain/wasm_cc_binary.bzl | 155 --- emscripten_toolchain/wasm_rules.bzl | 6 - tfjs-backend-wasm/scripts/build-wasm.sh | 2 +- .../scripts/create-worker-module.js | 1 + tfjs-backend-wasm/src/cc/BUILD | 2 +- 16 files changed, 16 insertions(+), 1619 deletions(-) delete mode 100644 emscripten_toolchain/BUILD.bazel delete mode 100644 emscripten_toolchain/crosstool.bzl delete mode 100755 emscripten_toolchain/emar.sh delete mode 100755 emscripten_toolchain/emcc.sh delete mode 100755 emscripten_toolchain/emcc_link.sh delete mode 100644 emscripten_toolchain/emscripten.BUILD delete mode 100644 emscripten_toolchain/emscripten_config delete mode 100755 emscripten_toolchain/env.sh delete mode 100644 emscripten_toolchain/link_wrapper.py delete mode 100644 emscripten_toolchain/wasm_binary.py delete mode 100644 emscripten_toolchain/wasm_cc_binary.bzl delete mode 100644 emscripten_toolchain/wasm_rules.bzl diff --git a/WORKSPACE b/WORKSPACE index 6ad88f1eb5..d5e19bcb9b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -12,33 +12,28 @@ yarn_install( yarn_lock = "//:yarn.lock", ) -# npm_install used for emscripten dependencies -load("@build_bazel_rules_nodejs//:index.bzl", "npm_install") - -# emscripten 2.0.14 +# Emscripten toolchain http_archive( - name = "emscripten", - sha256 = "e466cd47ddd4bf0acd645412fdf08eda6d232484e48e5a2643e08062a7a4cf56", - strip_prefix = "install", - url = "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/fc5562126762ab26c4757147a3b4c24e85a7289e/wasm-binaries.tbz2", - build_file = "//emscripten_toolchain:emscripten.BUILD", - type = "tar.bz2", + name = "emsdk", + strip_prefix = "emsdk-c1589b55641787d55d53e883852035beea9aec3f/bazel", + url = "https://github.com/emscripten-core/emsdk/archive/c1589b55641787d55d53e883852035beea9aec3f.tar.gz", + sha256 = "7a58a9996b113d3e0675df30b5f17e28aa47de2e684a844f05394fe2f6f12e8e", ) -# Install emscripten dependencies -npm_install( - name = "emsdk_npm", - package_json = "@emscripten//:emscripten/package.json", - package_lock_json = "@emscripten//:emscripten/package-lock.json", -) +load("@emsdk//:deps.bzl", emsdk_deps = "deps") +emsdk_deps() + +load("@emsdk//:emscripten_deps.bzl", emsdk_emscripten_deps = "emscripten_deps") +emsdk_emscripten_deps() + load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") # xnnpack used for fast vectorized wasm operations git_repository( name = "xnnpack", - commit = "f69ac5d218a4f2a68bc2981d61603d43966e8068", + commit = "3bfbdaf00211b313b143af39279bb6bf1f7effc0", remote = "https://github.com/google/XNNPACK.git", - shallow_since = "1614818900 -0800", + shallow_since = "1617056836 -0700", ) # The libraries below are transitive dependencies of XNNPACK that we need to diff --git a/emscripten_toolchain/BUILD.bazel b/emscripten_toolchain/BUILD.bazel deleted file mode 100644 index a676ad7cd6..0000000000 --- a/emscripten_toolchain/BUILD.bazel +++ /dev/null @@ -1,80 +0,0 @@ -load(":crosstool.bzl", "emscripten_cc_toolchain_config_rule") - -package(default_visibility = ["//visibility:public"]) - -filegroup( - name = "common-script-includes", - srcs = [ - "emar.sh", - "emcc.sh", - "emscripten_config", - "env.sh", - "@emscripten//:all", - "@nodejs//:node_files", - "@emsdk_npm//:node_modules", - ], -) - -filegroup( - name = "compile-emscripten", - srcs = [":common-script-includes"], -) - -filegroup( - name = "link-emscripten", - srcs = [ - "emcc_link.sh", - "link_wrapper.py", - ":common-script-includes", - "@emscripten//:all", - "@nodejs//:node_files", - ], -) - -filegroup( - name = "every-file", - srcs = [ - ":compile-emscripten", - ":link-emscripten", - "@emscripten//:all", - "@nodejs//:node_files", - ], -) - -filegroup(name = "empty") - -# dlmalloc.bc is implictly added by the emscripten toolchain -cc_library(name = "malloc") - -emscripten_cc_toolchain_config_rule( - name = "wasm", - cpu = "wasm", - emscripten_version = "emscripten", -) - -cc_toolchain( - name = "cc-compiler-wasm", - all_files = ":every-file", - ar_files = ":common-script-includes", - as_files = ":empty", - compiler_files = ":compile-emscripten", - dwp_files = ":empty", - linker_files = ":link-emscripten", - objcopy_files = ":empty", - strip_files = ":empty", - toolchain_config = "wasm", - toolchain_identifier = "emscripten-wasm", -) - -cc_toolchain_suite( - name = "everything", - toolchains = { - "wasm": ":cc-compiler-wasm", - "wasm|emscripten": ":cc-compiler-wasm", - }, -) - -py_binary( - name = "wasm_binary", - srcs = ["wasm_binary.py"], -) diff --git a/emscripten_toolchain/crosstool.bzl b/emscripten_toolchain/crosstool.bzl deleted file mode 100644 index 0da1cb0175..0000000000 --- a/emscripten_toolchain/crosstool.bzl +++ /dev/null @@ -1,1090 +0,0 @@ -"""This module encapsulates logic to create emscripten_cc_toolchain_config rule.""" - -load( - "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", - "action_config", - "env_entry", - "env_set", - "feature", - "feature_set", - "flag_group", - "tool", - "tool_path", - "variable_with_value", - "with_feature_set", - _flag_set = "flag_set", -) -load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") - -def flag_set(flags = None, features = None, not_features = None, **kwargs): - """Extension to flag_set which allows for a "simple" form. - - The simple form allows specifying flags as a simple list instead of a flag_group - if enable_if or expand_if semantics are not required. - - Similarly, the simple form allows passing features/not_features if they are a simple - list of semantically "and" features. - (i.e. "asan" and "dbg", rather than "asan" or "dbg") - - Args: - flags: list, set of flags - features: list, set of features required to be enabled. - not_features: list, set of features required to not be enabled. - **kwargs: The rest of the args for flag_set. - - Returns: - flag_set - """ - if flags: - if kwargs.get("flag_groups"): - fail("Cannot set flags and flag_groups") - else: - kwargs["flag_groups"] = [flag_group(flags = flags)] - - if features or not_features: - if kwargs.get("with_features"): - fail("Cannot set features/not_feature and with_features") - kwargs["with_features"] = [with_feature_set( - features = features or [], - not_features = not_features or [], - )] - return _flag_set(**kwargs) - -CROSSTOOL_DEFAULT_WARNINGS = [ - "-Wall", -] - -def _impl(ctx): - target_cpu = ctx.attr.cpu - toolchain_identifier = "emscripten-" + target_cpu - target_system_name = target_cpu + "-unknown-emscripten" - - host_system_name = "i686-unknown-linux-gnu" - - target_libc = "musl/js" - - abi_version = "emscripten_syscalls" - - compiler = "emscripten" - abi_libc_version = "default" - - cc_target_os = "emscripten" - emscripten_version = ctx.attr.emscripten_version - - builtin_sysroot = "external/emscripten/emscripten/cache/sysroot" - - ################################################################ - # Tools - ################################################################ - clang_tool = tool(path = "emcc.sh") - clif_match_tool = tool(path = "dummy_clif_matcher") - link_tool = tool(path = "emcc_link.sh") - archive_tool = tool(path = "emar.sh") - strip_tool = tool(path = "NOT_USED_STRIP_TOOL") - - #### Legacy tool paths (much of this is redundant with action_configs, but - #### these are still used for some things) - tool_paths = [ - tool_path(name = "ar", path = "emar.sh"), - tool_path(name = "cpp", path = "/bin/false"), - tool_path(name = "gcc", path = "emcc.sh"), - tool_path(name = "gcov", path = "/bin/false"), - tool_path(name = "ld", path = "emcc_link.sh"), - tool_path(name = "nm", path = "NOT_USED"), - tool_path(name = "objdump", path = "/bin/false"), - tool_path(name = "strip", path = "NOT_USED"), - ] - - ################################################################ - # Action Configs - ################################################################ - - cpp_compile_action = action_config( - action_name = ACTION_NAMES.cpp_compile, - tools = [clang_tool], - ) - - cpp_module_compile_action = action_config( - action_name = ACTION_NAMES.cpp_module_compile, - tools = [clang_tool], - ) - - cpp_module_codegen_action = action_config( - action_name = ACTION_NAMES.cpp_module_codegen, - tools = [clang_tool], - ) - - clif_match_action = action_config( - action_name = ACTION_NAMES.clif_match, - tools = [clif_match_tool], - ) - - cpp_link_dynamic_library_action = action_config( - action_name = ACTION_NAMES.cpp_link_dynamic_library, - tools = [link_tool], - ) - - strip_action = action_config( - action_name = ACTION_NAMES.strip, - tools = [strip_tool], - ) - - preprocess_assemble_action = action_config( - action_name = ACTION_NAMES.preprocess_assemble, - tools = [clang_tool], - ) - - cpp_header_parsing_action = action_config( - action_name = ACTION_NAMES.cpp_header_parsing, - tools = [clang_tool], - ) - - cpp_link_static_library_action = action_config( - action_name = ACTION_NAMES.cpp_link_static_library, - enabled = True, - flag_sets = [ - flag_set( - flag_groups = [ - flag_group( - flags = ["rcsD", "%{output_execpath}"], - expand_if_available = "output_execpath", - ), - ], - ), - flag_set( - flag_groups = [ - flag_group( - iterate_over = "libraries_to_link", - flag_groups = [ - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file", - ), - ), - flag_group( - flags = ["%{libraries_to_link.object_files}"], - iterate_over = "libraries_to_link.object_files", - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - ], - expand_if_available = "libraries_to_link", - ), - ], - ), - flag_set( - flag_groups = [ - flag_group( - flags = ["@%{linker_param_file}"], - expand_if_available = "linker_param_file", - ), - ], - ), - ], - tools = [archive_tool], - ) - - c_compile_action = action_config( - action_name = ACTION_NAMES.c_compile, - tools = [clang_tool], - ) - - linkstamp_compile_action = action_config( - action_name = ACTION_NAMES.linkstamp_compile, - tools = [clang_tool], - ) - - assemble_action = action_config( - action_name = ACTION_NAMES.assemble, - tools = [clang_tool], - ) - - cpp_link_executable_action = action_config( - action_name = ACTION_NAMES.cpp_link_executable, - tools = [link_tool], - ) - - cpp_link_nodeps_dynamic_library_action = action_config( - action_name = ACTION_NAMES.cpp_link_nodeps_dynamic_library, - tools = [link_tool], - ) - - action_configs = [ - strip_action, - c_compile_action, - cpp_compile_action, - linkstamp_compile_action, - assemble_action, - preprocess_assemble_action, - cpp_header_parsing_action, - cpp_module_compile_action, - cpp_module_codegen_action, - cpp_link_executable_action, - cpp_link_dynamic_library_action, - cpp_link_nodeps_dynamic_library_action, - cpp_link_static_library_action, - clif_match_action, - ] - - all_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ACTION_NAMES.lto_backend, - ] - - all_cpp_compile_actions = [ - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ] - - preprocessor_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, - ] - - all_link_actions = [ - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ] - - ################################################################ - # Features - ################################################################ - - features = [ - # This set of magic "feature"s are important configuration information for blaze. - feature(name = "no_legacy_features", enabled = True), - feature( - name = "has_configured_linker_path", - enabled = True, - ), - - # Blaze requests this feature by default, but we don't care. - feature(name = "dependency_file"), - - # Blaze requests this feature by default, but we don't care. - feature(name = "random_seed"), - - # Formerly "needsPic" attribute - feature(name = "supports_pic", enabled = False), - - # Blaze requests this feature by default. - # Blaze also tests if this feature is supported, before setting the "pic" build-variable. - feature(name = "pic"), - - # Blaze requests this feature if fission is requested - # Blaze also tests if it's supported to see if we support fission. - feature(name = "per_object_debug_info"), - - # Blaze requests this feature by default. - # Blaze also tests if this feature is supported before setting preprocessor_defines - # (...but why?) - feature(name = "preprocessor_defines"), - - # Blaze requests this feature by default. - # Blaze also tests if this feature is supported before setting includes. (...but why?) - feature(name = "include_paths"), - - # Blaze tests if this feature is enabled in order to create implicit - # "nodeps" .so outputs from cc_library rules. - feature(name = "supports_dynamic_linker", enabled = False), - - # Blaze requests this feature when linking a cc_binary which is - # "dynamic" aka linked against nodeps-dynamic-library cc_library - # outputs. - feature(name = "dynamic_linking_mode"), - - #### Configuration features - feature( - name = "crosstool_cpu", - enabled = True, - implies = ["crosstool_cpu_" + target_cpu], - ), - feature( - name = "crosstool_cpu_asmjs", - provides = ["variant:crosstool_cpu"], - ), - feature( - name = "crosstool_cpu_wasm", - provides = ["variant:crosstool_cpu"], - ), - - # These 3 features will be automatically enabled by blaze in the - # corresponding build mode. - feature( - name = "opt", - provides = ["variant:crosstool_build_mode"], - ), - feature( - name = "dbg", - provides = ["variant:crosstool_build_mode"], - ), - feature( - name = "fastbuild", - provides = ["variant:crosstool_build_mode"], - ), - - #### User-settable features - - # Set if enabling exceptions. - feature(name = "exceptions"), - - # This feature overrides the default optimization to prefer execution speed - # over binary size (like clang -O3). - feature( - name = "optimized_for_speed", - provides = ["variant:crosstool_optimization_mode"], - ), - - # This feature overrides the default optimization to prefer binary size over - # execution speed (like clang -Oz). - feature( - name = "optimized_for_size", - provides = ["variant:crosstool_optimization_mode"], - ), - - # Convenience aliases / alt-spellings. - feature( - name = "optimize_for_speed", - implies = ["optimized_for_speed"], - ), - feature( - name = "optimize_for_size", - implies = ["optimized_for_size"], - ), - - # This feature allows easier use of profiling tools by preserving mangled - # C++ names. This does everything profiling_funcs does and more. - feature(name = "profiling"), - - # This feature emits only enough debug info for function names to appear - # in profiles. - feature(name = "profiling_funcs"), - - # This feature allows source maps to be generated. - feature( - name = "source_maps", - implies = ["full_debug_info"], - ), - feature( - name = "dwarf_debug_info", - implies = ["profiling"], - ), - - # Turns on full debug info (-g4). - feature(name = "full_debug_info"), - - # Enables the use of "Emscripten" Pthread implementation. - # https://kripken.github.io/emscripten-site/docs/porting/pthreads.html - # https://github.com/kripken/emscripten/wiki/Pthreads-with-WebAssembly - feature(name = "use_pthreads"), - - # If enabled, the runtime will exit when main() completes. - feature(name = "exit_runtime"), - - # Primarily for toolchain maintainers: - feature(name = "emcc_debug"), - feature(name = "emcc_debug_link"), - feature( - name = "llvm_backend", - requires = [feature_set(features = ["crosstool_cpu_wasm"])], - enabled = True, - ), - - # Remove once flag is flipped. - # See https://github.com/bazelbuild/bazel/issues/7687 - feature( - name = "do_not_split_linking_cmdline", - ), - - # Adds simd support, only available with the llvm backend. - feature( - name = "wasm_simd", - requires = [feature_set(features = ["llvm_backend"])], - ), - feature( - name = "precise_long_double_printf", - enabled = True, - ), - feature( - name = "wasm_warnings_as_errors", - enabled = True, - ), - - # ASan and UBSan. See also: - # https://emscripten.org/docs/debugging/Sanitizers.html - feature(name = "wasm_asan"), - feature(name = "wasm_ubsan"), - ] - - crosstool_default_flag_sets = [ - # Compile, Link, and CC_FLAGS make variable - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ], - flag_groups = [ - flag_group( - flags = ["--sysroot=%{sysroot}"], - expand_if_available = "sysroot", - ), - ], - ), - # Compile + Link - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ], - # This forces color diagnostics even on Forge (where we don't have an - # attached terminal). - flags = [ - "-fdiagnostics-color", - ], - ), - # C++ compiles (and implicitly link) - flag_set( - actions = all_cpp_compile_actions, - flags = [ - "-fno-exceptions", - ], - not_features = ["exceptions"], - ), - flag_set( - actions = all_cpp_compile_actions, - flags = [ - "-fexceptions", - ], - features = ["exceptions"], - ), - # All compiles (and implicitly link) - flag_set( - actions = all_compile_actions + - all_link_actions, - flags = [ - "-fno-strict-aliasing", - "-funsigned-char", - "-no-canonical-prefixes", - ], - ), - # Language Features - flag_set( - actions = all_cpp_compile_actions, - flags = ["-std=gnu++17", "-nostdinc", "-nostdinc++",], - ), - - # Emscripten-specific settings: - flag_set( - actions = all_compile_actions + all_link_actions, - flags = ["-s", "WASM=0"], - features = ["crosstool_cpu_asmjs"], - ), - flag_set( - actions = all_compile_actions + - all_link_actions, - flags = ["-s", "USE_PTHREADS=1"], - features = ["use_pthreads"], - ), - flag_set( - actions = all_link_actions, - flags = ["-s", "EXIT_RUNTIME=1"], - features = ["exit_runtime"], - ), - flag_set( - actions = all_compile_actions + all_link_actions, - flags = ["-pthread"], - features = ["llvm_backend", "use_pthreads"], - ), - flag_set( - actions = all_compile_actions + all_link_actions, - flags = ["-msimd128"], - features = ["wasm_simd"], - ), - flag_set( - actions = all_link_actions, - flags = ["-s", "PRINTF_LONG_DOUBLE=1"], - features = ["precise_long_double_printf"], - ), - - # Opt - flag_set( - actions = preprocessor_compile_actions, - flags = ["-DNDEBUG"], - features = ["opt"], - ), - flag_set( - actions = all_compile_actions, - flags = ["-fomit-frame-pointer"], - features = ["opt"], - ), - flag_set( - actions = all_compile_actions + - all_link_actions, - flags = ["-O3"], - features = ["opt"], - ), - # Users can override opt-level with semantic names... - flag_set( - actions = all_compile_actions + - all_link_actions, - flags = ["-Oz"], - features = ["optimized_for_size", "opt"], - ), - flag_set( - actions = all_compile_actions + - all_link_actions, - flags = ["-O3"], - features = ["optimized_for_speed", "opt"], - ), - - # Fastbuild - flag_set( - actions = all_compile_actions, - flags = ["-fomit-frame-pointer"], - features = ["fastbuild"], - ), - flag_set( - actions = all_compile_actions + - all_link_actions, - flags = ["-O2"], - features = ["fastbuild"], - ), - - # Dbg - flag_set( - actions = all_compile_actions, - flags = ["-fno-omit-frame-pointer"], - features = ["dbg"], - ), - flag_set( - actions = all_compile_actions + - all_link_actions, - flags = ["-g", "-O0"], - features = ["dbg"], - ), - flag_set( - actions = all_compile_actions + - all_link_actions, - flags = [ - "-g4", - "-fsanitize=address", - "-O1", - "-DADDRESS_SANITIZER=1", - "-fno-omit-frame-pointer", - ], - features = ["wasm_asan"], - ), - flag_set( - actions = all_compile_actions + - all_link_actions, - flags = [ - "-g4", - "-fsanitize=undefined", - "-O1", - "-DUNDEFINED_BEHAVIOR_SANITIZER=1", - "-fno-omit-frame-pointer", - "-fno-sanitize=vptr", - ], - features = ["wasm_ubsan"], - ), - - # Profiling provides full debug info and a special --profiling flag - # to control name mangling - flag_set( - actions = all_link_actions, - flags = ["--profiling"], - features = ["profiling"], - ), - flag_set( - actions = all_link_actions, - flags = ["--profiling_funcs"], - features = ["profiling_funcs"], - ), - flag_set( - actions = all_compile_actions + - all_link_actions, - flags = ["-g4"], - features = ["full_debug_info"], - ), - flag_set( - actions = all_link_actions, - flags = ["-gseparate-dwarf"], - features = ["dwarf_debug_info"], - ), - flag_set( - actions = all_compile_actions + - all_link_actions, - flags = ["-fdebug-compilation-dir=."], - features = ["dwarf_debug_info"], - ), - # Generic warning flag list - flag_set( - actions = all_compile_actions, - flags = CROSSTOOL_DEFAULT_WARNINGS, - ), - - # Defines and Includes and Paths and such - flag_set( - actions = all_compile_actions, - flag_groups = [ - flag_group(flags = ["-fPIC"], expand_if_available = "pic"), - ], - ), - flag_set( - actions = preprocessor_compile_actions, - flag_groups = [ - flag_group( - flags = ["-D%{preprocessor_defines}"], - iterate_over = "preprocessor_defines", - ), - ], - ), - flag_set( - actions = preprocessor_compile_actions, - flag_groups = [ - flag_group( - flags = ["-include", "%{includes}"], - iterate_over = "includes", - expand_if_available = "includes", - ), - ], - ), - flag_set( - actions = preprocessor_compile_actions, - flag_groups = [ - flag_group( - flags = ["-iquote", "%{quote_include_paths}"], - iterate_over = "quote_include_paths", - ), - flag_group( - flags = ["-I%{include_paths}"], - iterate_over = "include_paths", - ), - flag_group( - flags = ["-isystem", "%{system_include_paths}"], - iterate_over = "system_include_paths", - ), - ], - ), - - ## Linking options (not libs -- those go last) - - # Generic link options - flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ], - flags = ["-shared"], - ), - - # Linker search paths and objects: - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - iterate_over = "runtime_library_search_directories", - flag_groups = [ - flag_group( - flags = [ - "-Wl,-rpath,$EXEC_ORIGIN/%{runtime_library_search_directories}", - ], - expand_if_true = "is_cc_test", - ), - flag_group( - flags = [ - "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", - ], - expand_if_false = "is_cc_test", - ), - ], - expand_if_available = "runtime_library_search_directories", - ), - ], - ), - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - flags = ["-L%{library_search_directories}"], - iterate_over = "library_search_directories", - expand_if_available = "library_search_directories", - ), - ], - ), - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - # This is actually a list of object files from the linkstamp steps - flags = ["%{linkstamp_paths}"], - iterate_over = "linkstamp_paths", - expand_if_available = "linkstamp_paths", - ), - ], - ), - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - flags = ["@%{thinlto_param_file}"], - expand_if_available = "libraries_to_link", - expand_if_true = "thinlto_param_file", - ), - flag_group( - iterate_over = "libraries_to_link", - flag_groups = [ - flag_group( - flags = ["-Wl,--start-lib"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - flag_group( - flags = ["-Wl,-whole-archive"], - expand_if_true = "libraries_to_link.is_whole_archive", - ), - flag_group( - flags = ["%{libraries_to_link.object_files}"], - iterate_over = "libraries_to_link.object_files", - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file", - ), - ), - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "interface_library", - ), - ), - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "static_library", - ), - ), - flag_group( - flags = ["-l%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "dynamic_library", - ), - ), - flag_group( - flags = ["-l:%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "versioned_dynamic_library", - ), - ), - flag_group( - flags = ["-Wl,-no-whole-archive"], - expand_if_true = "libraries_to_link.is_whole_archive", - ), - flag_group( - flags = ["-Wl,--end-lib"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - ], - expand_if_available = "libraries_to_link", - ), - ], - ), - - # Configure the header parsing and preprocessing. - flag_set( - actions = [ACTION_NAMES.cpp_header_parsing], - flags = ["-xc++-header", "-fsyntax-only"], - features = ["parse_headers"], - ), - - # Note: user compile flags should be nearly last -- you probably - # don't want to put any more features after this! - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ], - flag_groups = [ - flag_group( - flags = ["%{user_compile_flags}"], - iterate_over = "user_compile_flags", - expand_if_available = "user_compile_flags", - ), - ], - ), - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - flags = ["%{user_link_flags}"], - iterate_over = "user_link_flags", - expand_if_available = "user_link_flags", - ), - ], - ), - ## Options which need to go late -- after all the user options -- go here. - flag_set( - # One might hope that these options would only be needed for C++ - # compiles. But, sadly, users compile ".c" files with custom - # copts=["-x", "c++"], and expect that to be able to find C++ stdlib - # headers. It might be worth pondering how blaze could support this sort - # of use-case better. - actions = preprocessor_compile_actions + - [ACTION_NAMES.cc_flags_make_variable], - flags = [ - "-iwithsysroot" + "/include/c++/v1", - "-iwithsysroot" + "/include/compat", - "-iwithsysroot" + "/include", - "-isystem", "external/emscripten/lib/clang/13.0.0/include", - ], - ), - # Inputs and outputs - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ], - flag_groups = [ - flag_group( - flags = ["-MD", "-MF", "%{dependency_file}"], - expand_if_available = "dependency_file", - ), - ], - ), - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ], - flag_groups = [ - flag_group( - flags = ["-c", "%{source_file}"], - expand_if_available = "source_file", - ), - ], - ), - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ], - flag_groups = [ - flag_group( - flags = ["-S"], - expand_if_available = "output_assembly_file", - ), - flag_group( - flags = ["-E"], - expand_if_available = "output_preprocess_file", - ), - flag_group( - flags = ["-o", "%{output_file}"], - expand_if_available = "output_file", - ), - ], - ), - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - flags = ["-o", "%{output_execpath}"], - expand_if_available = "output_execpath", - ), - ], - ), - # And finally, the params file! - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - flags = ["@%{linker_param_file}"], - expand_if_available = "linker_param_file", - ), - ], - ), - flag_set( - actions = all_compile_actions, - flags = [ - "-Wno-builtin-macro-redefined", - # Genrules may not escape quotes enough for these, so - # don't put them into $(CC_FLAGS): - '-D__DATE__="redacted"', - '-D__TIMESTAMP__="redacted"', - '-D__TIME__="redacted"', - ], - ), - flag_set( - actions = all_compile_actions, - flags = ["-Werror"], - features = ["wasm_warnings_as_errors"], - ), - ] - - crosstool_default_env_sets = [ - # Use llvm backend. Off by default, enabled via --features=llvm_backend - env_set( - actions = all_compile_actions + - all_link_actions + - [ACTION_NAMES.cpp_link_static_library], - env_entries = [env_entry(key = "EMCC_WASM_BACKEND", value = "1")], - with_features = [with_feature_set(features = ["llvm_backend"])], - ), - # Debug compile and link. Off by default, enabled via --features=emcc_debug - env_set( - actions = all_compile_actions, - env_entries = [env_entry(key = "EMCC_DEBUG", value = "1")], - with_features = [with_feature_set(features = ["emcc_debug"])], - ), - - # Debug only link step. Off by default, enabled via --features=emcc_debug_link - env_set( - actions = all_link_actions, - env_entries = [env_entry(key = "EMCC_DEBUG", value = "1")], - with_features = [ - with_feature_set(features = ["emcc_debug"]), - with_feature_set(features = ["emcc_debug_link"]), - ], - ), - ] - - crosstool_default_flags_feature = feature( - name = "crosstool_default_flags", - enabled = True, - flag_sets = crosstool_default_flag_sets, - env_sets = crosstool_default_env_sets, - ) - - features.append(crosstool_default_flags_feature) - - cxx_builtin_include_directories = [ - "external/emscripten/emscripten/cache/sysroot/include/c++/v1", - "external/emscripten/emscripten/cache/sysroot/include/compat", - "external/emscripten/emscripten/cache/sysroot/include", - "external/emscripten/lib/clang/13.0.0/include", - ] - - artifact_name_patterns = [] - - make_variables = [] - - out = ctx.actions.declare_file(ctx.label.name) - ctx.actions.write(out, "Fake executable") - return [ - cc_common.create_cc_toolchain_config_info( - ctx = ctx, - features = features, - action_configs = action_configs, - artifact_name_patterns = artifact_name_patterns, - cxx_builtin_include_directories = cxx_builtin_include_directories, - toolchain_identifier = toolchain_identifier, - host_system_name = host_system_name, - target_system_name = target_system_name, - target_cpu = target_cpu, - target_libc = target_libc, - compiler = compiler, - abi_version = abi_version, - abi_libc_version = abi_libc_version, - tool_paths = tool_paths, - make_variables = make_variables, - builtin_sysroot = builtin_sysroot, - cc_target_os = cc_target_os, - ), - DefaultInfo( - executable = out, - ), - ] - -emscripten_cc_toolchain_config_rule = rule( - implementation = _impl, - attrs = { - "cpu": attr.string(mandatory = True, values = ["asmjs", "wasm"]), - "emscripten_version": attr.string(mandatory = True), - }, - provides = [CcToolchainConfigInfo], - executable = True, -) diff --git a/emscripten_toolchain/emar.sh b/emscripten_toolchain/emar.sh deleted file mode 100755 index 965442e34c..0000000000 --- a/emscripten_toolchain/emar.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -source emscripten_toolchain/env.sh - -exec python3 $EMSCRIPTEN/emar.py "$@" diff --git a/emscripten_toolchain/emcc.sh b/emscripten_toolchain/emcc.sh deleted file mode 100755 index 66d00b6c9a..0000000000 --- a/emscripten_toolchain/emcc.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -source emscripten_toolchain/env.sh - -exec python3 external/emscripten/emscripten/emcc.py "$@" diff --git a/emscripten_toolchain/emcc_link.sh b/emscripten_toolchain/emcc_link.sh deleted file mode 100755 index 9d0f8e208b..0000000000 --- a/emscripten_toolchain/emcc_link.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -source emscripten_toolchain/env.sh - -exec python3 emscripten_toolchain/link_wrapper.py "$@" diff --git a/emscripten_toolchain/emscripten.BUILD b/emscripten_toolchain/emscripten.BUILD deleted file mode 100644 index 6f11852874..0000000000 --- a/emscripten_toolchain/emscripten.BUILD +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ['//visibility:public']) - -filegroup( - name = "all", - srcs = glob(["**"]), -) diff --git a/emscripten_toolchain/emscripten_config b/emscripten_toolchain/emscripten_config deleted file mode 100644 index b7948aca41..0000000000 --- a/emscripten_toolchain/emscripten_config +++ /dev/null @@ -1,12 +0,0 @@ -import os -import platform - -ROOT_DIR = os.environ["ROOT_DIR"] -EMSCRIPTEN_ROOT = os.environ["EMSCRIPTEN"] -LLVM_ROOT = ROOT_DIR + "/external/emscripten/bin" -BINARYEN_ROOT = ROOT_DIR + "/external/emscripten" -FROZEN_CACHE = True - -system = platform.system() -nodejs_binary = "node.exe" if(system =="Windows") else "bin/node" -NODE_JS = ROOT_DIR + "/external/nodejs_{}_amd64/{}".format(system.lower(), nodejs_binary) diff --git a/emscripten_toolchain/env.sh b/emscripten_toolchain/env.sh deleted file mode 100755 index dfb4ddc29d..0000000000 --- a/emscripten_toolchain/env.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -export ROOT_DIR=`(pwd -P)` -export EMSCRIPTEN=${ROOT_DIR}/external/emscripten/emscripten -export EM_CONFIG=${ROOT_DIR}/emscripten_toolchain/emscripten_config diff --git a/emscripten_toolchain/link_wrapper.py b/emscripten_toolchain/link_wrapper.py deleted file mode 100644 index a746ae8fd3..0000000000 --- a/emscripten_toolchain/link_wrapper.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python -"""wrapper around emcc link step. - -This wrapper currently serves the following purposes. - -1. Ensures we always link to file with .js extension. The upstream default - it to link to an llvm bitcode file which is never (AFAICT) want to do that. - -2. When building with --config=wasm the final output is multiple files, usually - at least one .js and one .wasm file. Since the cc_binary link step only - allows a single output, we must tar up the outputs into a single file. - -3. Add quotes around arguments that need them in the response file to work - around a bazel quirk. -""" - -from __future__ import print_function - -import os -import subprocess -import sys - -# Only argument should be @path/to/parameter/file -assert sys.argv[1][0] == '@' -param_filename = sys.argv[1][1:] -param_file_args = [l.strip() for l in open(param_filename, 'r').readlines()] - -output_index = param_file_args.index('-o') + 1 -orig_output = js_output = param_file_args[output_index] -outdir = os.path.dirname(orig_output) - -# google3-only(TODO(b/139440956): Default to False once the bug is fixed) -replace_response_file = any(' ' in a for a in param_file_args) - -if not os.path.splitext(orig_output)[1]: - js_output = orig_output + '.js' - param_file_args[output_index] = js_output - replace_response_file = True - -# Re-write response file if needed. -if replace_response_file: - new_param_filename = param_filename + '.modified' - with open(new_param_filename, 'w') as f: - for param in param_file_args: - if ' ' in param: - f.write('"%s"' % param) - else: - f.write(param) - f.write('\n') - sys.argv[1] = '@' + new_param_filename - -emcc_py = os.path.join(os.environ['EMSCRIPTEN'], 'emcc.py') -rtn = subprocess.call(['python3', emcc_py] + sys.argv[1:]) -if rtn != 0: - sys.exit(1) - -js_name = os.path.basename(js_output) -base_name = os.path.splitext(js_name)[0] - -files = [] -extensions = [ - '.js', - '.wasm', - '.wasm.map', - '.js.mem', - '.fetch.js', - '.worker.js', - '.data', - '.js.symbols', - '.wasm.debug.wasm' -] - -for ext in extensions: - filename = base_name + ext - if os.path.exists(os.path.join(outdir, filename)): - files.append(filename) - -wasm_base = os.path.join(outdir, base_name + '.wasm') -if os.path.exists(wasm_base + '.debug.wasm') and os.path.exists(wasm_base): - # If we have a .wasm.debug.wasm file and a .wasm file, we need to rewrite the - # section in the .wasm file that refers to it. The path that's in there - # is the blaze output path; we want it to be just the filename. - - llvm_objcopy = os.path.join( - os.environ['EMSCRIPTEN'], 'llvm-bin/llvm-objcopy') - # First, check to make sure the .wasm file has the header that needs to be - # rewritten. - rtn = subprocess.call([ - llvm_objcopy, - '--dump-section=external_debug_info=/dev/null', - wasm_base], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if rtn == 0: - # If llvm-objcopy did not return an error, the external_debug_info section - # must exist, so we're good to continue. - - # Next we need to convert length of the filename to LEB128. - # Start by converting the length of the filename to a bit string. - bit_string = '{0:b}'.format(len(base_name + '.wasm.debug.wasm')) - - # Pad the bit string with 0s so that its length is a multiple of 7. - while len(bit_string) % 7 != 0: - bit_string = '0' + bit_string - - # Break up our bit string into chunks of 7. - # We do this backwards because the final format is little-endian. - final_bytes = bytearray() - for i in reversed(range(0, len(bit_string), 7)): - binary_part = bit_string[i:i + 7] - if i != 0: - # Every chunk except the last one needs to be prepended with '1'. - # The length of each chunk is 7, so that one has an implicit '0'. - binary_part = '1' + binary_part - final_bytes.append(int(binary_part, 2)) - # Finally, add the actual filename. - final_bytes.extend(base_name + '.wasm.debug.wasm') - - # Write our length + filename bytes to a temp file. - with open('debugsection.tmp', 'wb+') as f: - f.write(final_bytes) - f.close() - - # First delete the old section. - subprocess.check_call([ - llvm_objcopy, - wasm_base, - '--remove-section=external_debug_info']) - # Rewrite section with the new size and filename from the temp file. - subprocess.check_call([ - llvm_objcopy, - wasm_base, - '--add-section=external_debug_info=debugsection.tmp']) - -# If we have more than one output file then create tarball -if len(files) > 1: - cmd = ['tar', 'cf', 'tmp.tar'] + files - subprocess.check_call(cmd, cwd=outdir) - os.rename(os.path.join(outdir, 'tmp.tar'), orig_output) -elif len(files) == 1: - # Otherwise, if only have a single output than move it to the expected name - if files[0] != os.path.basename(orig_output): - os.rename(os.path.join(outdir, files[0]), orig_output) -else: - print('emcc.py did not appear to output any known files!') - sys.exit(1) - -sys.exit(0) diff --git a/emscripten_toolchain/wasm_binary.py b/emscripten_toolchain/wasm_binary.py deleted file mode 100644 index 466e789e61..0000000000 --- a/emscripten_toolchain/wasm_binary.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Unpackages a bazel emscripten archive for use in a bazel BUILD rule. - -This script will take a tar archive containing the output of the emscripten -toolchain. This file contains any output files produced by a wasm_cc_binary or a -cc_binary built with --config=wasm. The files are extracted into the given -output path. - -The name of archive is expected to be of the format `foo` or `foo.XXX` and -the contents are expected to be foo.js and foo.wasm. - -Several optional files may also be in the archive, including but not limited to -foo.js.mem, pthread-main.js, and foo.wasm.map. - -If the file is not a tar archive, the passed file will simply be copied to its -destination. - -This script and its accompanying Bazel rule should allow you to extract a -WebAssembly binary into a larger web application. -""" - -import os -import subprocess -import sys - -from absl import app -from absl import flags - -FLAGS = flags.FLAGS -flags.DEFINE_string('archive', None, 'The the archive to extract from.') -flags.DEFINE_string('output_path', None, 'The path to extract into.') - - -def ensure(f): - if not os.path.exists(f): - with open(f, 'w'): - pass - - -def check(f): - if not os.path.exists(f): - raise Exception('Expected file in archive: %s' % f) - - -def main(argv): - basename = os.path.basename(FLAGS.archive) - stem = basename.split('.')[0] - - # Check the type of the input file - mimetype_bytes = subprocess.check_output(['file', '-Lb', '--mime-type', '--mime-encoding', FLAGS.archive]) - mimetype = mimetype_bytes.decode(sys.stdout.encoding) - - # If we have a tar, extract all files. If we have just a single file, copy it. - if 'tar' in mimetype: - subprocess.check_call( - ['tar', 'xf', FLAGS.archive, '-C', FLAGS.output_path]) - elif 'binary' in mimetype: - subprocess.check_call([ - 'cp', - FLAGS.archive, - os.path.join(FLAGS.output_path, stem + '.wasm')]) - elif 'text' in mimetype: - subprocess.check_call([ - 'cp', - FLAGS.archive, - os.path.join(FLAGS.output_path, stem + '.js')]) - else: - subprocess.check_call(['cp', FLAGS.archive, FLAGS.output_path]) - - # At least one of these two files should exist at this point. - ensure(os.path.join(FLAGS.output_path, stem + '.js')) - ensure(os.path.join(FLAGS.output_path, stem + '.wasm')) - - # And can optionally contain these extra files. - ensure(os.path.join(FLAGS.output_path, stem + '.wasm.map')) - ensure(os.path.join(FLAGS.output_path, stem + '.worker.js')) - ensure(os.path.join(FLAGS.output_path, stem + '.js.mem')) - ensure(os.path.join(FLAGS.output_path, stem + '.data')) - ensure(os.path.join(FLAGS.output_path, stem + '.fetch.js')) - ensure(os.path.join(FLAGS.output_path, stem + '.js.symbols')) - ensure(os.path.join(FLAGS.output_path, stem + '.wasm.debug.wasm')) - - -if __name__ == '__main__': - app.run(main) diff --git a/emscripten_toolchain/wasm_cc_binary.bzl b/emscripten_toolchain/wasm_cc_binary.bzl deleted file mode 100644 index ec8d8a7765..0000000000 --- a/emscripten_toolchain/wasm_cc_binary.bzl +++ /dev/null @@ -1,155 +0,0 @@ -"""wasm_cc_binary rule for compiling C++ targets to WebAssembly. -""" - -def _wasm_transition_impl(settings, attr): - _ignore = (settings, attr) - - features = list(settings["//command_line_option:features"]) - linkopts = list(settings["//command_line_option:linkopt"]) - copts = list(settings["//command_line_option:copt"]) - - if attr.threads == "emscripten": - # threads enabled - features.append("use_pthreads") - elif attr.threads == "off": - # threads disabled - features.append("-use_pthreads") - - if attr.exit_runtime == True: - features.append("exit_runtime") - - if attr.backend == "llvm": - features.append("llvm_backend") - elif attr.backend == "emscripten": - features.append("-llvm_backend") - - if attr.simd: - features.append("wasm_simd") - copts.append("-msimd128") - - return { - "//command_line_option:compiler": "emscripten", - "//command_line_option:crosstool_top": "//emscripten_toolchain:everything", - "//command_line_option:cpu": "wasm", - "//command_line_option:features": features, - "//command_line_option:dynamic_mode": "off", - "//command_line_option:linkopt": linkopts, - "//command_line_option:copt": copts, - "//command_line_option:platforms": [], - "//command_line_option:custom_malloc": "//emscripten_toolchain:malloc", - } - -_wasm_transition = transition( - implementation = _wasm_transition_impl, - inputs = [ - "//command_line_option:features", - "//command_line_option:linkopt", - "//command_line_option:copt", - ], - outputs = [ - "//command_line_option:compiler", - "//command_line_option:cpu", - "//command_line_option:crosstool_top", - "//command_line_option:features", - "//command_line_option:dynamic_mode", - "//command_line_option:linkopt", - "//command_line_option:copt", - "//command_line_option:platforms", - "//command_line_option:custom_malloc", - ], -) - -def _wasm_binary_impl(ctx): - cc_target = ctx.attr.cc_target[0] - - args = [ - "--output_path={}".format(ctx.outputs.loader.dirname), - ] + [ - ctx.expand_location("--archive=$(location {})".format( - cc_target.label, - ), [cc_target]), - ] - outputs = [ - ctx.outputs.loader, - ctx.outputs.wasm, - ctx.outputs.map, - ctx.outputs.mem, - ctx.outputs.fetch, - ctx.outputs.worker, - ctx.outputs.data, - ctx.outputs.symbols, - ctx.outputs.dwarf, - ] - - ctx.actions.run( - inputs = ctx.files.cc_target, - outputs = outputs, - arguments = args, - executable = ctx.executable._wasm_binary_extractor, - ) - - return DefaultInfo( - files = depset(outputs), - # This is needed since rules like web_test usually have a data - # dependency on this target. - data_runfiles = ctx.runfiles(transitive_files = depset(outputs)), - ) - -def _wasm_binary_outputs(name, cc_target): - basename = cc_target.name - basename = basename.split(".")[0] - outputs = { - "loader": "{}/{}.js".format(name, basename), - "wasm": "{}/{}.wasm".format(name, basename), - "map": "{}/{}.wasm.map".format(name, basename), - "mem": "{}/{}.js.mem".format(name, basename), - "fetch": "{}/{}.fetch.js".format(name, basename), - "worker": "{}/{}.worker.js".format(name, basename), - "data": "{}/{}.data".format(name, basename), - "symbols": "{}/{}.js.symbols".format(name, basename), - "dwarf": "{}/{}.wasm.debug.wasm".format(name, basename), - } - - return outputs - -# Wraps a C++ Blaze target, extracting the appropriate files. -# -# This rule will transition to the emscripten toolchain in order -# to build the the cc_target as a WebAssembly binary. -# -# Args: -# name: The name of the rule. -# cc_target: The cc_binary or cc_library to extract files from. -wasm_cc_binary = rule( - implementation = _wasm_binary_impl, - attrs = { - "backend": attr.string( - default = "_default", - values = ["_default", "emscripten", "llvm"], - ), - "cc_target": attr.label( - cfg = _wasm_transition, - mandatory = True, - ), - "exit_runtime": attr.bool( - default = False, - ), - "threads": attr.string( - default = "_default", - values = ["_default", "emscripten", "off"], - ), - "simd": attr.bool( - default = False, - ), - "_allowlist_function_transition": attr.label( - default = "@bazel_tools//tools/allowlists/function_transition_allowlist", - ), - "_wasm_binary_extractor": attr.label( - executable = True, - allow_files = True, - cfg = "exec", - default = Label("//emscripten_toolchain:wasm_binary"), - ), - }, - outputs = _wasm_binary_outputs, -) diff --git a/emscripten_toolchain/wasm_rules.bzl b/emscripten_toolchain/wasm_rules.bzl deleted file mode 100644 index 1c1c409ed5..0000000000 --- a/emscripten_toolchain/wasm_rules.bzl +++ /dev/null @@ -1,6 +0,0 @@ -"""Rules related to C++ and WebAssembly. -""" - -load("//emscripten_toolchain:wasm_cc_binary.bzl", _wasm_cc_binary = "wasm_cc_binary") - -wasm_cc_binary = _wasm_cc_binary diff --git a/tfjs-backend-wasm/scripts/build-wasm.sh b/tfjs-backend-wasm/scripts/build-wasm.sh index 617975f38f..73e950a2d5 100755 --- a/tfjs-backend-wasm/scripts/build-wasm.sh +++ b/tfjs-backend-wasm/scripts/build-wasm.sh @@ -33,7 +33,7 @@ cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm/tfjs-backend-was if [[ "$1" != "--dev" ]]; then # SIMD and threaded + SIMD builds. - yarn bazel build $BAZEL_REMOTE -c opt //tfjs-backend-wasm/src/cc:tfjs-backend-wasm-simd \ + yarn bazel build $BAZEL_REMOTE -c opt --copt="-msimd128" //tfjs-backend-wasm/src/cc:tfjs-backend-wasm-simd \ //tfjs-backend-wasm/src/cc:tfjs-backend-wasm-threaded-simd # Copy SIMD cp -f ../../dist/bin/tfjs-backend-wasm/src/cc/tfjs-backend-wasm-simd/tfjs-backend-wasm.wasm \ diff --git a/tfjs-backend-wasm/scripts/create-worker-module.js b/tfjs-backend-wasm/scripts/create-worker-module.js index b8b9fe47ca..a264ace177 100755 --- a/tfjs-backend-wasm/scripts/create-worker-module.js +++ b/tfjs-backend-wasm/scripts/create-worker-module.js @@ -26,5 +26,6 @@ const BASE_PATH = '../wasm-out/'; const WORKER_PATH = `${BASE_PATH}tfjs-backend-wasm-threaded-simd.worker.js`; const workerContents = fs.readFileSync(WORKER_PATH, "utf8"); +fs.chmodSync(WORKER_PATH, 0o644); fs.writeFileSync(`${WORKER_PATH}`, `export const wasmWorkerContents = '${workerContents.trim()}';`); diff --git a/tfjs-backend-wasm/src/cc/BUILD b/tfjs-backend-wasm/src/cc/BUILD index 509045b585..70d9c0dfcc 100644 --- a/tfjs-backend-wasm/src/cc/BUILD +++ b/tfjs-backend-wasm/src/cc/BUILD @@ -1,4 +1,4 @@ -load("//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") +load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") load(":build_defs.bzl", "tfjs_cc_library", "tfjs_unit_test") # Emcripten produces a much larger wasm bundle unless the cc_binary has srcs From dde7a906ce3d1f50f5c07677cd2335dfd5653997 Mon Sep 17 00:00:00 2001 From: Matthew Soulanille Date: Mon, 29 Mar 2021 16:04:13 -0700 Subject: [PATCH 24/24] Remove unused variables --- tfjs-backend-wasm/src/cc/kernels/All.cc | 1 - tfjs-backend-wasm/src/cc/kernels/Any.cc | 1 - 2 files changed, 2 deletions(-) diff --git a/tfjs-backend-wasm/src/cc/kernels/All.cc b/tfjs-backend-wasm/src/cc/kernels/All.cc index 474f2ea34e..5bd1c7f69a 100644 --- a/tfjs-backend-wasm/src/cc/kernels/All.cc +++ b/tfjs-backend-wasm/src/cc/kernels/All.cc @@ -33,7 +33,6 @@ void All(const size_t x_id, const size_t reduce_size, const size_t out_id) { auto& out_info = backend::get_tensor_info_out(out_id); const bool* x_buf = x_info.b(); - const size_t x_size = x_info.size; bool* out_buf = out_info.b_write(); const size_t out_size = out_info.size; diff --git a/tfjs-backend-wasm/src/cc/kernels/Any.cc b/tfjs-backend-wasm/src/cc/kernels/Any.cc index ac09d7eb34..6e27d22a99 100644 --- a/tfjs-backend-wasm/src/cc/kernels/Any.cc +++ b/tfjs-backend-wasm/src/cc/kernels/Any.cc @@ -33,7 +33,6 @@ void Any(const size_t x_id, const size_t reduce_size, const size_t out_id) { auto& out_info = backend::get_tensor_info_out(out_id); const bool* x_buf = x_info.b(); - const size_t x_size = x_info.size; bool* out_buf = out_info.b_write(); const size_t out_size = out_info.size;