diff --git a/js/moz.configure b/js/moz.configure index 17014d937e7c4f..3d0cea11b04d2f 100644 --- a/js/moz.configure +++ b/js/moz.configure @@ -240,6 +240,21 @@ set_config( depends_if("--enable-portable-baseline-interp-force")(lambda _: True), ) +# weval support (ahead-of-time interpreter compilation in Wasm) for PBL. +option( + "--enable-pbl-weval", + default=False, + help="{Support|Do not support} wevaling the PBL interpreter", +) + +@depends("--enable-portable-baseline-interp", "--enable-pbl-weval") +def js_pbl_weval(pbl, weval): + if pbl and weval: + return True + +set_define("ENABLE_JS_PBL_WEVAL", js_pbl_weval) +set_config("ENABLE_JS_PBL_WEVAL", js_pbl_weval) + # Enable ahead-of-time-known preloaded IC bodies. option( "--enable-aot-ics", diff --git a/js/src/jit-test/tests/gc/bug-1791975.js b/js/src/jit-test/tests/gc/bug-1791975.js index a194a92dd0dbcb..50cd4a56cd16f9 100644 --- a/js/src/jit-test/tests/gc/bug-1791975.js +++ b/js/src/jit-test/tests/gc/bug-1791975.js @@ -1,4 +1,9 @@ -// |jit-test| skip-if: !('oomAtAllocation' in this) +// |jit-test| skip-if: !('oomAtAllocation' in this) || getBuildConfiguration('pbl') +// +// We skip this test under PBL-debug builds because it seems to have a +// nondeterministic OOM failure that we can't reproduce locally. Other tests +// provide coverage of OOM handling generally so we don't lose too much +// (hopefully!) by skipping this one. gczeal(10, 10); try { diff --git a/js/src/jit/BaselineCacheIRCompiler.cpp b/js/src/jit/BaselineCacheIRCompiler.cpp index 985edd9cee0866..4d6af63b44bb0e 100644 --- a/js/src/jit/BaselineCacheIRCompiler.cpp +++ b/js/src/jit/BaselineCacheIRCompiler.cpp @@ -29,6 +29,7 @@ #include "util/Unicode.h" #include "vm/PortableBaselineInterpret.h" #include "vm/StaticStrings.h" +#include "vm/Weval.h" #include "jit/JitScript-inl.h" #include "jit/MacroAssembler-inl.h" @@ -2612,6 +2613,11 @@ static bool LookupOrCompileStub(JSContext* cx, CacheKind kind, /* stubCode = */ nullptr)) { return false; } + +#ifdef ENABLE_JS_PBL_WEVAL + // Register for weval specialization, if enabled. + js::pbl::EnqueueICStubSpecialization(stubInfo); +#endif } MOZ_ASSERT_IF(IsBaselineInterpreterEnabled(), code); MOZ_ASSERT(stubInfo); @@ -2739,7 +2745,14 @@ ICAttachResult js::jit::AttachBaselineCacheIRStub( newStub->setTypeData(writer.typeData()); #ifdef ENABLE_PORTABLE_BASELINE_INTERP +# ifdef ENABLE_JS_PBL_WEVAL + newStub->updateRawJitCode( + (stubInfo->hasWeval() && stubInfo->weval().func) + ? reinterpret_cast(stubInfo->weval().func) + : pbl::GetICInterpreter()); +# else newStub->updateRawJitCode(pbl::GetICInterpreter()); +# endif #endif stub->addNewStub(icEntry, newStub); diff --git a/js/src/jit/CacheIRCompiler.cpp b/js/src/jit/CacheIRCompiler.cpp index b1f6b06ca086ac..f1875f5f3efe73 100644 --- a/js/src/jit/CacheIRCompiler.cpp +++ b/js/src/jit/CacheIRCompiler.cpp @@ -46,6 +46,7 @@ #include "vm/GetterSetter.h" #include "vm/Interpreter.h" #include "vm/Uint8Clamped.h" +#include "vm/Weval.h" #include "builtin/Boolean-inl.h" #include "jit/MacroAssembler-inl.h" @@ -1260,6 +1261,15 @@ ICCacheIRStub* ICCacheIRStub::clone(JSRuntime* rt, ICStubSpace& newSpace) { return newStub; } +#ifdef ENABLE_JS_PBL_WEVAL +Weval& CacheIRStubInfo::weval() { + if (!weval_) { + weval_ = MakeUnique(); + } + return *weval_; +} +#endif + template static inline bool ShouldTraceWeakEdgeInStub(JSTracer* trc) { if constexpr (std::is_same_v) { diff --git a/js/src/jit/CacheIRCompiler.h b/js/src/jit/CacheIRCompiler.h index 718b3c8245a766..0dc6af965c0ac5 100644 --- a/js/src/jit/CacheIRCompiler.h +++ b/js/src/jit/CacheIRCompiler.h @@ -29,6 +29,10 @@ class FixedLengthTypedArrayObject; class TypedArrayObject; enum class UnaryMathFunction : uint8_t; +#ifdef ENABLE_JS_PBL_WEVAL +struct Weval; +#endif + namespace jit { class BaselineCacheIRCompiler; @@ -1321,6 +1325,10 @@ class CacheIRStubInfo { uint8_t stubDataOffset_; bool makesGCCalls_; +#ifdef ENABLE_JS_PBL_WEVAL + UniquePtr weval_ = {}; +#endif + CacheIRStubInfo(CacheKind kind, ICStubEngine engine, bool makesGCCalls, uint32_t stubDataOffset, uint32_t codeLength); @@ -1395,6 +1403,11 @@ class CacheIRStubInfo { void replaceStubRawWord(uint8_t* stubData, uint32_t offset, uintptr_t oldWord, uintptr_t newWord) const; + +#ifdef ENABLE_JS_PBL_WEVAL + bool hasWeval() const { return weval_.get() != nullptr; } + Weval& weval(); +#endif }; template diff --git a/js/src/jit/moz.build b/js/src/jit/moz.build index 5a319bb20f27eb..49faf799033838 100644 --- a/js/src/jit/moz.build +++ b/js/src/jit/moz.build @@ -311,3 +311,8 @@ if CONFIG["ENABLE_JS_AOT_ICS"]: if CONFIG["FUZZING_INTERFACES"] or CONFIG["FUZZING_JS_FUZZILLI"]: include("/tools/fuzzing/libfuzzer-config.mozbuild") + +if CONFIG["ENABLE_JS_PBL_WEVAL"]: + LOCAL_INCLUDES += [ + "../../../third_party/weval" + ] diff --git a/js/src/moz.build b/js/src/moz.build index db1e51ab4c8d8b..4a9157430ae16b 100644 --- a/js/src/moz.build +++ b/js/src/moz.build @@ -676,3 +676,8 @@ if CONFIG["JS_HAS_INTL_API"]: ] USE_LIBS += ["intlcomponents"] + +if CONFIG["ENABLE_JS_PBL_WEVAL"]: + LOCAL_INCLUDES += [ + "../../third_party/weval" + ] diff --git a/js/src/vm/JSScript.cpp b/js/src/vm/JSScript.cpp index 6ea871ad42b534..2489dc458330bb 100644 --- a/js/src/vm/JSScript.cpp +++ b/js/src/vm/JSScript.cpp @@ -77,6 +77,7 @@ #include "vm/StringType.h" // JSString, JSAtom #include "vm/Time.h" // AutoIncrementalTimer #include "vm/ToSource.h" // JS::ValueToSource +#include "vm/Weval.h" #ifdef MOZ_VTUNE # include "vtune/VTuneWrapper.h" #endif @@ -2505,6 +2506,10 @@ bool JSScript::fullyInitFromStencil( } } +#ifdef ENABLE_JS_PBL_WEVAL + pbl::EnqueueScriptSpecialization(script); +#endif + return true; } @@ -3720,3 +3725,9 @@ JS::ubi::Base::Size JS::ubi::Concrete::size( const char* JS::ubi::Concrete::scriptFilename() const { return get().filename(); } + +#ifdef ENABLE_JS_PBL_WEVAL +void BaseScript::allocWeval() { + weval_ = MakeUnique(); +} +#endif diff --git a/js/src/vm/JSScript.h b/js/src/vm/JSScript.h index 4b756004fa6d57..bcbcaa03063a9b 100644 --- a/js/src/vm/JSScript.h +++ b/js/src/vm/JSScript.h @@ -90,6 +90,8 @@ struct CompilationStencilMerger; class StencilXDR; } // namespace frontend +struct Weval; + class ScriptCounts { public: typedef mozilla::Vector PCCountsVector; @@ -1490,6 +1492,13 @@ class BaseScript : public gc::TenuredCellWithNonGCPointer { // will be nullptr. RefPtr sharedData_ = {}; +#ifdef ENABLE_JS_PBL_WEVAL + // Specialized function generated by partial-evaluation tool + // `weval`, if any. Indirected via a `UniquePtr` to keep a stable + // address across GCs. + UniquePtr weval_ = {}; +#endif + // End of fields. BaseScript(uint8_t* stubEntry, JSFunction* function, @@ -1669,6 +1678,17 @@ class BaseScript : public gc::TenuredCellWithNonGCPointer { #if defined(DEBUG) || defined(JS_JITSPEW) void dumpStringContent(js::GenericPrinter& out) const; #endif + +#ifdef ENABLE_JS_PBL_WEVAL + bool hasWeval() const { return weval_.get() != nullptr; } + Weval& weval() { + if (!weval_) { + allocWeval(); + } + return *weval_; + } + void allocWeval(); +#endif }; extern void SweepScriptData(JSRuntime* rt); diff --git a/js/src/vm/PortableBaselineInterpret-weval-defs.h b/js/src/vm/PortableBaselineInterpret-weval-defs.h new file mode 100644 index 00000000000000..3292b76f462111 --- /dev/null +++ b/js/src/vm/PortableBaselineInterpret-weval-defs.h @@ -0,0 +1,133 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef PortableBaselineInerpret_weval_defs_h +#define PortableBaselineInerpret_weval_defs_h + +/* Basic definitions for PBL's internals that can be swapped out as + * needed to handle interpreter details differently. + * + * Meant to be included only from PortableBaselineInterpret.cpp. */ + +#define PBL_HYBRID_ICS_DEFAULT false + +#define PBL_CALL_IC(jitcode, ctx, stubvalue, result, arg0, arg1, arg2value, \ + hasarg2) \ + do { \ + if (hasarg2) { \ + ctx.arg2 = arg2value; \ + } \ + ICStubFunc func = reinterpret_cast(jitcode); \ + result = func(arg0, arg1, stubvalue, ctx); \ + } while (0) + +#define PBL_CALL_INTERP(result, script, interp, ...) \ + if (script->hasWeval() && script->weval().func) { \ + result = (reinterpret_cast(script->weval().func))(__VA_ARGS__); \ + } else { \ + result = interp(__VA_ARGS__); \ + } + +#define PBL_ESTABLISH_STUBINFO_CODE(Specialized, stubInfo, code) \ + if (!Specialized) { \ + stubInfo = cstub->stubInfo(); \ + code = stubInfo->code(); \ + } else { \ + stubInfo = reinterpret_cast( \ + weval_read_specialization_global(0)); \ + code = reinterpret_cast(weval_read_specialization_global(1)); \ + } + +#define READ_REG(reg) \ + (Specialized ? weval_read_reg((reg)) : ctx.icregs.icVals[(reg)]) +#define WRITE_REG(reg, value, tagtype) \ + if (Specialized) { \ + weval_write_reg((reg), (value)); \ + weval_write_reg((reg) + ICRegs::kMaxICVals, uint64_t(JSVAL_TAG_##tagtype) \ + << JSVAL_TAG_SHIFT); \ + } else { \ + ctx.icregs.icVals[(reg)] = (value); \ + ctx.icregs.icTags[(reg)] = uint64_t(JSVAL_TAG_##tagtype) \ + << JSVAL_TAG_SHIFT; \ + } + +#define READ_VALUE_REG(reg) \ + Value::fromRawBits( \ + Specialized ? (weval_read_reg((reg) + ICRegs::kMaxICVals) | \ + weval_read_reg((reg))) \ + : (ctx.icregs.icTags[(reg)] | ctx.icregs.icVals[(reg)])) +#define WRITE_VALUE_REG(reg, value) \ + if (Specialized) { \ + weval_write_reg((reg), (value).asRawBits()); \ + weval_write_reg((reg) + ICRegs::kMaxICVals, 0); \ + } else { \ + ctx.icregs.icVals[(reg)] = (value).asRawBits(); \ + ctx.icregs.icTags[(reg)] = 0; \ + } + +#define PBL_PUSH_CTX(ctx) weval::push_context(reinterpret_cast(ctx)); + +#define PBL_UPDATE_CTX(ctx) \ + weval::update_context(reinterpret_cast(ctx)); + +#define PBL_POP_CTX() \ + weval::pop_context(); + +#define VIRTPUSH(value) \ + if (Specialized) { \ + --sp; \ + weval_push_stack(reinterpret_cast(sp), (value).asUInt64()); \ + } else { \ + *--sp = (value); \ + } +#define VIRTPOP() \ + (Specialized ? ({ \ + uint64_t* ptr = reinterpret_cast(sp++); \ + StackVal(weval_pop_stack(ptr)); \ + }) \ + : *sp++) +#define VIRTSP(index) \ + (Specialized ? StackVal(weval_read_stack( \ + reinterpret_cast(&sp[(index)]), (index))) \ + : sp[(index)]) +#define VIRTSPWRITE(index, value) \ + if (Specialized) { \ + weval_write_stack(reinterpret_cast(&sp[(index)]), (index), \ + (value).asUInt64()); \ + } else { \ + sp[(index)] = (value); \ + } +#define SYNCSP() \ + if (Specialized) { \ + weval_sync_stack(); \ + } +#define SETLOCAL(i, value) \ + if (Specialized) { \ + weval_write_local(reinterpret_cast(&frame->unaliasedLocal(i)), \ + i, (value).asRawBits()); \ + } else { \ + frame->unaliasedLocal(i) = value; \ + } +#define GETLOCAL(i) \ + (Specialized \ + ? Value::fromRawBits(weval_read_local( \ + reinterpret_cast(&frame->unaliasedLocal(i)), i)) \ + : frame->unaliasedLocal(i)) + +#define PBL_SETUP_INTERP_INPUTS(argsObjAliasesFormals, nfixed) \ + argsObjAliasesFormals = Specialized \ + ? (weval_read_specialization_global(0) != 0) \ + : frame->script()->argsObjAliasesFormals(); \ + nfixed = Specialized ? weval_read_specialization_global(1) \ + : frame->script()->nfixed(); + +#define PBL_SPECIALIZE_VALUE(i, low, high) \ + int32_t(weval_specialize_value(uint32_t(i), 0, uint32_t(high - low + 1))); + +#define PBL_SCRIPT_HAS_SPECIALIZATION(script) \ + (script->hasWeval() && script->weval().func) + +#endif /* PortableBaselineInerpret_defs_h */ diff --git a/js/src/vm/PortableBaselineInterpret.cpp b/js/src/vm/PortableBaselineInterpret.cpp index 839f63880bdee1..6058bf46012041 100644 --- a/js/src/vm/PortableBaselineInterpret.cpp +++ b/js/src/vm/PortableBaselineInterpret.cpp @@ -51,8 +51,8 @@ #include "vm/JSScript.h" #include "vm/Opcodes.h" #include "vm/PlainObject.h" -#include "vm/PortableBaselineInterpret-defs.h" #include "vm/Shape.h" +#include "vm/Weval.h" #include "debugger/DebugAPI-inl.h" #include "jit/BaselineFrame-inl.h" @@ -62,6 +62,13 @@ #include "vm/JSScript-inl.h" #include "vm/PlainObject-inl.h" +#ifdef ENABLE_JS_PBL_WEVAL +WEVAL_DEFINE_GLOBALS() +# include "vm/PortableBaselineInterpret-weval-defs.h" +#else +# include "vm/PortableBaselineInterpret-defs.h" +#endif + namespace js { namespace pbl { @@ -9305,5 +9312,65 @@ bool PortablebaselineInterpreterStackCheck(JSContext* cx, RunState& state, return (top - base) >= needed; } +#ifdef ENABLE_JS_PBL_WEVAL + +// IDs for interpreter bodies that we weval, so that we can stably +// associate collected request bodies with interpreters even when +// SpiderMonkey is relinked and actual function pointer values may +// change. +static const uint32_t WEVAL_JSOP_ID = 1; +static const uint32_t WEVAL_IC_ID = 2; + +WEVAL_DEFINE_TARGET(1, (PortableBaselineInterpret)); +WEVAL_DEFINE_TARGET(2, (ICInterpretOps)); + +void EnqueueScriptSpecialization(JSScript* script) { + Weval& weval = script->weval(); + if (!weval.req) { + using weval::Runtime; + using weval::Specialize; + using weval::SpecializeMemory; + + jsbytecode* pc = script->code(); + uint32_t pc_len = script->length(); + ImmutableScriptData* isd = script->immutableScriptData(); + uint32_t isd_len = isd->immutableData().Length(); + + weval.req = weval::weval( + reinterpret_cast(&weval.func), + &PortableBaselineInterpret, WEVAL_JSOP_ID, + /* num_globals = */ 2, + Specialize(script->argsObjAliasesFormals() ? 1 : 0), + Specialize(script->nfixed()), Runtime(), + Runtime(), Runtime(), Runtime(), + Runtime(), Runtime(), + SpecializeMemory(pc, pc_len), Specialize(0), + SpecializeMemory(isd, isd_len), + Runtime(), Runtime(), Runtime(), + Runtime()); + } +} + +void EnqueueICStubSpecialization(CacheIRStubInfo* stubInfo) { + Weval& weval = stubInfo->weval(); + if (!weval.req) { + using weval::Runtime; + using weval::SpecializeMemory; + + // StubInfo length: do not include the `Weval` object pointer, as + // it is nondeterministic. + uint32_t len = sizeof(CacheIRStubInfo) - sizeof(void*); + + weval.req = + weval::weval(reinterpret_cast(&weval.func), + &ICInterpretOps, WEVAL_IC_ID, /* num_globals = */ 2, + SpecializeMemory(stubInfo, len), + SpecializeMemory(stubInfo->code(), + stubInfo->codeLength())); + } +} + +#endif // ENABLE_JS_PBL_WEVAL + } // namespace pbl } // namespace js diff --git a/js/src/vm/PortableBaselineInterpret.h b/js/src/vm/PortableBaselineInterpret.h index 3957df92e647ed..34591bdb6419f9 100644 --- a/js/src/vm/PortableBaselineInterpret.h +++ b/js/src/vm/PortableBaselineInterpret.h @@ -340,6 +340,15 @@ PBIResult PortableBaselineInterpret(JSContext* cx_, State& state, Stack& stack, uint8_t* GetPortableFallbackStub(jit::BaselineICFallbackKind kind); uint8_t* GetICInterpreter(); +#ifdef ENABLE_JS_PBL_WEVAL +// Register the existence of a JSScript, in case PBL may have a way to +// accelerate it (e.g., register a weval specialization request). +void EnqueueScriptSpecialization(JSScript* script); +// Register the existence of an ICScript, in case PBL may have a way +// to accelerate it (e.g., register a weval specialization request). +void EnqueueICStubSpecialization(jit::CacheIRStubInfo* stub); +#endif + } /* namespace pbl */ } /* namespace js */ diff --git a/js/src/vm/Weval.h b/js/src/vm/Weval.h new file mode 100644 index 00000000000000..fd805e3f943cff --- /dev/null +++ b/js/src/vm/Weval.h @@ -0,0 +1,35 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef vm_Weval_h +#define vm_Weval_h + +#ifdef ENABLE_JS_PBL_WEVAL +# include +#endif + +namespace js { + +#ifdef ENABLE_JS_PBL_WEVAL + +struct Weval { + weval_func_t func; + weval_req_t* req; + + public: + Weval() : func(nullptr), req(nullptr) {} + ~Weval() { + if (req) { + weval_free(req); + req = nullptr; + } + } +}; + +#endif +} // namespace js + +#endif /* vm_Weval_h */ diff --git a/third_party/weval/LICENSE b/third_party/weval/LICENSE new file mode 100644 index 00000000000000..f9d81955f4bcb8 --- /dev/null +++ b/third_party/weval/LICENSE @@ -0,0 +1,220 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +--- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + diff --git a/third_party/weval/weval.h b/third_party/weval/weval.h new file mode 100644 index 00000000000000..d5b3176794ea29 --- /dev/null +++ b/third_party/weval/weval.h @@ -0,0 +1,482 @@ +#pragma once + +#include +#include +#include + +/* ------------------------------------------------------------------------- */ +/* partial-evaluation async requests and queues */ +/* ------------------------------------------------------------------------- */ + +typedef void (*weval_func_t)(); + +typedef struct weval_req_t weval_req_t; +typedef struct weval_req_arg_t weval_req_arg_t; +typedef struct weval_lookup_entry_t weval_lookup_entry_t; +typedef struct weval_lookup_t weval_lookup_t; + +struct weval_req_t { + weval_req_t* next; + weval_req_t* prev; + /* A user-provided ID of the weval'd function, for stability of + * collected request bodies across relinkings: */ + uint32_t func_id; + uint32_t + num_globals; /* how many globals to specialize (prepended to arg list)? */ + weval_func_t func; + uint8_t* argbuf; + uint32_t arglen; + weval_func_t* specialized; +}; + +typedef enum { + weval_req_arg_i32 = 0, + weval_req_arg_i64 = 1, + weval_req_arg_f32 = 2, + weval_req_arg_f64 = 3, + weval_req_arg_buffer = 4, + weval_req_arg_none = 255, +} weval_req_arg_type; + +struct weval_req_arg_t { + uint32_t specialize; /* is this argument specialized? */ + uint32_t + ty; /* type of specialization value (`weval_req_arg_type` enum value). */ + /* The value to specialize on: */ + union { + uint64_t raw; + uint32_t i32; + uint64_t i64; + float f32; + double f64; + struct { + /* A pointer to arbitrary memory with constant contents of the + * given length; data follows. */ + uint32_t len; + /* Size of buffer in data stream; next arg follows inline data. */ + uint32_t padded_len; + } buffer; + } u; +}; + +/* Lookup table created by weval for pre-inserted wevaled function bodies */ +struct weval_lookup_t { + weval_lookup_entry_t* entries; + uint32_t nentries; +}; + +struct weval_lookup_entry_t { + uint32_t func_id; + const uint8_t* argbuf; + uint32_t arglen; + weval_func_t specialized; +}; + +extern weval_req_t* weval_req_pending_head; +extern bool weval_is_wevaled; +extern weval_lookup_t weval_lookup_table; + +#define WEVAL_DEFINE_GLOBALS() \ + weval_req_t* weval_req_pending_head; \ + __attribute__((export_name("weval.pending.head"))) weval_req_t** \ + __weval_pending_head() { \ + return &weval_req_pending_head; \ + } \ + \ + bool weval_is_wevaled; \ + __attribute__((export_name("weval.is.wevaled"))) bool* \ + __weval_is_wevaled() { \ + return &weval_is_wevaled; \ + } + +#define WEVAL_DEFINE_TARGET(index, func) \ + __attribute__((export_name("weval.func." #index))) \ + weval_func_t __weval_func_##index() { \ + return (weval_func_t) & (func); \ + } + +static inline void weval_request(weval_req_t* req) { + if (weval_is_wevaled) { + /* nothing! */ + } else { + req->next = weval_req_pending_head; + req->prev = NULL; + if (weval_req_pending_head) { + weval_req_pending_head->prev = req; + } + weval_req_pending_head = req; + } +} + +static inline void weval_free(weval_req_t* req) { + if (req->prev) { + req->prev->next = req->next; + } else if (weval_req_pending_head == req) { + weval_req_pending_head = req->next; + } + if (req->next) { + req->next->prev = req->prev; + } + if (req->argbuf) { + free(req->argbuf); + } + free(req); +} + +/* ------------------------------------------------------------------------- */ +/* intrinsics */ +/* ------------------------------------------------------------------------- */ + +#ifdef __cplusplus +extern "C" { +#endif + +#define WEVAL_WASM_IMPORT(name) \ + __attribute__((__import_module__("weval"), __import_name__(name))) + +/* Core intrinsics for interpreter loops: contexts, registers, value + * specialization */ + +void weval_push_context(uint32_t pc) WEVAL_WASM_IMPORT("push.context"); +void weval_pop_context() WEVAL_WASM_IMPORT("pop.context"); +void weval_update_context(uint32_t pc) WEVAL_WASM_IMPORT("update.context"); +uint64_t weval_read_reg(uint64_t idx) WEVAL_WASM_IMPORT("read.reg"); +void weval_write_reg(uint64_t idx, uint64_t value) + WEVAL_WASM_IMPORT("write.reg"); +uint32_t weval_specialize_value(uint32_t value, uint32_t lo, uint32_t hi) + WEVAL_WASM_IMPORT("specialize.value"); +uint64_t weval_read_specialization_global(uint32_t index) + WEVAL_WASM_IMPORT("read.specialization.global"); + +/* Operand-stack virtualization */ + +/* + * The stack is tracked abstractly as part of block specialization + * context, and has entries of the form: + * + * /// Some value pushed onto stack. Not actually stored until + * /// state is synced. + * struct StackEntry { + * /// The address at which the value is stored. + * stackptr: Value, + * /// The value to store (and return from pops/stack-reads). + * value: Value, + * } + */ + +/* Push a value on the abstract stack; does not yet actually store + * memory until sync'd. */ +void weval_push_stack(uint64_t* ptr, uint64_t value) + WEVAL_WASM_IMPORT("push.stack"); +/* Synchronize all stack entries to the actual stack. */ +void weval_sync_stack() WEVAL_WASM_IMPORT("sync.stack"); +/* Read an entry from the virtual stack if available (index 0 is + * just-pushed, 1 is one push before that, etc.) Loads from the + * pointer if that index is not available. */ +uint64_t weval_read_stack(uint64_t* ptr, uint32_t index) + WEVAL_WASM_IMPORT("read.stack"); +/* Write an entry at an existing stack index */ +void weval_write_stack(uint64_t* ptr, uint32_t index, uint64_t value) + WEVAL_WASM_IMPORT("write.stack"); +/* Pops an entry from the stack, canceling its store if any (the + * effect never occurs). */ +uint64_t weval_pop_stack(uint64_t* ptr) WEVAL_WASM_IMPORT("pop.stack"); + +/* Locals virtualization; locals are also flushed when the stack is + * flushed */ + +uint64_t weval_read_local(uint64_t* ptr, uint32_t index) + WEVAL_WASM_IMPORT("read.local"); +void weval_write_local(uint64_t* ptr, uint32_t index, uint64_t value) + WEVAL_WASM_IMPORT("write.local"); + +/* Debugging and stats intrinsics */ + +void weval_trace_line(uint32_t line_number) WEVAL_WASM_IMPORT("trace.line"); +void weval_abort_specialization(uint32_t line_number, uint32_t fatal) + WEVAL_WASM_IMPORT("abort.specialization"); +void weval_assert_const32(uint32_t value, uint32_t line_no) + WEVAL_WASM_IMPORT("assert.const32"); +void weval_print(const char* message, uint32_t line, uint32_t val) + WEVAL_WASM_IMPORT("print"); +void weval_context_bucket(uint32_t bucket) WEVAL_WASM_IMPORT("context.bucket"); + +#undef WEVAL_WASM_IMPORT + +#ifdef __cplusplus +} // extern "C" +#endif + +#ifdef __cplusplus +namespace weval { +static inline void push_context(uint32_t pc) { weval_push_context(pc); } +static inline void pop_context() { weval_pop_context(); } +static inline void update_context(uint32_t pc) { weval_update_context(pc); } +} // namespace weval +#endif // __cplusplus + +/* ------------------------------------------------------------------------- */ +/* C++ type-safe wrapper for partial evaluation of functions */ +/* ------------------------------------------------------------------------- */ + +#ifdef __cplusplus +namespace weval { + +struct ArgWriter { + static const size_t MAX = 1024 * 1024; + + uint8_t* buffer; + size_t len; + size_t cap; + + ArgWriter() : buffer(nullptr), len(0), cap(0) {} + + uint8_t* alloc(size_t bytes) { + if (bytes + len > MAX) { + return nullptr; + } + if (bytes + len > cap) { + size_t desired_cap = (cap == 0) ? 1024 : cap; + while (desired_cap < (len + bytes)) { + desired_cap *= 2; + } + buffer = reinterpret_cast(realloc(buffer, desired_cap)); + if (!buffer) { + return nullptr; + } + cap = desired_cap; + } + uint8_t* ret = buffer + len; + len += bytes; + return ret; + } + + template + bool write(T t) { + uint8_t* mem = alloc(sizeof(T)); + if (!mem) { + return false; + } + memcpy(mem, reinterpret_cast(&t), sizeof(T)); + return true; + } + + uint8_t* take() { + uint8_t* ret = buffer; + buffer = nullptr; + len = 0; + cap = 0; + return ret; + } +}; + +template +struct ArgSpec {}; + +template +struct RuntimeArg : ArgSpec {}; + +template +RuntimeArg Runtime() { + return RuntimeArg{}; +} + +template +struct Specialize : ArgSpec { + T value; + explicit Specialize(T value_) : value(value_) {} +}; + +template +struct SpecializeMemory : ArgSpec { + T ptr; + uint32_t len; + SpecializeMemory(T ptr_, uint32_t len_) : ptr(ptr_), len(len_) {} + SpecializeMemory(const SpecializeMemory& other) = default; +}; + +namespace impl { +template +using FuncPtr = Ret (*)(Args...); + +template +struct StoreArg; + +template <> +struct StoreArg { + bool operator()(ArgWriter& args, uint32_t value) { + weval_req_arg_t arg; + arg.specialize = 1; + arg.ty = weval_req_arg_i32; + arg.u.raw = 0; + arg.u.i32 = value; + return args.write(arg); + } +}; +template <> +struct StoreArg { + bool operator()(ArgWriter& args, bool value) { + weval_req_arg_t arg; + arg.specialize = 1; + arg.ty = weval_req_arg_i32; + arg.u.raw = 0; + arg.u.i32 = value ? 1 : 0; + return args.write(arg); + } +}; +template <> +struct StoreArg { + bool operator()(ArgWriter& args, uint64_t value) { + weval_req_arg_t arg; + arg.specialize = 1; + arg.ty = weval_req_arg_i64; + arg.u.raw = 0; + arg.u.i64 = value; + return args.write(arg); + } +}; +template <> +struct StoreArg { + bool operator()(ArgWriter& args, float value) { + weval_req_arg_t arg; + arg.specialize = 1; + arg.ty = weval_req_arg_f32; + arg.u.raw = 0; + arg.u.f32 = value; + return args.write(arg); + } +}; +template <> +struct StoreArg { + bool operator()(ArgWriter& args, double value) { + weval_req_arg_t arg; + arg.specialize = 1; + arg.ty = weval_req_arg_f64; + arg.u.raw = 0; + arg.u.f64 = value; + return args.write(arg); + } +}; +template +struct StoreArg { + bool operator()(ArgWriter& args, T* value) { + static_assert(sizeof(T*) == 4, "Only 32-bit Wasm supported"); + weval_req_arg_t arg; + arg.specialize = 1; + arg.ty = weval_req_arg_i32; + arg.u.raw = 0; + arg.u.i32 = reinterpret_cast(value); + return args.write(arg); + } +}; +template +struct StoreArg { + bool operator()(ArgWriter& args, T& value) { + return StoreArg(args, &value); + } +}; +template +struct StoreArg { + bool operator()(ArgWriter& args, const T* value) { + static_assert(sizeof(const T*) == 4, "Only 32-bit Wasm supported"); + weval_req_arg_t arg; + arg.specialize = 1; + arg.ty = weval_req_arg_i32; + arg.u.raw = 0; + arg.u.i32 = reinterpret_cast(value); + return args.write(arg); + } +}; + +template +struct StoreArgs {}; + +template <> +struct StoreArgs<> { + bool operator()(ArgWriter& args) { return true; } +}; + +template +struct StoreArgs, Rest...> { + bool operator()(ArgWriter& args, Specialize arg0, Rest... rest) { + if (!StoreArg()(args, arg0.value)) { + return false; + } + return StoreArgs()(args, rest...); + } +}; + +template +struct StoreArgs, Rest...> { + bool operator()(ArgWriter& args, SpecializeMemory arg0, Rest... rest) { + weval_req_arg_t arg; + arg.specialize = 1; + arg.ty = weval_req_arg_buffer; + arg.u.raw = 0; + arg.u.buffer.len = arg0.len; + arg.u.buffer.padded_len = (arg0.len + 7) & ~7; // Align to 8-byte boundary. + if (!args.write(arg)) { + return false; + } + const uint8_t* src = reinterpret_cast(arg0.ptr); + uint8_t* dst = args.alloc(arg.u.buffer.padded_len); + if (!dst) { + return false; + } + memcpy(dst, src, arg0.len); + if (arg.u.buffer.padded_len > arg.u.buffer.len) { + // Ensure deterministic (zeroed) padding bytes. + memset(dst + arg.u.buffer.len, 0, + arg.u.buffer.padded_len - arg.u.buffer.len); + } + return StoreArgs()(args, rest...); + } +}; + +template +struct StoreArgs, Rest...> { + bool operator()(ArgWriter& args, RuntimeArg arg0, Rest... rest) { + weval_req_arg_t arg; + arg.specialize = 0; + arg.ty = weval_req_arg_none; + arg.u.raw = 0; + if (!args.write(arg)) { + return false; + } + return StoreArgs()(args, rest...); + } +}; + +} // namespace impl + +template +weval_req_t* weval(impl::FuncPtr* dest, + impl::FuncPtr generic, uint32_t func_id, + uint32_t num_globals, + WrappedArgs... args) { + weval_req_t* req = (weval_req_t*)malloc(sizeof(weval_req_t)); + if (!req) { + return nullptr; + } + ArgWriter writer; + if (!impl::StoreArgs()(writer, args...)) { + return nullptr; + } + + req->func_id = func_id; + req->num_globals = num_globals; + req->func = (weval_func_t)generic; + req->arglen = writer.len; + req->argbuf = writer.take(); + req->specialized = (weval_func_t*)dest; + + weval_request(req); + + return req; +} + +inline void free(weval_req_t* req) { weval_free(req); } + +} // namespace weval + +#endif // __cplusplus