LibJS: Join locals, constants and registers into single vector

Merging registers, constants and locals into single vector means:
- Better data locality
- No need to check type in Interpreter::get() and Interpreter::set()
  which are very hot functions

Performance improvement is visible in almost all Octane and Kraken
tests.

Author: kalenikaliaksandr

Userland/Libraries/LibJS/AST.cpp

@@ -1624,7 +1624,10 @@ void ScopeNode::block_declaration_instantiation(VM& vm, Environment* environment
 
             // iii. Perform ! env.InitializeBinding(fn, fo). NOTE: This step is replaced in section B.3.2.6.
             if (function_declaration.name_identifier()->is_local()) {
-                vm.running_execution_context().local(function_declaration.name_identifier()->local_variable_index()) = function;
+                auto& running_execution_context = vm.running_execution_context();
+                auto number_of_registers = running_execution_context.executable->number_of_registers;
+                auto number_of_constants = running_execution_context.executable->constants.size();
+                running_execution_context.local(function_declaration.name_identifier()->local_variable_index() + number_of_registers + number_of_constants) = function;
             } else {
                 VERIFY(is<DeclarativeEnvironment>(*environment));
                 static_cast<DeclarativeEnvironment&>(*environment).initialize_or_set_mutable_binding({}, vm, function_declaration.name(), function);

Userland/Libraries/LibJS/Bytecode/Generator.cpp

@@ -266,6 +266,19 @@ CodeGenerationErrorOr<NonnullGCPtr<Executable>> Generator::emit_function_body_by
 
     HashMap<size_t, SourceRecord> source_map;
 
+    for (auto& block : generator.m_root_basic_blocks) {
+        if (!block->is_terminated()) {
+            // NOTE: We must ensure that the "undefined" constant, which will be used by the not yet
+            // emitted End instruction, is taken into account while shifting local operands by the
+            // number of constants.
+            (void)generator.add_constant(js_undefined());
+            break;
+        }
+    }
+
+    auto number_of_registers = generator.m_next_register;
+    auto number_of_constants = generator.m_constants.size();
+
     for (auto& block : generator.m_root_basic_blocks) {
         basic_block_start_offsets.append(bytecode.size());
         if (block->handler() || block->finalizer()) {
@@ -287,6 +300,21 @@ CodeGenerationErrorOr<NonnullGCPtr<Executable>> Generator::emit_function_body_by
         while (!it.at_end()) {
             auto& instruction = const_cast<Instruction&>(*it);
 
+            instruction.visit_operands([number_of_registers, number_of_constants](Operand& operand) {
+                switch (operand.type()) {
+                case Operand::Type::Register:
+                    break;
+                case Operand::Type::Local:
+                    operand.offset_index_by(number_of_registers + number_of_constants);
+                    break;
+                case Operand::Type::Constant:
+                    operand.offset_index_by(number_of_registers);
+                    break;
+                default:
+                    VERIFY_NOT_REACHED();
+                }
+            });
+
             // OPTIMIZATION: Don't emit jumps that just jump to the next block.
             if (instruction.type() == Instruction::Type::Jump) {
                 auto& jump = static_cast<Bytecode::Op::Jump&>(instruction);
@@ -333,6 +361,20 @@ CodeGenerationErrorOr<NonnullGCPtr<Executable>> Generator::emit_function_body_by
         }
         if (!block->is_terminated()) {
             Op::End end(generator.add_constant(js_undefined()));
+            end.visit_operands([number_of_registers, number_of_constants](Operand& operand) {
+                switch (operand.type()) {
+                case Operand::Type::Register:
+                    break;
+                case Operand::Type::Local:
+                    operand.offset_index_by(number_of_registers + number_of_constants);
+                    break;
+                case Operand::Type::Constant:
+                    operand.offset_index_by(number_of_registers);
+                    break;
+                default:
+                    VERIFY_NOT_REACHED();
+                }
+            });
             bytecode.append(reinterpret_cast<u8 const*>(&end), end.length());
         }
         if (block->handler() || block->finalizer()) {

Userland/Libraries/LibJS/Bytecode/Instruction.cpp

@@ -42,6 +42,22 @@ void Instruction::visit_labels(Function<void(JS::Bytecode::Label&)> visitor)
 #undef __BYTECODE_OP
 }
 
+void Instruction::visit_operands(Function<void(JS::Bytecode::Operand&)> visitor)
+{
+#define __BYTECODE_OP(op)                                               \
+    case Type::op:                                                      \
+        static_cast<Op::op&>(*this).visit_operands_impl(move(visitor)); \
+        return;
+
+    switch (type()) {
+        ENUMERATE_BYTECODE_OPS(__BYTECODE_OP)
+    default:
+        VERIFY_NOT_REACHED();
+    }
+
+#undef __BYTECODE_OP
+}
+
 template<typename Op>
 concept HasVariableLength = Op::IsVariableLength;
 

Userland/Libraries/LibJS/Bytecode/Instruction.h

@@ -157,6 +157,7 @@ class alignas(void*) Instruction {
     size_t length() const;
     ByteString to_byte_string(Bytecode::Executable const&) const;
     void visit_labels(Function<void(Label&)> visitor);
+    void visit_operands(Function<void(Operand&)> visitor);
     static void destroy(Instruction&);
 
 protected:
@@ -166,6 +167,7 @@ class alignas(void*) Instruction {
     }
 
     void visit_labels_impl(Function<void(Label&)>) { }
+    void visit_operands_impl(Function<void(Operand&)>) { }
 
 private:
     Type m_type {};

Userland/Libraries/LibJS/Bytecode/Interpreter.cpp

@@ -54,7 +54,7 @@ static ByteString format_operand(StringView name, Operand operand, Bytecode::Exe
         break;
     case Operand::Type::Constant: {
         builder.append("\033[36m"sv);
-        auto value = executable.constants[operand.index()];
+        auto value = executable.constants[operand.index() - executable.number_of_registers];
         if (value.is_empty())
             builder.append("<Empty>"sv);
         else if (value.is_boolean())
@@ -153,30 +153,12 @@ Interpreter::~Interpreter()
 
 ALWAYS_INLINE Value Interpreter::get(Operand op) const
 {
-    switch (op.type()) {
-    case Operand::Type::Register:
-        return m_registers.data()[op.index()];
-    case Operand::Type::Local:
-        return m_locals.data()[op.index()];
-    case Operand::Type::Constant:
-        return m_constants.data()[op.index()];
-    }
-    __builtin_unreachable();
+    return m_registers_and_constants_and_locals.data()[op.index()];
 }
 
 ALWAYS_INLINE void Interpreter::set(Operand op, Value value)
 {
-    switch (op.type()) {
-    case Operand::Type::Register:
-        m_registers.data()[op.index()] = value;
-        return;
-    case Operand::Type::Local:
-        m_locals.data()[op.index()] = value;
-        return;
-    case Operand::Type::Constant:
-        break;
-    }
-    __builtin_unreachable();
+    m_registers_and_constants_and_locals.data()[op.index()] = value;
 }
 
 // 16.1.6 ScriptEvaluation ( scriptRecord ), https://tc39.es/ecma262/#sec-runtime-semantics-scriptevaluation
@@ -346,7 +328,7 @@ FLATTEN_ON_CLANG void Interpreter::run_bytecode(size_t entry_point)
 
     auto& running_execution_context = this->running_execution_context();
     auto* arguments = running_execution_context.arguments.data();
-    auto* locals = running_execution_context.locals.data();
+    auto* registers_and_constants_and_locals = running_execution_context.registers_and_constants_and_locals.data();
     auto& accumulator = this->accumulator();
     auto& executable = current_executable();
     auto const* bytecode = executable.bytecode.data();
@@ -384,7 +366,7 @@ FLATTEN_ON_CLANG void Interpreter::run_bytecode(size_t entry_point)
 
         handle_SetLocal: {
             auto& instruction = *reinterpret_cast<Op::SetLocal const*>(&bytecode[program_counter]);
-            locals[instruction.index()] = get(instruction.src());
+            registers_and_constants_and_locals[instruction.index()] = get(instruction.src());
             DISPATCH_NEXT(SetLocal);
         }
 
@@ -715,31 +697,35 @@ Interpreter::ResultAndReturnRegister Interpreter::run_executable(Executable& exe
     VERIFY(!vm().execution_context_stack().is_empty());
 
     auto& running_execution_context = vm().running_execution_context();
-    if (running_execution_context.registers.size() < executable.number_of_registers)
-        running_execution_context.registers.resize(executable.number_of_registers);
+    u32 registers_and_contants_count = executable.number_of_registers + executable.constants.size();
+    if (running_execution_context.registers_and_constants_and_locals.size() < registers_and_contants_count)
+        running_execution_context.registers_and_constants_and_locals.resize(registers_and_contants_count);
 
     TemporaryChange restore_running_execution_context { m_running_execution_context, &running_execution_context };
     TemporaryChange restore_arguments { m_arguments, running_execution_context.arguments.span() };
-    TemporaryChange restore_registers { m_registers, running_execution_context.registers.span() };
-    TemporaryChange restore_locals { m_locals, running_execution_context.locals.span() };
-    TemporaryChange restore_constants { m_constants, executable.constants.span() };
+    TemporaryChange restore_registers_and_constants_and_locals { m_registers_and_constants_and_locals, running_execution_context.registers_and_constants_and_locals.span() };
 
     reg(Register::accumulator()) = initial_accumulator_value;
     reg(Register::return_value()) = {};
 
     running_execution_context.executable = &executable;
 
+    for (size_t i = 0; i < executable.constants.size(); ++i) {
+        running_execution_context.registers_and_constants_and_locals[executable.number_of_registers + i] = executable.constants[i];
+    }
+
     run_bytecode(entry_point.value_or(0));
 
     dbgln_if(JS_BYTECODE_DEBUG, "Bytecode::Interpreter did run unit {:p}", &executable);
 
     if constexpr (JS_BYTECODE_DEBUG) {
-        for (size_t i = 0; i < registers().size(); ++i) {
+        auto const& registers_and_constants_and_locals = running_execution_context.registers_and_constants_and_locals;
+        for (size_t i = 0; i < executable.number_of_registers; ++i) {
             String value_string;
-            if (registers()[i].is_empty())
+            if (registers_and_constants_and_locals[i].is_empty())
                 value_string = "(empty)"_string;
             else
-                value_string = registers()[i].to_string_without_side_effects();
+                value_string = registers_and_constants_and_locals[i].to_string_without_side_effects();
             dbgln("[{:3}] {}", i, value_string);
         }
     }
@@ -758,8 +744,8 @@ Interpreter::ResultAndReturnRegister Interpreter::run_executable(Executable& exe
     vm().finish_execution_generation();
 
     if (!exception.is_empty())
-        return { throw_completion(exception), running_execution_context.registers[0] };
-    return { return_value, running_execution_context.registers[0] };
+        return { throw_completion(exception), running_execution_context.registers_and_constants_and_locals[0] };
+    return { return_value, running_execution_context.registers_and_constants_and_locals[0] };
 }
 
 void Interpreter::enter_unwind_context()

Userland/Libraries/LibJS/Bytecode/Interpreter.h

@@ -49,11 +49,11 @@ class Interpreter {
     ALWAYS_INLINE Value& saved_return_value() { return reg(Register::saved_return_value()); }
     Value& reg(Register const& r)
     {
-        return m_registers.data()[r.index()];
+        return m_registers_and_constants_and_locals.data()[r.index()];
     }
     Value reg(Register const& r) const
     {
-        return m_registers.data()[r.index()];
+        return m_registers_and_constants_and_locals.data()[r.index()];
     }
 
     [[nodiscard]] Value get(Operand) const;
@@ -77,9 +77,6 @@ class Interpreter {
     Executable const& current_executable() const { return *m_current_executable; }
     Optional<size_t> program_counter() const { return m_program_counter; }
 
-    Vector<Value>& registers() { return vm().running_execution_context().registers; }
-    Vector<Value> const& registers() const { return vm().running_execution_context().registers; }
-
     ExecutionContext& running_execution_context() { return *m_running_execution_context; }
 
 private:
@@ -99,9 +96,7 @@ class Interpreter {
     GCPtr<DeclarativeEnvironment> m_global_declarative_environment { nullptr };
     Optional<size_t&> m_program_counter;
     Span<Value> m_arguments;
-    Span<Value> m_registers;
-    Span<Value> m_locals;
-    Span<Value> m_constants;
+    Span<Value> m_registers_and_constants_and_locals;
     ExecutionContext* m_running_execution_context { nullptr };
 };