From 2123d6fe138723c45f0d3bc19de39af763152271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gustavsson?= Date: Fri, 15 Dec 2023 08:00:50 +0100 Subject: [PATCH] Optimize matching out a bitstring --- erts/emulator/beam/jit/arm/instr_bs.cpp | 20 ++++++--- erts/emulator/beam/jit/x86/instr_bs.cpp | 22 ++++++---- erts/emulator/test/bs_match_bin_SUITE.erl | 52 +++++++++++++++++++++-- 3 files changed, 76 insertions(+), 18 deletions(-) diff --git a/erts/emulator/beam/jit/arm/instr_bs.cpp b/erts/emulator/beam/jit/arm/instr_bs.cpp index f49188280811..4c714ef4aa2f 100644 --- a/erts/emulator/beam/jit/arm/instr_bs.cpp +++ b/erts/emulator/beam/jit/arm/instr_bs.cpp @@ -3730,9 +3730,6 @@ static std::vector opt_bsm_segments( case BsmSegment::action::GET_BITSTRING: if (seg.size > 64) { read_action_pos = -1; - } else if (seg.action == BsmSegment::action::GET_BITSTRING && - seg.size % 8 != 0) { - read_action_pos = -1; } else { if ((seg.flags & BSF_LITTLE) != 0 || read_action_pos < 0 || seg.size + segs.at(read_action_pos).size > 64) { @@ -4122,16 +4119,25 @@ void BeamModuleAssembler::emit_i_bs_match_test_heap(ArgLabel const &Fail, } case BsmSegment::action::GET_BITSTRING: { auto Live = seg.live; + ERTS_ASSERT(seg.size > 64); comment("get binary %ld", seg.size); auto ctx = load_source(Ctx, TMP1); + if (position_is_valid) { + a.mov(ARG5, bin_position); + } else { + a.ldur(ARG5, emit_boxed_val(ctx.reg, start_offset)); + } lea(ARG1, arm::Mem(c_p, offsetof(Process, htop))); - a.ldur(ARG2, emit_boxed_val(ctx.reg, orig_offset)); - a.and_(ARG3, ARG2, imm(~TAG_PTR_MASK__)); - a.and_(ARG2, ARG2, imm(TAG_PTR_MASK__)); + if (seg.size <= ERL_ONHEAP_BITS_LIMIT) { + comment("skipped setting registers not used for heap binary"); + } else { + a.ldur(ARG2, emit_boxed_val(ctx.reg, orig_offset)); + a.and_(ARG3, ARG2, imm(~TAG_PTR_MASK__)); + a.and_(ARG2, ARG2, imm(TAG_PTR_MASK__)); + } a.ldur(ARG4, emit_boxed_val(ctx.reg, base_offset)); a.and_(ARG4, ARG4, imm(~ERL_SUB_BITS_FLAG_MASK)); - a.ldur(ARG5, emit_boxed_val(ctx.reg, start_offset)); mov_imm(ARG6, seg.size); a.add(TMP2, ARG5, ARG6); a.stur(TMP2, emit_boxed_val(ctx.reg, start_offset)); diff --git a/erts/emulator/beam/jit/x86/instr_bs.cpp b/erts/emulator/beam/jit/x86/instr_bs.cpp index fd3f32ba7e42..1fc2af02b5f0 100644 --- a/erts/emulator/beam/jit/x86/instr_bs.cpp +++ b/erts/emulator/beam/jit/x86/instr_bs.cpp @@ -3894,9 +3894,6 @@ static std::vector opt_bsm_segments( if (seg.size > 64) { read_action_pos = -1; - } else if (seg.action == BsmSegment::action::GET_BITSTRING && - seg.size % 8 != 0) { - read_action_pos = -1; } else if ((seg.flags & BSF_LITTLE) != 0 && is_common_size) { seg.action = BsmSegment::action::READ_INTEGER; read_action_pos = -1; @@ -4367,6 +4364,7 @@ void BeamModuleAssembler::emit_i_bs_match_test_heap(ArgLabel const &Fail, break; } case BsmSegment::action::GET_BITSTRING: { + ERTS_ASSERT(seg.size > 64); comment("get binary %ld", seg.size); if (is_ctx_valid) { a.mov(RET, ctx); @@ -4374,14 +4372,22 @@ void BeamModuleAssembler::emit_i_bs_match_test_heap(ArgLabel const &Fail, mov_arg(RET, Ctx); } emit_enter_runtime(); + if (is_position_valid) { + a.mov(ARG5, bin_position); + } else { + a.mov(ARG5, emit_boxed_val(RET, start_offset)); + } a.lea(ARG1, x86::qword_ptr(c_p, offsetof(Process, htop))); - a.mov(ARG2, emit_boxed_val(RET, orig_offset)); - a.mov(ARG3, ARG2); - a.and_(ARG2, imm(TAG_PTR_MASK__)); - a.and_(ARG3, imm(~TAG_PTR_MASK__)); + if (seg.size <= ERL_ONHEAP_BITS_LIMIT) { + comment("skipped setting registers not used for heap binary"); + } else { + a.mov(ARG2, emit_boxed_val(RET, orig_offset)); + a.mov(ARG3, ARG2); + a.and_(ARG2, imm(TAG_PTR_MASK__)); + a.and_(ARG3, imm(~TAG_PTR_MASK__)); + } a.mov(ARG4, emit_boxed_val(RET, base_offset)); a.and_(ARG4, imm(~ERL_SUB_BITS_FLAG_MASK)); - a.mov(ARG5, emit_boxed_val(RET, start_offset)); mov_imm(ARG6, seg.size); a.add(emit_boxed_val(RET, start_offset), ARG6); diff --git a/erts/emulator/test/bs_match_bin_SUITE.erl b/erts/emulator/test/bs_match_bin_SUITE.erl index 0ea354b01ee1..a80592f1df24 100644 --- a/erts/emulator/test/bs_match_bin_SUITE.erl +++ b/erts/emulator/test/bs_match_bin_SUITE.erl @@ -24,7 +24,8 @@ init_per_group/2,end_per_group/2, byte_split_binary/1,bit_split_binary/1,match_huge_bin/1, bs_match_string_edge_case/1,contexts/1, - empty_binary/1,small_bitstring/1]). + empty_binary/1,small_bitstring/1, + known_position/1]). -include_lib("common_test/include/ct.hrl"). @@ -33,7 +34,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [byte_split_binary, bit_split_binary, match_huge_bin, bs_match_string_edge_case, contexts, empty_binary, - small_bitstring]. + small_bitstring,known_position]. groups() -> []. @@ -281,7 +282,9 @@ small_bitstring(_Config) -> %% heap space for small bitstrings. rand_seed(), Bin = rand:bytes(10_000), - ok = small_bitstring_1(id(Bin), id(Bin)). + ok = small_bitstring_1(id(Bin), id(Bin)), + ok = small_bitstring_2(id(Bin), id(7)), + ok = small_bitstring_3(id(Bin), id(64)). small_bitstring_1(<>, @@ -291,6 +294,49 @@ small_bitstring_1(<>, <<>>) -> ok. +small_bitstring_2(<<>>, _) -> + ok; +small_bitstring_2(Bin, N7) -> + %% Ensure that matching fixed sizes gives the same result as + %% matching dynamic sizes. + + <> = Bin, + <> = Bin, + + <> = Bin, + <> = Bin, + + small_bitstring_2(As, N7). + +small_bitstring_3(<<>>, _) -> + ok; +small_bitstring_3(Bin, N64) -> + %% Ensure that matching fixed sizes gives the same result as + %% matching dynamic sizes for larger sizes. + + <> = Bin, + <> = Bin, + + <> = Bin, + <> = Bin, + + <> = Bin, + <> = Bin, + + small_bitstring_3(As, N64). + +known_position(_Config) -> + %% Cover the case of an extracted bitstring having a known position. + <> = id(<<42:8,"abcdefghij">>), + 42 = Int, + <<"abcdefghi">> = BitString, + + ok. + %%% %%% Common utilities. %%%