Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Do some minor enhancements of the code generation in the JIT #7956

Merged
merged 12 commits into from
Dec 20, 2023
Merged

Do some minor enhancements of the code generation in the JIT #7956

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
213ba49
Simplify instructions with known not-small operands
bjorng Oct 13, 2023
ae8fe47
AArch64: Optimize binary matching
bjorng Oct 14, 2023
0a3a1f3
AArch64: Optimize binary construction
bjorng Oct 15, 2023
c0dcdd4
AArch64: Combine move + move + trim
bjorng Oct 19, 2023
c653961
AArch64: Enhance move_call_last
bjorng Oct 20, 2023
7ca660b
x86_64: Optimize arithmetic instructions
bjorng Nov 16, 2023
5cf90cf
Add helpers is_cons() and is_not_cons()
bjorng Oct 21, 2023
be78def
Optimize tag tests in is_eq_exact and is_ne_exact
bjorng Oct 21, 2023
c76f8c2
Optimize test for arithmetic equality
bjorng Oct 22, 2023
8cd93d0
Optimize equality tests with known literals
bjorng Nov 25, 2023
9699a0d
Optimize matching out a bitstring
bjorng Dec 15, 2023
1e3099e
Aarch64: Combine put_list + deallocate
bjorng Dec 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 78 additions & 9 deletions erts/emulator/beam/jit/arm/beam_asm.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,20 @@ struct BeamAssembler : public BeamAssemblerCommon {
}
}

void emit_is_cons(Label Fail, arm::Gp Src) {
const int bitNumber = 1;
ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST ==
(1 << bitNumber));
a.tbnz(Src, imm(bitNumber), Fail);
}

void emit_is_not_cons(Label Fail, arm::Gp Src) {
const int bitNumber = 1;
ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST ==
(1 << bitNumber));
a.tbz(Src, imm(bitNumber), Fail);
}

void emit_is_boxed(Label Fail, arm::Gp Src) {
const int bitNumber = 0;
ERTS_CT_ASSERT(_TAG_PRIMARY_MASK - TAG_PRIMARY_BOXED ==
Expand Down Expand Up @@ -660,14 +674,6 @@ struct BeamAssembler : public BeamAssemblerCommon {
}
}

/* Set the Z flag if Reg1 and Reg2 are both immediates. */
void emit_are_both_immediate(arm::Gp Reg1, arm::Gp Reg2) {
ERTS_CT_ASSERT(TAG_PRIMARY_IMMED1 == _TAG_PRIMARY_MASK);
a.and_(SUPER_TMP, Reg1, Reg2);
a.and_(SUPER_TMP, SUPER_TMP, imm(_TAG_PRIMARY_MASK));
a.cmp(SUPER_TMP, imm(TAG_PRIMARY_IMMED1));
}

/* Set the Z flag if Reg1 and Reg2 are definitely not equal based
* on their tags alone. (They may still be equal if both are
* immediates and all other bits are equal too.) */
Expand Down Expand Up @@ -1227,12 +1233,15 @@ class BeamModuleAssembler : public BeamAssembler,
const arm::Gp bin_base,
const arm::Gp bitdata);

void emit_extract_integer(const arm::Gp bitdata,
void emit_extract_integer(const arm::Gp &bitdata,
const arm::Gp &small_tag,
Uint flags,
Uint position,
Uint bits,
const ArgRegister &Dst);

void emit_extract_bitstring(const arm::Gp bitdata,
Uint position,
Uint bits,
const ArgRegister &Dst);

Expand Down Expand Up @@ -1758,6 +1767,66 @@ class BeamModuleAssembler : public BeamAssembler,
a.ldp(gp1, gp2, arm::Mem(SUPER_TMP));
}
}

/* Set the Z flag if Reg1 and Reg2 are definitely not equal based
* on their tags alone. (They may still be equal if both are
* immediates and all other bits are equal too.) */
void emit_is_unequal_based_on_tags(Label Unequal,
const ArgVal &Src1,
arm::Gp Reg1,
const ArgVal &Src2,
arm::Gp Reg2) {
ERTS_CT_ASSERT(TAG_PRIMARY_IMMED1 == _TAG_PRIMARY_MASK);
ERTS_CT_ASSERT((TAG_PRIMARY_LIST | TAG_PRIMARY_BOXED) ==
TAG_PRIMARY_IMMED1);

if (always_one_of<BeamTypeId::AlwaysBoxed>(Src1)) {
emit_is_boxed(Unequal, Reg2);
} else if (always_one_of<BeamTypeId::AlwaysBoxed>(Src2)) {
emit_is_boxed(Unequal, Reg1);
} else if (exact_type<BeamTypeId::Cons>(Src1)) {
emit_is_cons(Unequal, Reg2);
} else if (exact_type<BeamTypeId::Cons>(Src2)) {
emit_is_cons(Unequal, Reg1);
} else {
a.orr(SUPER_TMP, Reg1, Reg2);

if (never_one_of<BeamTypeId::Cons>(Src1) ||
never_one_of<BeamTypeId::Cons>(Src2)) {
emit_is_boxed(Unequal, SUPER_TMP);
} else if (never_one_of<BeamTypeId::AlwaysBoxed>(Src1) ||
never_one_of<BeamTypeId::AlwaysBoxed>(Src2)) {
emit_is_cons(Unequal, SUPER_TMP);
} else {
a.and_(SUPER_TMP, SUPER_TMP, imm(_TAG_PRIMARY_MASK));
a.cmp(SUPER_TMP, imm(TAG_PRIMARY_IMMED1));

/*
* SUPER_TMP will now be TAG_PRIMARY_IMMED1 if either
* one or both registers are immediates, or if one
* register is a list and the other a boxed.
*/
a.b_eq(Unequal);
}
}
}

/* Set the Z flag if Reg1 and Reg2 are both immediates. */
void emit_are_both_immediate(const ArgVal &Src1,
arm::Gp Reg1,
const ArgVal &Src2,
arm::Gp Reg2) {
ERTS_CT_ASSERT(TAG_PRIMARY_IMMED1 == _TAG_PRIMARY_MASK);
if (always_immediate(Src1)) {
a.and_(SUPER_TMP, Reg2, imm(_TAG_PRIMARY_MASK));
} else if (always_immediate(Src2)) {
a.and_(SUPER_TMP, Reg1, imm(_TAG_PRIMARY_MASK));
} else {
a.and_(SUPER_TMP, Reg1, Reg2);
a.and_(SUPER_TMP, SUPER_TMP, imm(_TAG_PRIMARY_MASK));
}
a.cmp(SUPER_TMP, imm(TAG_PRIMARY_IMMED1));
}
};

void beamasm_metadata_update(std::string module_name,
Expand Down
2 changes: 2 additions & 0 deletions erts/emulator/beam/jit/arm/beam_asm_global.hpp.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@
int128_to_big_shared
int_div_rem_body_shared
int_div_rem_guard_shared
is_eq_exact_list_shared
is_eq_exact_shallow_boxed_shared
is_in_range_shared
is_ge_lt_shared
minus_body_shared
Expand Down
77 changes: 50 additions & 27 deletions erts/emulator/beam/jit/arm/instr_arith.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ void BeamModuleAssembler::emit_add_sub_types(bool is_small_result,
if (is_small_result) {
comment("skipped overflow test because the result is always small");
emit_are_both_small(LHS, lhs_reg, RHS, rhs_reg, next);
} else if (RHS.isLiteral()) {
/* Skipping test for small */
} else {
if (always_small(RHS)) {
a.and_(TMP1, lhs_reg, imm(_TAG_IMMED1_MASK));
Expand All @@ -60,8 +62,11 @@ void BeamModuleAssembler::emit_are_both_small(const ArgSource &LHS,
const ArgSource &RHS,
const a64::Gp rhs_reg,
const Label next) {
if (always_small(RHS) &&
always_one_of<BeamTypeId::Integer, BeamTypeId::AlwaysBoxed>(LHS)) {
if (RHS.isLiteral()) {
comment("skipped test for small because one operand is never small");
} else if (always_small(RHS) &&
always_one_of<BeamTypeId::Integer, BeamTypeId::AlwaysBoxed>(
LHS)) {
comment("simplified test for small operand since other types are "
"boxed");
emit_is_boxed(next, lhs_reg);
Expand Down Expand Up @@ -165,7 +170,9 @@ void BeamModuleAssembler::emit_i_plus(const ArgLabel &Fail,

auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3);

if (rhs_is_arm_literal) {
if (RHS.isLiteral()) {
comment("skipped test for small because one operand is never small");
} else if (rhs_is_arm_literal) {
Uint cleared_tag = RHS.as<ArgSmall>().get() & ~_TAG_IMMED1_MASK;
a.adds(ARG1, lhs.reg, imm(cleared_tag));
} else {
Expand Down Expand Up @@ -349,10 +356,11 @@ void BeamModuleAssembler::emit_i_minus(const ArgLabel &Fail,
}

Label next = a.newLabel();

auto [lhs, rhs] = load_sources(LHS, ARG2, RHS, ARG3);

if (rhs_is_arm_literal) {
if (RHS.isLiteral()) {
comment("skipped test for small because one operand is never small");
} else if (rhs_is_arm_literal) {
Uint cleared_tag = RHS.as<ArgSmall>().get() & ~_TAG_IMMED1_MASK;
a.subs(ARG1, lhs.reg, imm(cleared_tag));
} else {
Expand Down Expand Up @@ -535,6 +543,7 @@ void BeamModuleAssembler::emit_i_mul_add(const ArgLabel &Fail,
const ArgRegister &Dst) {
bool is_product_small = is_product_small_if_args_are_small(Src1, Src2);
bool is_sum_small = is_sum_small_if_args_are_small(Src3, Src4);
bool sometimes_small = !(Src2.isLiteral() || Src4.isLiteral());
bool is_increment_zero =
Src4.isSmall() && Src4.as<ArgSmall>().getSigned() == 0;
Sint factor = 0;
Expand Down Expand Up @@ -585,7 +594,7 @@ void BeamModuleAssembler::emit_i_mul_add(const ArgLabel &Fail,
emit_are_both_small(Src1, src1.reg, Src2, src2.reg, small);
} else if (always_small(Src2)) {
emit_are_both_small(Src1, src1.reg, Src4, src4.reg, small);
} else {
} else if (sometimes_small) {
ASSERT(!is_increment_zero);
ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
a.and_(TMP1, src1.reg, src2.reg);
Expand All @@ -602,6 +611,9 @@ void BeamModuleAssembler::emit_i_mul_add(const ArgLabel &Fail,
a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
a.b_eq(small);
}
} else {
comment("skipped test for small because one operand is never "
"small");
}

mov_var(ARG2, src1);
Expand All @@ -625,14 +637,18 @@ void BeamModuleAssembler::emit_i_mul_add(const ArgLabel &Fail,
}
}

a.b(store_result);
if (sometimes_small) {
a.b(store_result);
}
}

a.bind(small);
if (is_increment_zero) {
comment("multiply smalls");
} else {
comment("multiply and add smalls");
if (sometimes_small) {
if (is_increment_zero) {
comment("multiply smalls");
} else {
comment("multiply and add smalls");
}
}

if (is_product_small && is_sum_small) {
Expand All @@ -657,7 +673,7 @@ void BeamModuleAssembler::emit_i_mul_add(const ArgLabel &Fail,
}

comment("skipped test for small result");
} else {
} else if (sometimes_small) {
auto min_increment = std::get<0>(getClampedRange(Src4));

a.and_(TMP3, src1.reg, imm(~_TAG_IMMED1_MASK));
Expand Down Expand Up @@ -1181,22 +1197,29 @@ void BeamModuleAssembler::emit_i_band(const ArgLabel &Fail,
} else {
Label next = a.newLabel();

/* TAG & TAG = TAG, so we don't need to tag it again. */
a.and_(ARG1, lhs.reg, rhs.reg);

ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
if (always_one_of<BeamTypeId::Integer, BeamTypeId::AlwaysBoxed>(LHS) &&
always_one_of<BeamTypeId::Integer, BeamTypeId::AlwaysBoxed>(RHS)) {
comment("simplified test for small operands since other types are "
"boxed");
emit_is_boxed(next, ARG1);
if (RHS.isLiteral()) {
comment("skipped test for small because one operand is never "
"small");
} else {
/* All other term types has at least one zero in the low 4
* bits. Therefore, the result will be a small iff both
* operands are small. */
a.and_(TMP1, ARG1, imm(_TAG_IMMED1_MASK));
a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
a.b_eq(next);
/* TAG & TAG = TAG, so we don't need to tag it again. */
a.and_(ARG1, lhs.reg, rhs.reg);

ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK);
if (always_one_of<BeamTypeId::Integer, BeamTypeId::AlwaysBoxed>(
LHS) &&
always_one_of<BeamTypeId::Integer, BeamTypeId::AlwaysBoxed>(
RHS)) {
comment("simplified test for small operands since other types "
"are boxed");
emit_is_boxed(next, ARG1);
} else {
/* All other term types has at least one zero in the low 4
* bits. Therefore, the result will be a small iff both
* operands are small. */
a.and_(TMP1, ARG1, imm(_TAG_IMMED1_MASK));
a.cmp(TMP1, imm(_TAG_IMMED1_SMALL));
a.b_eq(next);
}
}

mov_var(ARG2, lhs);
Expand Down
Loading
Loading