Skip to content

8350459: MontgomeryIntegerPolynomialP256 multiply intrinsic with AVX2 on x86_64 #23719

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 10 commits into from
Closed

8350459: MontgomeryIntegerPolynomialP256 multiply intrinsic with AVX2 on x86_64 #23719

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7328768
split up ASM and Math changes
vpaprotsk Feb 12, 2025
e080395
whitespace
vpaprotsk Feb 21, 2025
1bc0b8c
comments from Sandhya
vpaprotsk Mar 4, 2025
bb45013
whitespace, correct vpxor, missed comment
vpaprotsk Mar 4, 2025
9d13cef
more comment improvements
vpaprotsk Mar 5, 2025
72650cd
improve test comment
vpaprotsk Mar 17, 2025
56fd168
whitespace
vpaprotsk Mar 17, 2025
c6c597f
prettify test
vpaprotsk Mar 24, 2025
a7f756a
whitespace
vpaprotsk Mar 24, 2025
0a4230a
Fix copyright stmt
vpaprotsk Mar 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions src/hotspot/cpu/x86/assembler_x86.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3498,6 +3498,30 @@ void Assembler::vmovdqu(Address dst, XMMRegister src) {
emit_operand(src, dst, 0);
}

// Move Aligned 256bit Vector
void Assembler::vmovdqa(XMMRegister dst, Address src) {
assert(UseAVX > 0, "");
InstructionMark im(this);
InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
emit_int8(0x6F);
emit_operand(dst, src, 0);
}

void Assembler::vmovdqa(Address dst, XMMRegister src) {
assert(UseAVX > 0, "");
InstructionMark im(this);
InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
attributes.reset_is_clear_context();
// swap src<->dst for encoding
assert(src != xnoreg, "sanity");
vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
emit_int8(0x7F);
emit_operand(src, dst, 0);
}

void Assembler::vpmaskmovd(XMMRegister dst, XMMRegister mask, Address src, int vector_len) {
assert((VM_Version::supports_avx2() && vector_len == AVX_256bit), "");
InstructionMark im(this);
Expand Down Expand Up @@ -3760,6 +3784,27 @@ void Assembler::evmovdquq(XMMRegister dst, KRegister mask, Address src, bool mer
emit_operand(dst, src, 0);
}

// Move Aligned 512bit Vector
void Assembler::evmovdqaq(XMMRegister dst, Address src, int vector_len) {
// Unmasked instruction
evmovdqaq(dst, k0, src, /*merge*/ false, vector_len);
}

void Assembler::evmovdqaq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) {
assert(VM_Version::supports_evex(), "");
InstructionMark im(this);
InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
attributes.set_embedded_opmask_register_specifier(mask);
attributes.set_is_evex_instruction();
if (merge) {
attributes.reset_is_clear_context();
}
vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
emit_int8(0x6F);
emit_operand(dst, src, 0);
}

void Assembler::evmovntdquq(Address dst, XMMRegister src, int vector_len) {
// Unmasked instruction
evmovntdquq(dst, k0, src, /*merge*/ true, vector_len);
Expand Down
10 changes: 9 additions & 1 deletion src/hotspot/cpu/x86/assembler_x86.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -1755,6 +1755,10 @@ class Assembler : public AbstractAssembler {
void vmovdqu(XMMRegister dst, Address src);
void vmovdqu(XMMRegister dst, XMMRegister src);

// Move Aligned 256bit Vector
void vmovdqa(XMMRegister dst, Address src);
void vmovdqa(Address dst, XMMRegister src);

// Move Unaligned 512bit Vector
void evmovdqub(XMMRegister dst, XMMRegister src, int vector_len);
void evmovdqub(XMMRegister dst, Address src, int vector_len);
Expand Down Expand Up @@ -1788,6 +1792,10 @@ class Assembler : public AbstractAssembler {
void evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len);
void evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len);

// Move Aligned 512bit Vector
void evmovdqaq(XMMRegister dst, Address src, int vector_len);
void evmovdqaq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len);

// Move lower 64bit to high 64bit in 128bit register
void movlhps(XMMRegister dst, XMMRegister src);

Expand Down
77 changes: 77 additions & 0 deletions src/hotspot/cpu/x86/macroAssembler_x86.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2696,6 +2696,60 @@ void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len
}
}

void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src, int vector_len) {
if (vector_len == AVX_512bit) {
evmovdquq(dst, src, AVX_512bit);
} else if (vector_len == AVX_256bit) {
vmovdqu(dst, src);
} else {
movdqu(dst, src);
}
}

void MacroAssembler::vmovdqu(Address dst, XMMRegister src, int vector_len) {
if (vector_len == AVX_512bit) {
evmovdquq(dst, src, AVX_512bit);
} else if (vector_len == AVX_256bit) {
vmovdqu(dst, src);
} else {
movdqu(dst, src);
}
}

void MacroAssembler::vmovdqu(XMMRegister dst, Address src, int vector_len) {
if (vector_len == AVX_512bit) {
evmovdquq(dst, src, AVX_512bit);
} else if (vector_len == AVX_256bit) {
vmovdqu(dst, src);
} else {
movdqu(dst, src);
}
}

void MacroAssembler::vmovdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");

if (reachable(src)) {
vmovdqa(dst, as_Address(src));
}
else {
lea(rscratch, src);
vmovdqa(dst, Address(rscratch, 0));
}
}

void MacroAssembler::vmovdqa(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");

if (vector_len == AVX_512bit) {
evmovdqaq(dst, src, AVX_512bit, rscratch);
} else if (vector_len == AVX_256bit) {
vmovdqa(dst, src, rscratch);
} else {
movdqa(dst, src, rscratch);
}
}

void MacroAssembler::kmov(KRegister dst, Address src) {
if (VM_Version::supports_avx512bw()) {
kmovql(dst, src);
Expand Down Expand Up @@ -2820,6 +2874,29 @@ void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_l
}
}

void MacroAssembler::evmovdqaq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");

if (reachable(src)) {
Assembler::evmovdqaq(dst, mask, as_Address(src), merge, vector_len);
} else {
lea(rscratch, src);
Assembler::evmovdqaq(dst, mask, Address(rscratch, 0), merge, vector_len);
}
}

void MacroAssembler::evmovdqaq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");

if (reachable(src)) {
Assembler::evmovdqaq(dst, as_Address(src), vector_len);
} else {
lea(rscratch, src);
Assembler::evmovdqaq(dst, Address(rscratch, 0), vector_len);
}
}


void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");

Expand Down
10 changes: 10 additions & 0 deletions src/hotspot/cpu/x86/macroAssembler_x86.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1347,6 +1347,14 @@ class MacroAssembler: public Assembler {
void vmovdqu(XMMRegister dst, XMMRegister src);
void vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
void vmovdqu(XMMRegister dst, XMMRegister src, int vector_len);
void vmovdqu(XMMRegister dst, Address src, int vector_len);
void vmovdqu(Address dst, XMMRegister src, int vector_len);

// AVX Aligned forms
using Assembler::vmovdqa;
void vmovdqa(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void vmovdqa(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);

// AVX512 Unaligned
void evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len);
Expand Down Expand Up @@ -1403,6 +1411,7 @@ class MacroAssembler: public Assembler {
void evmovdquq(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdquq(dst, src, vector_len); }
void evmovdquq(Address dst, XMMRegister src, int vector_len) { Assembler::evmovdquq(dst, src, vector_len); }
void evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
void evmovdqaq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);

void evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
if (dst->encoding() != src->encoding() || mask != k0) {
Expand All @@ -1412,6 +1421,7 @@ class MacroAssembler: public Assembler {
void evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdquq(dst, mask, src, merge, vector_len); }
void evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdquq(dst, mask, src, merge, vector_len); }
void evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
void evmovdqaq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);

// Move Aligned Double Quadword
void movdqa(XMMRegister dst, XMMRegister src) { Assembler::movdqa(dst, src); }
Expand Down
Loading