Skip to content

8274242: Implement fast-path for ASCII-compatible CharsetEncoders on x86 #310

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cl4es wants to merge 1 commit into from
Closed

8274242: Implement fast-path for ASCII-compatible CharsetEncoders on x86 #310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/hotspot/cpu/aarch64/aarch64.ad
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16830,6 +16830,7 @@ instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
iRegI_R0 result, rFlagsReg cr)
%{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(USE_KILL src, USE_KILL dst, USE_KILL len,
KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
Expand Down
3 changes: 3 additions & 0 deletions src/hotspot/cpu/aarch64/matcher_aarch64.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,4 +155,7 @@
return true;
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = false;

#endif // CPU_AARCH64_MATCHER_AARCH64_HPP
3 changes: 3 additions & 0 deletions src/hotspot/cpu/arm/matcher_arm.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,4 +147,7 @@
return false;
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = false;

#endif // CPU_ARM_MATCHER_ARM_HPP
2 changes: 2 additions & 0 deletions src/hotspot/cpu/ppc/matcher_ppc.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,5 +156,7 @@
return VM_Version::has_fcfids();
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = false;

#endif // CPU_PPC_MATCHER_PPC_HPP
1 change: 1 addition & 0 deletions src/hotspot/cpu/ppc/ppc.ad
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12793,6 +12793,7 @@ instruct has_negatives(rarg1RegP ary1, iRegIsrc len, iRegIdst result, iRegLdst t
// encode char[] to byte[] in ISO_8859_1
instruct encode_iso_array(rarg1RegP src, rarg2RegP dst, iRegIsrc len, iRegIdst result, iRegLdst tmp1,
iRegLdst tmp2, iRegLdst tmp3, iRegLdst tmp4, iRegLdst tmp5, regCTR ctr, flagsRegCR0 cr0) %{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP_DEF result, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
USE_KILL src, USE_KILL dst, KILL ctr, KILL cr0);
Expand Down
3 changes: 3 additions & 0 deletions src/hotspot/cpu/s390/matcher_s390.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,4 +144,7 @@
return true;
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = false;

#endif // CPU_S390_MATCHER_S390_HPP
1 change: 1 addition & 0 deletions src/hotspot/cpu/s390/s390.ad
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10277,6 +10277,7 @@ instruct has_negatives(rarg5RegP ary1, iRegI len, iRegI result, roddRegI oddReg,

// encode char[] to byte[] in ISO_8859_1
instruct encode_iso_array(iRegP src, iRegP dst, iRegI result, iRegI len, iRegI tmp, flagsReg cr) %{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP_DEF result, TEMP tmp, KILL cr); // R0, R1 are killed, too.
ins_cost(300);
Expand Down
30 changes: 23 additions & 7 deletions src/hotspot/cpu/x86/macroAssembler_x86.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5614,7 +5614,7 @@ void MacroAssembler::generate_fill(BasicType t, bool aligned,
BIND(L_exit);
}

// encode char[] to byte[] in ISO_8859_1
// encode char[] to byte[] in ISO_8859_1 or ASCII
//@IntrinsicCandidate
//private static int implEncodeISOArray(byte[] sa, int sp,
//byte[] da, int dp, int len) {
Expand All @@ -5627,10 +5627,23 @@ void MacroAssembler::generate_fill(BasicType t, bool aligned,
// }
// return i;
//}
//
//@IntrinsicCandidate
//private static int implEncodeAsciiArray(char[] sa, int sp,
// byte[] da, int dp, int len) {
// int i = 0;
// for (; i < len; i++) {
// char c = sa[sp++];
// if (c >= '\u0080')
// break;
// da[dp++] = (byte)c;
// }
// return i;
//}
void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
XMMRegister tmp1Reg, XMMRegister tmp2Reg,
XMMRegister tmp3Reg, XMMRegister tmp4Reg,
Register tmp5, Register result) {
Register tmp5, Register result, bool ascii) {

// rsi: src
// rdi: dst
Expand All @@ -5641,6 +5654,9 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
assert_different_registers(src, dst, len, tmp5, result);
Label L_done, L_copy_1_char, L_copy_1_char_exit;

int mask = ascii ? 0xff80ff80 : 0xff00ff00;
int short_mask = ascii ? 0xff80 : 0xff00;

// set result
xorl(result, result);
// check for zero length
Expand All @@ -5660,7 +5676,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,

if (UseAVX >= 2) {
Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vector
movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector
movdl(tmp1Reg, tmp5);
vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
jmp(L_chars_32_check);
Expand All @@ -5669,7 +5685,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
vptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector
vptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector
jccb(Assembler::notZero, L_copy_32_chars_exit);
vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
Expand All @@ -5684,7 +5700,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
jccb(Assembler::greater, L_copy_16_chars_exit);

} else if (UseSSE42Intrinsics) {
movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vector
movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector
movdl(tmp1Reg, tmp5);
pshufd(tmp1Reg, tmp1Reg, 0);
jmpb(L_chars_16_check);
Expand All @@ -5708,7 +5724,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
por(tmp2Reg, tmp4Reg);
}
ptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector
ptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector
jccb(Assembler::notZero, L_copy_16_chars_exit);
packuswb(tmp3Reg, tmp4Reg);
}
Expand Down Expand Up @@ -5746,7 +5762,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,

bind(L_copy_1_char);
load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
testl(tmp5, 0xff00); // check if Unicode char
testl(tmp5, short_mask); // check if Unicode or non-ASCII char
jccb(Assembler::notZero, L_copy_1_char_exit);
movb(Address(dst, len, Address::times_1, 0), tmp5);
addptr(len, 1);
Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/cpu/x86/macroAssembler_x86.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1734,7 +1734,7 @@ class MacroAssembler: public Assembler {

void encode_iso_array(Register src, Register dst, Register len,
XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
XMMRegister tmp4, Register tmp5, Register result);
XMMRegister tmp4, Register tmp5, Register result, bool ascii);

#ifdef _LP64
void add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2);
Expand Down
3 changes: 3 additions & 0 deletions src/hotspot/cpu/x86/matcher_x86.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,4 +191,7 @@
return true;
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = true;

#endif // CPU_X86_MATCHER_X86_HPP
21 changes: 19 additions & 2 deletions src/hotspot/cpu/x86/x86_32.ad
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12205,18 +12205,35 @@ instruct string_inflate_evex(Universe dummy, eSIRegP src, eDIRegP dst, eDXRegI l
instruct encode_iso_array(eSIRegP src, eDIRegP dst, eDXRegI len,
regD tmp1, regD tmp2, regD tmp3, regD tmp4,
eCXRegI tmp5, eAXRegI result, eFlagsReg cr) %{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);

format %{ "Encode array $src,$dst,$len -> $result // KILL ECX, EDX, $tmp1, $tmp2, $tmp3, $tmp4, ESI, EDI " %}
format %{ "Encode iso array $src,$dst,$len -> $result // KILL ECX, EDX, $tmp1, $tmp2, $tmp3, $tmp4, ESI, EDI " %}
ins_encode %{
__ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
$tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
%}
ins_pipe( pipe_slow );
%}

// encode char[] to byte[] in ASCII
instruct encode_ascii_array(eSIRegP src, eDIRegP dst, eDXRegI len,
regD tmp1, regD tmp2, regD tmp3, regD tmp4,
eCXRegI tmp5, eAXRegI result, eFlagsReg cr) %{
predicate(((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);

format %{ "Encode ascii array $src,$dst,$len -> $result // KILL ECX, EDX, $tmp1, $tmp2, $tmp3, $tmp4, ESI, EDI " %}
ins_encode %{
__ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
$tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
%}
ins_pipe( pipe_slow );
%}

//----------Control Flow Instructions------------------------------------------
// Signed compare Instructions
Expand Down
22 changes: 20 additions & 2 deletions src/hotspot/cpu/x86/x86_64.ad
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11748,14 +11748,32 @@ instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_Reg
instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);

format %{ "Encode array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
format %{ "Encode iso array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
ins_encode %{
__ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
$tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
%}
ins_pipe( pipe_slow );
%}

// encode char[] to byte[] in ASCII
instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
predicate(((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);

format %{ "Encode ascii array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
ins_encode %{
__ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
$tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
%}
ins_pipe( pipe_slow );
%}
Expand Down
1 change: 1 addition & 0 deletions src/hotspot/share/classfile/vmIntrinsics.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,7 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) {
if (!SpecialArraysEquals) return true;
break;
case vmIntrinsics::_encodeISOArray:
case vmIntrinsics::_encodeAsciiArray:
case vmIntrinsics::_encodeByteISOArray:
if (!SpecialEncodeISOArray) return true;
break;
Expand Down
3 changes: 3 additions & 0 deletions src/hotspot/share/classfile/vmIntrinsics.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,9 @@ class methodHandle;
\
do_intrinsic(_encodeByteISOArray, java_lang_StringCoding, encodeISOArray_name, indexOfI_signature, F_S) \
\
do_intrinsic(_encodeAsciiArray, java_lang_StringCoding, encodeAsciiArray_name, encodeISOArray_signature, F_S) \
do_name( encodeAsciiArray_name, "implEncodeAsciiArray") \
\
do_class(java_math_BigInteger, "java/math/BigInteger") \
do_intrinsic(_multiplyToLen, java_math_BigInteger, multiplyToLen_name, multiplyToLen_signature, F_S) \
do_name( multiplyToLen_name, "implMultiplyToLen") \
Expand Down
3 changes: 3 additions & 0 deletions src/hotspot/share/opto/c2compiler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,9 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method, bool is_virt
case vmIntrinsics::_copyMemory:
if (StubRoutines::unsafe_arraycopy() == NULL) return false;
break;
case vmIntrinsics::_encodeAsciiArray:
if (!Matcher::match_rule_supported(Op_EncodeISOArray) || !Matcher::supports_encode_ascii_array) return false;
break;
case vmIntrinsics::_encodeISOArray:
case vmIntrinsics::_encodeByteISOArray:
if (!Matcher::match_rule_supported(Op_EncodeISOArray)) return false;
Expand Down
8 changes: 6 additions & 2 deletions src/hotspot/share/opto/intrinsicnode.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,10 +168,14 @@ class HasNegativesNode: public StrIntrinsicNode {


//------------------------------EncodeISOArray--------------------------------
// encode char[] to byte[] in ISO_8859_1
// encode char[] to byte[] in ISO_8859_1 or ASCII
class EncodeISOArrayNode: public Node {
bool ascii;
public:
EncodeISOArrayNode(Node* control, Node* arymem, Node* s1, Node* s2, Node* c): Node(control, arymem, s1, s2, c) {};
EncodeISOArrayNode(Node* control, Node* arymem, Node* s1, Node* s2, Node* c, bool ascii)
: Node(control, arymem, s1, s2, c), ascii(ascii) {}

bool is_ascii() { return ascii; }
virtual int Opcode() const;
virtual bool depends_only_on_test() const { return false; }
virtual const Type* bottom_type() const { return TypeInt::INT; }
Expand Down
10 changes: 6 additions & 4 deletions src/hotspot/share/opto/library_call.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -588,7 +588,9 @@ bool LibraryCallKit::try_to_inline(int predicate) {

case vmIntrinsics::_encodeISOArray:
case vmIntrinsics::_encodeByteISOArray:
return inline_encodeISOArray();
return inline_encodeISOArray(false);
case vmIntrinsics::_encodeAsciiArray:
return inline_encodeISOArray(true);

case vmIntrinsics::_updateCRC32:
return inline_updateCRC32();
Expand Down Expand Up @@ -4846,8 +4848,8 @@ LibraryCallKit::tightly_coupled_allocation(Node* ptr) {
}

//-------------inline_encodeISOArray-----------------------------------
// encode char[] to byte[] in ISO_8859_1
bool LibraryCallKit::inline_encodeISOArray() {
// encode char[] to byte[] in ISO_8859_1 or ASCII
bool LibraryCallKit::inline_encodeISOArray(bool ascii) {
assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters");
// no receiver since it is static method
Node *src = argument(0);
Expand Down Expand Up @@ -4882,7 +4884,7 @@ bool LibraryCallKit::inline_encodeISOArray() {
// 'dst_start' points to dst array + scaled offset

const TypeAryPtr* mtype = TypeAryPtr::BYTES;
Node* enc = new EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length);
Node* enc = new EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length, ascii);
enc = _gvn.transform(enc);
Node* res_mem = _gvn.transform(new SCMemProjNode(enc));
set_memory(res_mem, mtype);
Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/share/opto/library_call.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ class LibraryCallKit : public GraphKit {
Node* get_state_from_digest_object(Node *digestBase_object, const char* state_type);
Node* get_digest_length_from_digest_object(Node *digestBase_object);
Node* inline_digestBase_implCompressMB_predicate(int predicate);
bool inline_encodeISOArray();
bool inline_encodeISOArray(bool ascii);
bool inline_updateCRC32();
bool inline_updateBytesCRC32();
bool inline_updateByteBufferCRC32();
Expand Down
16 changes: 15 additions & 1 deletion src/java.base/share/classes/java/lang/StringCoding.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public static boolean hasNegatives(byte[] ba, int off, int len) {

@IntrinsicCandidate
public static int implEncodeISOArray(byte[] sa, int sp,
byte[] da, int dp, int len) {
byte[] da, int dp, int len) {
int i = 0;
for (; i < len; i++) {
char c = StringUTF16.getChar(sa, sp++);
Expand All @@ -57,4 +57,18 @@ public static int implEncodeISOArray(byte[] sa, int sp,
return i;
}

@IntrinsicCandidate
public static int implEncodeAsciiArray(char[] sa, int sp,
byte[] da, int dp, int len)
{
int i = 0;
for (; i < len; i++) {
char c = sa[sp++];
if (c >= '\u0080')
break;
da[dp++] = (byte)c;
}
return i;
}

}
4 changes: 4 additions & 0 deletions src/java.base/share/classes/java/lang/System.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2410,6 +2410,10 @@ public int decodeASCII(byte[] src, int srcOff, char[] dst, int dstOff, int len)
return String.decodeASCII(src, srcOff, dst, dstOff, len);
}

public int encodeASCII(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
return StringCoding.implEncodeAsciiArray(src, srcOff, dst, dstOff, len);
}

public void setCause(Throwable t, Throwable cause) {
t.setCause(cause);
}
Expand Down
9 changes: 9 additions & 0 deletions src/java.base/share/classes/jdk/internal/access/JavaLangAccess.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,15 @@ public interface JavaLangAccess {
*/
int decodeASCII(byte[] src, int srcOff, char[] dst, int dstOff, int len);

/**
* Encodes ASCII codepoints as possible from the source array into
* the destination byte array, assuming that the encoding is ASCII
* compatible
*
* @return the number of bytes successfully encoded, or 0 if none
*/
int encodeASCII(char[] src, int srcOff, byte[] dst, int dstOff, int len);

/**
* Set the cause of Throwable
* @param cause set t's cause to new value
Expand Down
Loading