Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

8274242: Implement fast-path for ASCII-compatible CharsetEncoders on x86 #5621

Closed
wants to merge 15 commits into from
Closed
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
@@ -16855,6 +16855,7 @@ instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
iRegI_R0 result, rFlagsReg cr)
%{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(USE_KILL src, USE_KILL dst, USE_KILL len,
KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
@@ -160,4 +160,7 @@
return true;
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = false;

#endif // CPU_AARCH64_MATCHER_AARCH64_HPP
@@ -152,4 +152,7 @@
return false;
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = false;

#endif // CPU_ARM_MATCHER_ARM_HPP
@@ -161,5 +161,7 @@
return VM_Version::has_fcfids();
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = false;

#endif // CPU_PPC_MATCHER_PPC_HPP
@@ -12789,6 +12789,7 @@ instruct has_negatives(rarg1RegP ary1, iRegIsrc len, iRegIdst result, iRegLdst t
// encode char[] to byte[] in ISO_8859_1
instruct encode_iso_array(rarg1RegP src, rarg2RegP dst, iRegIsrc len, iRegIdst result, iRegLdst tmp1,
iRegLdst tmp2, iRegLdst tmp3, iRegLdst tmp4, iRegLdst tmp5, regCTR ctr, flagsRegCR0 cr0) %{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP_DEF result, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
USE_KILL src, USE_KILL dst, KILL ctr, KILL cr0);
@@ -149,4 +149,7 @@
return true;
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = false;

#endif // CPU_S390_MATCHER_S390_HPP
@@ -10282,6 +10282,7 @@ instruct has_negatives(rarg5RegP ary1, iRegI len, iRegI result, roddRegI oddReg,

// encode char[] to byte[] in ISO_8859_1
instruct encode_iso_array(iRegP src, iRegP dst, iRegI result, iRegI len, iRegI tmp, flagsReg cr) %{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP_DEF result, TEMP tmp, KILL cr); // R0, R1 are killed, too.
ins_cost(300);
@@ -5414,7 +5414,7 @@ void MacroAssembler::generate_fill(BasicType t, bool aligned,
BIND(L_exit);
}

// encode char[] to byte[] in ISO_8859_1
// encode char[] to byte[] in ISO_8859_1 or ASCII
//@IntrinsicCandidate
//private static int implEncodeISOArray(byte[] sa, int sp,
//byte[] da, int dp, int len) {
@@ -5427,10 +5427,23 @@ void MacroAssembler::generate_fill(BasicType t, bool aligned,
// }
// return i;
//}
//
//@IntrinsicCandidate
//private static int implEncodeAsciiArray(char[] sa, int sp,
// byte[] da, int dp, int len) {
// int i = 0;
// for (; i < len; i++) {
// char c = sa[sp++];
// if (c >= '\u0080')
// break;
// da[dp++] = (byte)c;
// }
// return i;
//}
void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
XMMRegister tmp1Reg, XMMRegister tmp2Reg,
XMMRegister tmp3Reg, XMMRegister tmp4Reg,
Register tmp5, Register result) {
Register tmp5, Register result, bool ascii) {

// rsi: src
// rdi: dst
@@ -5441,6 +5454,9 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
assert_different_registers(src, dst, len, tmp5, result);
Label L_done, L_copy_1_char, L_copy_1_char_exit;

int mask = ascii ? 0xff80ff80 : 0xff00ff00;
int short_mask = ascii ? 0xff80 : 0xff00;

// set result
xorl(result, result);
// check for zero length
@@ -5460,7 +5476,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,

if (UseAVX >= 2) {
Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vector
movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector
movdl(tmp1Reg, tmp5);
vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
jmp(L_chars_32_check);
@@ -5469,7 +5485,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
vptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector
vptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector
jccb(Assembler::notZero, L_copy_32_chars_exit);
vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
@@ -5484,7 +5500,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
jccb(Assembler::greater, L_copy_16_chars_exit);

} else if (UseSSE42Intrinsics) {
movl(tmp5, 0xff00ff00); // create mask to test for Unicode chars in vector
movl(tmp5, mask); // create mask to test for Unicode or non-ASCII chars in vector
movdl(tmp1Reg, tmp5);
pshufd(tmp1Reg, tmp1Reg, 0);
jmpb(L_chars_16_check);
@@ -5508,7 +5524,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
por(tmp2Reg, tmp4Reg);
}
ptest(tmp2Reg, tmp1Reg); // check for Unicode chars in vector
ptest(tmp2Reg, tmp1Reg); // check for Unicode or non-ASCII chars in vector
jccb(Assembler::notZero, L_copy_16_chars_exit);
packuswb(tmp3Reg, tmp4Reg);
}
@@ -5546,7 +5562,7 @@ void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,

bind(L_copy_1_char);
load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
testl(tmp5, 0xff00); // check if Unicode char
testl(tmp5, short_mask); // check if Unicode or non-ASCII char
jccb(Assembler::notZero, L_copy_1_char_exit);
movb(Address(dst, len, Address::times_1, 0), tmp5);
addptr(len, 1);
@@ -1724,7 +1724,7 @@ class MacroAssembler: public Assembler {

void encode_iso_array(Register src, Register dst, Register len,
XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
XMMRegister tmp4, Register tmp5, Register result);
XMMRegister tmp4, Register tmp5, Register result, bool ascii);

#ifdef _LP64
void add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2);
@@ -192,4 +192,7 @@
return true;
}

// Implements a variant of EncodeISOArrayNode that encode ASCII only
static const bool supports_encode_ascii_array = true;

#endif // CPU_X86_MATCHER_X86_HPP
@@ -12199,18 +12199,35 @@ instruct string_inflate_evex(Universe dummy, eSIRegP src, eDIRegP dst, eDXRegI l
instruct encode_iso_array(eSIRegP src, eDIRegP dst, eDXRegI len,
regD tmp1, regD tmp2, regD tmp3, regD tmp4,
eCXRegI tmp5, eAXRegI result, eFlagsReg cr) %{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);

format %{ "Encode array $src,$dst,$len -> $result // KILL ECX, EDX, $tmp1, $tmp2, $tmp3, $tmp4, ESI, EDI " %}
format %{ "Encode iso array $src,$dst,$len -> $result // KILL ECX, EDX, $tmp1, $tmp2, $tmp3, $tmp4, ESI, EDI " %}
ins_encode %{
__ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
$tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
%}
ins_pipe( pipe_slow );
%}

// encode char[] to byte[] in ASCII
instruct encode_ascii_array(eSIRegP src, eDIRegP dst, eDXRegI len,
regD tmp1, regD tmp2, regD tmp3, regD tmp4,
eCXRegI tmp5, eAXRegI result, eFlagsReg cr) %{
predicate(((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);

format %{ "Encode ascii array $src,$dst,$len -> $result // KILL ECX, EDX, $tmp1, $tmp2, $tmp3, $tmp4, ESI, EDI " %}
ins_encode %{
__ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
$tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
%}
ins_pipe( pipe_slow );
%}

//----------Control Flow Instructions------------------------------------------
// Signed compare Instructions
@@ -11770,14 +11770,32 @@ instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_Reg
instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
predicate(!((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);

format %{ "Encode array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
format %{ "Encode iso array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
ins_encode %{
__ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
$tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
%}
ins_pipe( pipe_slow );
%}

// encode char[] to byte[] in ASCII
instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
predicate(((EncodeISOArrayNode*)n)->is_ascii());
match(Set result (EncodeISOArray src (Binary dst len)));
effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);

format %{ "Encode ascii array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
ins_encode %{
__ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
$tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
$tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
%}
ins_pipe( pipe_slow );
%}
@@ -505,6 +505,7 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) {
if (!SpecialArraysEquals) return true;
break;
case vmIntrinsics::_encodeISOArray:
case vmIntrinsics::_encodeAsciiArray:
case vmIntrinsics::_encodeByteISOArray:
if (!SpecialEncodeISOArray) return true;
break;
@@ -353,6 +353,9 @@ class methodHandle;
\
do_intrinsic(_encodeByteISOArray, java_lang_StringCoding, encodeISOArray_name, indexOfI_signature, F_S) \
\
do_intrinsic(_encodeAsciiArray, java_lang_StringCoding, encodeAsciiArray_name, encodeISOArray_signature, F_S) \
do_name( encodeAsciiArray_name, "implEncodeAsciiArray") \
\
do_class(java_math_BigInteger, "java/math/BigInteger") \
do_intrinsic(_multiplyToLen, java_math_BigInteger, multiplyToLen_name, multiplyToLen_signature, F_S) \
do_name( multiplyToLen_name, "implMultiplyToLen") \
@@ -216,6 +216,9 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method, bool is_virt
case vmIntrinsics::_copyMemory:
if (StubRoutines::unsafe_arraycopy() == NULL) return false;
break;
case vmIntrinsics::_encodeAsciiArray:
if (!Matcher::match_rule_supported(Op_EncodeISOArray) || !Matcher::supports_encode_ascii_array) return false;
break;
case vmIntrinsics::_encodeISOArray:
case vmIntrinsics::_encodeByteISOArray:
if (!Matcher::match_rule_supported(Op_EncodeISOArray)) return false;
@@ -168,10 +168,14 @@ class HasNegativesNode: public StrIntrinsicNode {


//------------------------------EncodeISOArray--------------------------------
// encode char[] to byte[] in ISO_8859_1
// encode char[] to byte[] in ISO_8859_1 or ASCII
class EncodeISOArrayNode: public Node {
bool ascii;
public:
EncodeISOArrayNode(Node* control, Node* arymem, Node* s1, Node* s2, Node* c): Node(control, arymem, s1, s2, c) {};
EncodeISOArrayNode(Node* control, Node* arymem, Node* s1, Node* s2, Node* c, bool ascii)
: Node(control, arymem, s1, s2, c), ascii(ascii) {}

bool is_ascii() { return ascii; }
virtual int Opcode() const;
virtual bool depends_only_on_test() const { return false; }
virtual const Type* bottom_type() const { return TypeInt::INT; }
@@ -591,7 +591,9 @@ bool LibraryCallKit::try_to_inline(int predicate) {

case vmIntrinsics::_encodeISOArray:
case vmIntrinsics::_encodeByteISOArray:
return inline_encodeISOArray();
return inline_encodeISOArray(false);
case vmIntrinsics::_encodeAsciiArray:
return inline_encodeISOArray(true);

case vmIntrinsics::_updateCRC32:
return inline_updateCRC32();
@@ -4882,8 +4884,8 @@ LibraryCallKit::tightly_coupled_allocation(Node* ptr) {
}

//-------------inline_encodeISOArray-----------------------------------
// encode char[] to byte[] in ISO_8859_1
bool LibraryCallKit::inline_encodeISOArray() {
// encode char[] to byte[] in ISO_8859_1 or ASCII
bool LibraryCallKit::inline_encodeISOArray(bool ascii) {
assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters");
// no receiver since it is static method
Node *src = argument(0);
@@ -4918,7 +4920,7 @@ bool LibraryCallKit::inline_encodeISOArray() {
// 'dst_start' points to dst array + scaled offset

const TypeAryPtr* mtype = TypeAryPtr::BYTES;
Node* enc = new EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length);
Node* enc = new EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length, ascii);
enc = _gvn.transform(enc);
Node* res_mem = _gvn.transform(new SCMemProjNode(enc));
set_memory(res_mem, mtype);
@@ -285,7 +285,7 @@ class LibraryCallKit : public GraphKit {
Node* get_state_from_digest_object(Node *digestBase_object, const char* state_type);
Node* get_digest_length_from_digest_object(Node *digestBase_object);
Node* inline_digestBase_implCompressMB_predicate(int predicate);
bool inline_encodeISOArray();
bool inline_encodeISOArray(bool ascii);
bool inline_updateCRC32();
bool inline_updateBytesCRC32();
bool inline_updateByteBufferCRC32();
@@ -46,7 +46,7 @@ public static boolean hasNegatives(byte[] ba, int off, int len) {

@IntrinsicCandidate
public static int implEncodeISOArray(byte[] sa, int sp,
byte[] da, int dp, int len) {
byte[] da, int dp, int len) {
int i = 0;
for (; i < len; i++) {
char c = StringUTF16.getChar(sa, sp++);
@@ -57,4 +57,18 @@ public static int implEncodeISOArray(byte[] sa, int sp,
return i;
}

@IntrinsicCandidate
public static int implEncodeAsciiArray(char[] sa, int sp,
byte[] da, int dp, int len)
{
int i = 0;
for (; i < len; i++) {
char c = sa[sp++];
if (c >= '\u0080')
break;
da[dp++] = (byte)c;
}
return i;
}

}
@@ -2419,6 +2419,10 @@ public int decodeASCII(byte[] src, int srcOff, char[] dst, int dstOff, int len)
return String.decodeASCII(src, srcOff, dst, dstOff, len);
}

public int encodeASCII(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
return StringCoding.implEncodeAsciiArray(src, srcOff, dst, dstOff, len);
}

public void setCause(Throwable t, Throwable cause) {
t.setCause(cause);
}
@@ -356,6 +356,15 @@ public interface JavaLangAccess {
*/
int decodeASCII(byte[] src, int srcOff, char[] dst, int dstOff, int len);

/**
* Encodes ASCII codepoints as possible from the source array into
* the destination byte array, assuming that the encoding is ASCII
* compatible
*
* @return the number of bytes successfully encoded, or 0 if none
*/
int encodeASCII(char[] src, int srcOff, byte[] dst, int dstOff, int len);

/**
* Set the cause of Throwable
* @param cause set t's cause to new value