Skip to content

Commit 5ec1b80

Browse files
RealFYangYanhong Zhu
RealFYang
and
Yanhong Zhu
committed
8255287: aarch64: fix SVE patterns for vector shift count
Co-authored-by: Yanhong Zhu <zhuyanhong2@huawei.com> Reviewed-by: adinn
1 parent 12daf2b commit 5ec1b80

File tree

7 files changed

+149
-40
lines changed

7 files changed

+149
-40
lines changed

‎src/hotspot/cpu/aarch64/aarch64_sve.ad

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1329,7 +1329,7 @@ instruct vlsrL(vReg dst, vReg shift) %{
13291329

13301330
instruct vasrB_imm(vReg dst, vReg src, immI shift) %{
13311331
predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
1332-
match(Set dst (RShiftVB src shift));
1332+
match(Set dst (RShiftVB src (RShiftCntV shift)));
13331333
ins_cost(SVE_COST);
13341334
format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (B)" %}
13351335
ins_encode %{
@@ -1348,7 +1348,7 @@ instruct vasrB_imm(vReg dst, vReg src, immI shift) %{
13481348

13491349
instruct vasrS_imm(vReg dst, vReg src, immI shift) %{
13501350
predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
1351-
match(Set dst (RShiftVS src shift));
1351+
match(Set dst (RShiftVS src (RShiftCntV shift)));
13521352
ins_cost(SVE_COST);
13531353
format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (H)" %}
13541354
ins_encode %{
@@ -1367,7 +1367,7 @@ instruct vasrS_imm(vReg dst, vReg src, immI shift) %{
13671367

13681368
instruct vasrI_imm(vReg dst, vReg src, immI shift) %{
13691369
predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
1370-
match(Set dst (RShiftVI src shift));
1370+
match(Set dst (RShiftVI src (RShiftCntV shift)));
13711371
ins_cost(SVE_COST);
13721372
format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (S)" %}
13731373
ins_encode %{
@@ -1385,7 +1385,7 @@ instruct vasrI_imm(vReg dst, vReg src, immI shift) %{
13851385

13861386
instruct vasrL_imm(vReg dst, vReg src, immI shift) %{
13871387
predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
1388-
match(Set dst (RShiftVL src shift));
1388+
match(Set dst (RShiftVL src (RShiftCntV shift)));
13891389
ins_cost(SVE_COST);
13901390
format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (D)" %}
13911391
ins_encode %{
@@ -1403,7 +1403,7 @@ instruct vasrL_imm(vReg dst, vReg src, immI shift) %{
14031403

14041404
instruct vlsrB_imm(vReg dst, vReg src, immI shift) %{
14051405
predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
1406-
match(Set dst (URShiftVB src shift));
1406+
match(Set dst (URShiftVB src (RShiftCntV shift)));
14071407
ins_cost(SVE_COST);
14081408
format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (B)" %}
14091409
ins_encode %{
@@ -1426,7 +1426,7 @@ instruct vlsrB_imm(vReg dst, vReg src, immI shift) %{
14261426

14271427
instruct vlsrS_imm(vReg dst, vReg src, immI shift) %{
14281428
predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
1429-
match(Set dst (URShiftVS src shift));
1429+
match(Set dst (URShiftVS src (RShiftCntV shift)));
14301430
ins_cost(SVE_COST);
14311431
format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (H)" %}
14321432
ins_encode %{
@@ -1436,7 +1436,7 @@ instruct vlsrS_imm(vReg dst, vReg src, immI shift) %{
14361436
as_FloatRegister($src$$reg));
14371437
return;
14381438
}
1439-
if (con >= 8) {
1439+
if (con >= 16) {
14401440
__ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
14411441
as_FloatRegister($src$$reg));
14421442
return;
@@ -1449,7 +1449,7 @@ instruct vlsrS_imm(vReg dst, vReg src, immI shift) %{
14491449

14501450
instruct vlsrI_imm(vReg dst, vReg src, immI shift) %{
14511451
predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
1452-
match(Set dst (URShiftVI src shift));
1452+
match(Set dst (URShiftVI src (RShiftCntV shift)));
14531453
ins_cost(SVE_COST);
14541454
format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (S)" %}
14551455
ins_encode %{
@@ -1467,7 +1467,7 @@ instruct vlsrI_imm(vReg dst, vReg src, immI shift) %{
14671467

14681468
instruct vlsrL_imm(vReg dst, vReg src, immI shift) %{
14691469
predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
1470-
match(Set dst (URShiftVL src shift));
1470+
match(Set dst (URShiftVL src (RShiftCntV shift)));
14711471
ins_cost(SVE_COST);
14721472
format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (D)" %}
14731473
ins_encode %{
@@ -1485,7 +1485,7 @@ instruct vlsrL_imm(vReg dst, vReg src, immI shift) %{
14851485

14861486
instruct vlslB_imm(vReg dst, vReg src, immI shift) %{
14871487
predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
1488-
match(Set dst (LShiftVB src shift));
1488+
match(Set dst (LShiftVB src (LShiftCntV shift)));
14891489
ins_cost(SVE_COST);
14901490
format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (B)" %}
14911491
ins_encode %{
@@ -1503,12 +1503,12 @@ instruct vlslB_imm(vReg dst, vReg src, immI shift) %{
15031503

15041504
instruct vlslS_imm(vReg dst, vReg src, immI shift) %{
15051505
predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
1506-
match(Set dst (LShiftVS src shift));
1506+
match(Set dst (LShiftVS src (LShiftCntV shift)));
15071507
ins_cost(SVE_COST);
15081508
format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (H)" %}
15091509
ins_encode %{
15101510
int con = (int)$shift$$constant;
1511-
if (con >= 8) {
1511+
if (con >= 16) {
15121512
__ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
15131513
as_FloatRegister($src$$reg));
15141514
return;
@@ -1521,7 +1521,7 @@ instruct vlslS_imm(vReg dst, vReg src, immI shift) %{
15211521

15221522
instruct vlslI_imm(vReg dst, vReg src, immI shift) %{
15231523
predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
1524-
match(Set dst (LShiftVI src shift));
1524+
match(Set dst (LShiftVI src (LShiftCntV shift)));
15251525
ins_cost(SVE_COST);
15261526
format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (S)" %}
15271527
ins_encode %{
@@ -1534,7 +1534,7 @@ instruct vlslI_imm(vReg dst, vReg src, immI shift) %{
15341534

15351535
instruct vlslL_imm(vReg dst, vReg src, immI shift) %{
15361536
predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
1537-
match(Set dst (LShiftVL src shift));
1537+
match(Set dst (LShiftVL src (LShiftCntV shift)));
15381538
ins_cost(SVE_COST);
15391539
format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (D)" %}
15401540
ins_encode %{

‎src/hotspot/cpu/aarch64/aarch64_sve_ad.m4

Lines changed: 27 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -727,14 +727,14 @@ instruct $1(vReg dst, vReg shift) %{
727727
ins_pipe(pipe_slow);
728728
%}')dnl
729729
dnl
730-
dnl VSHIFT_IMM_UNPREDICATE($1, $2, $3, $4, $5 )
731-
dnl VSHIFT_IMM_UNPREDICATE(insn_name, op_name, size, min_vec_len, insn)
730+
dnl VSHIFT_IMM_UNPREDICATE($1, $2, $3, $4, $5, $6 )
731+
dnl VSHIFT_IMM_UNPREDICATE(insn_name, op_name, op_name2, size, min_vec_len, insn)
732732
define(`VSHIFT_IMM_UNPREDICATE', `
733733
instruct $1(vReg dst, vReg src, immI shift) %{
734-
predicate(UseSVE > 0 && n->as_Vector()->length() >= $4);
735-
match(Set dst ($2 src shift));
734+
predicate(UseSVE > 0 && n->as_Vector()->length() >= $5);
735+
match(Set dst ($2 src ($3 shift)));
736736
ins_cost(SVE_COST);
737-
format %{ "$5 $dst, $src, $shift\t# vector (sve) ($3)" %}
737+
format %{ "$6 $dst, $src, $shift\t# vector (sve) ($4)" %}
738738
ins_encode %{
739739
int con = (int)$shift$$constant;dnl
740740
ifelse(eval(index(`$1', `vasr') == 0 || index(`$1', `vlsr') == 0), 1, `
@@ -743,16 +743,21 @@ ifelse(eval(index(`$1', `vasr') == 0 || index(`$1', `vlsr') == 0), 1, `
743743
as_FloatRegister($src$$reg));
744744
return;
745745
}')dnl
746-
ifelse(eval(index(`$1', `vasr') == 0), 1, `ifelse(eval(index(`$3', `B') == 0), 1, `
747-
if (con >= 8) con = 7;')ifelse(eval(index(`$3', `H') == 0), 1, `
746+
ifelse(eval(index(`$1', `vasr') == 0), 1, `ifelse(eval(index(`$4', `B') == 0), 1, `
747+
if (con >= 8) con = 7;')ifelse(eval(index(`$4', `H') == 0), 1, `
748748
if (con >= 16) con = 15;')')dnl
749-
ifelse(eval((index(`$1', `vlsl') == 0 || index(`$1', `vlsr') == 0) && (index(`$3', `B') == 0 || index(`$3', `H') == 0)), 1, `
749+
ifelse(eval(index(`$1', `vlsl') == 0 || index(`$1', `vlsr') == 0), 1, `ifelse(eval(index(`$4', `B') == 0), 1, `
750750
if (con >= 8) {
751751
__ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
752752
as_FloatRegister($src$$reg));
753753
return;
754-
}')
755-
__ $5(as_FloatRegister($dst$$reg), __ $3,
754+
}')ifelse(eval(index(`$4', `H') == 0), 1, `
755+
if (con >= 16) {
756+
__ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
757+
as_FloatRegister($src$$reg));
758+
return;
759+
}')')
760+
__ $6(as_FloatRegister($dst$$reg), __ $4,
756761
as_FloatRegister($src$$reg), con);
757762
%}
758763
ins_pipe(pipe_slow);
@@ -786,18 +791,18 @@ VSHIFT_TRUE_PREDICATE(vlsrB, URShiftVB, B, 16, sve_lsr)
786791
VSHIFT_TRUE_PREDICATE(vlsrS, URShiftVS, H, 8, sve_lsr)
787792
VSHIFT_TRUE_PREDICATE(vlsrI, URShiftVI, S, 4, sve_lsr)
788793
VSHIFT_TRUE_PREDICATE(vlsrL, URShiftVL, D, 2, sve_lsr)
789-
VSHIFT_IMM_UNPREDICATE(vasrB_imm, RShiftVB, B, 16, sve_asr)
790-
VSHIFT_IMM_UNPREDICATE(vasrS_imm, RShiftVS, H, 8, sve_asr)
791-
VSHIFT_IMM_UNPREDICATE(vasrI_imm, RShiftVI, S, 4, sve_asr)
792-
VSHIFT_IMM_UNPREDICATE(vasrL_imm, RShiftVL, D, 2, sve_asr)
793-
VSHIFT_IMM_UNPREDICATE(vlsrB_imm, URShiftVB, B, 16, sve_lsr)
794-
VSHIFT_IMM_UNPREDICATE(vlsrS_imm, URShiftVS, H, 8, sve_lsr)
795-
VSHIFT_IMM_UNPREDICATE(vlsrI_imm, URShiftVI, S, 4, sve_lsr)
796-
VSHIFT_IMM_UNPREDICATE(vlsrL_imm, URShiftVL, D, 2, sve_lsr)
797-
VSHIFT_IMM_UNPREDICATE(vlslB_imm, LShiftVB, B, 16, sve_lsl)
798-
VSHIFT_IMM_UNPREDICATE(vlslS_imm, LShiftVS, H, 8, sve_lsl)
799-
VSHIFT_IMM_UNPREDICATE(vlslI_imm, LShiftVI, S, 4, sve_lsl)
800-
VSHIFT_IMM_UNPREDICATE(vlslL_imm, LShiftVL, D, 2, sve_lsl)
794+
VSHIFT_IMM_UNPREDICATE(vasrB_imm, RShiftVB, RShiftCntV, B, 16, sve_asr)
795+
VSHIFT_IMM_UNPREDICATE(vasrS_imm, RShiftVS, RShiftCntV, H, 8, sve_asr)
796+
VSHIFT_IMM_UNPREDICATE(vasrI_imm, RShiftVI, RShiftCntV, S, 4, sve_asr)
797+
VSHIFT_IMM_UNPREDICATE(vasrL_imm, RShiftVL, RShiftCntV, D, 2, sve_asr)
798+
VSHIFT_IMM_UNPREDICATE(vlsrB_imm, URShiftVB, RShiftCntV, B, 16, sve_lsr)
799+
VSHIFT_IMM_UNPREDICATE(vlsrS_imm, URShiftVS, RShiftCntV, H, 8, sve_lsr)
800+
VSHIFT_IMM_UNPREDICATE(vlsrI_imm, URShiftVI, RShiftCntV, S, 4, sve_lsr)
801+
VSHIFT_IMM_UNPREDICATE(vlsrL_imm, URShiftVL, RShiftCntV, D, 2, sve_lsr)
802+
VSHIFT_IMM_UNPREDICATE(vlslB_imm, LShiftVB, LShiftCntV, B, 16, sve_lsl)
803+
VSHIFT_IMM_UNPREDICATE(vlslS_imm, LShiftVS, LShiftCntV, H, 8, sve_lsl)
804+
VSHIFT_IMM_UNPREDICATE(vlslI_imm, LShiftVI, LShiftCntV, S, 4, sve_lsl)
805+
VSHIFT_IMM_UNPREDICATE(vlslL_imm, LShiftVL, LShiftCntV, D, 2, sve_lsl)
801806
VSHIFT_COUNT(vshiftcntB, B, 16, T_BYTE)
802807
VSHIFT_COUNT(vshiftcntS, H, 8, T_SHORT)
803808
VSHIFT_COUNT(vshiftcntI, S, 4, T_INT)

‎src/hotspot/cpu/aarch64/vm_version_aarch64.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,7 @@ void VM_Version::initialize() {
401401
warning("SVE does not support vector length less than 16 bytes. Disabling SVE.");
402402
UseSVE = 0;
403403
} else if ((MaxVectorSize % 16) == 0 && is_power_of_2(MaxVectorSize)) {
404-
int new_vl = set_and_get_current_sve_vector_lenght(MaxVectorSize);
404+
int new_vl = set_and_get_current_sve_vector_length(MaxVectorSize);
405405
_initial_sve_vector_length = new_vl;
406406
// Update MaxVectorSize to the largest supported value.
407407
if (new_vl < 0) {

‎src/hotspot/cpu/aarch64/vm_version_aarch64.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ class VM_Version : public Abstract_VM_Version {
5151
// Sets the SVE length and returns a new actual value or negative on error.
5252
// If the len is larger than the system largest supported SVE vector length,
5353
// the function sets the largest supported value.
54-
static int set_and_get_current_sve_vector_lenght(int len);
54+
static int set_and_get_current_sve_vector_length(int len);
5555
static int get_current_sve_vector_length();
5656

5757
public:

‎src/hotspot/os_cpu/linux_aarch64/vm_version_linux_aarch64.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ int VM_Version::get_current_sve_vector_length() {
8686
return prctl(PR_SVE_GET_VL);
8787
}
8888

89-
int VM_Version::set_and_get_current_sve_vector_lenght(int length) {
89+
int VM_Version::set_and_get_current_sve_vector_length(int length) {
9090
assert(_features & CPU_SVE, "should not call this");
9191
int new_length = prctl(PR_SVE_SET_VL, length);
9292
return new_length;

‎src/hotspot/os_cpu/windows_aarch64/vm_version_windows_aarch64.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ int VM_Version::get_current_sve_vector_length() {
3232
return 0;
3333
}
3434

35-
int VM_Version::set_and_get_current_sve_vector_lenght(int length) {
35+
int VM_Version::set_and_get_current_sve_vector_length(int length) {
3636
assert(_features & CPU_SVE, "should not call this");
3737
ShouldNotReachHere();
3838
return 0;

‎test/hotspot/jtreg/compiler/c2/aarch64/TestVectorShiftShorts.java

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/*
2+
* Copyright (c) 2020, Huawei Technologies Co. Ltd. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* This code is free software; you can redistribute it and/or modify it
6+
* under the terms of the GNU General Public License version 2 only, as
7+
* published by the Free Software Foundation.
8+
*
9+
* This code is distributed in the hope that it will be useful, but WITHOUT
10+
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11+
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12+
* version 2 for more details (a copy is included in the LICENSE file that
13+
* accompanied this code).
14+
*
15+
* You should have received a copy of the GNU General Public License version
16+
* 2 along with this work; if not, write to the Free Software Foundation,
17+
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18+
*
19+
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20+
* or visit www.oracle.com if you need additional information or have any
21+
* questions.
22+
*/
23+
24+
25+
/**
26+
* @test
27+
* @bug 8255287
28+
* @summary aarch64: fix SVE patterns for vector shift count
29+
*
30+
* @requires os.arch == "aarch64" & vm.compiler2.enabled
31+
* @run main/othervm -XX:UseSVE=2 -Xbatch -XX:-TieredCompilation
32+
* -XX:CompileCommand=compileonly,compiler.c2.aarch64.TestVectorShiftShorts::test_*
33+
* compiler.c2.aarch64.TestVectorShiftShorts
34+
*/
35+
36+
package compiler.c2.aarch64;
37+
38+
public class TestVectorShiftShorts {
39+
40+
private static final int ARRLEN = 1000;
41+
private static final int ITERS = 20000;
42+
43+
public static void main(String args[]) {
44+
short[] a0 = new short[ARRLEN];
45+
short[] a1 = new short[ARRLEN];
46+
47+
// Initialize
48+
test_init(a0, a1);
49+
50+
// Warmup
51+
for (int i = 0; i < ITERS; i++) {
52+
test_lshift(a0, a1);
53+
test_urshift(a0, a1);
54+
}
55+
56+
// Test and verify results
57+
test_init(a0, a1);
58+
test_lshift(a0, a1);
59+
verify_lshift(a0, a1);
60+
61+
test_init(a0, a1);
62+
test_urshift(a0, a1);
63+
verify_urshift(a0, a1);
64+
65+
// Finish
66+
System.out.println("Test passed");
67+
}
68+
69+
static void test_init(short[] a0, short[] a1) {
70+
for (int i = 0; i < ARRLEN; i++) {
71+
a0[i] = (short)(i & 3);
72+
a1[i] = (short)i;
73+
}
74+
}
75+
76+
static void test_lshift(short[] a0, short[] a1) {
77+
for (int i = 0; i < ARRLEN; i++) {
78+
a0[i] = (short)(a1[i] << 10);
79+
}
80+
}
81+
82+
static void verify_lshift(short[] a0, short[] a1) {
83+
for (int i = 0; i < ARRLEN; i++) {
84+
if (a0[i] != (short)(a1[i] << 10)) {
85+
throw new RuntimeException("LShift test failed.");
86+
}
87+
}
88+
}
89+
90+
static void test_urshift(short[] a0, short[] a1) {
91+
for (int i = 0; i < ARRLEN; i++) {
92+
a0[i] = (short)(a1[i] >>> 10);
93+
}
94+
}
95+
96+
static void verify_urshift(short[] a0, short[] a1) {
97+
for (int i = 0; i < ARRLEN; i++) {
98+
if (a0[i] != (short)(a1[i] >>> 10)) {
99+
throw new RuntimeException("URshift test failed.");
100+
}
101+
}
102+
}
103+
104+
}

0 commit comments

Comments
 (0)