Skip to content

Commit 37c6b23

Browse files
author
Fei Gao
committed
8308340: C2: Idealize Fma nodes
Reviewed-by: kvn, epeter
1 parent 583cb75 commit 37c6b23

20 files changed

+571
-220
lines changed

‎src/hotspot/cpu/aarch64/aarch64.ad

+28-21
Original file line numberDiff line numberDiff line change
@@ -2289,26 +2289,33 @@ bool Matcher::match_rule_supported(int opcode) {
22892289
if (!has_match_rule(opcode))
22902290
return false;
22912291

2292-
bool ret_value = true;
22932292
switch (opcode) {
22942293
case Op_OnSpinWait:
22952294
return VM_Version::supports_on_spin_wait();
22962295
case Op_CacheWB:
22972296
case Op_CacheWBPreSync:
22982297
case Op_CacheWBPostSync:
22992298
if (!VM_Version::supports_data_cache_line_flush()) {
2300-
ret_value = false;
2299+
return false;
23012300
}
23022301
break;
23032302
case Op_ExpandBits:
23042303
case Op_CompressBits:
23052304
if (!VM_Version::supports_svebitperm()) {
2306-
ret_value = false;
2305+
return false;
2306+
}
2307+
break;
2308+
case Op_FmaF:
2309+
case Op_FmaD:
2310+
case Op_FmaVF:
2311+
case Op_FmaVD:
2312+
if (!UseFMA) {
2313+
return false;
23072314
}
23082315
break;
23092316
}
23102317

2311-
return ret_value; // Per default match rules are supported.
2318+
return true; // Per default match rules are supported.
23122319
}
23132320

23142321
const RegMask* Matcher::predicate_reg_mask(void) {
@@ -14305,12 +14312,12 @@ instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
1430514312

1430614313
// src1 * src2 + src3
1430714314
instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14308-
predicate(UseFMA);
1430914315
match(Set dst (FmaF src3 (Binary src1 src2)));
1431014316

1431114317
format %{ "fmadds $dst, $src1, $src2, $src3" %}
1431214318

1431314319
ins_encode %{
14320+
assert(UseFMA, "Needs FMA instructions support.");
1431414321
__ fmadds(as_FloatRegister($dst$$reg),
1431514322
as_FloatRegister($src1$$reg),
1431614323
as_FloatRegister($src2$$reg),
@@ -14322,12 +14329,12 @@ instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
1432214329

1432314330
// src1 * src2 + src3
1432414331
instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14325-
predicate(UseFMA);
1432614332
match(Set dst (FmaD src3 (Binary src1 src2)));
1432714333

1432814334
format %{ "fmaddd $dst, $src1, $src2, $src3" %}
1432914335

1433014336
ins_encode %{
14337+
assert(UseFMA, "Needs FMA instructions support.");
1433114338
__ fmaddd(as_FloatRegister($dst$$reg),
1433214339
as_FloatRegister($src1$$reg),
1433314340
as_FloatRegister($src2$$reg),
@@ -14337,15 +14344,15 @@ instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
1433714344
ins_pipe(pipe_class_default);
1433814345
%}
1433914346

14340-
// -src1 * src2 + src3
14347+
// src1 * (-src2) + src3
14348+
// "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
1434114349
instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14342-
predicate(UseFMA);
14343-
match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
1434414350
match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
1434514351

1434614352
format %{ "fmsubs $dst, $src1, $src2, $src3" %}
1434714353

1434814354
ins_encode %{
14355+
assert(UseFMA, "Needs FMA instructions support.");
1434914356
__ fmsubs(as_FloatRegister($dst$$reg),
1435014357
as_FloatRegister($src1$$reg),
1435114358
as_FloatRegister($src2$$reg),
@@ -14355,15 +14362,15 @@ instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
1435514362
ins_pipe(pipe_class_default);
1435614363
%}
1435714364

14358-
// -src1 * src2 + src3
14365+
// src1 * (-src2) + src3
14366+
// "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
1435914367
instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14360-
predicate(UseFMA);
14361-
match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
1436214368
match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
1436314369

1436414370
format %{ "fmsubd $dst, $src1, $src2, $src3" %}
1436514371

1436614372
ins_encode %{
14373+
assert(UseFMA, "Needs FMA instructions support.");
1436714374
__ fmsubd(as_FloatRegister($dst$$reg),
1436814375
as_FloatRegister($src1$$reg),
1436914376
as_FloatRegister($src2$$reg),
@@ -14373,15 +14380,15 @@ instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
1437314380
ins_pipe(pipe_class_default);
1437414381
%}
1437514382

14376-
// -src1 * src2 - src3
14383+
// src1 * (-src2) - src3
14384+
// "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
1437714385
instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14378-
predicate(UseFMA);
14379-
match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
1438014386
match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
1438114387

1438214388
format %{ "fnmadds $dst, $src1, $src2, $src3" %}
1438314389

1438414390
ins_encode %{
14391+
assert(UseFMA, "Needs FMA instructions support.");
1438514392
__ fnmadds(as_FloatRegister($dst$$reg),
1438614393
as_FloatRegister($src1$$reg),
1438714394
as_FloatRegister($src2$$reg),
@@ -14391,15 +14398,15 @@ instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
1439114398
ins_pipe(pipe_class_default);
1439214399
%}
1439314400

14394-
// -src1 * src2 - src3
14401+
// src1 * (-src2) - src3
14402+
// "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
1439514403
instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14396-
predicate(UseFMA);
14397-
match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
1439814404
match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
1439914405

1440014406
format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
1440114407

1440214408
ins_encode %{
14409+
assert(UseFMA, "Needs FMA instructions support.");
1440314410
__ fnmaddd(as_FloatRegister($dst$$reg),
1440414411
as_FloatRegister($src1$$reg),
1440514412
as_FloatRegister($src2$$reg),
@@ -14411,12 +14418,12 @@ instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
1441114418

1441214419
// src1 * src2 - src3
1441314420
instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14414-
predicate(UseFMA);
1441514421
match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
1441614422

1441714423
format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
1441814424

1441914425
ins_encode %{
14426+
assert(UseFMA, "Needs FMA instructions support.");
1442014427
__ fnmsubs(as_FloatRegister($dst$$reg),
1442114428
as_FloatRegister($src1$$reg),
1442214429
as_FloatRegister($src2$$reg),
@@ -14428,13 +14435,13 @@ instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zer
1442814435

1442914436
// src1 * src2 - src3
1443014437
instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14431-
predicate(UseFMA);
1443214438
match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
1443314439

1443414440
format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
1443514441

1443614442
ins_encode %{
14437-
// n.b. insn name should be fnmsubd
14443+
assert(UseFMA, "Needs FMA instructions support.");
14444+
// n.b. insn name should be fnmsubd
1443814445
__ fnmsub(as_FloatRegister($dst$$reg),
1443914446
as_FloatRegister($src1$$reg),
1444014447
as_FloatRegister($src2$$reg),

‎src/hotspot/cpu/aarch64/aarch64_vector.ad

+27-54
Original file line numberDiff line numberDiff line change
@@ -2131,14 +2131,14 @@ instruct vmla_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
21312131
%}
21322132

21332133
// vector fmla
2134-
// dst_src1 = dst_src1 + src2 * src3
2134+
// dst_src1 = src2 * src3 + dst_src1
21352135

21362136
instruct vfmla(vReg dst_src1, vReg src2, vReg src3) %{
2137-
predicate(UseFMA);
21382137
match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
21392138
match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
21402139
format %{ "vfmla $dst_src1, $src2, $src3" %}
21412140
ins_encode %{
2141+
assert(UseFMA, "Needs FMA instructions support.");
21422142
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
21432143
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
21442144
__ fmla($dst_src1$$FloatRegister, get_arrangement(this),
@@ -2157,11 +2157,12 @@ instruct vfmla(vReg dst_src1, vReg src2, vReg src3) %{
21572157
// dst_src1 = dst_src1 * src2 + src3
21582158

21592159
instruct vfmad_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
2160-
predicate(UseFMA && UseSVE > 0);
2160+
predicate(UseSVE > 0);
21612161
match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 pg)));
21622162
match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 pg)));
21632163
format %{ "vfmad_masked $dst_src1, $pg, $src2, $src3" %}
21642164
ins_encode %{
2165+
assert(UseFMA, "Needs FMA instructions support.");
21652166
BasicType bt = Matcher::vector_element_basic_type(this);
21662167
__ sve_fmad($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
21672168
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
@@ -2221,34 +2222,14 @@ instruct vmls_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
22212222

22222223
// vector fmls
22232224

2224-
// dst_src1 = dst_src1 + -src2 * src3
2225-
instruct vfmls1(vReg dst_src1, vReg src2, vReg src3) %{
2226-
predicate(UseFMA);
2227-
match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3)));
2228-
match(Set dst_src1 (FmaVD dst_src1 (Binary (NegVD src2) src3)));
2229-
format %{ "vfmls1 $dst_src1, $src2, $src3" %}
2230-
ins_encode %{
2231-
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
2232-
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
2233-
__ fmls($dst_src1$$FloatRegister, get_arrangement(this),
2234-
$src2$$FloatRegister, $src3$$FloatRegister);
2235-
} else {
2236-
assert(UseSVE > 0, "must be sve");
2237-
BasicType bt = Matcher::vector_element_basic_type(this);
2238-
__ sve_fmls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
2239-
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
2240-
}
2241-
%}
2242-
ins_pipe(pipe_slow);
2243-
%}
2244-
2245-
// dst_src1 = dst_src1 + src2 * -src3
2246-
instruct vfmls2(vReg dst_src1, vReg src2, vReg src3) %{
2247-
predicate(UseFMA);
2225+
// dst_src1 = src2 * (-src3) + dst_src1
2226+
// "(-src2) * src3 + dst_src1" has been idealized to "src3 * (-src2) + dst_src1"
2227+
instruct vfmls(vReg dst_src1, vReg src2, vReg src3) %{
22482228
match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
22492229
match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
2250-
format %{ "vfmls2 $dst_src1, $src2, $src3" %}
2230+
format %{ "vfmls $dst_src1, $src2, $src3" %}
22512231
ins_encode %{
2232+
assert(UseFMA, "Needs FMA instructions support.");
22522233
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
22532234
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
22542235
__ fmls($dst_src1$$FloatRegister, get_arrangement(this),
@@ -2265,13 +2246,14 @@ instruct vfmls2(vReg dst_src1, vReg src2, vReg src3) %{
22652246

22662247
// vector fmsb - predicated
22672248

2268-
// dst_src1 = dst_src1 * -src2 + src3
2249+
// dst_src1 = dst_src1 * (-src2) + src3
22692250
instruct vfmsb_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
2270-
predicate(UseFMA && UseSVE > 0);
2251+
predicate(UseSVE > 0);
22712252
match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary src3 pg)));
22722253
match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary src3 pg)));
22732254
format %{ "vfmsb_masked $dst_src1, $pg, $src2, $src3" %}
22742255
ins_encode %{
2256+
assert(UseFMA, "Needs FMA instructions support.");
22752257
BasicType bt = Matcher::vector_element_basic_type(this);
22762258
__ sve_fmsb($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
22772259
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
@@ -2281,27 +2263,15 @@ instruct vfmsb_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
22812263

22822264
// vector fnmla (sve)
22832265

2284-
// dst_src1 = -dst_src1 + -src2 * src3
2285-
instruct vfnmla1(vReg dst_src1, vReg src2, vReg src3) %{
2286-
predicate(UseFMA && UseSVE > 0);
2287-
match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary (NegVF src2) src3)));
2288-
match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary (NegVD src2) src3)));
2289-
format %{ "vfnmla1 $dst_src1, $src2, $src3" %}
2290-
ins_encode %{
2291-
BasicType bt = Matcher::vector_element_basic_type(this);
2292-
__ sve_fnmla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
2293-
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
2294-
%}
2295-
ins_pipe(pipe_slow);
2296-
%}
2297-
2298-
// dst_src1 = -dst_src1 + src2 * -src3
2299-
instruct vfnmla2(vReg dst_src1, vReg src2, vReg src3) %{
2300-
predicate(UseFMA && UseSVE > 0);
2266+
// dst_src1 = src2 * (-src3) - dst_src1
2267+
// "(-src2) * src3 - dst_src1" has been idealized to "src3 * (-src2) - dst_src1"
2268+
instruct vfnmla(vReg dst_src1, vReg src2, vReg src3) %{
2269+
predicate(UseSVE > 0);
23012270
match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
23022271
match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
2303-
format %{ "vfnmla2 $dst_src1, $src2, $src3" %}
2272+
format %{ "vfnmla $dst_src1, $src2, $src3" %}
23042273
ins_encode %{
2274+
assert(UseFMA, "Needs FMA instructions support.");
23052275
BasicType bt = Matcher::vector_element_basic_type(this);
23062276
__ sve_fnmla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
23072277
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
@@ -2311,13 +2281,14 @@ instruct vfnmla2(vReg dst_src1, vReg src2, vReg src3) %{
23112281

23122282
// vector fnmad - predicated
23132283

2314-
// dst_src1 = -src3 + dst_src1 * -src2
2284+
// dst_src1 = dst_src1 * (-src2) - src3
23152285
instruct vfnmad_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
2316-
predicate(UseFMA && UseSVE > 0);
2286+
predicate(UseSVE > 0);
23172287
match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary (NegVF src3) pg)));
23182288
match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary (NegVD src3) pg)));
23192289
format %{ "vfnmad_masked $dst_src1, $pg, $src2, $src3" %}
23202290
ins_encode %{
2291+
assert(UseFMA, "Needs FMA instructions support.");
23212292
BasicType bt = Matcher::vector_element_basic_type(this);
23222293
__ sve_fnmad($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
23232294
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
@@ -2327,13 +2298,14 @@ instruct vfnmad_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
23272298

23282299
// vector fnmls (sve)
23292300

2330-
// dst_src1 = -dst_src1 + src2 * src3
2301+
// dst_src1 = src2 * src3 - dst_src1
23312302
instruct vfnmls(vReg dst_src1, vReg src2, vReg src3) %{
2332-
predicate(UseFMA && UseSVE > 0);
2303+
predicate(UseSVE > 0);
23332304
match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
23342305
match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
23352306
format %{ "vfnmls $dst_src1, $src2, $src3" %}
23362307
ins_encode %{
2308+
assert(UseFMA, "Needs FMA instructions support.");
23372309
BasicType bt = Matcher::vector_element_basic_type(this);
23382310
__ sve_fnmls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
23392311
ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
@@ -2343,13 +2315,14 @@ instruct vfnmls(vReg dst_src1, vReg src2, vReg src3) %{
23432315

23442316
// vector fnmsb - predicated
23452317

2346-
// dst_src1 = -src3 + dst_src1 * src2
2318+
// dst_src1 = dst_src1 * src2 - src3
23472319
instruct vfnmsb_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
2348-
predicate(UseFMA && UseSVE > 0);
2320+
predicate(UseSVE > 0);
23492321
match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary (NegVF src3) pg)));
23502322
match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary (NegVD src3) pg)));
23512323
format %{ "vfnmsb_masked $dst_src1, $pg, $src2, $src3" %}
23522324
ins_encode %{
2325+
assert(UseFMA, "Needs FMA instructions support.");
23532326
BasicType bt = Matcher::vector_element_basic_type(this);
23542327
__ sve_fnmsb($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
23552328
$pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);

0 commit comments

Comments
 (0)