Skip to content

Commit 2d64c2a

Browse files
gergo-gergo-
committed
[GR-65520] [GR-65033] Allow Long128Mask/Double128Mask in frame states
PullRequest: graal/20988
2 parents ab75b44 + d53f96b commit 2d64c2a

File tree

10 files changed

+103
-40
lines changed

10 files changed

+103
-40
lines changed

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotBackendFactory.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ protected HotSpotSuitesProvider createSuites(GraalHotSpotVMConfig config, HotSpo
126126
protected HotSpotLoweringProvider createLowerer(HotSpotGraalRuntimeProvider runtime, MetaAccessProvider metaAccess, HotSpotHostForeignCallsProvider foreignCalls,
127127
HotSpotRegistersProvider registers, HotSpotConstantReflectionProvider constantReflection, HotSpotPlatformConfigurationProvider platformConfig,
128128
HotSpotMetaAccessExtensionProvider metaAccessExtensionProvider, TargetDescription target) {
129-
boolean enableObjectVectorization = false;
129+
boolean enableObjectVectorization = true;
130130
VectorAMD64 varch = new VectorAMD64((AMD64) target.arch, metaAccess.getArrayIndexScale(JavaKind.Object), runtime.getVMConfig().useCompressedOops, runtime.getVMConfig().objectAlignment,
131131
runtime.getVMConfig().maxVectorSize, enableObjectVectorization);
132132
return new AMD64HotSpotLoweringProvider(runtime, metaAccess, foreignCalls, registers, constantReflection, platformConfig, metaAccessExtensionProvider, target, varch);

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/vector/lir/aarch64/AArch64ASIMDMove.java

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,8 @@ private static boolean tryEncodeSimdConst(AArch64MacroAssembler masm, Value resu
9191
int length = result.getPlatformKind().getVectorLength();
9292
assert length > 0 : length;
9393
int vectorLength = simdConstant.getVectorLength();
94-
assert length <= vectorLength : "length>=" + vectorLength;
94+
boolean specialCaseTwoBytes = result.getPlatformKind().equals(AArch64Kind.V32_BYTE) && simdConstant.getSerializedSize() == 2;
95+
assert specialCaseTwoBytes || length <= vectorLength : "length>=" + vectorLength;
9596

9697
if (simdConstant.isAllSame()) {
9798
ElementSize eSize = ElementSize.fromKind(elementKind);
@@ -143,7 +144,11 @@ private static void simdConst2Reg(CompilationResultBuilder crb, AArch64MacroAsse
143144
/* Encode constant in data section. */
144145
try (AArch64MacroAssembler.ScratchRegister scr = masm.getScratchRegister()) {
145146
Register scratch = scr.getRegister();
146-
DataSection.Data data = crb.dataBuilder.createMultiDataItem(simdConstant.getValues());
147+
SimdConstant constantInMemory = simdConstant;
148+
if (size.bytes() <= ASIMDSize.HalfReg.bytes() && constantInMemory.getSerializedSize() < size.bytes()) {
149+
constantInMemory = padConstant(constantInMemory, size.bytes());
150+
}
151+
DataSection.Data data = crb.dataBuilder.createMultiDataItem(constantInMemory.getValues());
147152
crb.dataBuilder.updateAlignment(data, size.bytes());
148153
crb.recordDataSectionReference(data);
149154
masm.adrpAdd(scratch);
@@ -153,6 +158,20 @@ private static void simdConst2Reg(CompilationResultBuilder crb, AArch64MacroAsse
153158

154159
}
155160

161+
/** Pad the given constant to the given number of bytes. */
162+
private static SimdConstant padConstant(SimdConstant simdConstant, int toBytes) {
163+
GraalError.guarantee(simdConstant.getSerializedSize() < toBytes, "can't pad %s to %s bytes, it's already big enough", simdConstant, toBytes);
164+
Constant[] constants = new Constant[simdConstant.getVectorLength() + (toBytes - simdConstant.getSerializedSize())];
165+
int i;
166+
for (i = 0; i < simdConstant.getVectorLength(); i++) {
167+
constants[i] = simdConstant.getValue(i);
168+
}
169+
for (; i < constants.length; i++) {
170+
constants[i] = JavaConstant.forByte((byte) (i % 2 == 0 ? 0xc0 : 0xfe));
171+
}
172+
return new SimdConstant(constants);
173+
}
174+
156175
@Override
157176
public boolean canRematerializeToStack() {
158177
return false;

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/vector/lir/amd64/AMD64AVX512ArithmeticLIRGenerator.java

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
*/
2525
package jdk.graal.compiler.vector.lir.amd64;
2626

27-
import static jdk.graal.compiler.vector.lir.amd64.AMD64VectorNodeMatchRules.getRegisterSize;
2827
import static jdk.graal.compiler.asm.amd64.AMD64Assembler.AMD64SIMDInstructionEncoding.EVEX;
2928
import static jdk.graal.compiler.asm.amd64.AMD64Assembler.EvexGatherOp.EVGATHERDPD;
3029
import static jdk.graal.compiler.asm.amd64.AMD64Assembler.EvexGatherOp.EVGATHERDPS;
@@ -200,6 +199,7 @@
200199
import static jdk.graal.compiler.asm.amd64.AMD64Assembler.VexShiftOp.EVPSRLD;
201200
import static jdk.graal.compiler.asm.amd64.AMD64Assembler.VexShiftOp.EVPSRLQ;
202201
import static jdk.graal.compiler.asm.amd64.AMD64Assembler.VexShiftOp.EVPSRLW;
202+
import static jdk.graal.compiler.vector.lir.amd64.AMD64VectorNodeMatchRules.getRegisterSize;
203203

204204
import java.util.Arrays;
205205
import java.util.EnumSet;
@@ -1848,12 +1848,22 @@ public Value emitMaskedMergeOp(LIRKind resultKind, MaskedOpMetaData meta, Value
18481848
Variable result = getLIRGen().newVariable(resultKind);
18491849

18501850
NormalizedMaskedOp normalizedOp = NormalizedMaskedOp.make(getMaskedOpcode(getArchitecture(), meta, eKind, srcEKind), avxSize, src1, src2);
1851+
Value normalizedSrc2 = normalizedOp.src2;
1852+
if (normalizedOp.opcode == EVPERMILPD) {
1853+
/*
1854+
* EVPERMILPD uses the SECOND bit in each element as the index. See also
1855+
* AMD64VectorShuffle.
1856+
*/
1857+
Variable tmp = getLIRGen().newVariable(normalizedOp.src2.getValueKind());
1858+
getLIRGen().append(new AMD64VectorBinary.AVXBinaryConstOp(EVPSLLQ, getRegisterSize(tmp), tmp, asAllocatable(src2), 1));
1859+
normalizedSrc2 = tmp;
1860+
}
18511861
AVX512MaskedOp.AVX512MaskedMergeOp lirOp;
1852-
if (normalizedOp.src2() == null) {
1862+
if (normalizedSrc2 == null) {
18531863
lirOp = new AVX512MaskedOp.AVX512MaskedMergeOp(normalizedOp.opcode(), avxSize, result, asAllocatable(background), asAllocatable(mask), Value.ILLEGAL, asAllocatable(normalizedOp.src1()));
18541864
} else {
18551865
lirOp = new AVX512MaskedOp.AVX512MaskedMergeOp(normalizedOp.opcode(), avxSize, result, asAllocatable(background), asAllocatable(mask), asAllocatable(normalizedOp.src1()),
1856-
asAllocatable(normalizedOp.src2()));
1866+
asAllocatable(normalizedSrc2));
18571867
}
18581868
getLIRGen().append(lirOp);
18591869
return result;
@@ -1883,12 +1893,22 @@ public Value emitMaskedZeroOp(LIRKind resultKind, MaskedOpMetaData meta, Value m
18831893
GraalError.guarantee(predicate != null, "%s", meta);
18841894
}
18851895

1896+
Value normalizedSrc2 = normalizedOp.src2;
1897+
if (normalizedOp.opcode == EVPERMILPD) {
1898+
/*
1899+
* EVPERMILPD uses the SECOND bit in each element as the index. See also
1900+
* AMD64VectorShuffle.
1901+
*/
1902+
Variable tmp = getLIRGen().newVariable(normalizedOp.src2.getValueKind());
1903+
getLIRGen().append(new AMD64VectorBinary.AVXBinaryConstOp(EVPSLLQ, getRegisterSize(tmp), tmp, asAllocatable(src2), 1));
1904+
normalizedSrc2 = tmp;
1905+
}
18861906
AVX512MaskedOp.AVX512MaskedZeroOp lirOp;
1887-
if (normalizedOp.src2() == null) {
1907+
if (normalizedSrc2 == null) {
18881908
lirOp = new AVX512MaskedOp.AVX512MaskedZeroOp(normalizedOp.opcode(), predicate, avxSize, result, asAllocatable(mask), Value.ILLEGAL, asAllocatable(normalizedOp.src1()));
18891909
} else {
18901910
lirOp = new AVX512MaskedOp.AVX512MaskedZeroOp(normalizedOp.opcode(), predicate, avxSize, result, asAllocatable(mask), asAllocatable(normalizedOp.src1()),
1891-
asAllocatable(normalizedOp.src2()));
1911+
asAllocatable(normalizedSrc2));
18921912
}
18931913
getLIRGen().append(lirOp);
18941914
return result;

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/vector/lir/amd64/AMD64VectorMoveFactory.java

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import jdk.vm.ci.meta.AllocatableValue;
4444
import jdk.vm.ci.meta.Constant;
4545
import jdk.vm.ci.meta.JavaConstant;
46+
import jdk.vm.ci.meta.JavaKind;
4647
import jdk.vm.ci.meta.Value;
4748

4849
public class AMD64VectorMoveFactory extends AMD64MoveFactoryBase {
@@ -117,13 +118,24 @@ private LIRInstruction createLoad(AllocatableValue result, Constant constant, bo
117118

118119
SimdConstant simd = (SimdConstant) constant;
119120
AMD64Kind kind = (AMD64Kind) result.getPlatformKind();
120-
GraalError.guarantee(kind.getVectorLength() <= simd.getVectorLength(), "vector length mismatch: %s vs. %s", kind, simd);
121+
122+
/*
123+
* JVMCI doesn't have a 16-bit vector kind. Two-byte constants are assigned a 32-bit
124+
* kind instead. If we construct the constant value in a register (for all zeros or all
125+
* ones), this doesn't matter. If we emit a load from the constant area, we make sure to
126+
* pad two-byte constants to four bytes and load them with a four-byte load.
127+
*/
128+
boolean specialCaseTwoBytes = kind.equals(AMD64Kind.V32_BYTE) && simd.getSerializedSize() == 2;
129+
GraalError.guarantee(specialCaseTwoBytes || kind.getVectorLength() <= simd.getVectorLength(), "vector length mismatch: %s vs. %s", kind, simd);
121130

122131
if (simd.isDefaultForKind()) {
123132
return new AVXClearVectorConstant(result, simd, encoding);
124133
} else if (SimdConstant.isAllOnes(simd)) {
125134
return new AVXAllOnesOp(result, simd);
126135
} else {
136+
if (specialCaseTwoBytes) {
137+
simd = padTwoByteConstant(simd);
138+
}
127139
return new AVXLoadConstantVectorOp(result, simd, kind.getScalar(), encoding);
128140
}
129141
} else if (constant instanceof JavaConstant) {
@@ -140,6 +152,16 @@ private LIRInstruction createLoad(AllocatableValue result, Constant constant, bo
140152
return null;
141153
}
142154

155+
/**
156+
* Pad the given constant, which must have a size of two bytes, to four bytes by appending
157+
* bytes. The values of the padding bytes are irrelevant, as they should never be used.
158+
*/
159+
private static SimdConstant padTwoByteConstant(SimdConstant simd) {
160+
GraalError.guarantee(simd.getVectorLength() == 2 && simd.getValue(0) instanceof JavaConstant javaConstant && javaConstant.getJavaKind() == JavaKind.Byte, "expected exactly two bytes: %s",
161+
simd);
162+
return new SimdConstant(new Constant[]{simd.getValue(0), simd.getValue(1), JavaConstant.forByte((byte) 0xc0), JavaConstant.forByte((byte) 0xfe)});
163+
}
164+
143165
@Override
144166
public LIRInstruction createLoad(AllocatableValue result, Constant input) {
145167
LIRInstruction load = createLoad(result, input, false);

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/vector/lir/hotspot/aarch64/AArch64HotSpotVectorLIRGenerator.java

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import jdk.graal.compiler.lir.gen.MoveFactory;
3838
import jdk.graal.compiler.vector.lir.aarch64.AArch64VectorArithmeticLIRGenerator;
3939
import jdk.graal.compiler.vector.nodes.simd.SimdConstant;
40+
import jdk.vm.ci.aarch64.AArch64Kind;
4041
import jdk.vm.ci.code.Register;
4142
import jdk.vm.ci.meta.Constant;
4243
import jdk.vm.ci.meta.PlatformKind;
@@ -79,9 +80,14 @@ public Value emitConstant(LIRKind kind, Constant constant) {
7980
int length = kind.getPlatformKind().getVectorLength();
8081
if (length == 1) {
8182
return super.emitConstant(kind, constant);
82-
} else if (constant instanceof SimdConstant) {
83-
assert ((SimdConstant) constant).getVectorLength() == length : constant + " " + length;
84-
return super.emitConstant(kind, constant);
83+
} else if (constant instanceof SimdConstant simd) {
84+
/*
85+
* JVMCI doesn't have a 16-bit vector kind. Two-byte constants are assigned a 32-bit
86+
* kind instead.
87+
*/
88+
boolean specialCaseTwoBytes = kind.getPlatformKind().equals(AArch64Kind.V32_BYTE) && simd.getSerializedSize() == 2;
89+
assert specialCaseTwoBytes || simd.getVectorLength() == length : constant + " " + length;
90+
return super.emitConstant(kind, simd);
8591
} else {
8692
return super.emitConstant(kind, SimdConstant.broadcast(constant, length));
8793
}

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/vector/lir/hotspot/amd64/AMD64HotSpotVectorLIRGenerator.java

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,9 +149,14 @@ public Value emitConstant(LIRKind kind, Constant constant) {
149149
int length = kind.getPlatformKind().getVectorLength();
150150
if (length == 1) {
151151
return super.emitConstant(kind, constant);
152-
} else if (constant instanceof SimdConstant) {
153-
assert ((SimdConstant) constant).getVectorLength() == length : constant + " " + length;
154-
return super.emitConstant(kind, constant);
152+
} else if (constant instanceof SimdConstant simd) {
153+
/*
154+
* JVMCI doesn't have a 16-bit vector kind. Two-byte constants are assigned a 32-bit
155+
* kind instead.
156+
*/
157+
boolean specialCaseTwoBytes = kind.getPlatformKind().equals(AMD64Kind.V32_BYTE) && simd.getSerializedSize() == 2;
158+
assert specialCaseTwoBytes || simd.getVectorLength() == length : constant + " " + length;
159+
return super.emitConstant(kind, simd);
155160
} else {
156161
return super.emitConstant(kind, SimdConstant.broadcast(constant, length));
157162
}

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/vector/replacements/vectorapi/VectorAPIExpansionPhase.java

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@ private static NodeUnionFind collectNodes(StructuredGraph graph, NodeFlood flood
257257
* frame states but not to other macros. We don't want to lose these usages.
258258
*/
259259
for (Node node : flood) {
260-
if (node instanceof VectorAPIMacroNode || node instanceof ValuePhiNode || node instanceof ValueProxyNode) {
260+
if ((node instanceof VectorAPIMacroNode && !(node instanceof VectorAPISinkNode)) || node instanceof ValuePhiNode || node instanceof ValueProxyNode) {
261261
for (Node usage : node.usages()) {
262262
if (usage instanceof ValuePhiNode || usage instanceof ValueProxyNode) {
263263
unionFind.union(node, usage);
@@ -382,15 +382,6 @@ private static Iterable<ConnectedComponent> buildConnectedComponents(StructuredG
382382
* deoptimization, this will materialize the SIMD value as a vector object
383383
* on the heap.
384384
*/
385-
if (node instanceof ValueNode valueNode) {
386-
ResolvedJavaType javaType = valueNode.stamp(NodeView.DEFAULT).javaType(context.getMetaAccess());
387-
VectorAPIType vectorType = VectorAPIType.ofType(javaType, context.getProviders());
388-
if (vectorType == null || (vectorType.isMask && !VectorAPIBoxingUtils.canConvertLogicToBooleans(vectorType.stamp, VectorAPIUtils.vectorArchitecture(context)))) {
389-
graph.getDebug().log(DebugContext.DETAILED_LEVEL, "bad (mask) type %s at %s: %s", vectorType, valueNode, valueNode.stamp(NodeView.DEFAULT));
390-
component.canExpand = false;
391-
break;
392-
}
393-
}
394385
continue;
395386
} else {
396387
/*

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/vector/replacements/vectorapi/VectorAPIUtils.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,6 @@
2727

2828
import org.graalvm.word.LocationIdentity;
2929

30-
import jdk.graal.compiler.vector.nodes.simd.SimdPrimitiveCompareNode;
31-
3230
import jdk.graal.compiler.core.common.calc.CanonicalCondition;
3331
import jdk.graal.compiler.core.common.type.IntegerStamp;
3432
import jdk.graal.compiler.core.common.type.ObjectStamp;
@@ -51,6 +49,7 @@
5149
import jdk.graal.compiler.vector.architecture.VectorArchitecture;
5250
import jdk.graal.compiler.vector.architecture.VectorLoweringProvider;
5351
import jdk.graal.compiler.vector.nodes.simd.SimdConstant;
52+
import jdk.graal.compiler.vector.nodes.simd.SimdPrimitiveCompareNode;
5453
import jdk.graal.compiler.vector.nodes.simd.SimdStamp;
5554
import jdk.vm.ci.meta.ConstantReflectionProvider;
5655
import jdk.vm.ci.meta.JavaConstant;
@@ -122,7 +121,7 @@ public static SimdStamp stampForVectorClass(ResolvedJavaType vmType, ResolvedJav
122121
*/
123122
return null;
124123
}
125-
if (vectorArchitecture(providers).getSupportedVectorMoveLength(vectorType.payloadStamp.getComponent(0), vectorType.vectorLength) != vectorType.vectorLength) {
124+
if (!vectorType.isMask && vectorArchitecture(providers).getSupportedVectorMoveLength(vectorType.payloadStamp.getComponent(0), vectorType.vectorLength) != vectorType.vectorLength) {
126125
/*
127126
* This vector type is not natively supported by the target. Don't try to intrinsify
128127
* operations on vectors of this type.

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/vector/replacements/vectorapi/nodes/VectorAPICompareNode.java

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,10 @@
3030

3131
import org.graalvm.collections.EconomicMap;
3232

33-
import jdk.graal.compiler.vector.architecture.VectorArchitecture;
34-
import jdk.graal.compiler.vector.nodes.simd.SimdPrimitiveCompareNode;
35-
import jdk.graal.compiler.vector.replacements.vectorapi.VectorAPIOperations;
36-
import jdk.graal.compiler.vector.replacements.vectorapi.VectorAPIUtils;
37-
3833
import jdk.graal.compiler.core.common.calc.CanonicalCondition;
3934
import jdk.graal.compiler.core.common.calc.Condition;
4035
import jdk.graal.compiler.core.common.calc.Condition.CanonicalizedCondition;
36+
import jdk.graal.compiler.core.common.type.ArithmeticStamp;
4137
import jdk.graal.compiler.core.common.type.ObjectStamp;
4238
import jdk.graal.compiler.core.common.type.Stamp;
4339
import jdk.graal.compiler.graph.Node;
@@ -52,7 +48,11 @@
5248
import jdk.graal.compiler.nodes.spi.Canonicalizable;
5349
import jdk.graal.compiler.nodes.spi.CanonicalizerTool;
5450
import jdk.graal.compiler.nodes.spi.CoreProviders;
51+
import jdk.graal.compiler.vector.architecture.VectorArchitecture;
52+
import jdk.graal.compiler.vector.nodes.simd.SimdPrimitiveCompareNode;
5553
import jdk.graal.compiler.vector.nodes.simd.SimdStamp;
54+
import jdk.graal.compiler.vector.replacements.vectorapi.VectorAPIOperations;
55+
import jdk.graal.compiler.vector.replacements.vectorapi.VectorAPIUtils;
5656
import jdk.vm.ci.meta.JavaKind;
5757

5858
/**
@@ -170,12 +170,13 @@ public boolean canExpand(VectorArchitecture vectorArch, EconomicMap<ValueNode, S
170170
return false;
171171
}
172172

173-
/*
174-
* Strictly speaking, if the canonicalized condition requires a negation, we should also
175-
* check if negation of logic vectors is supported by the architecture. There is no
176-
* convenient way to do this, and also every realistic architecture allows negation.
177-
* Therefore, only check for support of the actual comparison.
178-
*/
173+
if (canonicalizedCondition.mustNegate()) {
174+
SimdStamp resultStamp = this.vectorStamp();
175+
ArithmeticStamp resultElementStamp = (ArithmeticStamp) resultStamp.getComponent(0);
176+
if (vectorArch.getSupportedVectorArithmeticLength(resultElementStamp, length, resultElementStamp.getOps().getNot()) != length) {
177+
return false;
178+
}
179+
}
179180
return vectorArch.getSupportedVectorComparisonLength(elementStamp, canonicalizedCondition.getCanonicalCondition(), length) == length;
180181
}
181182

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/vector/replacements/vectorapi/nodes/VectorAPIMacroNode.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ public static SimdConstant maybeConstantValue(ValueNode node, CoreProviders prov
150150
GraalError.guarantee(providers != null, "must only be called with non-null providers unless isSimdConstant(node)");
151151
if (node.isJavaConstant()) {
152152
ValueNode readConstant = VectorAPIBoxingUtils.tryReadSimdConstant(node.asJavaConstant(), providers);
153-
if (readConstant.isConstant() && readConstant.asConstant() instanceof SimdConstant simdConstant) {
153+
if (readConstant != null && readConstant.isConstant() && readConstant.asConstant() instanceof SimdConstant simdConstant) {
154154
return simdConstant;
155155
}
156156
}

0 commit comments

Comments
 (0)