[AArch64] Add target features for Armv9-A Scalable Matrix Extension (SME)

c-rhodes · c-rhodes · commit 9e42675103e2 · 2021-07-12T13:28:10.000Z
First patch in a series adding MC layer support for the Arm Scalable Matrix Extension. This patch adds the following features: sme, sme-i64, sme-f64 The sme-i64 and sme-f64 flags are for the optional I16I64 and F64F64 features. If a target supports I16I64 then the following instructions are implemented: * 64-bit integer ADDHA and ADDVA variants (D105570). * SMOPA, SMOPS, SUMOPA, SUMOPS, UMOPA, UMOPS, USMOPA, and USMOPS instructions that accumulate 16-bit integer outer products into 64-bit integer tiles. If a target supports F64F64 then the FMOPA and FMOPS instructions that accumulate double-precision floating-point outer products into double-precision tiles are implemented. Outer products are implemented in D105571. The reference can be found here: https://developer.arm.com/documentation/ddi0602/2021-06 Reviewed By: CarolineConcatto Differential Revision: https://reviews.llvm.org/D105569
diff --git a/llvm/include/llvm/Support/AArch64TargetParser.def b/llvm/include/llvm/Support/AArch64TargetParser.def
@@ -110,6 +110,9 @@ AARCH64_ARCH_EXT_NAME("ls64",         AArch64::AEK_LS64,        "+ls64",  "-ls64
 AARCH64_ARCH_EXT_NAME("brbe",         AArch64::AEK_BRBE,        "+brbe",  "-brbe")
 AARCH64_ARCH_EXT_NAME("pauth",        AArch64::AEK_PAUTH,       "+pauth", "-pauth")
 AARCH64_ARCH_EXT_NAME("flagm",        AArch64::AEK_FLAGM,       "+flagm", "-flagm")
+AARCH64_ARCH_EXT_NAME("sme",          AArch64::AEK_SME,         "+sme",   "-sme")
+AARCH64_ARCH_EXT_NAME("sme-f64",      AArch64::AEK_SMEF64,      "+sme-f64", "-sme-f64")
+AARCH64_ARCH_EXT_NAME("sme-i64",      AArch64::AEK_SMEI64,      "+sme-i64", "-sme-i64")
 #undef AARCH64_ARCH_EXT_NAME
 
 #ifndef AARCH64_CPU_NAME
diff --git a/llvm/include/llvm/Support/AArch64TargetParser.h b/llvm/include/llvm/Support/AArch64TargetParser.h
@@ -66,6 +66,9 @@ enum ArchExtKind : uint64_t {
   AEK_BRBE =        1ULL << 34,
   AEK_PAUTH =       1ULL << 35,
   AEK_FLAGM =       1ULL << 36,
+  AEK_SME =         1ULL << 37,
+  AEK_SMEF64 =      1ULL << 38,
+  AEK_SMEI64 =      1ULL << 39,
 };
 
 enum class ArchKind {
diff --git a/llvm/lib/Support/AArch64TargetParser.cpp b/llvm/lib/Support/AArch64TargetParser.cpp
@@ -106,6 +106,12 @@ bool AArch64::getExtensionFeatures(uint64_t Extensions,
     Features.push_back("+pauth");
   if (Extensions & AEK_FLAGM)
     Features.push_back("+flagm");
+  if (Extensions & AArch64::AEK_SME)
+    Features.push_back("+sme");
+  if (Extensions & AArch64::AEK_SMEF64)
+    Features.push_back("+sme-f64");
+  if (Extensions & AArch64::AEK_SMEI64)
+    Features.push_back("+sme-i64");
 
   return true;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
@@ -429,6 +429,17 @@ def FeatureEnhancedCounterVirtualization :
 def FeatureRME : SubtargetFeature<"rme", "HasRME",
     "true", "Enable Realm Management Extension">;
 
+// FIXME: SME should only imply the subset of SVE(2) instructions that are
+// legal in streaming mode.
+def FeatureSME : SubtargetFeature<"sme", "HasSME", "true",
+  "Enable Scalable Matrix Extension (SME)", [FeatureSVE2, FeatureBF16]>;
+
+def FeatureSMEF64 : SubtargetFeature<"sme-f64", "HasSMEF64", "true",
+  "Enable Scalable Matrix Extension (SME) F64F64 instructions", [FeatureSME]>;
+
+def FeatureSMEI64 : SubtargetFeature<"sme-i64", "HasSMEI64", "true",
+  "Enable Scalable Matrix Extension (SME) I16I64 instructions", [FeatureSME]>;
+
 //===----------------------------------------------------------------------===//
 // Architectures.
 //
@@ -549,6 +560,10 @@ def PAUnsupported : AArch64Unsupported {
   let F = [HasPAuth];
 }
 
+def SMEUnsupported : AArch64Unsupported {
+  let F = [HasSME, HasSMEF64, HasSMEI64];
+}
+
 include "AArch64SchedA53.td"
 include "AArch64SchedA55.td"
 include "AArch64SchedA57.td"
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -122,6 +122,12 @@ def HasSVE2SHA3      : Predicate<"Subtarget->hasSVE2SHA3()">,
                                  AssemblerPredicate<(all_of FeatureSVE2SHA3), "sve2-sha3">;
 def HasSVE2BitPerm   : Predicate<"Subtarget->hasSVE2BitPerm()">,
                                  AssemblerPredicate<(all_of FeatureSVE2BitPerm), "sve2-bitperm">;
+def HasSME           : Predicate<"Subtarget->hasSME()">,
+                                 AssemblerPredicate<(all_of FeatureSME), "sme">;
+def HasSMEF64        : Predicate<"Subtarget->hasSMEF64()">,
+                                 AssemblerPredicate<(all_of FeatureSMEF64), "sme-f64">;
+def HasSMEI64        : Predicate<"Subtarget->hasSMEI64()">,
+                                 AssemblerPredicate<(all_of FeatureSMEI64), "sme-i64">;
 def HasRCPC          : Predicate<"Subtarget->hasRCPC()">,
                                  AssemblerPredicate<(all_of FeatureRCPC), "rcpc">;
 def HasAltNZCV       : Predicate<"Subtarget->hasAlternativeNZCV()">,
diff --git a/llvm/lib/Target/AArch64/AArch64SchedA53.td b/llvm/lib/Target/AArch64/AArch64SchedA53.td
@@ -27,7 +27,8 @@ def CortexA53Model : SchedMachineModel {
   let CompleteModel = 1;
 
   list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,
-                                                    PAUnsupported.F);
+                                                    PAUnsupported.F,
+                                                    SMEUnsupported.F);
 }
 
 
diff --git a/llvm/lib/Target/AArch64/AArch64SchedA57.td b/llvm/lib/Target/AArch64/AArch64SchedA57.td
@@ -32,7 +32,8 @@ def CortexA57Model : SchedMachineModel {
   let CompleteModel = 1;
 
   list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,
-                                                    PAUnsupported.F);
+                                                    PAUnsupported.F,
+                                                    SMEUnsupported.F);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/AArch64SchedCyclone.td b/llvm/lib/Target/AArch64/AArch64SchedCyclone.td
@@ -19,7 +19,8 @@ def CycloneModel : SchedMachineModel {
   let CompleteModel = 1;
 
   list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,
-                                                    PAUnsupported.F);
+                                                    PAUnsupported.F,
+                                                    SMEUnsupported.F);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/AArch64SchedExynosM3.td b/llvm/lib/Target/AArch64/AArch64SchedExynosM3.td
@@ -25,7 +25,8 @@ def ExynosM3Model : SchedMachineModel {
   let CompleteModel         =   1; // Use the default model otherwise.
 
   list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,
-                                                    PAUnsupported.F);
+                                                    PAUnsupported.F,
+                                                    SMEUnsupported.F);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/AArch64SchedExynosM4.td b/llvm/lib/Target/AArch64/AArch64SchedExynosM4.td
@@ -25,7 +25,8 @@ def ExynosM4Model : SchedMachineModel {
   let CompleteModel         =   1; // Use the default model otherwise.
 
   list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,
-                                                    PAUnsupported.F);
+                                                    PAUnsupported.F,
+                                                    SMEUnsupported.F);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/AArch64SchedExynosM5.td b/llvm/lib/Target/AArch64/AArch64SchedExynosM5.td
@@ -25,7 +25,8 @@ def ExynosM5Model : SchedMachineModel {
   let CompleteModel         =   1; // Use the default model otherwise.
 
   list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,
-                                                    PAUnsupported.F);
+                                                    PAUnsupported.F,
+                                                    SMEUnsupported.F);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/AArch64SchedFalkor.td b/llvm/lib/Target/AArch64/AArch64SchedFalkor.td
@@ -24,7 +24,8 @@ def FalkorModel : SchedMachineModel {
   let CompleteModel = 1;
 
   list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,
-                                                    PAUnsupported.F);
+                                                    PAUnsupported.F,
+                                                    SMEUnsupported.F);
   // FIXME: Remove when all errors have been fixed.
   let FullInstRWOverlapCheck = 0;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64SchedKryo.td b/llvm/lib/Target/AArch64/AArch64SchedKryo.td
@@ -28,7 +28,8 @@ def KryoModel : SchedMachineModel {
   let CompleteModel = 1;
 
   list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,
-                                                    PAUnsupported.F);
+                                                    PAUnsupported.F,
+                                                    SMEUnsupported.F);
   // FIXME: Remove when all errors have been fixed.
   let FullInstRWOverlapCheck = 0;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64SchedThunderX.td b/llvm/lib/Target/AArch64/AArch64SchedThunderX.td
@@ -26,7 +26,8 @@ def ThunderXT8XModel : SchedMachineModel {
   let CompleteModel = 1;
 
   list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,
-                                                    PAUnsupported.F);
+                                                    PAUnsupported.F,
+                                                    SMEUnsupported.F);
   // FIXME: Remove when all errors have been fixed.
   let FullInstRWOverlapCheck = 0;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64SchedThunderX2T99.td b/llvm/lib/Target/AArch64/AArch64SchedThunderX2T99.td
@@ -26,7 +26,8 @@ def ThunderX2T99Model : SchedMachineModel {
   let CompleteModel         =   1;
 
   list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,
-                                                    PAUnsupported.F);
+                                                    PAUnsupported.F,
+                                                    SMEUnsupported.F);
   // FIXME: Remove when all errors have been fixed.
   let FullInstRWOverlapCheck = 0;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -186,6 +186,11 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
   // Armv9-A Extensions
   bool HasRME = false;
 
+  // Arm Scalable Matrix Extension (SME)
+  bool HasSME = false;
+  bool HasSMEF64 = false;
+  bool HasSMEI64 = false;
+
   // Future architecture extensions.
   bool HasETE = false;
   bool HasTRBE = false;
@@ -485,6 +490,11 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
     return HasEnhancedCounterVirtualization;
   }
 
+  // Arm Scalable Matrix Extension (SME)
+  bool hasSME() const { return HasSME; }
+  bool hasSMEF64() const { return HasSMEF64; }
+  bool hasSMEI64() const { return HasSMEI64; }
+
   bool isLittleEndian() const { return IsLittle; }
 
   bool isTargetDarwin() const { return TargetTriple.isOSDarwin(); }
diff --git a/llvm/test/MC/AArch64/SME/feature.s b/llvm/test/MC/AArch64/SME/feature.s
@@ -0,0 +1,11 @@
+// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+sme < %s | FileCheck %s
+// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+sme-f64 < %s | FileCheck %s
+// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+sme-i64 < %s | FileCheck %s
+
+// Verify +sme flags imply +sve2
+tbx z0.b, z1.b, z2.b
+// CHECK: tbx z0.b, z1.b, z2.b
+
+// Verify +sme flags imply +bf16
+bfdot z0.s, z1.h, z2.h
+// CHECK-INST: bfdot z0.s, z1.h, z2.h
diff --git a/llvm/unittests/Support/TargetParserTest.cpp b/llvm/unittests/Support/TargetParserTest.cpp
@@ -1360,7 +1360,9 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
     AArch64::AEK_SVE,      AArch64::AEK_SVE2,
     AArch64::AEK_SVE2AES,  AArch64::AEK_SVE2SM4,
     AArch64::AEK_SVE2SHA3, AArch64::AEK_SVE2BITPERM,
-    AArch64::AEK_RCPC,     AArch64::AEK_FP16FML };
+    AArch64::AEK_RCPC,     AArch64::AEK_FP16FML,
+    AArch64::AEK_SME,      AArch64::AEK_SMEF64,
+    AArch64::AEK_SMEI64 };
 
   std::vector<StringRef> Features;
 
@@ -1392,6 +1394,9 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
   EXPECT_TRUE(llvm::is_contained(Features, "+sve2-sm4"));
   EXPECT_TRUE(llvm::is_contained(Features, "+sve2-sha3"));
   EXPECT_TRUE(llvm::is_contained(Features, "+sve2-bitperm"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+sme"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+sme-f64"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+sme-i64"));
 }
 
 TEST(TargetParserTest, AArch64ArchFeatures) {
@@ -1437,6 +1442,9 @@ TEST(TargetParserTest, AArch64ArchExtFeature) {
                               {"i8mm", "noi8mm", "+i8mm", "-i8mm"},
                               {"f32mm", "nof32mm", "+f32mm", "-f32mm"},
                               {"f64mm", "nof64mm", "+f64mm", "-f64mm"},
+                              {"sme", "nosme", "+sme", "-sme"},
+                              {"sme-f64", "nosme-f64", "+sme-f64", "-sme-f64"},
+                              {"sme-i64", "nosme-i64", "+sme-i64", "-sme-i64"},
 };
 
   for (unsigned i = 0; i < array_lengthof(ArchExt); i++) {

Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,8 @@ def CortexA53Model : SchedMachineModel {`
`27`	`27`	`let CompleteModel = 1;`
`28`	`28`
`29`	`29`	`list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,`
`30`		`- PAUnsupported.F);`
	`30`	`+ PAUnsupported.F,`
	`31`	`+ SMEUnsupported.F);`
`31`	`32`	`}`
`32`	`33`
`33`	`34`
Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,8 @@ def CortexA57Model : SchedMachineModel {`
`32`	`32`	`let CompleteModel = 1;`
`33`	`33`
`34`	`34`	`list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,`
`35`		`- PAUnsupported.F);`
	`35`	`+ PAUnsupported.F,`
	`36`	`+ SMEUnsupported.F);`
`36`	`37`	`}`
`37`	`38`
`38`	`39`	`//===----------------------------------------------------------------------===//`
Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,8 @@ def CycloneModel : SchedMachineModel {`
`19`	`19`	`let CompleteModel = 1;`
`20`	`20`
`21`	`21`	`list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,`
`22`		`- PAUnsupported.F);`
	`22`	`+ PAUnsupported.F,`
	`23`	`+ SMEUnsupported.F);`
`23`	`24`	`}`
`24`	`25`
`25`	`26`	`//===----------------------------------------------------------------------===//`
Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,8 @@ def ExynosM3Model : SchedMachineModel {`
`25`	`25`	`let CompleteModel = 1; // Use the default model otherwise.`
`26`	`26`
`27`	`27`	`list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,`
`28`		`- PAUnsupported.F);`
	`28`	`+ PAUnsupported.F,`
	`29`	`+ SMEUnsupported.F);`
`29`	`30`	`}`
`30`	`31`
`31`	`32`	`//===----------------------------------------------------------------------===//`
Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,8 @@ def ExynosM4Model : SchedMachineModel {`
`25`	`25`	`let CompleteModel = 1; // Use the default model otherwise.`
`26`	`26`
`27`	`27`	`list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,`
`28`		`- PAUnsupported.F);`
	`28`	`+ PAUnsupported.F,`
	`29`	`+ SMEUnsupported.F);`
`29`	`30`	`}`
`30`	`31`
`31`	`32`	`//===----------------------------------------------------------------------===//`