[AArch64][llvm] Add instructions for FEAT_MOPS_GO

clang/test/Driver/aarch64-vfat.c

@@ -13,3 +13,7 @@
 // RUN: %clang -target aarch64 -march=armv9.7a+btie -### -c %s 2>&1 | FileCheck -check-prefix=VFAT-BTIE %s
 // RUN: %clang -target aarch64 -march=armv9.7-a+btie -### -c %s 2>&1 | FileCheck -check-prefix=VFAT-BTIE %s
 // VFAT-BTIE: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+v9.7a"{{.*}} "-target-feature" "+btie"
+
+// RUN: %clang -target aarch64 -march=armv9.7a+mops-go -### -c %s 2>&1 | FileCheck -check-prefix=VFAT-MOPS-GO %s
+// RUN: %clang -target aarch64 -march=armv9.7-a+mops-go -### -c %s 2>&1 | FileCheck -check-prefix=VFAT-MOPS-GO %s
+// VFAT-MOPS-GO: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-cpu" "generic" "-target-feature" "+v9.7a"{{.*}} "-target-feature" "+mops-go"

clang/test/Driver/print-supported-extensions-aarch64.c

@@ -50,6 +50,7 @@
 // CHECK-NEXT:     lsui                FEAT_LSUI                                              Enable Armv9.6-A unprivileged load/store instructions
 // CHECK-NEXT:     lut                 FEAT_LUT                                               Enable Lookup Table instructions
 // CHECK-NEXT:     mops                FEAT_MOPS                                              Enable Armv8.8-A memcpy and memset acceleration instructions
+// CHECK-NEXT:     mops-go             FEAT_MOPS_GO                                           Enable memset acceleration granule only
 // CHECK-NEXT:     mpamv2              FEAT_MPAMv2                                            Enable Armv9.7-A MPAMv2 Lookaside Buffer Invalidate instructions
 // CHECK-NEXT:     memtag              FEAT_MTE, FEAT_MTE2                                    Enable Memory Tagging Extension
 // CHECK-NEXT:     mtetc               FEAT_MTETC                                             Enable Virtual Memory Tagging Extension

llvm/lib/Target/AArch64/AArch64Features.td

@@ -638,6 +638,9 @@ def FeatureS1POE2: ExtensionWithMArch<"poe2", "POE2", "FEAT_S1POE2",
 def FeatureTEV: ExtensionWithMArch<"tev", "TEV", "FEAT_TEV",
   "Enable TIndex Exception-like Vector instructions">;
 
+def FeatureMOPS_GO: ExtensionWithMArch<"mops-go", "MOPS_GO", "FEAT_MOPS_GO",
+  "Enable memset acceleration granule only">;
+
 //===----------------------------------------------------------------------===//
 //  Other Features
 //===----------------------------------------------------------------------===//

llvm/lib/Target/AArch64/AArch64InstrFormats.td

@@ -12604,7 +12604,7 @@ class MOPSMemoryMove<bits<2> opcode, bits<2> op1, bits<2> op2, string asm>
   : MOPSMemoryCopyMoveBase<1, opcode, op1, op2, asm>;
 
 class MOPSMemorySetBase<bit isTagging, bits<2> opcode, bit op1, bit op2,
-                        string asm>
+                        bit op3, string asm>
   : I<(outs GPR64common:$Rd_wb, GPR64:$Rn_wb),
       (ins GPR64common:$Rd, GPR64:$Rn, GPR64:$Rm),
       asm, "\t[$Rd]!, $Rn!, $Rm",
@@ -12620,7 +12620,8 @@ class MOPSMemorySetBase<bit isTagging, bits<2> opcode, bit op1, bit op2,
   let Inst{15-14} = opcode;
   let Inst{13} = op2;
   let Inst{12} = op1;
-  let Inst{11-10} = 0b01;
+  let Inst{11} = 0b0;
+  let Inst{10} = op3;
   let Inst{9-5} = Rn;
   let Inst{4-0} = Rd;
 
@@ -12629,11 +12630,11 @@ class MOPSMemorySetBase<bit isTagging, bits<2> opcode, bit op1, bit op2,
   let mayStore = 1;
 }
 
-class MOPSMemorySet<bits<2> opcode, bit op1, bit op2, string asm>
-  : MOPSMemorySetBase<0, opcode, op1, op2, asm>;
+class MOPSMemorySet<bits<2> opcode, bit op1, bit op2, bit op3, string asm>
+  : MOPSMemorySetBase<0, opcode, op1, op2, op3, asm>;
 
-class MOPSMemorySetTagging<bits<2> opcode, bit op1, bit op2, string asm>
-  : MOPSMemorySetBase<1, opcode, op1, op2, asm>;
+class MOPSMemorySetTagging<bits<2> opcode, bit op1, bit op2, bit op3, string asm>
+  : MOPSMemorySetBase<1, opcode, op1, op2, op3, asm>;
 
 multiclass MOPSMemoryCopyInsns<bits<2> opcode, string asm> {
   def ""   : MOPSMemoryCopy<opcode, 0b00, 0b00, asm>;
@@ -12674,17 +12675,27 @@ multiclass MOPSMemoryMoveInsns<bits<2> opcode, string asm> {
 }
 
 multiclass MOPSMemorySetInsns<bits<2> opcode, string asm> {
-  def "" : MOPSMemorySet<opcode, 0, 0, asm>;
-  def T  : MOPSMemorySet<opcode, 1, 0, asm # "t">;
-  def N  : MOPSMemorySet<opcode, 0, 1, asm # "n">;
-  def TN : MOPSMemorySet<opcode, 1, 1, asm # "tn">;
+  def "" : MOPSMemorySet<opcode, 0, 0, 1, asm>;
+  def T  : MOPSMemorySet<opcode, 1, 0, 1, asm # "t">;
+  def N  : MOPSMemorySet<opcode, 0, 1, 1, asm # "n">;
+  def TN : MOPSMemorySet<opcode, 1, 1, 1, asm # "tn">;
 }
 
 multiclass MOPSMemorySetTaggingInsns<bits<2> opcode, string asm> {
-  def "" : MOPSMemorySetTagging<opcode, 0, 0, asm>;
-  def T  : MOPSMemorySetTagging<opcode, 1, 0, asm # "t">;
-  def N  : MOPSMemorySetTagging<opcode, 0, 1, asm # "n">;
-  def TN : MOPSMemorySetTagging<opcode, 1, 1, asm # "tn">;
+  def "" : MOPSMemorySetTagging<opcode, 0, 0, 1, asm>;
+  def T  : MOPSMemorySetTagging<opcode, 1, 0, 1, asm # "t">;
+  def N  : MOPSMemorySetTagging<opcode, 0, 1, 1, asm # "n">;
+  def TN : MOPSMemorySetTagging<opcode, 1, 1, 1, asm # "tn">;
+}
+
+//----------------------------------------------------------------------------
+// MOPS Granule Only - FEAT_MOPS_GO
+//----------------------------------------------------------------------------
+multiclass MOPSGoMemorySetTaggingInsns<bits<2> opcode, string asm> {
+  def "" : MOPSMemorySetTagging<opcode, 0, 0, 0, asm>;
+  def T  : MOPSMemorySetTagging<opcode, 1, 0, 0, asm # "t">;
+  def N  : MOPSMemorySetTagging<opcode, 0, 1, 0, asm # "n">;
+  def TN : MOPSMemorySetTagging<opcode, 1, 1, 0, asm # "tn">;
 }
 
 //----------------------------------------------------------------------------

llvm/lib/Target/AArch64/AArch64InstrInfo.td

@@ -411,6 +411,8 @@ def HasS1POE2        : Predicate<"Subtarget->hasS1POE2()">,
                                  AssemblerPredicateWithAll<(all_of FeatureS1POE2), "poe2">;
 def HasTEV           : Predicate<"Subtarget->hasTEV()">,
                                  AssemblerPredicateWithAll<(all_of FeatureTEV), "tev">;
+def HasMOPS_GO       : Predicate<"Subtarget->hasMOPS_GO()">,
+                                 AssemblerPredicateWithAll<(all_of FeatureMOPS_GO), "mops-go">;
 def IsLE             : Predicate<"Subtarget->isLittleEndian()">;
 def IsBE             : Predicate<"!Subtarget->isLittleEndian()">;
 def IsWindows        : Predicate<"Subtarget->isTargetWindows()">;
@@ -10867,6 +10869,15 @@ let Predicates = [HasMOPS, HasMTE], Defs = [NZCV], Size = 12, mayLoad = 0, maySt
                                           [], "$Rd = $Rd_wb,$Rn = $Rn_wb">, Sched<[]>;
 }
 
+//-----------------------------------------------------------------------------
+// MOPS Granule Only Protection (FEAT_MOPS_GO)
+
+let Predicates = [HasMOPS_GO, HasMTE] in {
+    defm SETGOP : MOPSGoMemorySetTaggingInsns<0b00, "setgop">;
+    defm SETGOM : MOPSGoMemorySetTaggingInsns<0b01, "setgom">;
+    defm SETGOE : MOPSGoMemorySetTaggingInsns<0b10, "setgoe">;
+}
+
 //-----------------------------------------------------------------------------
 // v8.3 Pointer Authentication late patterns
 

llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp

@@ -3957,6 +3957,7 @@ static const struct Extension {
     {"poe2", {AArch64::FeatureS1POE2}},
     {"tev", {AArch64::FeatureTEV}},
     {"btie", {AArch64::FeatureBTIE}},
+    {"mops-go", {AArch64::FeatureMOPS_GO}},
 };
 
 static void setRequiredFeatureString(FeatureBitset FBS, std::string &Str) {
@@ -6050,7 +6051,19 @@ bool AArch64AsmParser::validateInstruction(MCInst &Inst, SMLoc &IDLoc,
   case AArch64::MOPSSETGE:
   case AArch64::MOPSSETGET:
   case AArch64::MOPSSETGEN:
-  case AArch64::MOPSSETGETN: {
+  case AArch64::MOPSSETGETN:
+  case AArch64::SETGOP:
+  case AArch64::SETGOPT:
+  case AArch64::SETGOPN:
+  case AArch64::SETGOPTN:
+  case AArch64::SETGOM:
+  case AArch64::SETGOMT:
+  case AArch64::SETGOMN:
+  case AArch64::SETGOMTN:
+  case AArch64::SETGOE:
+  case AArch64::SETGOET:
+  case AArch64::SETGOEN:
+  case AArch64::SETGOETN: {
     MCRegister Xd_wb = Inst.getOperand(0).getReg();
     MCRegister Xn_wb = Inst.getOperand(1).getReg();
     MCRegister Xd = Inst.getOperand(2).getReg();

llvm/test/MC/AArch64/arm-mops-go-diagnostics.s

@@ -0,0 +1,56 @@
+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+mops-go,+mte < %s 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+// All operand must be different from each other
+
+// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
+setgop [x0]!, x0!, x1
+setgop [x0]!, x1!, x0
+setgop [x1]!, x0!, x0
+
+// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
+setgom [x0]!, x0!, x1
+setgom [x0]!, x1!, x0
+setgom [x1]!, x0!, x0
+
+// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
+setgoe [x0]!, x0!, x1
+setgoe [x0]!, x1!, x0
+setgoe [x1]!, x0!, x0
+
+// SP cannot be used as argument at any position
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+setgop [sp]!, x1!, x2
+setgop [x0]!, sp!, x2
+setgop [x0]!, x1!, sp
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+setgom [sp]!, x1!, x2
+setgom [x0]!, sp!, x2
+setgom [x0]!, x1!, sp
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+setgoe [sp]!, x1!, x2
+setgoe [x0]!, sp!, x2
+setgoe [x0]!, x1!, sp
+
+// CHECK-ERROR: error: invalid operand for instruction
+setgop [xzr]!, x1!, x2
+
+// CHECK-ERROR: error: invalid operand for instruction
+setgom [xzr]!, x1!, x2
+
+// CHECK-ERROR: error: invalid operand for instruction
+setgoe [xzr]!, x1!, x2

llvm/test/MC/AArch64/arm-mops-go.s

@@ -0,0 +1,89 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+mops-go,+mte < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+all < %s \
+// RUN:        | llvm-objdump -d --mattr=+mops-go,+mte --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+all < %s \
+// RUN:        | llvm-objdump -d --mattr=-mops-go,-mte --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+mops-go,+mte < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+mops-go,+mte -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+//------------------------------------------------------------------------------
+// FEAT_MOPS_GO Extension instructions
+//------------------------------------------------------------------------------
+
+setgop [x3]!, x2!, x1
+// CHECK-INST:    setgop [x3]!, x2!, x1
+// CHECK-ENCODING: [0x43,0x00,0xc1,0x1d]
+// CHECK-UNKNOWN: 1dc10043
+// CHECK-ERROR: instruction requires: mops-go mte
+
+setgom [x3]!, x2!, x1
+// CHECK-INST:    setgom [x3]!, x2!, x1
+// CHECK-ENCODING: [0x43,0x40,0xc1,0x1d]
+// CHECK-UNKNOWN: 1dc14043
+// CHECK-ERROR: instruction requires: mops-go mte
+
+setgoe [x3]!, x2!, x1
+// CHECK-INST:    setgoe [x3]!, x2!, x1
+// CHECK-ENCODING: [0x43,0x80,0xc1,0x1d]
+// CHECK-UNKNOWN: 1dc18043
+// CHECK-ERROR: instruction requires: mops-go mte
+
+setgopn [x3]!, x2!, x1
+// CHECK-INST:    setgopn [x3]!, x2!, x1
+// CHECK-ENCODING: [0x43,0x20,0xc1,0x1d]
+// CHECK-UNKNOWN: 1dc12043
+// CHECK-ERROR: instruction requires: mops-go mte
+
+setgomn [x3]!, x2!, x1
+// CHECK-INST:    setgomn [x3]!, x2!, x1
+// CHECK-ENCODING: [0x43,0x60,0xc1,0x1d]
+// CHECK-UNKNOWN: 1dc16043
+// CHECK-ERROR: instruction requires: mops-go mte
+
+setgoen [x3]!, x2!, x1
+// CHECK-INST:    setgoen [x3]!, x2!, x1
+// CHECK-ENCODING: [0x43,0xa0,0xc1,0x1d]
+// CHECK-UNKNOWN: 1dc1a043
+// CHECK-ERROR: instruction requires: mops-go mte
+
+setgopt [x3]!, x2!, x1
+// CHECK-INST:    setgopt [x3]!, x2!, x1
+// CHECK-ENCODING: [0x43,0x10,0xc1,0x1d]
+// CHECK-UNKNOWN: 1dc11043
+// CHECK-ERROR: instruction requires: mops-go mte
+
+setgomt [x3]!, x2!, x1
+// CHECK-INST:    setgomt [x3]!, x2!, x1
+// CHECK-ENCODING: [0x43,0x50,0xc1,0x1d]
+// CHECK-UNKNOWN: 1dc15043
+// CHECK-ERROR: instruction requires: mops-go mte
+
+setgoet [x3]!, x2!, x1
+// CHECK-INST:    setgoet [x3]!, x2!, x1
+// CHECK-ENCODING: [0x43,0x90,0xc1,0x1d]
+// CHECK-UNKNOWN: 1dc19043
+// CHECK-ERROR: instruction requires: mops-go mte
+
+setgoptn [x3]!, x2!, x1
+// CHECK-INST:    setgoptn [x3]!, x2!, x1
+// CHECK-ENCODING: [0x43,0x30,0xc1,0x1d]
+// CHECK-UNKNOWN: 1dc13043
+// CHECK-ERROR: instruction requires: mops-go mte
+
+setgomtn [x3]!, x2!, x1
+// CHECK-INST:    setgomtn [x3]!, x2!, x1
+// CHECK-ENCODING: [0x43,0x70,0xc1,0x1d]
+// CHECK-UNKNOWN: 1dc17043
+// CHECK-ERROR: instruction requires: mops-go mte
+
+setgoetn [x3]!, x2!, x1
+// CHECK-INST:    setgoetn [x3]!, x2!, x1
+// CHECK-ENCODING: [0x43,0xb0,0xc1,0x1d]
+// CHECK-UNKNOWN: 1dc1b043
+// CHECK-ERROR: instruction requires: mops-go mte

llvm/unittests/TargetParser/TargetParserTest.cpp

@@ -1454,6 +1454,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
       AArch64::AEK_F16MM,        AArch64::AEK_F16F32DOT,
       AArch64::AEK_F16F32MM,     AArch64::AEK_POE2,
       AArch64::AEK_TEV,          AArch64::AEK_BTIE,
+      AArch64::AEK_MOPS_GO,
   };
 
   std::vector<StringRef> Features;
@@ -1580,6 +1581,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
   EXPECT_TRUE(llvm::is_contained(Features, "+poe2"));
   EXPECT_TRUE(llvm::is_contained(Features, "+tev"));
   EXPECT_TRUE(llvm::is_contained(Features, "+btie"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+mops-go"));
 
   // Assuming we listed every extension above, this should produce the same
   // result.
@@ -1761,6 +1763,7 @@ TEST(TargetParserTest, AArch64ArchExtFeature) {
       {"poe2", "nopoe2", "+poe2", "-poe2"},
       {"tev", "notev", "+tev", "-tev"},
       {"btie", "nobtie", "+btie", "-btie"},
+      {"mops-go", "nomops-go", "+mops-go", "-mops-go"},
   };
 
   for (unsigned i = 0; i < std::size(ArchExt); i++) {