diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ab2b53f9d060e..ca6f1aa202918 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15173,6 +15173,9 @@ static SDValue performVecReduceAddCombineWithUADDLP(SDNode *N,
 // Turn a v8i8/v16i8 extended vecreduce into a udot/sdot and vecreduce
 //   vecreduce.add(ext(A)) to vecreduce.add(DOT(zero, A, one))
 //   vecreduce.add(mul(ext(A), ext(B))) to vecreduce.add(DOT(zero, A, B))
+// If we have vectors larger than v16i8 we extract v16i8 vectors,
+// Follow the same steps above to get DOT instructions concatenate them
+// and generate vecreduce.add(concat_vector(DOT, DOT2, ..)).
 static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG,
                                           const AArch64Subtarget *ST) {
   if (!ST->hasDotProd())
@@ -15198,7 +15201,9 @@ static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   EVT Op0VT = A.getOperand(0).getValueType();
-  if (Op0VT != MVT::v8i8 && Op0VT != MVT::v16i8)
+  bool IsValidElementCount = Op0VT.getVectorNumElements() % 8 == 0;
+  bool IsValidSize = Op0VT.getScalarSizeInBits() == 8;
+  if (!IsValidElementCount || !IsValidSize)
     return SDValue();
 
   SDLoc DL(Op0);
@@ -15209,13 +15214,65 @@ static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG,
   else
     B = B.getOperand(0);
 
-  SDValue Zeros =
-      DAG.getConstant(0, DL, Op0VT == MVT::v8i8 ? MVT::v2i32 : MVT::v4i32);
+  unsigned IsMultipleOf16 = Op0VT.getVectorNumElements() % 16 == 0;
+  unsigned NumOfVecReduce;
+  EVT TargetType;
+  if (IsMultipleOf16) {
+    NumOfVecReduce = Op0VT.getVectorNumElements() / 16;
+    TargetType = MVT::v4i32;
+  } else {
+    NumOfVecReduce = Op0VT.getVectorNumElements() / 8;
+    TargetType = MVT::v2i32;
+  }
   auto DotOpcode =
       (ExtOpcode == ISD::ZERO_EXTEND) ? AArch64ISD::UDOT : AArch64ISD::SDOT;
-  SDValue Dot = DAG.getNode(DotOpcode, DL, Zeros.getValueType(), Zeros,
-                            A.getOperand(0), B);
-  return DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0), Dot);
+  // Handle the case where we need to generate only one Dot operation.
+  if (NumOfVecReduce == 1) {
+    SDValue Zeros = DAG.getConstant(0, DL, TargetType);
+    SDValue Dot = DAG.getNode(DotOpcode, DL, Zeros.getValueType(), Zeros,
+                              A.getOperand(0), B);
+    return DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0), Dot);
+  }
+  // Generate Dot instructions that are multiple of 16.
+  unsigned VecReduce16Num = Op0VT.getVectorNumElements() / 16;
+  SmallVector<SDValue, 4> SDotVec16;
+  unsigned I = 0;
+  for (; I < VecReduce16Num; I += 1) {
+    SDValue Zeros = DAG.getConstant(0, DL, MVT::v4i32);
+    SDValue Op0 =
+        DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v16i8, A.getOperand(0),
+                    DAG.getConstant(I * 16, DL, MVT::i64));
+    SDValue Op1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v16i8, B,
+                              DAG.getConstant(I * 16, DL, MVT::i64));
+    SDValue Dot =
+        DAG.getNode(DotOpcode, DL, Zeros.getValueType(), Zeros, Op0, Op1);
+    SDotVec16.push_back(Dot);
+  }
+  // Concatenate dot operations.
+  EVT SDot16EVT =
+      EVT::getVectorVT(*DAG.getContext(), MVT::i32, 4 * VecReduce16Num);
+  SDValue ConcatSDot16 =
+      DAG.getNode(ISD::CONCAT_VECTORS, DL, SDot16EVT, SDotVec16);
+  SDValue VecReduceAdd16 =
+      DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0), ConcatSDot16);
+  unsigned VecReduce8Num = (Op0VT.getVectorNumElements() % 16) / 8;
+  if (VecReduce8Num == 0)
+    return VecReduceAdd16;
+
+  // Generate the remainder Dot operation that is multiple of 8.
+  SmallVector<SDValue, 4> SDotVec8;
+  SDValue Zeros = DAG.getConstant(0, DL, MVT::v2i32);
+  SDValue Vec8Op0 =
+      DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, A.getOperand(0),
+                  DAG.getConstant(I * 16, DL, MVT::i64));
+  SDValue Vec8Op1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, B,
+                                DAG.getConstant(I * 16, DL, MVT::i64));
+  SDValue Dot =
+      DAG.getNode(DotOpcode, DL, Zeros.getValueType(), Zeros, Vec8Op0, Vec8Op1);
+  SDValue VecReudceAdd8 =
+      DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0), Dot);
+  return DAG.getNode(ISD::ADD, DL, N->getValueType(0), VecReduceAdd16,
+                     VecReudceAdd8);
 }
 
 // Given an (integer) vecreduce, we know the order of the inputs does not
diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index 870438505a3f2..00b44a8ecd578 100644
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -583,25 +583,19 @@ entry:
 define i32 @test_udot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
 ; CHECK-LABEL: test_udot_v24i8:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr q0, [x1]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-NEXT:    ldr q1, [x0]
+; CHECK-NEXT:    movi v3.2d, #0000000000000000
 ; CHECK-NEXT:    ldr d2, [x0, #16]
-; CHECK-NEXT:    ushll v5.8h, v0.8b, #0
-; CHECK-NEXT:    ldr d3, [x1, #16]
-; CHECK-NEXT:    ushll v4.8h, v1.8b, #0
-; CHECK-NEXT:    ushll v2.8h, v2.8b, #0
-; CHECK-NEXT:    umull2 v6.4s, v5.8h, v4.8h
-; CHECK-NEXT:    umull v4.4s, v5.4h, v4.4h
-; CHECK-NEXT:    ushll v3.8h, v3.8b, #0
-; CHECK-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-NEXT:    umlal2 v6.4s, v3.8h, v2.8h
-; CHECK-NEXT:    umlal v4.4s, v3.4h, v2.4h
-; CHECK-NEXT:    ushll2 v0.8h, v0.16b, #0
-; CHECK-NEXT:    umlal2 v6.4s, v0.8h, v1.8h
-; CHECK-NEXT:    umlal v4.4s, v0.4h, v1.4h
-; CHECK-NEXT:    add v0.4s, v4.4s, v6.4s
-; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    ldr d4, [x1, #16]
+; CHECK-NEXT:    ldr q5, [x1]
+; CHECK-NEXT:    udot v0.2s, v4.8b, v2.8b
+; CHECK-NEXT:    udot v3.4s, v5.16b, v1.16b
+; CHECK-NEXT:    addp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT:    addv s1, v3.4s
 ; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov w9, s1
+; CHECK-NEXT:    add w8, w9, w8
 ; CHECK-NEXT:    add w0, w8, w2
 ; CHECK-NEXT:    ret
 entry:
@@ -618,18 +612,19 @@ entry:
 define i32 @test_udot_v24i8_nomla(ptr nocapture readonly %a1) {
 ; CHECK-LABEL: test_udot_v24i8_nomla:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr d0, [x0, #16]
-; CHECK-NEXT:    ldr q1, [x0]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v2.8h, v1.8b, #0
-; CHECK-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-NEXT:    uaddl2 v3.4s, v2.8h, v0.8h
-; CHECK-NEXT:    uaddl v0.4s, v2.4h, v0.4h
-; CHECK-NEXT:    uaddw2 v2.4s, v3.4s, v1.8h
-; CHECK-NEXT:    uaddw v0.4s, v0.4s, v1.4h
-; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    movi v0.8b, #1
+; CHECK-NEXT:    ldr d4, [x0, #16]
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    ldr q5, [x0]
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    movi v3.16b, #1
+; CHECK-NEXT:    udot v2.2s, v4.8b, v0.8b
+; CHECK-NEXT:    udot v1.4s, v5.16b, v3.16b
+; CHECK-NEXT:    addp v0.2s, v2.2s, v2.2s
+; CHECK-NEXT:    addv s1, v1.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov w9, s1
+; CHECK-NEXT:    add w0, w9, w8
 ; CHECK-NEXT:    ret
 entry:
   %0 = load <24 x i8>, ptr %a1
@@ -640,25 +635,19 @@ entry:
 define i32 @test_sdot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
 ; CHECK-LABEL: test_sdot_v24i8:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr q0, [x1]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
 ; CHECK-NEXT:    ldr q1, [x0]
+; CHECK-NEXT:    movi v3.2d, #0000000000000000
 ; CHECK-NEXT:    ldr d2, [x0, #16]
-; CHECK-NEXT:    sshll v5.8h, v0.8b, #0
-; CHECK-NEXT:    ldr d3, [x1, #16]
-; CHECK-NEXT:    sshll v4.8h, v1.8b, #0
-; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
-; CHECK-NEXT:    smull2 v6.4s, v5.8h, v4.8h
-; CHECK-NEXT:    smull v4.4s, v5.4h, v4.4h
-; CHECK-NEXT:    sshll v3.8h, v3.8b, #0
-; CHECK-NEXT:    sshll2 v1.8h, v1.16b, #0
-; CHECK-NEXT:    smlal2 v6.4s, v3.8h, v2.8h
-; CHECK-NEXT:    smlal v4.4s, v3.4h, v2.4h
-; CHECK-NEXT:    sshll2 v0.8h, v0.16b, #0
-; CHECK-NEXT:    smlal2 v6.4s, v0.8h, v1.8h
-; CHECK-NEXT:    smlal v4.4s, v0.4h, v1.4h
-; CHECK-NEXT:    add v0.4s, v4.4s, v6.4s
-; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    ldr d4, [x1, #16]
+; CHECK-NEXT:    ldr q5, [x1]
+; CHECK-NEXT:    sdot v0.2s, v4.8b, v2.8b
+; CHECK-NEXT:    sdot v3.4s, v5.16b, v1.16b
+; CHECK-NEXT:    addp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT:    addv s1, v3.4s
 ; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov w9, s1
+; CHECK-NEXT:    add w8, w9, w8
 ; CHECK-NEXT:    add w0, w8, w2
 ; CHECK-NEXT:    ret
 entry:
@@ -675,208 +664,202 @@ entry:
 define i32 @test_sdot_v24i8_double(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
 ; CHECK-LABEL: test_sdot_v24i8_double:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr b0, [sp]
-; CHECK-NEXT:    add x8, sp, #8
-; CHECK-NEXT:    ldr b3, [sp, #64]
-; CHECK-NEXT:    add x9, sp, #16
-; CHECK-NEXT:    ldr b1, [sp, #192]
-; CHECK-NEXT:    add x10, sp, #200
+; CHECK-NEXT:    ldr b0, [sp, #128]
+; CHECK-NEXT:    add x8, sp, #136
+; CHECK-NEXT:    fmov s1, w0
+; CHECK-NEXT:    add x9, sp, #152
+; CHECK-NEXT:    ldr b2, [sp, #256]
+; CHECK-NEXT:    add x11, sp, #264
 ; CHECK-NEXT:    ld1 { v0.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #72
-; CHECK-NEXT:    ldr b5, [sp, #256]
-; CHECK-NEXT:    fmov s6, w0
-; CHECK-NEXT:    ld1 { v1.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #264
-; CHECK-NEXT:    ld1 { v3.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #24
-; CHECK-NEXT:    ld1 { v0.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #80
-; CHECK-NEXT:    ld1 { v5.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #208
-; CHECK-NEXT:    ldr b16, [sp, #128]
-; CHECK-NEXT:    add x11, sp, #56
-; CHECK-NEXT:    ld1 { v3.b }[2], [x9]
+; CHECK-NEXT:    add x8, sp, #144
+; CHECK-NEXT:    mov v1.b[1], w1
+; CHECK-NEXT:    add x10, sp, #168
+; CHECK-NEXT:    ld1 { v2.b }[1], [x11]
+; CHECK-NEXT:    add x11, sp, #520
+; CHECK-NEXT:    ldr b4, [sp, #512]
+; CHECK-NEXT:    add x12, sp, #72
+; CHECK-NEXT:    ld1 { v0.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #160
+; CHECK-NEXT:    mov v1.b[2], w2
+; CHECK-NEXT:    ldr b5, [sp, #320]
+; CHECK-NEXT:    ld1 { v4.b }[1], [x11]
+; CHECK-NEXT:    add x11, sp, #328
+; CHECK-NEXT:    ldr b3, [sp, #64]
+; CHECK-NEXT:    add x13, sp, #80
+; CHECK-NEXT:    ld1 { v0.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #176
+; CHECK-NEXT:    mov v1.b[3], w3
+; CHECK-NEXT:    ld1 { v5.b }[1], [x11]
+; CHECK-NEXT:    ld1 { v3.b }[1], [x12]
+; CHECK-NEXT:    add x12, sp, #184
+; CHECK-NEXT:    add x11, sp, #528
+; CHECK-NEXT:    ldr b7, [sp, #640]
+; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #272
+; CHECK-NEXT:    mov v1.b[4], w4
+; CHECK-NEXT:    ldr b16, [sp, #448]
+; CHECK-NEXT:    ld1 { v3.b }[2], [x13]
+; CHECK-NEXT:    ld1 { v2.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #336
+; CHECK-NEXT:    ld1 { v0.b }[5], [x10]
+; CHECK-NEXT:    add x10, sp, #280
+; CHECK-NEXT:    mov v1.b[5], w5
+; CHECK-NEXT:    ld1 { v4.b }[2], [x11]
+; CHECK-NEXT:    ld1 { v5.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #344
+; CHECK-NEXT:    ld1 { v2.b }[3], [x10]
+; CHECK-NEXT:    add x10, sp, #192
+; CHECK-NEXT:    ld1 { v0.b }[6], [x9]
 ; CHECK-NEXT:    add x9, sp, #88
-; CHECK-NEXT:    ld1 { v0.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #32
-; CHECK-NEXT:    ld1 { v1.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #136
-; CHECK-NEXT:    mov v6.b[1], w1
-; CHECK-NEXT:    ldr b2, [sp, #384]
+; CHECK-NEXT:    mov v1.b[6], w6
+; CHECK-NEXT:    add x11, sp, #536
+; CHECK-NEXT:    ld1 { v5.b }[3], [x8]
+; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    ld1 { v3.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #40
-; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #96
-; CHECK-NEXT:    ld1 { v16.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #144
-; CHECK-NEXT:    mov v6.b[2], w2
-; CHECK-NEXT:    ldr b4, [sp, #448]
-; CHECK-NEXT:    ld1 { v3.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #48
-; CHECK-NEXT:    ld1 { v0.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #104
-; CHECK-NEXT:    ld1 { v16.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #120
-; CHECK-NEXT:    mov v6.b[3], w3
-; CHECK-NEXT:    add x12, sp, #584
-; CHECK-NEXT:    ld1 { v3.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #216
-; CHECK-NEXT:    ld1 { v0.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #112
-; CHECK-NEXT:    ld1 { v1.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #272
-; CHECK-NEXT:    ld1 { v3.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #224
-; CHECK-NEXT:    ld1 { v0.b }[7], [x11]
-; CHECK-NEXT:    add x11, sp, #232
-; CHECK-NEXT:    ld1 { v5.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #152
-; CHECK-NEXT:    ld1 { v1.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #280
-; CHECK-NEXT:    ld1 { v3.b }[7], [x10]
-; CHECK-NEXT:    add x10, sp, #240
-; CHECK-NEXT:    ld1 { v16.b }[3], [x9]
 ; CHECK-NEXT:    add x9, sp, #288
-; CHECK-NEXT:    ld1 { v5.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #160
-; CHECK-NEXT:    ld1 { v1.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #248
-; CHECK-NEXT:    mov v6.b[4], w4
-; CHECK-NEXT:    ld1 { v16.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #168
-; CHECK-NEXT:    ld1 { v5.b }[4], [x9]
+; CHECK-NEXT:    ld1 { v0.b }[7], [x12]
+; CHECK-NEXT:    mov v1.b[7], w7
+; CHECK-NEXT:    ld1 { v4.b }[3], [x11]
+; CHECK-NEXT:    add x11, sp, #96
+; CHECK-NEXT:    ld1 { v2.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #544
+; CHECK-NEXT:    ld1 { v0.b }[8], [x10]
+; CHECK-NEXT:    add x10, sp, #200
+; CHECK-NEXT:    ld1 { v1.b }[8], [x8]
+; CHECK-NEXT:    add x8, sp, #8
+; CHECK-NEXT:    ld1 { v3.b }[4], [x11]
+; CHECK-NEXT:    add x11, sp, #352
+; CHECK-NEXT:    ld1 { v4.b }[4], [x9]
 ; CHECK-NEXT:    add x9, sp, #296
-; CHECK-NEXT:    ld1 { v1.b }[6], [x10]
-; CHECK-NEXT:    add x10, sp, #392
-; CHECK-NEXT:    mov v6.b[5], w5
-; CHECK-NEXT:    ld1 { v16.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #176
-; CHECK-NEXT:    ld1 { v2.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #400
-; CHECK-NEXT:    ld1 { v5.b }[5], [x9]
+; CHECK-NEXT:    ld1 { v0.b }[9], [x10]
+; CHECK-NEXT:    add x10, sp, #208
+; CHECK-NEXT:    ld1 { v1.b }[9], [x8]
+; CHECK-NEXT:    add x8, sp, #16
+; CHECK-NEXT:    ld1 { v5.b }[4], [x11]
+; CHECK-NEXT:    add x11, sp, #104
+; CHECK-NEXT:    ld1 { v2.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #552
+; CHECK-NEXT:    ld1 { v0.b }[10], [x10]
+; CHECK-NEXT:    add x10, sp, #216
+; CHECK-NEXT:    ld1 { v3.b }[5], [x11]
+; CHECK-NEXT:    add x11, sp, #360
+; CHECK-NEXT:    ld1 { v1.b }[10], [x8]
+; CHECK-NEXT:    add x8, sp, #24
+; CHECK-NEXT:    ld1 { v4.b }[5], [x9]
 ; CHECK-NEXT:    add x9, sp, #304
-; CHECK-NEXT:    mov v6.b[6], w6
-; CHECK-NEXT:    ld1 { v1.b }[7], [x11]
-; CHECK-NEXT:    ld1 { v16.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #184
-; CHECK-NEXT:    ld1 { v2.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #456
-; CHECK-NEXT:    ld1 { v5.b }[6], [x9]
+; CHECK-NEXT:    ld1 { v0.b }[11], [x10]
+; CHECK-NEXT:    add x10, sp, #224
+; CHECK-NEXT:    ld1 { v5.b }[5], [x11]
+; CHECK-NEXT:    add x11, sp, #112
+; CHECK-NEXT:    ld1 { v1.b }[11], [x8]
+; CHECK-NEXT:    add x8, sp, #32
+; CHECK-NEXT:    ld1 { v2.b }[6], [x9]
+; CHECK-NEXT:    add x9, sp, #560
+; CHECK-NEXT:    ld1 { v0.b }[12], [x10]
+; CHECK-NEXT:    add x10, sp, #232
+; CHECK-NEXT:    ld1 { v3.b }[6], [x11]
+; CHECK-NEXT:    add x11, sp, #368
+; CHECK-NEXT:    ld1 { v1.b }[12], [x8]
+; CHECK-NEXT:    add x8, sp, #40
+; CHECK-NEXT:    ld1 { v4.b }[6], [x9]
 ; CHECK-NEXT:    add x9, sp, #312
-; CHECK-NEXT:    mov v6.b[7], w7
-; CHECK-NEXT:    add x11, sp, #648
-; CHECK-NEXT:    ld1 { v16.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #408
-; CHECK-NEXT:    ld1 { v4.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #328
+; CHECK-NEXT:    ld1 { v5.b }[6], [x11]
+; CHECK-NEXT:    add x11, sp, #568
+; CHECK-NEXT:    ld1 { v0.b }[13], [x10]
+; CHECK-NEXT:    add x10, sp, #240
+; CHECK-NEXT:    ld1 { v1.b }[13], [x8]
+; CHECK-NEXT:    add x8, sp, #48
+; CHECK-NEXT:    ld1 { v2.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #376
+; CHECK-NEXT:    ld1 { v4.b }[7], [x11]
+; CHECK-NEXT:    add x11, sp, #576
+; CHECK-NEXT:    ld1 { v0.b }[14], [x10]
+; CHECK-NEXT:    add x10, sp, #248
 ; CHECK-NEXT:    ld1 { v5.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #416
-; CHECK-NEXT:    ld1 { v2.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #464
-; CHECK-NEXT:    sshll v6.8h, v6.8b, #0
-; CHECK-NEXT:    sshll v17.8h, v16.8b, #0
-; CHECK-NEXT:    ld1 { v4.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #472
-; CHECK-NEXT:    ld1 { v2.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #424
-; CHECK-NEXT:    sshll v7.8h, v3.8b, #0
-; CHECK-NEXT:    sshll v16.8h, v5.8b, #0
-; CHECK-NEXT:    smull2 v3.4s, v6.8h, v17.8h
-; CHECK-NEXT:    ld1 { v4.b }[3], [x8]
-; CHECK-NEXT:    smull v5.4s, v6.4h, v17.4h
-; CHECK-NEXT:    ldr b6, [sp, #320]
-; CHECK-NEXT:    ld1 { v2.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #480
+; CHECK-NEXT:    add x9, sp, #120
+; CHECK-NEXT:    ld1 { v1.b }[14], [x8]
+; CHECK-NEXT:    add x8, sp, #56
+; CHECK-NEXT:    ld1 { v4.b }[8], [x11]
+; CHECK-NEXT:    add x11, sp, #384
+; CHECK-NEXT:    ld1 { v0.b }[15], [x10]
+; CHECK-NEXT:    add x10, sp, #584
+; CHECK-NEXT:    ld1 { v3.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #648
+; CHECK-NEXT:    ld1 { v5.b }[8], [x11]
+; CHECK-NEXT:    add x11, sp, #592
+; CHECK-NEXT:    ld1 { v1.b }[15], [x8]
+; CHECK-NEXT:    add x8, sp, #392
+; CHECK-NEXT:    ld1 { v4.b }[9], [x10]
+; CHECK-NEXT:    add x10, sp, #456
+; CHECK-NEXT:    ld1 { v7.b }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #656
+; CHECK-NEXT:    ld1 { v5.b }[9], [x8]
+; CHECK-NEXT:    add x8, sp, #400
+; CHECK-NEXT:    ld1 { v16.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #464
+; CHECK-NEXT:    ld1 { v4.b }[10], [x11]
+; CHECK-NEXT:    add x11, sp, #600
+; CHECK-NEXT:    ld1 { v7.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #664
+; CHECK-NEXT:    ld1 { v5.b }[10], [x8]
+; CHECK-NEXT:    add x8, sp, #408
+; CHECK-NEXT:    ld1 { v16.b }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #472
+; CHECK-NEXT:    ld1 { v4.b }[11], [x11]
+; CHECK-NEXT:    add x11, sp, #608
+; CHECK-NEXT:    ld1 { v7.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #672
+; CHECK-NEXT:    ld1 { v5.b }[11], [x8]
+; CHECK-NEXT:    add x8, sp, #416
+; CHECK-NEXT:    ld1 { v16.b }[3], [x10]
+; CHECK-NEXT:    add x10, sp, #480
+; CHECK-NEXT:    ld1 { v4.b }[12], [x11]
+; CHECK-NEXT:    add x11, sp, #616
+; CHECK-NEXT:    ld1 { v7.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #680
+; CHECK-NEXT:    ld1 { v5.b }[12], [x8]
+; CHECK-NEXT:    add x8, sp, #424
+; CHECK-NEXT:    ld1 { v16.b }[4], [x10]
+; CHECK-NEXT:    add x10, sp, #488
+; CHECK-NEXT:    ld1 { v4.b }[13], [x11]
+; CHECK-NEXT:    add x11, sp, #624
+; CHECK-NEXT:    ld1 { v7.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #688
+; CHECK-NEXT:    ld1 { v5.b }[13], [x8]
 ; CHECK-NEXT:    add x8, sp, #432
-; CHECK-NEXT:    ldr b17, [sp, #512]
-; CHECK-NEXT:    ld1 { v6.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #520
-; CHECK-NEXT:    ld1 { v4.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #336
-; CHECK-NEXT:    ld1 { v2.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #488
-; CHECK-NEXT:    ld1 { v17.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #440
-; CHECK-NEXT:    ld1 { v6.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #528
-; CHECK-NEXT:    ld1 { v4.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #344
-; CHECK-NEXT:    smlal2 v3.4s, v7.8h, v16.8h
-; CHECK-NEXT:    ld1 { v2.b }[7], [x10]
-; CHECK-NEXT:    smlal v5.4s, v7.4h, v16.4h
-; CHECK-NEXT:    ldr b7, [sp, #640]
-; CHECK-NEXT:    ld1 { v17.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #352
-; CHECK-NEXT:    ld1 { v6.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #536
-; CHECK-NEXT:    ld1 { v7.b }[1], [x11]
-; CHECK-NEXT:    add x11, sp, #656
-; CHECK-NEXT:    ldr b16, [sp, #576]
-; CHECK-NEXT:    ld1 { v17.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #544
-; CHECK-NEXT:    ld1 { v6.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #360
-; CHECK-NEXT:    ld1 { v16.b }[1], [x12]
-; CHECK-NEXT:    add x12, sp, #592
-; CHECK-NEXT:    ld1 { v7.b }[2], [x11]
-; CHECK-NEXT:    add x11, sp, #664
-; CHECK-NEXT:    ld1 { v17.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #552
-; CHECK-NEXT:    ld1 { v6.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #368
-; CHECK-NEXT:    ld1 { v16.b }[2], [x12]
-; CHECK-NEXT:    add x12, sp, #600
-; CHECK-NEXT:    ld1 { v7.b }[3], [x11]
-; CHECK-NEXT:    add x11, sp, #672
-; CHECK-NEXT:    ld1 { v17.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #560
-; CHECK-NEXT:    ld1 { v6.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #376
-; CHECK-NEXT:    ld1 { v16.b }[3], [x12]
-; CHECK-NEXT:    add x12, sp, #608
-; CHECK-NEXT:    ld1 { v7.b }[4], [x11]
-; CHECK-NEXT:    add x11, sp, #680
-; CHECK-NEXT:    ld1 { v17.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #568
-; CHECK-NEXT:    ld1 { v6.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #496
-; CHECK-NEXT:    ld1 { v16.b }[4], [x12]
-; CHECK-NEXT:    add x12, sp, #616
-; CHECK-NEXT:    ld1 { v7.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #688
-; CHECK-NEXT:    ld1 { v4.b }[6], [x9]
+; CHECK-NEXT:    ld1 { v16.b }[5], [x10]
+; CHECK-NEXT:    add x10, sp, #496
+; CHECK-NEXT:    ld1 { v4.b }[14], [x11]
+; CHECK-NEXT:    add x11, sp, #632
+; CHECK-NEXT:    ld1 { v7.b }[6], [x9]
 ; CHECK-NEXT:    add x9, sp, #696
-; CHECK-NEXT:    ld1 { v17.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #504
-; CHECK-NEXT:    ld1 { v16.b }[5], [x12]
-; CHECK-NEXT:    ld1 { v7.b }[6], [x11]
-; CHECK-NEXT:    ld1 { v4.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #624
-; CHECK-NEXT:    sshll v6.8h, v6.8b, #0
-; CHECK-NEXT:    sshll v17.8h, v17.8b, #0
+; CHECK-NEXT:    ld1 { v5.b }[14], [x8]
+; CHECK-NEXT:    add x8, sp, #440
+; CHECK-NEXT:    ld1 { v16.b }[6], [x10]
+; CHECK-NEXT:    add x10, sp, #504
+; CHECK-NEXT:    movi v6.2d, #0000000000000000
+; CHECK-NEXT:    ld1 { v4.b }[15], [x11]
+; CHECK-NEXT:    movi v17.2d, #0000000000000000
 ; CHECK-NEXT:    ld1 { v7.b }[7], [x9]
-; CHECK-NEXT:    ld1 { v16.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #632
-; CHECK-NEXT:    smull v18.4s, v6.4h, v17.4h
-; CHECK-NEXT:    smull2 v6.4s, v6.8h, v17.8h
-; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
-; CHECK-NEXT:    sshll v7.8h, v7.8b, #0
-; CHECK-NEXT:    ld1 { v16.b }[7], [x8]
-; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-NEXT:    smlal2 v6.4s, v4.8h, v7.8h
-; CHECK-NEXT:    smlal v18.4s, v4.4h, v7.4h
-; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
-; CHECK-NEXT:    sshll v4.8h, v16.8b, #0
-; CHECK-NEXT:    smlal2 v3.4s, v0.8h, v1.8h
-; CHECK-NEXT:    smlal v5.4s, v0.4h, v1.4h
-; CHECK-NEXT:    smlal2 v6.4s, v2.8h, v4.8h
-; CHECK-NEXT:    smlal v18.4s, v2.4h, v4.4h
-; CHECK-NEXT:    add v0.4s, v5.4s, v3.4s
-; CHECK-NEXT:    add v1.4s, v18.4s, v6.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ld1 { v5.b }[15], [x8]
+; CHECK-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-NEXT:    ld1 { v16.b }[7], [x10]
+; CHECK-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-NEXT:    sdot v17.4s, v1.16b, v0.16b
+; CHECK-NEXT:    sdot v6.4s, v5.16b, v4.16b
+; CHECK-NEXT:    sdot v18.2s, v3.8b, v2.8b
+; CHECK-NEXT:    sdot v19.2s, v16.8b, v7.8b
+; CHECK-NEXT:    addv s0, v17.4s
+; CHECK-NEXT:    addv s2, v6.4s
+; CHECK-NEXT:    addp v1.2s, v18.2s, v18.2s
+; CHECK-NEXT:    addp v3.2s, v19.2s, v19.2s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov w10, s2
+; CHECK-NEXT:    fmov w9, s1
+; CHECK-NEXT:    fmov w11, s3
+; CHECK-NEXT:    add w8, w8, w9
+; CHECK-NEXT:    add w9, w10, w11
+; CHECK-NEXT:    add w0, w8, w9
 ; CHECK-NEXT:    ret
 entry:
   %az = sext <24 x i8> %a to <24 x i32>
@@ -894,108 +877,112 @@ entry:
 define i32 @test_sdot_v24i8_double_nomla(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) {
 ; CHECK-LABEL: test_sdot_v24i8_double_nomla:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr b0, [sp]
-; CHECK-NEXT:    add x8, sp, #8
-; CHECK-NEXT:    add x9, sp, #16
-; CHECK-NEXT:    ldr b1, [sp, #64]
-; CHECK-NEXT:    ldr b2, [sp, #384]
-; CHECK-NEXT:    add x10, sp, #24
-; CHECK-NEXT:    ld1 { v0.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #72
-; CHECK-NEXT:    ldr b3, [sp, #448]
-; CHECK-NEXT:    fmov s4, w0
-; CHECK-NEXT:    ldr b5, [sp, #320]
-; CHECK-NEXT:    add x12, sp, #472
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    ldr b1, [sp, #320]
+; CHECK-NEXT:    add x8, sp, #328
+; CHECK-NEXT:    add x9, sp, #376
+; CHECK-NEXT:    add x10, sp, #8
+; CHECK-NEXT:    ldr b2, [sp, #64]
+; CHECK-NEXT:    mov v0.b[1], w1
+; CHECK-NEXT:    add x13, sp, #72
 ; CHECK-NEXT:    ld1 { v1.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #456
-; CHECK-NEXT:    ld1 { v0.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #392
-; CHECK-NEXT:    mov v4.b[1], w1
-; CHECK-NEXT:    add x11, sp, #56
-; CHECK-NEXT:    ld1 { v3.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #32
-; CHECK-NEXT:    ld1 { v2.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #80
-; CHECK-NEXT:    ld1 { v0.b }[3], [x10]
-; CHECK-NEXT:    add x10, sp, #40
-; CHECK-NEXT:    mov v4.b[2], w2
-; CHECK-NEXT:    ld1 { v1.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #328
-; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #88
-; CHECK-NEXT:    ld1 { v5.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #48
-; CHECK-NEXT:    mov v4.b[3], w3
+; CHECK-NEXT:    add x8, sp, #336
+; CHECK-NEXT:    ldr b3, [sp, #448]
+; CHECK-NEXT:    add x14, sp, #456
+; CHECK-NEXT:    ld1 { v2.b }[1], [x13]
+; CHECK-NEXT:    add x11, sp, #16
+; CHECK-NEXT:    mov v0.b[2], w2
+; CHECK-NEXT:    add x13, sp, #80
+; CHECK-NEXT:    ld1 { v1.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #344
+; CHECK-NEXT:    ld1 { v3.b }[1], [x14]
+; CHECK-NEXT:    add x14, sp, #464
+; CHECK-NEXT:    ld1 { v2.b }[2], [x13]
+; CHECK-NEXT:    add x12, sp, #24
+; CHECK-NEXT:    mov v0.b[3], w3
+; CHECK-NEXT:    add x13, sp, #88
 ; CHECK-NEXT:    ld1 { v1.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #464
-; CHECK-NEXT:    ld1 { v0.b }[5], [x10]
-; CHECK-NEXT:    add x10, sp, #96
-; CHECK-NEXT:    ld1 { v3.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #104
-; CHECK-NEXT:    ld1 { v1.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #336
-; CHECK-NEXT:    ld1 { v0.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #112
-; CHECK-NEXT:    mov v4.b[4], w4
-; CHECK-NEXT:    ld1 { v5.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #344
-; CHECK-NEXT:    ld1 { v1.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #400
-; CHECK-NEXT:    ld1 { v3.b }[3], [x12]
-; CHECK-NEXT:    ld1 { v0.b }[7], [x11]
-; CHECK-NEXT:    add x11, sp, #480
-; CHECK-NEXT:    ld1 { v2.b }[2], [x8]
 ; CHECK-NEXT:    add x8, sp, #352
-; CHECK-NEXT:    ld1 { v5.b }[3], [x10]
-; CHECK-NEXT:    add x10, sp, #488
-; CHECK-NEXT:    ld1 { v1.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #408
-; CHECK-NEXT:    ld1 { v3.b }[4], [x11]
-; CHECK-NEXT:    add x11, sp, #416
-; CHECK-NEXT:    mov v4.b[5], w5
-; CHECK-NEXT:    ld1 { v2.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #120
-; CHECK-NEXT:    ld1 { v5.b }[4], [x8]
+; CHECK-NEXT:    ld1 { v3.b }[2], [x14]
+; CHECK-NEXT:    add x14, sp, #472
+; CHECK-NEXT:    ld1 { v2.b }[3], [x13]
+; CHECK-NEXT:    add x13, sp, #96
+; CHECK-NEXT:    mov v0.b[4], w4
+; CHECK-NEXT:    movi v6.8b, #1
+; CHECK-NEXT:    ld1 { v1.b }[4], [x8]
 ; CHECK-NEXT:    add x8, sp, #360
-; CHECK-NEXT:    ld1 { v3.b }[5], [x10]
-; CHECK-NEXT:    add x10, sp, #496
-; CHECK-NEXT:    ld1 { v1.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #424
-; CHECK-NEXT:    ld1 { v2.b }[4], [x11]
-; CHECK-NEXT:    ld1 { v5.b }[5], [x8]
+; CHECK-NEXT:    ld1 { v3.b }[3], [x14]
+; CHECK-NEXT:    add x14, sp, #480
+; CHECK-NEXT:    ld1 { v2.b }[4], [x13]
+; CHECK-NEXT:    add x13, sp, #104
+; CHECK-NEXT:    mov v0.b[5], w5
+; CHECK-NEXT:    ld1 { v1.b }[5], [x8]
 ; CHECK-NEXT:    add x8, sp, #368
-; CHECK-NEXT:    mov v4.b[6], w6
-; CHECK-NEXT:    ld1 { v3.b }[6], [x10]
-; CHECK-NEXT:    add x10, sp, #504
-; CHECK-NEXT:    ld1 { v2.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #432
-; CHECK-NEXT:    ld1 { v5.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #376
-; CHECK-NEXT:    mov v4.b[7], w7
-; CHECK-NEXT:    ld1 { v3.b }[7], [x10]
-; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-NEXT:    ld1 { v3.b }[4], [x14]
+; CHECK-NEXT:    add x14, sp, #488
+; CHECK-NEXT:    ld1 { v2.b }[5], [x13]
+; CHECK-NEXT:    add x13, sp, #496
+; CHECK-NEXT:    mov v0.b[6], w6
+; CHECK-NEXT:    ld1 { v1.b }[6], [x8]
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    ld1 { v3.b }[5], [x14]
+; CHECK-NEXT:    movi v4.16b, #1
+; CHECK-NEXT:    mov v0.b[7], w7
+; CHECK-NEXT:    ld1 { v1.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #384
+; CHECK-NEXT:    ld1 { v3.b }[6], [x13]
+; CHECK-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-NEXT:    ld1 { v0.b }[8], [x8]
+; CHECK-NEXT:    add x8, sp, #32
+; CHECK-NEXT:    ld1 { v1.b }[8], [x9]
+; CHECK-NEXT:    add x9, sp, #392
+; CHECK-NEXT:    movi v7.2d, #0000000000000000
+; CHECK-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-NEXT:    ld1 { v0.b }[9], [x10]
+; CHECK-NEXT:    add x10, sp, #40
+; CHECK-NEXT:    ld1 { v1.b }[9], [x9]
+; CHECK-NEXT:    add x9, sp, #400
+; CHECK-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-NEXT:    ld1 { v0.b }[10], [x11]
+; CHECK-NEXT:    add x11, sp, #48
+; CHECK-NEXT:    ld1 { v1.b }[10], [x9]
+; CHECK-NEXT:    add x9, sp, #408
+; CHECK-NEXT:    ld1 { v0.b }[11], [x12]
+; CHECK-NEXT:    add x12, sp, #56
+; CHECK-NEXT:    ld1 { v1.b }[11], [x9]
+; CHECK-NEXT:    add x9, sp, #416
+; CHECK-NEXT:    ld1 { v0.b }[12], [x8]
+; CHECK-NEXT:    add x8, sp, #424
+; CHECK-NEXT:    ld1 { v1.b }[12], [x9]
+; CHECK-NEXT:    add x9, sp, #112
+; CHECK-NEXT:    ld1 { v0.b }[13], [x10]
+; CHECK-NEXT:    add x10, sp, #120
+; CHECK-NEXT:    ld1 { v1.b }[13], [x8]
+; CHECK-NEXT:    add x8, sp, #432
 ; CHECK-NEXT:    ld1 { v2.b }[6], [x9]
-; CHECK-NEXT:    ld1 { v5.b }[7], [x8]
+; CHECK-NEXT:    add x9, sp, #504
+; CHECK-NEXT:    ld1 { v0.b }[14], [x11]
+; CHECK-NEXT:    ld1 { v1.b }[14], [x8]
 ; CHECK-NEXT:    add x8, sp, #440
-; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
-; CHECK-NEXT:    sshll v3.8h, v3.8b, #0
-; CHECK-NEXT:    ld1 { v2.b }[7], [x8]
-; CHECK-NEXT:    sshll v5.8h, v5.8b, #0
-; CHECK-NEXT:    saddl2 v6.4s, v4.8h, v1.8h
-; CHECK-NEXT:    saddl v1.4s, v4.4h, v1.4h
-; CHECK-NEXT:    saddl2 v4.4s, v5.8h, v3.8h
-; CHECK-NEXT:    saddl v3.4s, v5.4h, v3.4h
-; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
-; CHECK-NEXT:    saddw2 v5.4s, v6.4s, v0.8h
-; CHECK-NEXT:    saddw v0.4s, v1.4s, v0.4h
-; CHECK-NEXT:    saddw2 v1.4s, v4.4s, v2.8h
-; CHECK-NEXT:    saddw v2.4s, v3.4s, v2.4h
-; CHECK-NEXT:    add v0.4s, v0.4s, v5.4s
-; CHECK-NEXT:    add v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    addv s0, v0.4s
-; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ld1 { v2.b }[7], [x10]
+; CHECK-NEXT:    ld1 { v3.b }[7], [x9]
+; CHECK-NEXT:    ld1 { v0.b }[15], [x12]
+; CHECK-NEXT:    ld1 { v1.b }[15], [x8]
+; CHECK-NEXT:    sdot v7.2s, v2.8b, v6.8b
+; CHECK-NEXT:    sdot v5.2s, v3.8b, v6.8b
+; CHECK-NEXT:    sdot v16.4s, v0.16b, v4.16b
+; CHECK-NEXT:    sdot v17.4s, v1.16b, v4.16b
+; CHECK-NEXT:    addp v0.2s, v7.2s, v7.2s
+; CHECK-NEXT:    addp v1.2s, v5.2s, v5.2s
+; CHECK-NEXT:    addv s2, v16.4s
+; CHECK-NEXT:    addv s3, v17.4s
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    fmov w9, s2
+; CHECK-NEXT:    fmov w11, s1
+; CHECK-NEXT:    fmov w10, s3
+; CHECK-NEXT:    add w8, w9, w8
+; CHECK-NEXT:    add w9, w10, w11
+; CHECK-NEXT:    add w0, w8, w9
 ; CHECK-NEXT:    ret
 entry:
   %az = sext <24 x i8> %a to <24 x i32>
@@ -1483,27 +1470,12 @@ entry:
 define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
 ; CHECK-LABEL: test_udot_v32i8:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q0, q3, [x1]
-; CHECK-NEXT:    ushll2 v7.8h, v0.16b, #0
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ldp q2, q1, [x0]
-; CHECK-NEXT:    ushll v6.8h, v3.8b, #0
-; CHECK-NEXT:    ushll2 v3.8h, v3.16b, #0
-; CHECK-NEXT:    ushll2 v5.8h, v2.16b, #0
-; CHECK-NEXT:    ushll v2.8h, v2.8b, #0
-; CHECK-NEXT:    umull2 v16.4s, v7.8h, v5.8h
-; CHECK-NEXT:    umull2 v17.4s, v0.8h, v2.8h
-; CHECK-NEXT:    umull v5.4s, v7.4h, v5.4h
-; CHECK-NEXT:    umull v0.4s, v0.4h, v2.4h
-; CHECK-NEXT:    ushll v4.8h, v1.8b, #0
-; CHECK-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-NEXT:    umlal2 v17.4s, v6.8h, v4.8h
-; CHECK-NEXT:    umlal2 v16.4s, v3.8h, v1.8h
-; CHECK-NEXT:    umlal v5.4s, v3.4h, v1.4h
-; CHECK-NEXT:    umlal v0.4s, v6.4h, v4.4h
-; CHECK-NEXT:    add v1.4s, v17.4s, v16.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v5.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ldp q3, q2, [x0]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ldr q1, [x1, #16]
+; CHECK-NEXT:    udot v0.4s, v1.16b, v2.16b
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    udot v0.4s, v1.16b, v3.16b
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    add w0, w8, w2
@@ -1522,19 +1494,13 @@ entry:
 define i32 @test_udot_v32i8_nomla(ptr nocapture readonly %a1) {
 ; CHECK-LABEL: test_udot_v32i8_nomla:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q1, q0, [x0]
-; CHECK-NEXT:    ushll2 v3.8h, v1.16b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    ushll v2.8h, v0.8b, #0
-; CHECK-NEXT:    ushll2 v0.8h, v0.16b, #0
-; CHECK-NEXT:    uaddl2 v5.4s, v1.8h, v2.8h
-; CHECK-NEXT:    uaddl2 v4.4s, v3.8h, v0.8h
-; CHECK-NEXT:    uaddl v0.4s, v3.4h, v0.4h
-; CHECK-NEXT:    uaddl v1.4s, v1.4h, v2.4h
-; CHECK-NEXT:    add v2.4s, v5.4s, v4.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    movi v0.16b, #1
+; CHECK-NEXT:    ldr q2, [x0, #16]
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    udot v1.4s, v2.16b, v0.16b
+; CHECK-NEXT:    ldr q2, [x0]
+; CHECK-NEXT:    udot v1.4s, v2.16b, v0.16b
+; CHECK-NEXT:    addv s0, v1.4s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
 entry:
@@ -1546,27 +1512,12 @@ entry:
 define i32 @test_sdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
 ; CHECK-LABEL: test_sdot_v32i8:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q0, q3, [x1]
-; CHECK-NEXT:    sshll2 v7.8h, v0.16b, #0
-; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ldp q2, q1, [x0]
-; CHECK-NEXT:    sshll v6.8h, v3.8b, #0
-; CHECK-NEXT:    sshll2 v3.8h, v3.16b, #0
-; CHECK-NEXT:    sshll2 v5.8h, v2.16b, #0
-; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
-; CHECK-NEXT:    smull2 v16.4s, v7.8h, v5.8h
-; CHECK-NEXT:    smull2 v17.4s, v0.8h, v2.8h
-; CHECK-NEXT:    smull v5.4s, v7.4h, v5.4h
-; CHECK-NEXT:    smull v0.4s, v0.4h, v2.4h
-; CHECK-NEXT:    sshll v4.8h, v1.8b, #0
-; CHECK-NEXT:    sshll2 v1.8h, v1.16b, #0
-; CHECK-NEXT:    smlal2 v17.4s, v6.8h, v4.8h
-; CHECK-NEXT:    smlal2 v16.4s, v3.8h, v1.8h
-; CHECK-NEXT:    smlal v5.4s, v3.4h, v1.4h
-; CHECK-NEXT:    smlal v0.4s, v6.4h, v4.4h
-; CHECK-NEXT:    add v1.4s, v17.4s, v16.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v5.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ldp q3, q2, [x0]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ldr q1, [x1, #16]
+; CHECK-NEXT:    sdot v0.4s, v1.16b, v2.16b
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    sdot v0.4s, v1.16b, v3.16b
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    add w0, w8, w2
@@ -1585,45 +1536,13 @@ entry:
 define i32 @test_sdot_v32i8_double(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
 ; CHECK-LABEL: test_sdot_v32i8_double:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll2 v19.8h, v0.16b, #0
-; CHECK-NEXT:    sshll2 v20.8h, v2.16b, #0
-; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
-; CHECK-NEXT:    smull v21.4s, v19.4h, v20.4h
-; CHECK-NEXT:    smull2 v19.4s, v19.8h, v20.8h
-; CHECK-NEXT:    sshll v17.8h, v3.8b, #0
-; CHECK-NEXT:    sshll v18.8h, v1.8b, #0
-; CHECK-NEXT:    sshll2 v1.8h, v1.16b, #0
-; CHECK-NEXT:    smull2 v20.4s, v0.8h, v2.8h
-; CHECK-NEXT:    sshll2 v3.8h, v3.16b, #0
-; CHECK-NEXT:    smull v0.4s, v0.4h, v2.4h
-; CHECK-NEXT:    sshll v16.8h, v5.8b, #0
-; CHECK-NEXT:    smlal2 v19.4s, v1.8h, v3.8h
-; CHECK-NEXT:    smlal v21.4s, v1.4h, v3.4h
-; CHECK-NEXT:    sshll2 v1.8h, v5.16b, #0
-; CHECK-NEXT:    sshll2 v2.8h, v4.16b, #0
-; CHECK-NEXT:    sshll v3.8h, v4.8b, #0
-; CHECK-NEXT:    sshll v4.8h, v7.8b, #0
-; CHECK-NEXT:    sshll2 v5.8h, v7.16b, #0
-; CHECK-NEXT:    sshll2 v7.8h, v6.16b, #0
-; CHECK-NEXT:    sshll v6.8h, v6.8b, #0
-; CHECK-NEXT:    smlal2 v20.4s, v18.8h, v17.8h
-; CHECK-NEXT:    smlal v0.4s, v18.4h, v17.4h
-; CHECK-NEXT:    smull2 v17.4s, v2.8h, v7.8h
-; CHECK-NEXT:    smull2 v18.4s, v3.8h, v6.8h
-; CHECK-NEXT:    smull v2.4s, v2.4h, v7.4h
-; CHECK-NEXT:    smull v3.4s, v3.4h, v6.4h
-; CHECK-NEXT:    smlal2 v17.4s, v1.8h, v5.8h
-; CHECK-NEXT:    smlal2 v18.4s, v16.8h, v4.8h
-; CHECK-NEXT:    smlal v2.4s, v1.4h, v5.4h
-; CHECK-NEXT:    smlal v3.4s, v16.4h, v4.4h
-; CHECK-NEXT:    add v1.4s, v20.4s, v19.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v21.4s
-; CHECK-NEXT:    add v4.4s, v18.4s, v17.4s
-; CHECK-NEXT:    add v2.4s, v3.4s, v2.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    add v1.4s, v2.4s, v4.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-NEXT:    sdot v16.4s, v1.16b, v3.16b
+; CHECK-NEXT:    sdot v17.4s, v5.16b, v7.16b
+; CHECK-NEXT:    sdot v16.4s, v0.16b, v2.16b
+; CHECK-NEXT:    sdot v17.4s, v4.16b, v6.16b
+; CHECK-NEXT:    add v0.4s, v16.4s, v17.4s
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
@@ -1643,29 +1562,14 @@ entry:
 define i32 @test_sdot_v32i8_double_nomla(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
 ; CHECK-LABEL: test_sdot_v32i8_double_nomla:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v3.8h, v1.8b, #0
-; CHECK-NEXT:    sshll2 v1.8h, v1.16b, #0
-; CHECK-NEXT:    sshll2 v6.8h, v0.16b, #0
-; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-NEXT:    saddl2 v7.4s, v6.8h, v1.8h
-; CHECK-NEXT:    saddl2 v16.4s, v0.8h, v3.8h
-; CHECK-NEXT:    sshll v2.8h, v5.8b, #0
-; CHECK-NEXT:    saddl v0.4s, v0.4h, v3.4h
-; CHECK-NEXT:    sshll2 v3.8h, v5.16b, #0
-; CHECK-NEXT:    sshll2 v5.8h, v4.16b, #0
-; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
-; CHECK-NEXT:    saddl v1.4s, v6.4h, v1.4h
-; CHECK-NEXT:    add v6.4s, v16.4s, v7.4s
-; CHECK-NEXT:    saddl2 v7.4s, v5.8h, v3.8h
-; CHECK-NEXT:    saddl v3.4s, v5.4h, v3.4h
-; CHECK-NEXT:    saddl2 v5.4s, v4.8h, v2.8h
-; CHECK-NEXT:    saddl v2.4s, v4.4h, v2.4h
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    add v1.4s, v5.4s, v7.4s
-; CHECK-NEXT:    add v2.4s, v2.4s, v3.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v6.4s
-; CHECK-NEXT:    add v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    movi v2.16b, #1
+; CHECK-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-NEXT:    movi v6.2d, #0000000000000000
+; CHECK-NEXT:    sdot v3.4s, v1.16b, v2.16b
+; CHECK-NEXT:    sdot v6.4s, v5.16b, v2.16b
+; CHECK-NEXT:    sdot v3.4s, v0.16b, v2.16b
+; CHECK-NEXT:    sdot v6.4s, v4.16b, v2.16b
+; CHECK-NEXT:    add v0.4s, v3.4s, v6.4s
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
@@ -2277,37 +2181,15 @@ entry:
 define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
 ; CHECK-LABEL: test_udot_v48i8:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr q2, [x0]
-; CHECK-NEXT:    ldr q4, [x1]
-; CHECK-NEXT:    ldp q5, q0, [x1, #16]
-; CHECK-NEXT:    ushll2 v16.8h, v2.16b, #0
-; CHECK-NEXT:    ushll v2.8h, v2.8b, #0
-; CHECK-NEXT:    ushll2 v19.8h, v4.16b, #0
-; CHECK-NEXT:    ushll v4.8h, v4.8b, #0
-; CHECK-NEXT:    umull2 v20.4s, v19.8h, v16.8h
-; CHECK-NEXT:    ldp q3, q1, [x0, #16]
-; CHECK-NEXT:    umull2 v21.4s, v4.8h, v2.8h
-; CHECK-NEXT:    umull v16.4s, v19.4h, v16.4h
-; CHECK-NEXT:    umull v2.4s, v4.4h, v2.4h
-; CHECK-NEXT:    ushll v18.8h, v0.8b, #0
-; CHECK-NEXT:    ushll2 v0.8h, v0.16b, #0
-; CHECK-NEXT:    ushll v7.8h, v1.8b, #0
-; CHECK-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-NEXT:    umlal2 v21.4s, v18.8h, v7.8h
-; CHECK-NEXT:    umlal2 v20.4s, v0.8h, v1.8h
-; CHECK-NEXT:    umlal v16.4s, v0.4h, v1.4h
-; CHECK-NEXT:    umlal v2.4s, v18.4h, v7.4h
-; CHECK-NEXT:    ushll v6.8h, v3.8b, #0
-; CHECK-NEXT:    ushll2 v3.8h, v3.16b, #0
-; CHECK-NEXT:    ushll v17.8h, v5.8b, #0
-; CHECK-NEXT:    ushll2 v5.8h, v5.16b, #0
-; CHECK-NEXT:    umlal2 v21.4s, v17.8h, v6.8h
-; CHECK-NEXT:    umlal2 v20.4s, v5.8h, v3.8h
-; CHECK-NEXT:    umlal v16.4s, v5.4h, v3.4h
-; CHECK-NEXT:    umlal v2.4s, v17.4h, v6.4h
-; CHECK-NEXT:    add v0.4s, v21.4s, v20.4s
-; CHECK-NEXT:    add v1.4s, v2.4s, v16.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ldr q1, [x1, #32]
+; CHECK-NEXT:    ldr q2, [x0, #32]
+; CHECK-NEXT:    udot v0.4s, v1.16b, v2.16b
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    ldp q3, q2, [x0]
+; CHECK-NEXT:    udot v0.4s, v1.16b, v3.16b
+; CHECK-NEXT:    ldr q1, [x1, #16]
+; CHECK-NEXT:    udot v0.4s, v1.16b, v2.16b
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    add w0, w8, w2
@@ -2326,26 +2208,15 @@ entry:
 define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) {
 ; CHECK-LABEL: test_udot_v48i8_nomla:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q0, q1, [x0, #16]
-; CHECK-NEXT:    ushll v3.8h, v0.8b, #0
-; CHECK-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-NEXT:    movi v0.16b, #1
+; CHECK-NEXT:    ldr q2, [x0, #32]
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    udot v1.4s, v2.16b, v0.16b
 ; CHECK-NEXT:    ldr q2, [x0]
-; CHECK-NEXT:    ushll v4.8h, v1.8b, #0
-; CHECK-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-NEXT:    ushll v5.8h, v2.8b, #0
-; CHECK-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-NEXT:    uaddl2 v7.4s, v5.8h, v4.8h
-; CHECK-NEXT:    uaddl2 v6.4s, v2.8h, v1.8h
-; CHECK-NEXT:    uaddl v1.4s, v2.4h, v1.4h
-; CHECK-NEXT:    uaddl v2.4s, v5.4h, v4.4h
-; CHECK-NEXT:    uaddw2 v4.4s, v6.4s, v0.8h
-; CHECK-NEXT:    uaddw2 v5.4s, v7.4s, v3.8h
-; CHECK-NEXT:    uaddw v0.4s, v1.4s, v0.4h
-; CHECK-NEXT:    uaddw v1.4s, v2.4s, v3.4h
-; CHECK-NEXT:    add v2.4s, v5.4s, v4.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    addv s0, v0.4s
+; CHECK-NEXT:    udot v1.4s, v2.16b, v0.16b
+; CHECK-NEXT:    ldr q2, [x0, #16]
+; CHECK-NEXT:    udot v1.4s, v2.16b, v0.16b
+; CHECK-NEXT:    addv s0, v1.4s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
 entry:
@@ -2357,37 +2228,15 @@ entry:
 define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
 ; CHECK-LABEL: test_sdot_v48i8:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr q2, [x0]
-; CHECK-NEXT:    ldr q4, [x1]
-; CHECK-NEXT:    ldp q5, q0, [x1, #16]
-; CHECK-NEXT:    sshll2 v16.8h, v2.16b, #0
-; CHECK-NEXT:    sshll v2.8h, v2.8b, #0
-; CHECK-NEXT:    sshll2 v19.8h, v4.16b, #0
-; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
-; CHECK-NEXT:    smull2 v20.4s, v19.8h, v16.8h
-; CHECK-NEXT:    ldp q3, q1, [x0, #16]
-; CHECK-NEXT:    smull2 v21.4s, v4.8h, v2.8h
-; CHECK-NEXT:    smull v16.4s, v19.4h, v16.4h
-; CHECK-NEXT:    smull v2.4s, v4.4h, v2.4h
-; CHECK-NEXT:    sshll v18.8h, v0.8b, #0
-; CHECK-NEXT:    sshll2 v0.8h, v0.16b, #0
-; CHECK-NEXT:    sshll v7.8h, v1.8b, #0
-; CHECK-NEXT:    sshll2 v1.8h, v1.16b, #0
-; CHECK-NEXT:    smlal2 v21.4s, v18.8h, v7.8h
-; CHECK-NEXT:    smlal2 v20.4s, v0.8h, v1.8h
-; CHECK-NEXT:    smlal v16.4s, v0.4h, v1.4h
-; CHECK-NEXT:    smlal v2.4s, v18.4h, v7.4h
-; CHECK-NEXT:    sshll v6.8h, v3.8b, #0
-; CHECK-NEXT:    sshll2 v3.8h, v3.16b, #0
-; CHECK-NEXT:    sshll v17.8h, v5.8b, #0
-; CHECK-NEXT:    sshll2 v5.8h, v5.16b, #0
-; CHECK-NEXT:    smlal2 v21.4s, v17.8h, v6.8h
-; CHECK-NEXT:    smlal2 v20.4s, v5.8h, v3.8h
-; CHECK-NEXT:    smlal v16.4s, v5.4h, v3.4h
-; CHECK-NEXT:    smlal v2.4s, v17.4h, v6.4h
-; CHECK-NEXT:    add v0.4s, v21.4s, v20.4s
-; CHECK-NEXT:    add v1.4s, v2.4s, v16.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ldr q1, [x1, #32]
+; CHECK-NEXT:    ldr q2, [x0, #32]
+; CHECK-NEXT:    sdot v0.4s, v1.16b, v2.16b
+; CHECK-NEXT:    ldr q1, [x1]
+; CHECK-NEXT:    ldp q3, q2, [x0]
+; CHECK-NEXT:    sdot v0.4s, v1.16b, v3.16b
+; CHECK-NEXT:    ldr q1, [x1, #16]
+; CHECK-NEXT:    sdot v0.4s, v1.16b, v2.16b
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    add w0, w8, w2
@@ -2406,414 +2255,380 @@ entry:
 define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
 ; CHECK-LABEL: test_sdot_v48i8_double:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr b0, [sp, #64]
-; CHECK-NEXT:    add x8, sp, #72
-; CHECK-NEXT:    add x9, sp, #80
-; CHECK-NEXT:    ldr b1, [sp, #192]
-; CHECK-NEXT:    ldr b3, [sp, #128]
-; CHECK-NEXT:    add x10, sp, #88
-; CHECK-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-NEXT:    ldr b2, [sp, #576]
+; CHECK-NEXT:    add x8, sp, #584
+; CHECK-NEXT:    ldr b3, [sp, #192]
+; CHECK-NEXT:    add x10, sp, #328
+; CHECK-NEXT:    ldr b0, [sp, #320]
+; CHECK-NEXT:    add x9, sp, #592
+; CHECK-NEXT:    ld1 { v2.b }[1], [x8]
 ; CHECK-NEXT:    add x8, sp, #200
-; CHECK-NEXT:    ldr b4, [sp, #256]
-; CHECK-NEXT:    fmov s17, w0
-; CHECK-NEXT:    ldr b18, [sp]
-; CHECK-NEXT:    add x11, sp, #528
-; CHECK-NEXT:    ld1 { v1.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #264
-; CHECK-NEXT:    ld1 { v0.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #136
-; CHECK-NEXT:    ldr b5, [sp, #576]
-; CHECK-NEXT:    ld1 { v4.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #144
-; CHECK-NEXT:    ld1 { v3.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #208
-; CHECK-NEXT:    ld1 { v0.b }[3], [x10]
-; CHECK-NEXT:    add x10, sp, #96
-; CHECK-NEXT:    ldr b2, [sp, #448]
-; CHECK-NEXT:    ld1 { v1.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #8
-; CHECK-NEXT:    ld1 { v3.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #104
-; CHECK-NEXT:    ld1 { v0.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #216
-; CHECK-NEXT:    ld1 { v18.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #152
-; CHECK-NEXT:    ldr b21, [sp, #320]
-; CHECK-NEXT:    ld1 { v1.b }[3], [x10]
-; CHECK-NEXT:    add x10, sp, #272
-; CHECK-NEXT:    ld1 { v0.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #224
-; CHECK-NEXT:    ld1 { v3.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #112
-; CHECK-NEXT:    ld1 { v4.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #160
-; CHECK-NEXT:    ld1 { v1.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #280
-; CHECK-NEXT:    ld1 { v0.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #232
-; CHECK-NEXT:    ld1 { v3.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #120
-; CHECK-NEXT:    ld1 { v4.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #168
-; CHECK-NEXT:    ld1 { v1.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #288
-; CHECK-NEXT:    ld1 { v0.b }[7], [x10]
-; CHECK-NEXT:    add x10, sp, #16
-; CHECK-NEXT:    ld1 { v3.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #240
-; CHECK-NEXT:    ld1 { v4.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #176
-; CHECK-NEXT:    ld1 { v18.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #296
-; CHECK-NEXT:    ld1 { v1.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #24
-; CHECK-NEXT:    ld1 { v3.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #248
-; CHECK-NEXT:    ld1 { v4.b }[5], [x10]
-; CHECK-NEXT:    add x10, sp, #184
-; CHECK-NEXT:    ld1 { v18.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #304
-; CHECK-NEXT:    ld1 { v1.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #32
-; CHECK-NEXT:    ld1 { v3.b }[7], [x10]
-; CHECK-NEXT:    add x10, sp, #584
-; CHECK-NEXT:    ld1 { v4.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #312
-; CHECK-NEXT:    ld1 { v18.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #456
-; CHECK-NEXT:    ld1 { v5.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #600
-; CHECK-NEXT:    sshll v6.8h, v0.8b, #0
-; CHECK-NEXT:    ldr b0, [sp, #384]
-; CHECK-NEXT:    ld1 { v4.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #40
-; CHECK-NEXT:    ld1 { v2.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #464
-; CHECK-NEXT:    ldr b19, [sp, #512]
-; CHECK-NEXT:    ld1 { v18.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #592
-; CHECK-NEXT:    sshll v16.8h, v1.8b, #0
-; CHECK-NEXT:    ldr b1, [sp, #640]
+; CHECK-NEXT:    fmov s1, w0
+; CHECK-NEXT:    add x11, sp, #648
+; CHECK-NEXT:    ld1 { v0.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #336
+; CHECK-NEXT:    ld1 { v3.b }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #208
 ; CHECK-NEXT:    ld1 { v2.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #472
-; CHECK-NEXT:    ld1 { v5.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #48
-; CHECK-NEXT:    mov v17.b[1], w1
-; CHECK-NEXT:    sshll v7.8h, v3.8b, #0
-; CHECK-NEXT:    ld1 { v18.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #608
-; CHECK-NEXT:    ld1 { v5.b }[3], [x10]
-; CHECK-NEXT:    add x10, sp, #616
+; CHECK-NEXT:    add x9, sp, #600
+; CHECK-NEXT:    mov v1.b[1], w1
+; CHECK-NEXT:    ldr b18, [sp, #1344]
+; CHECK-NEXT:    ld1 { v0.b }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #224
+; CHECK-NEXT:    ld1 { v3.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #216
 ; CHECK-NEXT:    ld1 { v2.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #480
-; CHECK-NEXT:    mov v17.b[2], w2
-; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
-; CHECK-NEXT:    ld1 { v5.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #56
+; CHECK-NEXT:    add x9, sp, #608
+; CHECK-NEXT:    mov v1.b[2], w2
+; CHECK-NEXT:    ldr b19, [sp, #960]
+; CHECK-NEXT:    ldr b4, [sp, #448]
+; CHECK-NEXT:    ld1 { v3.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #344
 ; CHECK-NEXT:    ld1 { v2.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #488
-; CHECK-NEXT:    mov v17.b[3], w3
-; CHECK-NEXT:    ld1 { v18.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #624
-; CHECK-NEXT:    ld1 { v5.b }[5], [x10]
-; CHECK-NEXT:    add x10, sp, #648
+; CHECK-NEXT:    add x9, sp, #616
+; CHECK-NEXT:    mov v1.b[3], w3
+; CHECK-NEXT:    ldr b5, [sp, #64]
+; CHECK-NEXT:    ld1 { v0.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #352
+; CHECK-NEXT:    ld1 { v3.b }[4], [x10]
+; CHECK-NEXT:    add x10, sp, #232
 ; CHECK-NEXT:    ld1 { v2.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #496
-; CHECK-NEXT:    mov v17.b[4], w4
-; CHECK-NEXT:    ld1 { v1.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #656
-; CHECK-NEXT:    ld1 { v5.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #632
-; CHECK-NEXT:    ld1 { v2.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #504
-; CHECK-NEXT:    mov v17.b[5], w5
-; CHECK-NEXT:    ld1 { v1.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #664
-; CHECK-NEXT:    ld1 { v5.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #392
+; CHECK-NEXT:    add x9, sp, #360
+; CHECK-NEXT:    mov v1.b[4], w4
+; CHECK-NEXT:    ldr b16, [sp, #1088]
+; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #624
+; CHECK-NEXT:    ld1 { v3.b }[5], [x10]
+; CHECK-NEXT:    add x10, sp, #240
+; CHECK-NEXT:    movi v7.2d, #0000000000000000
+; CHECK-NEXT:    ldr b17, [sp, #704]
+; CHECK-NEXT:    ld1 { v2.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #368
+; CHECK-NEXT:    ld1 { v0.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #632
+; CHECK-NEXT:    mov v1.b[5], w5
+; CHECK-NEXT:    ld1 { v3.b }[6], [x10]
+; CHECK-NEXT:    add x10, sp, #376
 ; CHECK-NEXT:    ld1 { v2.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #520
-; CHECK-NEXT:    mov v17.b[6], w6
-; CHECK-NEXT:    ld1 { v0.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #328
-; CHECK-NEXT:    ld1 { v19.b }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #640
+; CHECK-NEXT:    ld1 { v0.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #248
+; CHECK-NEXT:    mov v1.b[6], w6
+; CHECK-NEXT:    movi v6.2d, #0000000000000000
+; CHECK-NEXT:    ld1 { v3.b }[7], [x8]
+; CHECK-NEXT:    add x8, sp, #256
+; CHECK-NEXT:    ld1 { v0.b }[7], [x10]
+; CHECK-NEXT:    mov x10, sp
+; CHECK-NEXT:    mov v1.b[7], w7
+; CHECK-NEXT:    ld1 { v2.b }[8], [x9]
+; CHECK-NEXT:    add x9, sp, #384
+; CHECK-NEXT:    ld1 { v3.b }[8], [x8]
+; CHECK-NEXT:    add x8, sp, #264
+; CHECK-NEXT:    ld1 { v1.b }[8], [x10]
+; CHECK-NEXT:    add x10, sp, #8
+; CHECK-NEXT:    ld1 { v0.b }[8], [x9]
+; CHECK-NEXT:    add x9, sp, #392
+; CHECK-NEXT:    ld1 { v3.b }[9], [x8]
+; CHECK-NEXT:    add x8, sp, #272
+; CHECK-NEXT:    ld1 { v2.b }[9], [x11]
+; CHECK-NEXT:    add x11, sp, #656
+; CHECK-NEXT:    ld1 { v1.b }[9], [x10]
+; CHECK-NEXT:    add x10, sp, #16
+; CHECK-NEXT:    ld1 { v0.b }[9], [x9]
 ; CHECK-NEXT:    add x9, sp, #400
-; CHECK-NEXT:    ld1 { v1.b }[3], [x10]
-; CHECK-NEXT:    add x10, sp, #672
-; CHECK-NEXT:    ld1 { v21.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #336
-; CHECK-NEXT:    ld1 { v0.b }[2], [x9]
+; CHECK-NEXT:    ld1 { v3.b }[10], [x8]
+; CHECK-NEXT:    add x8, sp, #280
+; CHECK-NEXT:    ld1 { v2.b }[10], [x11]
+; CHECK-NEXT:    add x11, sp, #664
+; CHECK-NEXT:    ld1 { v1.b }[10], [x10]
+; CHECK-NEXT:    add x10, sp, #24
+; CHECK-NEXT:    ld1 { v0.b }[10], [x9]
 ; CHECK-NEXT:    add x9, sp, #408
-; CHECK-NEXT:    ld1 { v19.b }[2], [x11]
-; CHECK-NEXT:    add x11, sp, #536
-; CHECK-NEXT:    ld1 { v1.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #680
-; CHECK-NEXT:    ld1 { v21.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #344
-; CHECK-NEXT:    ld1 { v0.b }[3], [x9]
+; CHECK-NEXT:    ld1 { v3.b }[11], [x8]
+; CHECK-NEXT:    add x8, sp, #288
+; CHECK-NEXT:    ld1 { v2.b }[11], [x11]
+; CHECK-NEXT:    add x11, sp, #672
+; CHECK-NEXT:    ld1 { v1.b }[11], [x10]
+; CHECK-NEXT:    add x10, sp, #32
+; CHECK-NEXT:    ld1 { v0.b }[11], [x9]
 ; CHECK-NEXT:    add x9, sp, #416
-; CHECK-NEXT:    sshll v3.8h, v18.8b, #0
-; CHECK-NEXT:    ld1 { v19.b }[3], [x11]
-; CHECK-NEXT:    ld1 { v1.b }[5], [x10]
-; CHECK-NEXT:    add x10, sp, #688
-; CHECK-NEXT:    ld1 { v21.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #352
-; CHECK-NEXT:    ld1 { v0.b }[4], [x9]
+; CHECK-NEXT:    ld1 { v3.b }[12], [x8]
+; CHECK-NEXT:    add x8, sp, #296
+; CHECK-NEXT:    ld1 { v2.b }[12], [x11]
+; CHECK-NEXT:    add x11, sp, #680
+; CHECK-NEXT:    ld1 { v1.b }[12], [x10]
+; CHECK-NEXT:    add x10, sp, #40
+; CHECK-NEXT:    ld1 { v0.b }[12], [x9]
 ; CHECK-NEXT:    add x9, sp, #424
-; CHECK-NEXT:    mov v17.b[7], w7
-; CHECK-NEXT:    add x11, sp, #544
-; CHECK-NEXT:    ld1 { v1.b }[6], [x10]
-; CHECK-NEXT:    add x10, sp, #696
-; CHECK-NEXT:    ld1 { v21.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #360
-; CHECK-NEXT:    ld1 { v0.b }[5], [x9]
+; CHECK-NEXT:    ld1 { v3.b }[13], [x8]
+; CHECK-NEXT:    add x8, sp, #304
+; CHECK-NEXT:    ld1 { v2.b }[13], [x11]
+; CHECK-NEXT:    add x11, sp, #688
+; CHECK-NEXT:    ld1 { v1.b }[13], [x10]
+; CHECK-NEXT:    add x10, sp, #48
+; CHECK-NEXT:    ld1 { v0.b }[13], [x9]
 ; CHECK-NEXT:    add x9, sp, #432
-; CHECK-NEXT:    sshll v18.8h, v2.8b, #0
-; CHECK-NEXT:    ld1 { v19.b }[4], [x11]
-; CHECK-NEXT:    sshll v20.8h, v5.8b, #0
-; CHECK-NEXT:    ld1 { v1.b }[7], [x10]
-; CHECK-NEXT:    ld1 { v21.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #368
-; CHECK-NEXT:    ld1 { v0.b }[6], [x9]
+; CHECK-NEXT:    ld1 { v3.b }[14], [x8]
+; CHECK-NEXT:    add x8, sp, #312
+; CHECK-NEXT:    ld1 { v2.b }[14], [x11]
+; CHECK-NEXT:    add x11, sp, #696
+; CHECK-NEXT:    ld1 { v1.b }[14], [x10]
+; CHECK-NEXT:    add x10, sp, #456
+; CHECK-NEXT:    ld1 { v0.b }[14], [x9]
 ; CHECK-NEXT:    add x9, sp, #440
-; CHECK-NEXT:    sshll v17.8h, v17.8b, #0
-; CHECK-NEXT:    add x10, sp, #1032
-; CHECK-NEXT:    sshll v22.8h, v1.8b, #0
-; CHECK-NEXT:    add x11, sp, #552
-; CHECK-NEXT:    ld1 { v21.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #376
-; CHECK-NEXT:    ld1 { v0.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #968
-; CHECK-NEXT:    ld1 { v19.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #560
-; CHECK-NEXT:    ld1 { v21.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #840
-; CHECK-NEXT:    sshll v5.8h, v0.8b, #0
-; CHECK-NEXT:    ld1 { v19.b }[6], [x11]
-; CHECK-NEXT:    add x11, sp, #568
-; CHECK-NEXT:    smull2 v0.4s, v3.8h, v5.8h
-; CHECK-NEXT:    sshll v2.8h, v21.8b, #0
-; CHECK-NEXT:    ldr b21, [sp, #832]
-; CHECK-NEXT:    smull v3.4s, v3.4h, v5.4h
-; CHECK-NEXT:    ldr b5, [sp, #960]
-; CHECK-NEXT:    smull2 v1.4s, v17.8h, v2.8h
-; CHECK-NEXT:    ld1 { v19.b }[7], [x11]
-; CHECK-NEXT:    ld1 { v21.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #848
-; CHECK-NEXT:    smull v2.4s, v17.4h, v2.4h
-; CHECK-NEXT:    ld1 { v5.b }[1], [x9]
-; CHECK-NEXT:    ldr b17, [sp, #1024]
-; CHECK-NEXT:    add x9, sp, #904
-; CHECK-NEXT:    smlal2 v0.4s, v4.8h, v22.8h
-; CHECK-NEXT:    add x11, sp, #856
-; CHECK-NEXT:    ld1 { v21.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #976
-; CHECK-NEXT:    smlal v3.4s, v4.4h, v22.4h
-; CHECK-NEXT:    ldr b4, [sp, #896]
-; CHECK-NEXT:    ld1 { v17.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #984
-; CHECK-NEXT:    ld1 { v5.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #1040
-; CHECK-NEXT:    ld1 { v4.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #912
-; CHECK-NEXT:    ld1 { v21.b }[3], [x11]
-; CHECK-NEXT:    add x11, sp, #864
-; CHECK-NEXT:    ld1 { v17.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #1048
-; CHECK-NEXT:    ld1 { v5.b }[3], [x10]
-; CHECK-NEXT:    add x10, sp, #992
-; CHECK-NEXT:    ld1 { v4.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #920
-; CHECK-NEXT:    ld1 { v21.b }[4], [x11]
-; CHECK-NEXT:    add x11, sp, #872
-; CHECK-NEXT:    ld1 { v17.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #1056
-; CHECK-NEXT:    ld1 { v5.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #1000
-; CHECK-NEXT:    ld1 { v4.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #928
-; CHECK-NEXT:    ld1 { v21.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #880
-; CHECK-NEXT:    ld1 { v17.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #1064
-; CHECK-NEXT:    ld1 { v5.b }[5], [x10]
-; CHECK-NEXT:    add x10, sp, #1008
-; CHECK-NEXT:    ld1 { v4.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #936
-; CHECK-NEXT:    sshll v19.8h, v19.8b, #0
-; CHECK-NEXT:    ld1 { v21.b }[6], [x11]
-; CHECK-NEXT:    ld1 { v17.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #1072
-; CHECK-NEXT:    ld1 { v5.b }[6], [x10]
-; CHECK-NEXT:    add x10, sp, #1016
-; CHECK-NEXT:    ld1 { v4.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #944
-; CHECK-NEXT:    smlal2 v1.4s, v16.8h, v20.8h
-; CHECK-NEXT:    add x11, sp, #888
-; CHECK-NEXT:    ld1 { v17.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #1080
-; CHECK-NEXT:    smlal2 v0.4s, v7.8h, v19.8h
-; CHECK-NEXT:    ld1 { v5.b }[7], [x10]
-; CHECK-NEXT:    ld1 { v4.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #952
-; CHECK-NEXT:    smlal v2.4s, v16.4h, v20.4h
-; CHECK-NEXT:    ldr b16, [sp, #768]
-; CHECK-NEXT:    add x10, sp, #776
-; CHECK-NEXT:    ld1 { v17.b }[7], [x8]
-; CHECK-NEXT:    smlal v3.4s, v7.4h, v19.4h
-; CHECK-NEXT:    ldr b19, [sp, #1152]
-; CHECK-NEXT:    add x8, sp, #1160
-; CHECK-NEXT:    ld1 { v4.b }[7], [x9]
-; CHECK-NEXT:    ldr b7, [sp, #704]
-; CHECK-NEXT:    add x9, sp, #712
-; CHECK-NEXT:    ld1 { v16.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #784
-; CHECK-NEXT:    ld1 { v19.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #1168
-; CHECK-NEXT:    smlal2 v1.4s, v6.8h, v18.8h
-; CHECK-NEXT:    ld1 { v21.b }[7], [x11]
-; CHECK-NEXT:    smlal v2.4s, v6.4h, v18.4h
-; CHECK-NEXT:    ldr b18, [sp, #1408]
-; CHECK-NEXT:    add x11, sp, #1416
-; CHECK-NEXT:    ld1 { v7.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #720
-; CHECK-NEXT:    ld1 { v16.b }[2], [x10]
-; CHECK-NEXT:    ld1 { v19.b }[2], [x8]
-; CHECK-NEXT:    add x10, sp, #792
-; CHECK-NEXT:    ld1 { v18.b }[1], [x11]
-; CHECK-NEXT:    add x11, sp, #1424
-; CHECK-NEXT:    add x8, sp, #1176
-; CHECK-NEXT:    ld1 { v7.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #728
-; CHECK-NEXT:    ld1 { v16.b }[3], [x10]
-; CHECK-NEXT:    add x10, sp, #800
-; CHECK-NEXT:    ld1 { v18.b }[2], [x11]
-; CHECK-NEXT:    add x11, sp, #1432
-; CHECK-NEXT:    ld1 { v19.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #1184
-; CHECK-NEXT:    ld1 { v7.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #736
-; CHECK-NEXT:    ld1 { v16.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #808
-; CHECK-NEXT:    ld1 { v18.b }[3], [x11]
-; CHECK-NEXT:    add x11, sp, #1440
-; CHECK-NEXT:    ld1 { v19.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #1192
-; CHECK-NEXT:    ld1 { v7.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #744
-; CHECK-NEXT:    ld1 { v16.b }[5], [x10]
-; CHECK-NEXT:    add x10, sp, #816
-; CHECK-NEXT:    ld1 { v18.b }[4], [x11]
-; CHECK-NEXT:    add x11, sp, #1448
-; CHECK-NEXT:    ld1 { v19.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #1200
-; CHECK-NEXT:    ld1 { v7.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #752
-; CHECK-NEXT:    ld1 { v16.b }[6], [x10]
-; CHECK-NEXT:    add x10, sp, #824
-; CHECK-NEXT:    ld1 { v18.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #1456
-; CHECK-NEXT:    ld1 { v19.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #1208
-; CHECK-NEXT:    ld1 { v7.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #760
-; CHECK-NEXT:    ld1 { v16.b }[7], [x10]
-; CHECK-NEXT:    add x10, sp, #1464
-; CHECK-NEXT:    ld1 { v18.b }[6], [x11]
-; CHECK-NEXT:    add x11, sp, #1288
-; CHECK-NEXT:    ld1 { v19.b }[7], [x8]
+; CHECK-NEXT:    ld1 { v3.b }[15], [x8]
+; CHECK-NEXT:    add x8, sp, #56
+; CHECK-NEXT:    ld1 { v4.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #72
+; CHECK-NEXT:    ld1 { v2.b }[15], [x11]
+; CHECK-NEXT:    add x11, sp, #464
+; CHECK-NEXT:    ld1 { v1.b }[15], [x8]
 ; CHECK-NEXT:    add x8, sp, #1352
-; CHECK-NEXT:    sshll v20.8h, v17.8b, #0
-; CHECK-NEXT:    ld1 { v7.b }[7], [x9]
-; CHECK-NEXT:    ldr b17, [sp, #1088]
-; CHECK-NEXT:    add x9, sp, #1096
-; CHECK-NEXT:    sshll v6.8h, v21.8b, #0
-; CHECK-NEXT:    ld1 { v18.b }[7], [x10]
-; CHECK-NEXT:    sshll v21.8h, v16.8b, #0
-; CHECK-NEXT:    ldr b16, [sp, #1344]
-; CHECK-NEXT:    sshll v19.8h, v19.8b, #0
-; CHECK-NEXT:    ld1 { v17.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #1104
-; CHECK-NEXT:    add x10, sp, #1224
-; CHECK-NEXT:    smull v22.4s, v21.4h, v19.4h
-; CHECK-NEXT:    ld1 { v16.b }[1], [x8]
-; CHECK-NEXT:    smull2 v19.4s, v21.8h, v19.8h
-; CHECK-NEXT:    ldr b21, [sp, #1216]
-; CHECK-NEXT:    sshll v18.8h, v18.8b, #0
-; CHECK-NEXT:    ld1 { v17.b }[2], [x9]
+; CHECK-NEXT:    ld1 { v0.b }[15], [x9]
+; CHECK-NEXT:    add x9, sp, #968
+; CHECK-NEXT:    ld1 { v5.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #80
+; CHECK-NEXT:    ld1 { v18.b }[1], [x8]
 ; CHECK-NEXT:    add x8, sp, #1360
-; CHECK-NEXT:    add x9, sp, #1112
-; CHECK-NEXT:    smlal v22.4s, v20.4h, v18.4h
-; CHECK-NEXT:    ld1 { v21.b }[1], [x10]
-; CHECK-NEXT:    smlal2 v19.4s, v20.8h, v18.8h
-; CHECK-NEXT:    ldr b18, [sp, #1280]
-; CHECK-NEXT:    ld1 { v16.b }[2], [x8]
-; CHECK-NEXT:    add x10, sp, #1232
-; CHECK-NEXT:    ld1 { v17.b }[3], [x9]
+; CHECK-NEXT:    ld1 { v19.b }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #976
+; CHECK-NEXT:    ld1 { v4.b }[2], [x11]
+; CHECK-NEXT:    add x11, sp, #472
+; CHECK-NEXT:    ld1 { v5.b }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #88
+; CHECK-NEXT:    ld1 { v18.b }[2], [x8]
 ; CHECK-NEXT:    add x8, sp, #1368
-; CHECK-NEXT:    ld1 { v18.b }[1], [x11]
-; CHECK-NEXT:    add x11, sp, #1296
-; CHECK-NEXT:    add x9, sp, #1120
-; CHECK-NEXT:    ld1 { v21.b }[2], [x10]
-; CHECK-NEXT:    ld1 { v16.b }[3], [x8]
-; CHECK-NEXT:    add x10, sp, #1240
+; CHECK-NEXT:    ld1 { v19.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #984
+; CHECK-NEXT:    ld1 { v4.b }[3], [x11]
+; CHECK-NEXT:    add x11, sp, #480
+; CHECK-NEXT:    ld1 { v5.b }[3], [x10]
+; CHECK-NEXT:    add x10, sp, #96
+; CHECK-NEXT:    ld1 { v18.b }[3], [x8]
 ; CHECK-NEXT:    add x8, sp, #1376
-; CHECK-NEXT:    ld1 { v18.b }[2], [x11]
-; CHECK-NEXT:    add x11, sp, #1304
-; CHECK-NEXT:    ld1 { v17.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #1128
-; CHECK-NEXT:    ld1 { v21.b }[3], [x10]
-; CHECK-NEXT:    add x10, sp, #1248
-; CHECK-NEXT:    ld1 { v16.b }[4], [x8]
+; CHECK-NEXT:    ld1 { v19.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #992
+; CHECK-NEXT:    ld1 { v4.b }[4], [x11]
+; CHECK-NEXT:    add x11, sp, #488
+; CHECK-NEXT:    ld1 { v5.b }[4], [x10]
+; CHECK-NEXT:    add x10, sp, #104
+; CHECK-NEXT:    ld1 { v18.b }[4], [x8]
 ; CHECK-NEXT:    add x8, sp, #1384
-; CHECK-NEXT:    ld1 { v18.b }[3], [x11]
-; CHECK-NEXT:    add x11, sp, #1312
-; CHECK-NEXT:    ld1 { v17.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #1136
-; CHECK-NEXT:    ld1 { v21.b }[4], [x10]
-; CHECK-NEXT:    add x10, sp, #1256
-; CHECK-NEXT:    ld1 { v16.b }[5], [x8]
+; CHECK-NEXT:    ld1 { v19.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #1000
+; CHECK-NEXT:    ld1 { v4.b }[5], [x11]
+; CHECK-NEXT:    add x11, sp, #496
+; CHECK-NEXT:    ld1 { v5.b }[5], [x10]
+; CHECK-NEXT:    add x10, sp, #112
+; CHECK-NEXT:    ld1 { v18.b }[5], [x8]
 ; CHECK-NEXT:    add x8, sp, #1392
-; CHECK-NEXT:    ld1 { v18.b }[4], [x11]
-; CHECK-NEXT:    add x11, sp, #1320
-; CHECK-NEXT:    ld1 { v17.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #1144
-; CHECK-NEXT:    ld1 { v21.b }[5], [x10]
-; CHECK-NEXT:    add x10, sp, #1264
-; CHECK-NEXT:    ld1 { v16.b }[6], [x8]
+; CHECK-NEXT:    ld1 { v19.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #1008
+; CHECK-NEXT:    ld1 { v4.b }[6], [x11]
+; CHECK-NEXT:    add x11, sp, #504
+; CHECK-NEXT:    ld1 { v5.b }[6], [x10]
+; CHECK-NEXT:    add x10, sp, #120
+; CHECK-NEXT:    ld1 { v18.b }[6], [x8]
 ; CHECK-NEXT:    add x8, sp, #1400
-; CHECK-NEXT:    ld1 { v18.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #1328
-; CHECK-NEXT:    ld1 { v17.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #1272
-; CHECK-NEXT:    sshll v7.8h, v7.8b, #0
-; CHECK-NEXT:    ld1 { v21.b }[6], [x10]
-; CHECK-NEXT:    ld1 { v16.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #1336
-; CHECK-NEXT:    ld1 { v18.b }[6], [x11]
-; CHECK-NEXT:    sshll v17.8h, v17.8b, #0
-; CHECK-NEXT:    sshll v5.8h, v5.8b, #0
-; CHECK-NEXT:    ld1 { v21.b }[7], [x9]
-; CHECK-NEXT:    smull2 v20.4s, v7.8h, v17.8h
-; CHECK-NEXT:    smull v7.4s, v7.4h, v17.4h
+; CHECK-NEXT:    ld1 { v19.b }[6], [x9]
+; CHECK-NEXT:    add x9, sp, #1016
+; CHECK-NEXT:    ld1 { v4.b }[7], [x11]
+; CHECK-NEXT:    add x11, sp, #512
+; CHECK-NEXT:    ld1 { v5.b }[7], [x10]
+; CHECK-NEXT:    add x10, sp, #128
 ; CHECK-NEXT:    ld1 { v18.b }[7], [x8]
-; CHECK-NEXT:    sshll v16.8h, v16.8b, #0
-; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
-; CHECK-NEXT:    smlal2 v20.4s, v5.8h, v16.8h
-; CHECK-NEXT:    smlal v7.4s, v5.4h, v16.4h
-; CHECK-NEXT:    sshll v17.8h, v21.8b, #0
-; CHECK-NEXT:    sshll v18.8h, v18.8b, #0
-; CHECK-NEXT:    smlal2 v20.4s, v6.8h, v17.8h
-; CHECK-NEXT:    smlal2 v19.4s, v4.8h, v18.8h
-; CHECK-NEXT:    smlal v22.4s, v4.4h, v18.4h
-; CHECK-NEXT:    smlal v7.4s, v6.4h, v17.4h
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v1.4s, v2.4s, v3.4s
-; CHECK-NEXT:    add v2.4s, v20.4s, v19.4s
-; CHECK-NEXT:    add v3.4s, v7.4s, v22.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v1.4s, v3.4s, v2.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    add x8, sp, #1408
+; CHECK-NEXT:    ld1 { v19.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #1024
+; CHECK-NEXT:    ld1 { v4.b }[8], [x11]
+; CHECK-NEXT:    add x11, sp, #520
+; CHECK-NEXT:    ld1 { v5.b }[8], [x10]
+; CHECK-NEXT:    add x10, sp, #136
+; CHECK-NEXT:    ld1 { v18.b }[8], [x8]
+; CHECK-NEXT:    add x8, sp, #1416
+; CHECK-NEXT:    ld1 { v19.b }[8], [x9]
+; CHECK-NEXT:    add x9, sp, #1032
+; CHECK-NEXT:    ld1 { v4.b }[9], [x11]
+; CHECK-NEXT:    add x11, sp, #528
+; CHECK-NEXT:    ld1 { v5.b }[9], [x10]
+; CHECK-NEXT:    add x10, sp, #144
+; CHECK-NEXT:    ld1 { v18.b }[9], [x8]
+; CHECK-NEXT:    add x8, sp, #1424
+; CHECK-NEXT:    ld1 { v19.b }[9], [x9]
+; CHECK-NEXT:    add x9, sp, #1040
+; CHECK-NEXT:    ld1 { v4.b }[10], [x11]
+; CHECK-NEXT:    add x11, sp, #536
+; CHECK-NEXT:    ld1 { v5.b }[10], [x10]
+; CHECK-NEXT:    add x10, sp, #152
+; CHECK-NEXT:    ld1 { v18.b }[10], [x8]
+; CHECK-NEXT:    add x8, sp, #1432
+; CHECK-NEXT:    ld1 { v19.b }[10], [x9]
+; CHECK-NEXT:    add x9, sp, #1048
+; CHECK-NEXT:    ld1 { v4.b }[11], [x11]
+; CHECK-NEXT:    add x11, sp, #544
+; CHECK-NEXT:    ld1 { v5.b }[11], [x10]
+; CHECK-NEXT:    add x10, sp, #160
+; CHECK-NEXT:    ld1 { v18.b }[11], [x8]
+; CHECK-NEXT:    add x8, sp, #1440
+; CHECK-NEXT:    ld1 { v19.b }[11], [x9]
+; CHECK-NEXT:    add x9, sp, #1056
+; CHECK-NEXT:    ld1 { v4.b }[12], [x11]
+; CHECK-NEXT:    add x11, sp, #552
+; CHECK-NEXT:    ld1 { v5.b }[12], [x10]
+; CHECK-NEXT:    add x10, sp, #168
+; CHECK-NEXT:    ld1 { v18.b }[12], [x8]
+; CHECK-NEXT:    add x8, sp, #1448
+; CHECK-NEXT:    ld1 { v19.b }[12], [x9]
+; CHECK-NEXT:    add x9, sp, #1064
+; CHECK-NEXT:    ld1 { v4.b }[13], [x11]
+; CHECK-NEXT:    add x11, sp, #1112
+; CHECK-NEXT:    ld1 { v5.b }[13], [x10]
+; CHECK-NEXT:    add x10, sp, #1096
+; CHECK-NEXT:    ld1 { v18.b }[13], [x8]
+; CHECK-NEXT:    add x8, sp, #1456
+; CHECK-NEXT:    ld1 { v19.b }[13], [x9]
+; CHECK-NEXT:    add x9, sp, #1072
+; CHECK-NEXT:    ld1 { v16.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #712
+; CHECK-NEXT:    sdot v7.4s, v3.16b, v2.16b
+; CHECK-NEXT:    ldr b2, [sp, #1216]
+; CHECK-NEXT:    ld1 { v18.b }[14], [x8]
+; CHECK-NEXT:    add x8, sp, #1464
+; CHECK-NEXT:    ld1 { v19.b }[14], [x9]
+; CHECK-NEXT:    add x9, sp, #1080
+; CHECK-NEXT:    ld1 { v17.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #840
+; CHECK-NEXT:    ldr b3, [sp, #832]
+; CHECK-NEXT:    ld1 { v18.b }[15], [x8]
+; CHECK-NEXT:    add x8, sp, #560
+; CHECK-NEXT:    ld1 { v19.b }[15], [x9]
+; CHECK-NEXT:    add x9, sp, #176
+; CHECK-NEXT:    ld1 { v3.b }[1], [x10]
+; CHECK-NEXT:    add x10, sp, #848
+; CHECK-NEXT:    ld1 { v4.b }[14], [x8]
+; CHECK-NEXT:    add x8, sp, #1104
+; CHECK-NEXT:    ld1 { v5.b }[14], [x9]
+; CHECK-NEXT:    add x9, sp, #1224
+; CHECK-NEXT:    sdot v6.4s, v19.16b, v18.16b
+; CHECK-NEXT:    ld1 { v16.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #720
+; CHECK-NEXT:    ld1 { v2.b }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #1232
+; CHECK-NEXT:    ld1 { v3.b }[2], [x10]
+; CHECK-NEXT:    add x10, sp, #856
+; CHECK-NEXT:    ld1 { v17.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #728
+; CHECK-NEXT:    ld1 { v16.b }[3], [x11]
+; CHECK-NEXT:    add x11, sp, #1120
+; CHECK-NEXT:    ld1 { v2.b }[2], [x9]
+; CHECK-NEXT:    add x9, sp, #1240
+; CHECK-NEXT:    ld1 { v3.b }[3], [x10]
+; CHECK-NEXT:    add x10, sp, #864
+; CHECK-NEXT:    ld1 { v17.b }[3], [x8]
+; CHECK-NEXT:    add x8, sp, #736
+; CHECK-NEXT:    ld1 { v16.b }[4], [x11]
+; CHECK-NEXT:    add x11, sp, #1128
+; CHECK-NEXT:    ld1 { v2.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #1248
+; CHECK-NEXT:    ld1 { v3.b }[4], [x10]
+; CHECK-NEXT:    add x10, sp, #872
+; CHECK-NEXT:    ld1 { v17.b }[4], [x8]
+; CHECK-NEXT:    add x8, sp, #744
+; CHECK-NEXT:    ld1 { v16.b }[5], [x11]
+; CHECK-NEXT:    add x11, sp, #1136
+; CHECK-NEXT:    ld1 { v2.b }[4], [x9]
+; CHECK-NEXT:    add x9, sp, #1256
+; CHECK-NEXT:    ld1 { v3.b }[5], [x10]
+; CHECK-NEXT:    add x10, sp, #880
+; CHECK-NEXT:    ld1 { v17.b }[5], [x8]
+; CHECK-NEXT:    add x8, sp, #752
+; CHECK-NEXT:    ld1 { v16.b }[6], [x11]
+; CHECK-NEXT:    add x11, sp, #1144
+; CHECK-NEXT:    ld1 { v2.b }[5], [x9]
+; CHECK-NEXT:    add x9, sp, #1264
+; CHECK-NEXT:    ld1 { v3.b }[6], [x10]
+; CHECK-NEXT:    add x10, sp, #888
+; CHECK-NEXT:    ld1 { v17.b }[6], [x8]
+; CHECK-NEXT:    add x8, sp, #760
+; CHECK-NEXT:    ld1 { v16.b }[7], [x11]
+; CHECK-NEXT:    add x11, sp, #1152
+; CHECK-NEXT:    ld1 { v2.b }[6], [x9]
+; CHECK-NEXT:    add x9, sp, #1272
+; CHECK-NEXT:    ld1 { v3.b }[7], [x10]
+; CHECK-NEXT:    add x10, sp, #896
+; CHECK-NEXT:    ld1 { v17.b }[7], [x8]
+; CHECK-NEXT:    add x8, sp, #768
+; CHECK-NEXT:    ld1 { v16.b }[8], [x11]
+; CHECK-NEXT:    add x11, sp, #1160
+; CHECK-NEXT:    ld1 { v2.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #1280
+; CHECK-NEXT:    ld1 { v3.b }[8], [x10]
+; CHECK-NEXT:    add x10, sp, #904
+; CHECK-NEXT:    ld1 { v17.b }[8], [x8]
+; CHECK-NEXT:    add x8, sp, #776
+; CHECK-NEXT:    ld1 { v16.b }[9], [x11]
+; CHECK-NEXT:    add x11, sp, #1168
+; CHECK-NEXT:    ld1 { v2.b }[8], [x9]
+; CHECK-NEXT:    add x9, sp, #1288
+; CHECK-NEXT:    ld1 { v3.b }[9], [x10]
+; CHECK-NEXT:    add x10, sp, #912
+; CHECK-NEXT:    ld1 { v17.b }[9], [x8]
+; CHECK-NEXT:    add x8, sp, #784
+; CHECK-NEXT:    ld1 { v16.b }[10], [x11]
+; CHECK-NEXT:    add x11, sp, #1176
+; CHECK-NEXT:    ld1 { v2.b }[9], [x9]
+; CHECK-NEXT:    add x9, sp, #1296
+; CHECK-NEXT:    ld1 { v3.b }[10], [x10]
+; CHECK-NEXT:    add x10, sp, #920
+; CHECK-NEXT:    ld1 { v17.b }[10], [x8]
+; CHECK-NEXT:    add x8, sp, #792
+; CHECK-NEXT:    ld1 { v16.b }[11], [x11]
+; CHECK-NEXT:    add x11, sp, #1184
+; CHECK-NEXT:    ld1 { v2.b }[10], [x9]
+; CHECK-NEXT:    add x9, sp, #1304
+; CHECK-NEXT:    ld1 { v3.b }[11], [x10]
+; CHECK-NEXT:    add x10, sp, #928
+; CHECK-NEXT:    ld1 { v17.b }[11], [x8]
+; CHECK-NEXT:    add x8, sp, #800
+; CHECK-NEXT:    ld1 { v16.b }[12], [x11]
+; CHECK-NEXT:    add x11, sp, #1192
+; CHECK-NEXT:    ld1 { v2.b }[11], [x9]
+; CHECK-NEXT:    add x9, sp, #1312
+; CHECK-NEXT:    ld1 { v3.b }[12], [x10]
+; CHECK-NEXT:    add x10, sp, #936
+; CHECK-NEXT:    ld1 { v17.b }[12], [x8]
+; CHECK-NEXT:    add x8, sp, #808
+; CHECK-NEXT:    ld1 { v16.b }[13], [x11]
+; CHECK-NEXT:    add x11, sp, #1200
+; CHECK-NEXT:    ld1 { v2.b }[12], [x9]
+; CHECK-NEXT:    add x9, sp, #1320
+; CHECK-NEXT:    ld1 { v3.b }[13], [x10]
+; CHECK-NEXT:    add x10, sp, #944
+; CHECK-NEXT:    ld1 { v17.b }[13], [x8]
+; CHECK-NEXT:    add x8, sp, #816
+; CHECK-NEXT:    ld1 { v16.b }[14], [x11]
+; CHECK-NEXT:    add x11, sp, #1208
+; CHECK-NEXT:    ld1 { v2.b }[13], [x9]
+; CHECK-NEXT:    add x9, sp, #1328
+; CHECK-NEXT:    ld1 { v3.b }[14], [x10]
+; CHECK-NEXT:    add x10, sp, #952
+; CHECK-NEXT:    ld1 { v17.b }[14], [x8]
+; CHECK-NEXT:    add x8, sp, #824
+; CHECK-NEXT:    ld1 { v16.b }[15], [x11]
+; CHECK-NEXT:    add x11, sp, #568
+; CHECK-NEXT:    ld1 { v2.b }[14], [x9]
+; CHECK-NEXT:    add x9, sp, #1336
+; CHECK-NEXT:    sdot v7.4s, v1.16b, v0.16b
+; CHECK-NEXT:    ld1 { v3.b }[15], [x10]
+; CHECK-NEXT:    ld1 { v17.b }[15], [x8]
+; CHECK-NEXT:    add x8, sp, #184
+; CHECK-NEXT:    ld1 { v4.b }[15], [x11]
+; CHECK-NEXT:    ld1 { v2.b }[15], [x9]
+; CHECK-NEXT:    ld1 { v5.b }[15], [x8]
+; CHECK-NEXT:    sdot v6.4s, v17.16b, v16.16b
+; CHECK-NEXT:    sdot v7.4s, v5.16b, v4.16b
+; CHECK-NEXT:    sdot v6.4s, v3.16b, v2.16b
+; CHECK-NEXT:    add v0.4s, v7.4s, v6.4s
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
@@ -2833,214 +2648,195 @@ entry:
 define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) {
 ; CHECK-LABEL: test_sdot_v48i8_double_nomla:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldr b0, [sp, #64]
-; CHECK-NEXT:    add x8, sp, #72
-; CHECK-NEXT:    ldr b1, [sp, #192]
-; CHECK-NEXT:    add x9, sp, #80
-; CHECK-NEXT:    ldr b4, [sp, #128]
-; CHECK-NEXT:    add x10, sp, #136
-; CHECK-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-NEXT:    ldr b0, [sp, #192]
 ; CHECK-NEXT:    add x8, sp, #200
-; CHECK-NEXT:    add x11, sp, #152
-; CHECK-NEXT:    ldr b6, [sp, #256]
-; CHECK-NEXT:    ld1 { v4.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #144
-; CHECK-NEXT:    ld1 { v1.b }[1], [x8]
-; CHECK-NEXT:    add x8, sp, #88
-; CHECK-NEXT:    ld1 { v0.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #208
-; CHECK-NEXT:    ldr b7, [sp]
-; CHECK-NEXT:    fmov s5, w0
-; CHECK-NEXT:    ld1 { v4.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #112
-; CHECK-NEXT:    ld1 { v1.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #96
-; CHECK-NEXT:    ld1 { v0.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #216
-; CHECK-NEXT:    mov v5.b[1], w1
+; CHECK-NEXT:    fmov s1, w0
+; CHECK-NEXT:    add x9, sp, #216
+; CHECK-NEXT:    ldr b2, [sp, #64]
+; CHECK-NEXT:    add x11, sp, #72
+; CHECK-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-NEXT:    add x8, sp, #208
+; CHECK-NEXT:    mov v1.b[1], w1
+; CHECK-NEXT:    add x10, sp, #232
+; CHECK-NEXT:    ld1 { v2.b }[1], [x11]
+; CHECK-NEXT:    add x11, sp, #712
+; CHECK-NEXT:    ldr b4, [sp, #704]
+; CHECK-NEXT:    add x12, sp, #968
+; CHECK-NEXT:    ld1 { v0.b }[2], [x8]
+; CHECK-NEXT:    add x8, sp, #224
+; CHECK-NEXT:    mov v1.b[2], w2
 ; CHECK-NEXT:    ldr b3, [sp, #960]
-; CHECK-NEXT:    ld1 { v4.b }[3], [x11]
-; CHECK-NEXT:    add x11, sp, #264
-; CHECK-NEXT:    ld1 { v1.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #104
-; CHECK-NEXT:    ld1 { v0.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #224
-; CHECK-NEXT:    ld1 { v6.b }[1], [x11]
-; CHECK-NEXT:    add x11, sp, #304
-; CHECK-NEXT:    mov v5.b[2], w2
-; CHECK-NEXT:    ldr b19, [sp, #768]
-; CHECK-NEXT:    ld1 { v1.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #232
-; CHECK-NEXT:    ld1 { v0.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #160
-; CHECK-NEXT:    mov v5.b[3], w3
-; CHECK-NEXT:    ld1 { v1.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #120
-; CHECK-NEXT:    ld1 { v0.b }[6], [x10]
-; CHECK-NEXT:    add x10, sp, #240
-; CHECK-NEXT:    ld1 { v4.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #272
-; CHECK-NEXT:    mov v5.b[4], w4
-; CHECK-NEXT:    ld1 { v1.b }[6], [x10]
-; CHECK-NEXT:    add x10, sp, #8
-; CHECK-NEXT:    ld1 { v0.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #248
-; CHECK-NEXT:    ld1 { v6.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #16
-; CHECK-NEXT:    ld1 { v7.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #168
-; CHECK-NEXT:    ld1 { v1.b }[7], [x9]
-; CHECK-NEXT:    add x9, sp, #280
-; CHECK-NEXT:    mov v5.b[5], w5
-; CHECK-NEXT:    ld1 { v4.b }[5], [x10]
-; CHECK-NEXT:    add x10, sp, #176
-; CHECK-NEXT:    ld1 { v6.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #288
-; CHECK-NEXT:    ld1 { v7.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #24
-; CHECK-NEXT:    sshll v2.8h, v1.8b, #0
-; CHECK-NEXT:    ldr b1, [sp, #832]
-; CHECK-NEXT:    ld1 { v4.b }[6], [x10]
+; CHECK-NEXT:    ldr b5, [sp, #832]
+; CHECK-NEXT:    add x13, sp, #80
+; CHECK-NEXT:    ld1 { v4.b }[1], [x11]
+; CHECK-NEXT:    add x11, sp, #840
+; CHECK-NEXT:    ld1 { v0.b }[3], [x9]
+; CHECK-NEXT:    add x9, sp, #240
+; CHECK-NEXT:    mov v1.b[3], w3
+; CHECK-NEXT:    ld1 { v3.b }[1], [x12]
+; CHECK-NEXT:    add x12, sp, #248
+; CHECK-NEXT:    ld1 { v5.b }[1], [x11]
+; CHECK-NEXT:    add x11, sp, #976
+; CHECK-NEXT:    ld1 { v2.b }[2], [x13]
+; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
+; CHECK-NEXT:    add x13, sp, #720
+; CHECK-NEXT:    mov v1.b[4], w4
+; CHECK-NEXT:    add x8, sp, #256
+; CHECK-NEXT:    ld1 { v3.b }[2], [x11]
+; CHECK-NEXT:    add x14, sp, #848
+; CHECK-NEXT:    ld1 { v4.b }[2], [x13]
+; CHECK-NEXT:    add x13, sp, #984
+; CHECK-NEXT:    ld1 { v0.b }[5], [x10]
+; CHECK-NEXT:    add x11, sp, #88
+; CHECK-NEXT:    mov v1.b[5], w5
+; CHECK-NEXT:    add x10, sp, #264
+; CHECK-NEXT:    ld1 { v3.b }[3], [x13]
+; CHECK-NEXT:    mov x13, sp
+; CHECK-NEXT:    ld1 { v5.b }[2], [x14]
+; CHECK-NEXT:    add x14, sp, #280
+; CHECK-NEXT:    ld1 { v0.b }[6], [x9]
+; CHECK-NEXT:    add x9, sp, #272
+; CHECK-NEXT:    mov v1.b[6], w6
+; CHECK-NEXT:    ld1 { v2.b }[3], [x11]
+; CHECK-NEXT:    add x11, sp, #856
+; CHECK-NEXT:    movi v6.16b, #1
+; CHECK-NEXT:    ld1 { v0.b }[7], [x12]
+; CHECK-NEXT:    add x12, sp, #728
+; CHECK-NEXT:    mov v1.b[7], w7
+; CHECK-NEXT:    ld1 { v5.b }[3], [x11]
+; CHECK-NEXT:    add x11, sp, #992
+; CHECK-NEXT:    ld1 { v4.b }[3], [x12]
+; CHECK-NEXT:    add x12, sp, #96
+; CHECK-NEXT:    ld1 { v0.b }[8], [x8]
+; CHECK-NEXT:    add x8, sp, #288
+; CHECK-NEXT:    ld1 { v1.b }[8], [x13]
+; CHECK-NEXT:    add x13, sp, #8
+; CHECK-NEXT:    ld1 { v3.b }[4], [x11]
+; CHECK-NEXT:    add x11, sp, #864
+; CHECK-NEXT:    ld1 { v2.b }[4], [x12]
+; CHECK-NEXT:    add x12, sp, #736
+; CHECK-NEXT:    ld1 { v0.b }[9], [x10]
 ; CHECK-NEXT:    add x10, sp, #296
-; CHECK-NEXT:    ld1 { v6.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #840
-; CHECK-NEXT:    ld1 { v7.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #32
-; CHECK-NEXT:    mov v5.b[6], w6
-; CHECK-NEXT:    ld1 { v1.b }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #848
-; CHECK-NEXT:    ld1 { v6.b }[5], [x10]
-; CHECK-NEXT:    add x10, sp, #968
-; CHECK-NEXT:    ld1 { v7.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #40
-; CHECK-NEXT:    mov v5.b[7], w7
-; CHECK-NEXT:    ld1 { v3.b }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #976
-; CHECK-NEXT:    ld1 { v1.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #856
-; CHECK-NEXT:    ld1 { v7.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #48
-; CHECK-NEXT:    ld1 { v6.b }[6], [x11]
-; CHECK-NEXT:    add x11, sp, #312
-; CHECK-NEXT:    ld1 { v3.b }[2], [x10]
-; CHECK-NEXT:    add x10, sp, #984
-; CHECK-NEXT:    ld1 { v1.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #864
-; CHECK-NEXT:    ld1 { v7.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #56
-; CHECK-NEXT:    ld1 { v6.b }[7], [x11]
-; CHECK-NEXT:    add x11, sp, #184
-; CHECK-NEXT:    ld1 { v3.b }[3], [x10]
-; CHECK-NEXT:    add x10, sp, #712
-; CHECK-NEXT:    ld1 { v1.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #992
-; CHECK-NEXT:    ld1 { v7.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #872
-; CHECK-NEXT:    sshll v16.8h, v6.8b, #0
-; CHECK-NEXT:    ldr b6, [sp, #704]
-; CHECK-NEXT:    ld1 { v3.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #1000
-; CHECK-NEXT:    ld1 { v1.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #880
-; CHECK-NEXT:    sshll v18.8h, v7.8b, #0
-; CHECK-NEXT:    ld1 { v6.b }[1], [x10]
-; CHECK-NEXT:    sshll v17.8h, v5.8b, #0
-; CHECK-NEXT:    add x10, sp, #776
-; CHECK-NEXT:    ld1 { v3.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #1008
-; CHECK-NEXT:    ld1 { v1.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #888
-; CHECK-NEXT:    saddl2 v5.4s, v18.8h, v16.8h
-; CHECK-NEXT:    ld1 { v19.b }[1], [x10]
-; CHECK-NEXT:    saddl v16.4s, v18.4h, v16.4h
-; CHECK-NEXT:    ldr b18, [sp, #1024]
-; CHECK-NEXT:    ld1 { v3.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #1032
-; CHECK-NEXT:    ld1 { v1.b }[7], [x8]
+; CHECK-NEXT:    ld1 { v1.b }[9], [x13]
+; CHECK-NEXT:    add x13, sp, #16
+; CHECK-NEXT:    ld1 { v5.b }[4], [x11]
+; CHECK-NEXT:    add x11, sp, #1000
+; CHECK-NEXT:    ld1 { v4.b }[4], [x12]
+; CHECK-NEXT:    add x12, sp, #104
+; CHECK-NEXT:    ld1 { v0.b }[10], [x9]
+; CHECK-NEXT:    add x9, sp, #304
+; CHECK-NEXT:    ld1 { v1.b }[10], [x13]
+; CHECK-NEXT:    add x13, sp, #24
+; CHECK-NEXT:    ld1 { v3.b }[5], [x11]
+; CHECK-NEXT:    add x11, sp, #872
+; CHECK-NEXT:    ld1 { v2.b }[5], [x12]
+; CHECK-NEXT:    add x12, sp, #744
+; CHECK-NEXT:    ld1 { v0.b }[11], [x14]
+; CHECK-NEXT:    add x14, sp, #312
+; CHECK-NEXT:    ld1 { v1.b }[11], [x13]
+; CHECK-NEXT:    add x13, sp, #32
+; CHECK-NEXT:    ld1 { v4.b }[5], [x12]
+; CHECK-NEXT:    add x12, sp, #40
+; CHECK-NEXT:    ld1 { v5.b }[5], [x11]
+; CHECK-NEXT:    add x11, sp, #112
+; CHECK-NEXT:    ld1 { v0.b }[12], [x8]
+; CHECK-NEXT:    add x8, sp, #1008
+; CHECK-NEXT:    ld1 { v1.b }[12], [x13]
+; CHECK-NEXT:    ld1 { v2.b }[6], [x11]
+; CHECK-NEXT:    add x11, sp, #48
+; CHECK-NEXT:    ld1 { v3.b }[6], [x8]
 ; CHECK-NEXT:    add x8, sp, #1016
-; CHECK-NEXT:    saddl2 v7.4s, v17.8h, v2.8h
-; CHECK-NEXT:    add x10, sp, #904
-; CHECK-NEXT:    saddl v2.4s, v17.4h, v2.4h
-; CHECK-NEXT:    ld1 { v18.b }[1], [x9]
+; CHECK-NEXT:    ld1 { v0.b }[13], [x10]
+; CHECK-NEXT:    add x10, sp, #752
+; CHECK-NEXT:    ld1 { v1.b }[13], [x12]
+; CHECK-NEXT:    movi v7.2d, #0000000000000000
 ; CHECK-NEXT:    ld1 { v3.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #720
-; CHECK-NEXT:    ldr b17, [sp, #896]
-; CHECK-NEXT:    add x9, sp, #1040
-; CHECK-NEXT:    ld1 { v4.b }[7], [x11]
-; CHECK-NEXT:    add x11, sp, #728
-; CHECK-NEXT:    ld1 { v6.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #784
-; CHECK-NEXT:    ld1 { v17.b }[1], [x10]
+; CHECK-NEXT:    add x8, sp, #1024
+; CHECK-NEXT:    ld1 { v4.b }[6], [x10]
+; CHECK-NEXT:    add x10, sp, #880
+; CHECK-NEXT:    ld1 { v0.b }[14], [x9]
+; CHECK-NEXT:    add x9, sp, #760
+; CHECK-NEXT:    ld1 { v1.b }[14], [x11]
+; CHECK-NEXT:    add x11, sp, #120
+; CHECK-NEXT:    ld1 { v3.b }[8], [x8]
+; CHECK-NEXT:    add x8, sp, #1032
+; CHECK-NEXT:    ld1 { v4.b }[7], [x9]
+; CHECK-NEXT:    add x9, sp, #768
+; CHECK-NEXT:    ld1 { v5.b }[6], [x10]
+; CHECK-NEXT:    add x10, sp, #888
+; CHECK-NEXT:    ld1 { v2.b }[7], [x11]
+; CHECK-NEXT:    add x11, sp, #128
+; CHECK-NEXT:    ld1 { v3.b }[9], [x8]
+; CHECK-NEXT:    add x8, sp, #1040
+; CHECK-NEXT:    ld1 { v4.b }[8], [x9]
+; CHECK-NEXT:    add x9, sp, #776
+; CHECK-NEXT:    ld1 { v5.b }[7], [x10]
+; CHECK-NEXT:    add x10, sp, #896
+; CHECK-NEXT:    ld1 { v2.b }[8], [x11]
+; CHECK-NEXT:    add x11, sp, #136
+; CHECK-NEXT:    ld1 { v3.b }[10], [x8]
+; CHECK-NEXT:    add x8, sp, #1048
+; CHECK-NEXT:    ld1 { v4.b }[9], [x9]
+; CHECK-NEXT:    add x9, sp, #784
+; CHECK-NEXT:    ld1 { v5.b }[8], [x10]
+; CHECK-NEXT:    add x10, sp, #904
+; CHECK-NEXT:    ld1 { v2.b }[9], [x11]
+; CHECK-NEXT:    add x11, sp, #144
+; CHECK-NEXT:    ld1 { v3.b }[11], [x8]
+; CHECK-NEXT:    add x8, sp, #1056
+; CHECK-NEXT:    ld1 { v4.b }[10], [x9]
+; CHECK-NEXT:    add x9, sp, #792
+; CHECK-NEXT:    ld1 { v5.b }[9], [x10]
 ; CHECK-NEXT:    add x10, sp, #912
-; CHECK-NEXT:    ld1 { v18.b }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #1048
-; CHECK-NEXT:    ld1 { v19.b }[2], [x8]
-; CHECK-NEXT:    add x8, sp, #792
-; CHECK-NEXT:    ld1 { v6.b }[3], [x11]
-; CHECK-NEXT:    add x11, sp, #736
-; CHECK-NEXT:    ld1 { v17.b }[2], [x10]
+; CHECK-NEXT:    ld1 { v2.b }[10], [x11]
+; CHECK-NEXT:    add x11, sp, #152
+; CHECK-NEXT:    ld1 { v3.b }[12], [x8]
+; CHECK-NEXT:    add x8, sp, #1064
+; CHECK-NEXT:    ld1 { v4.b }[11], [x9]
+; CHECK-NEXT:    add x9, sp, #800
+; CHECK-NEXT:    ld1 { v5.b }[10], [x10]
 ; CHECK-NEXT:    add x10, sp, #920
-; CHECK-NEXT:    ld1 { v18.b }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #1056
-; CHECK-NEXT:    ld1 { v19.b }[3], [x8]
-; CHECK-NEXT:    add x8, sp, #800
-; CHECK-NEXT:    ld1 { v6.b }[4], [x11]
-; CHECK-NEXT:    add x11, sp, #744
-; CHECK-NEXT:    ld1 { v17.b }[3], [x10]
+; CHECK-NEXT:    ld1 { v2.b }[11], [x11]
+; CHECK-NEXT:    add x11, sp, #160
+; CHECK-NEXT:    ld1 { v3.b }[13], [x8]
+; CHECK-NEXT:    add x8, sp, #1072
+; CHECK-NEXT:    ld1 { v4.b }[12], [x9]
+; CHECK-NEXT:    add x9, sp, #808
+; CHECK-NEXT:    ld1 { v5.b }[11], [x10]
 ; CHECK-NEXT:    add x10, sp, #928
-; CHECK-NEXT:    ld1 { v18.b }[4], [x9]
-; CHECK-NEXT:    add x9, sp, #1064
-; CHECK-NEXT:    ld1 { v19.b }[4], [x8]
-; CHECK-NEXT:    add x8, sp, #808
-; CHECK-NEXT:    ld1 { v6.b }[5], [x11]
-; CHECK-NEXT:    add x11, sp, #752
-; CHECK-NEXT:    ld1 { v17.b }[4], [x10]
+; CHECK-NEXT:    ld1 { v2.b }[12], [x11]
+; CHECK-NEXT:    add x11, sp, #168
+; CHECK-NEXT:    ld1 { v3.b }[14], [x8]
+; CHECK-NEXT:    add x8, sp, #1080
+; CHECK-NEXT:    ld1 { v4.b }[13], [x9]
+; CHECK-NEXT:    add x9, sp, #816
+; CHECK-NEXT:    ld1 { v5.b }[12], [x10]
 ; CHECK-NEXT:    add x10, sp, #936
-; CHECK-NEXT:    ld1 { v18.b }[5], [x9]
-; CHECK-NEXT:    add x9, sp, #1072
-; CHECK-NEXT:    ld1 { v19.b }[5], [x8]
-; CHECK-NEXT:    add x8, sp, #816
-; CHECK-NEXT:    ld1 { v6.b }[6], [x11]
-; CHECK-NEXT:    add x11, sp, #760
-; CHECK-NEXT:    ld1 { v17.b }[5], [x10]
+; CHECK-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-NEXT:    ld1 { v0.b }[15], [x14]
+; CHECK-NEXT:    ld1 { v2.b }[13], [x11]
+; CHECK-NEXT:    add x11, sp, #56
+; CHECK-NEXT:    ld1 { v3.b }[15], [x8]
+; CHECK-NEXT:    add x8, sp, #176
+; CHECK-NEXT:    ld1 { v4.b }[14], [x9]
+; CHECK-NEXT:    add x9, sp, #824
+; CHECK-NEXT:    ld1 { v5.b }[13], [x10]
 ; CHECK-NEXT:    add x10, sp, #944
-; CHECK-NEXT:    ld1 { v18.b }[6], [x9]
-; CHECK-NEXT:    add x9, sp, #1080
-; CHECK-NEXT:    ld1 { v19.b }[6], [x8]
-; CHECK-NEXT:    add x8, sp, #824
-; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ld1 { v6.b }[7], [x11]
-; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
-; CHECK-NEXT:    ld1 { v17.b }[6], [x10]
-; CHECK-NEXT:    ld1 { v18.b }[7], [x9]
-; CHECK-NEXT:    ld1 { v19.b }[7], [x8]
-; CHECK-NEXT:    add x8, sp, #952
-; CHECK-NEXT:    saddw2 v5.4s, v5.4s, v4.8h
-; CHECK-NEXT:    saddw2 v7.4s, v7.4s, v0.8h
-; CHECK-NEXT:    ld1 { v17.b }[7], [x8]
-; CHECK-NEXT:    saddw v0.4s, v2.4s, v0.4h
-; CHECK-NEXT:    add v5.4s, v7.4s, v5.4s
-; CHECK-NEXT:    sshll v2.8h, v3.8b, #0
-; CHECK-NEXT:    sshll v3.8h, v18.8b, #0
-; CHECK-NEXT:    sshll v7.8h, v19.8b, #0
-; CHECK-NEXT:    sshll v6.8h, v6.8b, #0
-; CHECK-NEXT:    saddw v4.4s, v16.4s, v4.4h
-; CHECK-NEXT:    saddl2 v16.4s, v7.8h, v3.8h
-; CHECK-NEXT:    saddl v3.4s, v7.4h, v3.4h
-; CHECK-NEXT:    saddl2 v7.4s, v6.8h, v2.8h
-; CHECK-NEXT:    saddl v2.4s, v6.4h, v2.4h
-; CHECK-NEXT:    sshll v6.8h, v17.8b, #0
-; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-NEXT:    saddw2 v16.4s, v16.4s, v6.8h
-; CHECK-NEXT:    saddw2 v7.4s, v7.4s, v1.8h
-; CHECK-NEXT:    saddw v3.4s, v3.4s, v6.4h
-; CHECK-NEXT:    saddw v1.4s, v2.4s, v1.4h
-; CHECK-NEXT:    add v0.4s, v0.4s, v4.4s
-; CHECK-NEXT:    add v2.4s, v7.4s, v16.4s
-; CHECK-NEXT:    add v1.4s, v1.4s, v3.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v5.4s
-; CHECK-NEXT:    add v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ld1 { v1.b }[15], [x11]
+; CHECK-NEXT:    sdot v16.4s, v0.16b, v6.16b
+; CHECK-NEXT:    ld1 { v2.b }[14], [x8]
+; CHECK-NEXT:    sdot v7.4s, v3.16b, v6.16b
+; CHECK-NEXT:    ld1 { v4.b }[15], [x9]
+; CHECK-NEXT:    ld1 { v5.b }[14], [x10]
+; CHECK-NEXT:    add x8, sp, #184
+; CHECK-NEXT:    add x9, sp, #952
+; CHECK-NEXT:    sdot v16.4s, v1.16b, v6.16b
+; CHECK-NEXT:    ld1 { v2.b }[15], [x8]
+; CHECK-NEXT:    sdot v7.4s, v4.16b, v6.16b
+; CHECK-NEXT:    ld1 { v5.b }[15], [x9]
+; CHECK-NEXT:    sdot v16.4s, v2.16b, v6.16b
+; CHECK-NEXT:    sdot v7.4s, v5.16b, v6.16b
+; CHECK-NEXT:    add v0.4s, v16.4s, v7.4s
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
@@ -3056,49 +2852,17 @@ entry:
 define i32 @test_udot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
 ; CHECK-LABEL: test_udot_v64i8:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q5, q6, [x0]
-; CHECK-NEXT:    ushll2 v16.8h, v5.16b, #0
-; CHECK-NEXT:    ushll v5.8h, v5.8b, #0
-; CHECK-NEXT:    ldp q18, q19, [x1]
-; CHECK-NEXT:    ushll2 v7.8h, v6.16b, #0
-; CHECK-NEXT:    ushll v6.8h, v6.8b, #0
-; CHECK-NEXT:    ushll2 v23.8h, v18.16b, #0
-; CHECK-NEXT:    ushll v18.8h, v18.8b, #0
-; CHECK-NEXT:    ldp q17, q2, [x1, #32]
-; CHECK-NEXT:    ushll2 v22.8h, v19.16b, #0
-; CHECK-NEXT:    ushll v19.8h, v19.8b, #0
-; CHECK-NEXT:    umull v24.4s, v18.4h, v5.4h
-; CHECK-NEXT:    umull v25.4s, v19.4h, v6.4h
-; CHECK-NEXT:    umull v26.4s, v23.4h, v16.4h
-; CHECK-NEXT:    ldp q4, q3, [x0, #32]
-; CHECK-NEXT:    umull v27.4s, v22.4h, v7.4h
-; CHECK-NEXT:    umull2 v7.4s, v22.8h, v7.8h
-; CHECK-NEXT:    umull2 v16.4s, v23.8h, v16.8h
-; CHECK-NEXT:    umull2 v6.4s, v19.8h, v6.8h
-; CHECK-NEXT:    umull2 v5.4s, v18.8h, v5.8h
-; CHECK-NEXT:    ushll v0.8h, v4.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v3.8b, #0
-; CHECK-NEXT:    ushll2 v4.8h, v4.16b, #0
-; CHECK-NEXT:    ushll2 v3.8h, v3.16b, #0
-; CHECK-NEXT:    ushll v20.8h, v17.8b, #0
-; CHECK-NEXT:    ushll v21.8h, v2.8b, #0
-; CHECK-NEXT:    ushll2 v17.8h, v17.16b, #0
-; CHECK-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-NEXT:    umlal2 v16.4s, v17.8h, v4.8h
-; CHECK-NEXT:    umlal2 v7.4s, v2.8h, v3.8h
-; CHECK-NEXT:    umlal2 v6.4s, v21.8h, v1.8h
-; CHECK-NEXT:    umlal2 v5.4s, v20.8h, v0.8h
-; CHECK-NEXT:    umlal v27.4s, v2.4h, v3.4h
-; CHECK-NEXT:    umlal v26.4s, v17.4h, v4.4h
-; CHECK-NEXT:    umlal v25.4s, v21.4h, v1.4h
-; CHECK-NEXT:    umlal v24.4s, v20.4h, v0.4h
-; CHECK-NEXT:    add v0.4s, v16.4s, v7.4s
-; CHECK-NEXT:    add v1.4s, v5.4s, v6.4s
-; CHECK-NEXT:    add v2.4s, v26.4s, v27.4s
-; CHECK-NEXT:    add v3.4s, v24.4s, v25.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v1.4s, v3.4s, v2.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    ldp q1, q4, [x1, #32]
+; CHECK-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ldp q2, q3, [x0, #32]
+; CHECK-NEXT:    udot v5.4s, v1.16b, v2.16b
+; CHECK-NEXT:    ldp q6, q7, [x0]
+; CHECK-NEXT:    udot v0.4s, v4.16b, v3.16b
+; CHECK-NEXT:    ldp q1, q16, [x1]
+; CHECK-NEXT:    udot v5.4s, v1.16b, v6.16b
+; CHECK-NEXT:    udot v0.4s, v16.16b, v7.16b
+; CHECK-NEXT:    add v0.4s, v5.4s, v0.4s
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    add w0, w8, w2
@@ -3117,31 +2881,16 @@ entry:
 define i32 @test_udot_v64i8_nomla(ptr nocapture readonly %a1) {
 ; CHECK-LABEL: test_udot_v64i8_nomla:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q1, q0, [x0, #32]
-; CHECK-NEXT:    ushll v4.8h, v1.8b, #0
-; CHECK-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-NEXT:    ldp q3, q2, [x0]
-; CHECK-NEXT:    ushll v5.8h, v0.8b, #0
-; CHECK-NEXT:    ushll2 v0.8h, v0.16b, #0
-; CHECK-NEXT:    ushll2 v7.8h, v3.16b, #0
-; CHECK-NEXT:    ushll v3.8h, v3.8b, #0
-; CHECK-NEXT:    ushll v6.8h, v2.8b, #0
-; CHECK-NEXT:    ushll2 v2.8h, v2.16b, #0
-; CHECK-NEXT:    uaddl2 v17.4s, v7.8h, v1.8h
-; CHECK-NEXT:    uaddl2 v16.4s, v2.8h, v0.8h
-; CHECK-NEXT:    uaddl2 v18.4s, v6.8h, v5.8h
-; CHECK-NEXT:    uaddl v0.4s, v2.4h, v0.4h
-; CHECK-NEXT:    add v16.4s, v17.4s, v16.4s
-; CHECK-NEXT:    uaddl2 v17.4s, v3.8h, v4.8h
-; CHECK-NEXT:    uaddl v1.4s, v7.4h, v1.4h
-; CHECK-NEXT:    uaddl v2.4s, v6.4h, v5.4h
-; CHECK-NEXT:    uaddl v3.4s, v3.4h, v4.4h
-; CHECK-NEXT:    add v4.4s, v17.4s, v18.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v1.4s, v3.4s, v2.4s
-; CHECK-NEXT:    add v2.4s, v4.4s, v16.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ldp q4, q3, [x0, #32]
+; CHECK-NEXT:    movi v0.16b, #1
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    udot v1.4s, v3.16b, v0.16b
+; CHECK-NEXT:    ldp q3, q5, [x0]
+; CHECK-NEXT:    udot v2.4s, v4.16b, v0.16b
+; CHECK-NEXT:    udot v2.4s, v3.16b, v0.16b
+; CHECK-NEXT:    udot v1.4s, v5.16b, v0.16b
+; CHECK-NEXT:    add v0.4s, v2.4s, v1.4s
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
@@ -3154,49 +2903,17 @@ entry:
 define i32 @test_sdot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) {
 ; CHECK-LABEL: test_sdot_v64i8:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp q5, q6, [x0]
-; CHECK-NEXT:    sshll2 v16.8h, v5.16b, #0
-; CHECK-NEXT:    sshll v5.8h, v5.8b, #0
-; CHECK-NEXT:    ldp q18, q19, [x1]
-; CHECK-NEXT:    sshll2 v7.8h, v6.16b, #0
-; CHECK-NEXT:    sshll v6.8h, v6.8b, #0
-; CHECK-NEXT:    sshll2 v23.8h, v18.16b, #0
-; CHECK-NEXT:    sshll v18.8h, v18.8b, #0
-; CHECK-NEXT:    ldp q17, q2, [x1, #32]
-; CHECK-NEXT:    sshll2 v22.8h, v19.16b, #0
-; CHECK-NEXT:    sshll v19.8h, v19.8b, #0
-; CHECK-NEXT:    smull v24.4s, v18.4h, v5.4h
-; CHECK-NEXT:    smull v25.4s, v19.4h, v6.4h
-; CHECK-NEXT:    smull v26.4s, v23.4h, v16.4h
-; CHECK-NEXT:    ldp q4, q3, [x0, #32]
-; CHECK-NEXT:    smull v27.4s, v22.4h, v7.4h
-; CHECK-NEXT:    smull2 v7.4s, v22.8h, v7.8h
-; CHECK-NEXT:    smull2 v16.4s, v23.8h, v16.8h
-; CHECK-NEXT:    smull2 v6.4s, v19.8h, v6.8h
-; CHECK-NEXT:    smull2 v5.4s, v18.8h, v5.8h
-; CHECK-NEXT:    sshll v0.8h, v4.8b, #0
-; CHECK-NEXT:    sshll v1.8h, v3.8b, #0
-; CHECK-NEXT:    sshll2 v4.8h, v4.16b, #0
-; CHECK-NEXT:    sshll2 v3.8h, v3.16b, #0
-; CHECK-NEXT:    sshll v20.8h, v17.8b, #0
-; CHECK-NEXT:    sshll v21.8h, v2.8b, #0
-; CHECK-NEXT:    sshll2 v17.8h, v17.16b, #0
-; CHECK-NEXT:    sshll2 v2.8h, v2.16b, #0
-; CHECK-NEXT:    smlal2 v16.4s, v17.8h, v4.8h
-; CHECK-NEXT:    smlal2 v7.4s, v2.8h, v3.8h
-; CHECK-NEXT:    smlal2 v6.4s, v21.8h, v1.8h
-; CHECK-NEXT:    smlal2 v5.4s, v20.8h, v0.8h
-; CHECK-NEXT:    smlal v27.4s, v2.4h, v3.4h
-; CHECK-NEXT:    smlal v26.4s, v17.4h, v4.4h
-; CHECK-NEXT:    smlal v25.4s, v21.4h, v1.4h
-; CHECK-NEXT:    smlal v24.4s, v20.4h, v0.4h
-; CHECK-NEXT:    add v0.4s, v16.4s, v7.4s
-; CHECK-NEXT:    add v1.4s, v5.4s, v6.4s
-; CHECK-NEXT:    add v2.4s, v26.4s, v27.4s
-; CHECK-NEXT:    add v3.4s, v24.4s, v25.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v1.4s, v3.4s, v2.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    ldp q1, q4, [x1, #32]
+; CHECK-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ldp q2, q3, [x0, #32]
+; CHECK-NEXT:    sdot v5.4s, v1.16b, v2.16b
+; CHECK-NEXT:    ldp q6, q7, [x0]
+; CHECK-NEXT:    sdot v0.4s, v4.16b, v3.16b
+; CHECK-NEXT:    ldp q1, q16, [x1]
+; CHECK-NEXT:    sdot v5.4s, v1.16b, v6.16b
+; CHECK-NEXT:    sdot v0.4s, v16.16b, v7.16b
+; CHECK-NEXT:    add v0.4s, v5.4s, v0.4s
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    add w0, w8, w2
@@ -3215,100 +2932,27 @@ entry:
 define i32 @test_sdot_v64i8_double(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
 ; CHECK-LABEL: test_sdot_v64i8_double:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    stp d11, d10, [sp, #-32]! // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    stp d9, d8, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset b8, -8
-; CHECK-NEXT:    .cfi_offset b9, -16
-; CHECK-NEXT:    .cfi_offset b10, -24
-; CHECK-NEXT:    .cfi_offset b11, -32
-; CHECK-NEXT:    sshll2 v26.8h, v1.16b, #0
-; CHECK-NEXT:    sshll2 v27.8h, v0.16b, #0
-; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-NEXT:    sshll2 v30.8h, v5.16b, #0
-; CHECK-NEXT:    sshll2 v31.8h, v4.16b, #0
-; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
-; CHECK-NEXT:    sshll v5.8h, v5.8b, #0
-; CHECK-NEXT:    ldp q16, q17, [sp, #96]
-; CHECK-NEXT:    smull v8.4s, v0.4h, v4.4h
-; CHECK-NEXT:    smull v9.4s, v1.4h, v5.4h
-; CHECK-NEXT:    smull v10.4s, v27.4h, v31.4h
-; CHECK-NEXT:    smull v11.4s, v26.4h, v30.4h
-; CHECK-NEXT:    smull2 v26.4s, v26.8h, v30.8h
+; CHECK-NEXT:    ldp q18, q19, [sp, #96]
+; CHECK-NEXT:    movi v22.2d, #0000000000000000
+; CHECK-NEXT:    movi v23.2d, #0000000000000000
+; CHECK-NEXT:    movi v24.2d, #0000000000000000
+; CHECK-NEXT:    movi v25.2d, #0000000000000000
+; CHECK-NEXT:    sdot v22.4s, v3.16b, v7.16b
 ; CHECK-NEXT:    ldp q20, q21, [sp, #32]
-; CHECK-NEXT:    smull2 v27.4s, v27.8h, v31.8h
-; CHECK-NEXT:    smull2 v1.4s, v1.8h, v5.8h
-; CHECK-NEXT:    smull2 v0.4s, v0.8h, v4.8h
-; CHECK-NEXT:    sshll v24.8h, v2.8b, #0
-; CHECK-NEXT:    sshll v25.8h, v3.8b, #0
-; CHECK-NEXT:    sshll2 v2.8h, v2.16b, #0
-; CHECK-NEXT:    sshll2 v3.8h, v3.16b, #0
-; CHECK-NEXT:    sshll v28.8h, v6.8b, #0
-; CHECK-NEXT:    sshll v29.8h, v7.8b, #0
-; CHECK-NEXT:    sshll2 v6.8h, v6.16b, #0
-; CHECK-NEXT:    sshll2 v7.8h, v7.16b, #0
-; CHECK-NEXT:    ldp q19, q18, [sp, #128]
-; CHECK-NEXT:    smlal2 v26.4s, v3.8h, v7.8h
-; CHECK-NEXT:    smlal2 v27.4s, v2.8h, v6.8h
-; CHECK-NEXT:    smlal2 v1.4s, v25.8h, v29.8h
-; CHECK-NEXT:    smlal2 v0.4s, v24.8h, v28.8h
-; CHECK-NEXT:    smlal v11.4s, v3.4h, v7.4h
-; CHECK-NEXT:    ldp q23, q22, [sp, #64]
-; CHECK-NEXT:    smlal v10.4s, v2.4h, v6.4h
-; CHECK-NEXT:    smlal v9.4s, v25.4h, v29.4h
-; CHECK-NEXT:    smlal v8.4s, v24.4h, v28.4h
-; CHECK-NEXT:    sshll2 v6.8h, v21.16b, #0
-; CHECK-NEXT:    sshll2 v7.8h, v20.16b, #0
-; CHECK-NEXT:    sshll v21.8h, v21.8b, #0
-; CHECK-NEXT:    sshll v20.8h, v20.8b, #0
-; CHECK-NEXT:    sshll2 v24.8h, v17.16b, #0
-; CHECK-NEXT:    sshll2 v25.8h, v16.16b, #0
-; CHECK-NEXT:    sshll v16.8h, v16.8b, #0
-; CHECK-NEXT:    sshll v17.8h, v17.8b, #0
-; CHECK-NEXT:    add v2.4s, v27.4s, v26.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    add v1.4s, v10.4s, v11.4s
-; CHECK-NEXT:    add v3.4s, v8.4s, v9.4s
-; CHECK-NEXT:    smull v26.4s, v20.4h, v16.4h
-; CHECK-NEXT:    smull v27.4s, v21.4h, v17.4h
-; CHECK-NEXT:    smull v28.4s, v7.4h, v25.4h
-; CHECK-NEXT:    smull v29.4s, v6.4h, v24.4h
-; CHECK-NEXT:    smull2 v6.4s, v6.8h, v24.8h
-; CHECK-NEXT:    smull2 v7.4s, v7.8h, v25.8h
-; CHECK-NEXT:    smull2 v17.4s, v21.8h, v17.8h
-; CHECK-NEXT:    smull2 v16.4s, v20.8h, v16.8h
-; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
-; CHECK-NEXT:    sshll v2.8h, v23.8b, #0
-; CHECK-NEXT:    sshll v3.8h, v22.8b, #0
-; CHECK-NEXT:    sshll2 v4.8h, v23.16b, #0
-; CHECK-NEXT:    sshll2 v5.8h, v22.16b, #0
-; CHECK-NEXT:    sshll v22.8h, v19.8b, #0
-; CHECK-NEXT:    sshll v23.8h, v18.8b, #0
-; CHECK-NEXT:    sshll2 v19.8h, v19.16b, #0
-; CHECK-NEXT:    sshll2 v18.8h, v18.16b, #0
-; CHECK-NEXT:    smlal2 v7.4s, v4.8h, v19.8h
-; CHECK-NEXT:    smlal2 v6.4s, v5.8h, v18.8h
-; CHECK-NEXT:    smlal2 v17.4s, v3.8h, v23.8h
-; CHECK-NEXT:    smlal2 v16.4s, v2.8h, v22.8h
-; CHECK-NEXT:    smlal v29.4s, v5.4h, v18.4h
-; CHECK-NEXT:    smlal v28.4s, v4.4h, v19.4h
-; CHECK-NEXT:    smlal v27.4s, v3.4h, v23.4h
-; CHECK-NEXT:    smlal v26.4s, v2.4h, v22.4h
-; CHECK-NEXT:    add v2.4s, v7.4s, v6.4s
-; CHECK-NEXT:    add v3.4s, v16.4s, v17.4s
-; CHECK-NEXT:    add v4.4s, v28.4s, v29.4s
-; CHECK-NEXT:    add v5.4s, v26.4s, v27.4s
-; CHECK-NEXT:    add v2.4s, v3.4s, v2.4s
-; CHECK-NEXT:    add v3.4s, v5.4s, v4.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v1.4s, v3.4s, v2.4s
-; CHECK-NEXT:    ldp d9, d8, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    sdot v23.4s, v2.16b, v6.16b
+; CHECK-NEXT:    sdot v22.4s, v1.16b, v5.16b
+; CHECK-NEXT:    sdot v25.4s, v20.16b, v18.16b
+; CHECK-NEXT:    sdot v23.4s, v0.16b, v4.16b
+; CHECK-NEXT:    ldp q16, q17, [sp, #64]
+; CHECK-NEXT:    sdot v24.4s, v21.16b, v19.16b
+; CHECK-NEXT:    add v0.4s, v23.4s, v22.4s
+; CHECK-NEXT:    ldp q26, q3, [sp]
+; CHECK-NEXT:    sdot v25.4s, v26.16b, v16.16b
+; CHECK-NEXT:    sdot v24.4s, v3.16b, v17.16b
+; CHECK-NEXT:    add v1.4s, v25.4s, v24.4s
 ; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ldp d11, d10, [sp], #32 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:
   %az = sext <64 x i8> %a to <64 x i32>
@@ -3326,54 +2970,23 @@ entry:
 define i32 @test_sdot_v64i8_double_nomla(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
 ; CHECK-LABEL: test_sdot_v64i8_double_nomla:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v16.8h, v2.8b, #0
-; CHECK-NEXT:    sshll v17.8h, v0.8b, #0
-; CHECK-NEXT:    sshll2 v2.8h, v2.16b, #0
-; CHECK-NEXT:    sshll2 v18.8h, v3.16b, #0
-; CHECK-NEXT:    sshll2 v19.8h, v1.16b, #0
-; CHECK-NEXT:    sshll2 v0.8h, v0.16b, #0
-; CHECK-NEXT:    saddl2 v20.4s, v19.8h, v18.8h
-; CHECK-NEXT:    saddl2 v21.4s, v0.8h, v2.8h
-; CHECK-NEXT:    sshll v3.8h, v3.8b, #0
-; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-NEXT:    ldp q5, q4, [sp]
-; CHECK-NEXT:    add v20.4s, v21.4s, v20.4s
-; CHECK-NEXT:    saddl2 v21.4s, v1.8h, v3.8h
-; CHECK-NEXT:    saddl2 v22.4s, v17.8h, v16.8h
-; CHECK-NEXT:    saddl v18.4s, v19.4h, v18.4h
-; CHECK-NEXT:    saddl v0.4s, v0.4h, v2.4h
-; CHECK-NEXT:    ldp q7, q6, [sp, #32]
-; CHECK-NEXT:    saddl v1.4s, v1.4h, v3.4h
-; CHECK-NEXT:    saddl v2.4s, v17.4h, v16.4h
-; CHECK-NEXT:    add v3.4s, v22.4s, v21.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v18.4s
-; CHECK-NEXT:    add v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    add v2.4s, v3.4s, v20.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    sshll v1.8h, v7.8b, #0
-; CHECK-NEXT:    sshll v3.8h, v5.8b, #0
-; CHECK-NEXT:    sshll2 v7.8h, v7.16b, #0
-; CHECK-NEXT:    sshll2 v16.8h, v6.16b, #0
-; CHECK-NEXT:    sshll2 v17.8h, v4.16b, #0
-; CHECK-NEXT:    sshll2 v5.8h, v5.16b, #0
-; CHECK-NEXT:    saddl2 v18.4s, v17.8h, v16.8h
-; CHECK-NEXT:    saddl2 v19.4s, v5.8h, v7.8h
-; CHECK-NEXT:    sshll v6.8h, v6.8b, #0
-; CHECK-NEXT:    sshll v4.8h, v4.8b, #0
-; CHECK-NEXT:    add v18.4s, v19.4s, v18.4s
-; CHECK-NEXT:    saddl2 v19.4s, v4.8h, v6.8h
-; CHECK-NEXT:    saddl2 v20.4s, v3.8h, v1.8h
-; CHECK-NEXT:    saddl v16.4s, v17.4h, v16.4h
-; CHECK-NEXT:    saddl v5.4s, v5.4h, v7.4h
-; CHECK-NEXT:    saddl v4.4s, v4.4h, v6.4h
-; CHECK-NEXT:    saddl v1.4s, v3.4h, v1.4h
-; CHECK-NEXT:    add v3.4s, v20.4s, v19.4s
-; CHECK-NEXT:    add v5.4s, v5.4s, v16.4s
-; CHECK-NEXT:    add v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    add v3.4s, v3.4s, v18.4s
-; CHECK-NEXT:    add v1.4s, v1.4s, v5.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    ldp q4, q5, [sp, #32]
+; CHECK-NEXT:    movi v6.16b, #1
+; CHECK-NEXT:    movi v7.2d, #0000000000000000
+; CHECK-NEXT:    movi v16.2d, #0000000000000000
+; CHECK-NEXT:    movi v17.2d, #0000000000000000
+; CHECK-NEXT:    movi v18.2d, #0000000000000000
+; CHECK-NEXT:    sdot v7.4s, v3.16b, v6.16b
+; CHECK-NEXT:    sdot v16.4s, v2.16b, v6.16b
+; CHECK-NEXT:    ldp q3, q2, [sp]
+; CHECK-NEXT:    sdot v17.4s, v5.16b, v6.16b
+; CHECK-NEXT:    sdot v18.4s, v4.16b, v6.16b
+; CHECK-NEXT:    sdot v7.4s, v1.16b, v6.16b
+; CHECK-NEXT:    sdot v16.4s, v0.16b, v6.16b
+; CHECK-NEXT:    sdot v17.4s, v2.16b, v6.16b
+; CHECK-NEXT:    sdot v18.4s, v3.16b, v6.16b
+; CHECK-NEXT:    add v0.4s, v16.4s, v7.4s
+; CHECK-NEXT:    add v1.4s, v18.4s, v17.4s
 ; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w0, s0