diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 11d01429485dcb..f6834eb3d4502b 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -2819,6 +2819,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     Observer.changedInstr(MI);
     return Legalized;
   }
+  case TargetOpcode::G_VECREDUCE_FADD:
   case TargetOpcode::G_VECREDUCE_FMIN:
   case TargetOpcode::G_VECREDUCE_FMAX:
   case TargetOpcode::G_VECREDUCE_FMINIMUM:
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 316a9eaa63d4bb..e665bf42a98de8 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -970,11 +970,19 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .legalFor(PackedVectorAllTypeList)
       .lowerIf(isScalar(0));
 
+  // For fadd reductions we have pairwise operations available. We treat the
+  // usual legal types as legal and handle the lowering to pairwise instructions
+  // later.
   getActionDefinitionsBuilder(G_VECREDUCE_FADD)
-      // We only have FADDP to do reduction-like operations. Lower the rest.
-      .legalFor({{s32, v2s32}, {s64, v2s64}})
+      .legalFor({{s32, v2s32}, {s32, v4s32}, {s64, v2s64}})
+      .legalIf([=](const LegalityQuery &Query) {
+        const auto &Ty = Query.Types[1];
+        return (Ty == v4s16 || Ty == v8s16) && HasFP16;
+      })
+      .minScalarOrElt(0, MinFPScalar)
       .clampMaxNumElements(1, s64, 2)
-      .clampMaxNumElements(1, s32, 2)
+      .clampMaxNumElements(1, s32, 4)
+      .clampMaxNumElements(1, s16, 8)
       .lower();
 
   getActionDefinitionsBuilder(G_VECREDUCE_ADD)
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
index e770def93aa4e6..875108c57e9735 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
@@ -1,223 +1,346 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=aarch64-eabi -aarch64-neon-syntax=generic -mattr=+fullfp16 < %s | FileCheck --check-prefixes=CHECK,FULLFP16 %s
-; RUN: llc --mtriple=aarch64-eabi -aarch64-neon-syntax=generic < %s | FileCheck %s --check-prefixes=CHECK,CHECKNOFP16
+; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16
+; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16
+; RUN: llc -mtriple=aarch64-none-eabi -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16
 
 define float @add_HalfS(<2 x float> %bin.rdx)  {
-; CHECK-LABEL: add_HalfS:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    faddp s0, v0.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: add_HalfS:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    faddp s0, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: add_HalfS:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    movi v1.2s, #128, lsl #24
+; CHECK-GI-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    ret
   %r = call fast float @llvm.vector.reduce.fadd.f32.v2f32(float -0.0, <2 x float> %bin.rdx)
   ret float %r
 }
 
 define half @add_HalfH(<4 x half> %bin.rdx)  {
-; FULLFP16-LABEL: add_HalfH:
-; FULLFP16:       // %bb.0:
-; FULLFP16-NEXT:    faddp v0.4h, v0.4h, v0.4h
-; FULLFP16-NEXT:    faddp h0, v0.2h
-; FULLFP16-NEXT:    ret
+; CHECK-SD-NOFP16-LABEL: add_HalfH:
+; CHECK-SD-NOFP16:       // %bb.0:
+; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s2, s1
+; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s1, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s1, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
 ;
-; CHECKNOFP16-LABEL: add_HalfH:
-; CHECKNOFP16:       // %bb.0:
-; CHECKNOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECKNOFP16-NEXT:    mov h1, v0.h[1]
-; CHECKNOFP16-NEXT:    fcvt s2, h0
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s1, s2, s1
-; CHECKNOFP16-NEXT:    mov h2, v0.h[2]
-; CHECKNOFP16-NEXT:    mov h0, v0.h[3]
-; CHECKNOFP16-NEXT:    fcvt h1, s1
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s1, s1, s2
-; CHECKNOFP16-NEXT:    fcvt h1, s1
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s0, s1, s0
-; CHECKNOFP16-NEXT:    fcvt h0, s0
-; CHECKNOFP16-NEXT:    ret
+; CHECK-SD-FP16-LABEL: add_HalfH:
+; CHECK-SD-FP16:       // %bb.0:
+; CHECK-SD-FP16-NEXT:    faddp v0.4h, v0.4h, v0.4h
+; CHECK-SD-FP16-NEXT:    faddp h0, v0.2h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: add_HalfH:
+; CHECK-GI-NOFP16:       // %bb.0:
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    mov w8, #32768 // =0x8000
+; CHECK-GI-NOFP16-NEXT:    fmov s1, w8
+; CHECK-GI-NOFP16-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: add_HalfH:
+; CHECK-GI-FP16:       // %bb.0:
+; CHECK-GI-FP16-NEXT:    faddp v0.4h, v0.4h, v0.4h
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI1_0
+; CHECK-GI-FP16-NEXT:    ldr h1, [x8, :lo12:.LCPI1_0]
+; CHECK-GI-FP16-NEXT:    faddp h0, v0.2h
+; CHECK-GI-FP16-NEXT:    fadd h0, h0, h1
+; CHECK-GI-FP16-NEXT:    ret
   %r = call fast half @llvm.vector.reduce.fadd.f16.v4f16(half -0.0, <4 x half> %bin.rdx)
   ret half %r
 }
 
 
 define half @add_H(<8 x half> %bin.rdx)  {
-; FULLFP16-LABEL: add_H:
-; FULLFP16:       // %bb.0:
-; FULLFP16-NEXT:    faddp v1.8h, v0.8h, v0.8h
-; FULLFP16-NEXT:    faddp v0.8h, v1.8h, v0.8h
-; FULLFP16-NEXT:    faddp h0, v0.2h
-; FULLFP16-NEXT:    ret
+; CHECK-SD-NOFP16-LABEL: add_H:
+; CHECK-SD-NOFP16:       // %bb.0:
+; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s2, s1
+; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s1, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s1, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s1, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s1, s2
+; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s1, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s1, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: add_H:
+; CHECK-SD-FP16:       // %bb.0:
+; CHECK-SD-FP16-NEXT:    faddp v1.8h, v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    faddp v0.8h, v1.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    faddp h0, v0.2h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: add_H:
+; CHECK-GI-NOFP16:       // %bb.0:
+; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    mov w8, #32768 // =0x8000
+; CHECK-GI-NOFP16-NEXT:    fadd v0.4s, v1.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fmov s1, w8
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    ret
 ;
-; CHECKNOFP16-LABEL: add_H:
-; CHECKNOFP16:       // %bb.0:
-; CHECKNOFP16-NEXT:    mov h1, v0.h[1]
-; CHECKNOFP16-NEXT:    fcvt s2, h0
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s1, s2, s1
-; CHECKNOFP16-NEXT:    mov h2, v0.h[2]
-; CHECKNOFP16-NEXT:    fcvt h1, s1
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s1, s1, s2
-; CHECKNOFP16-NEXT:    mov h2, v0.h[3]
-; CHECKNOFP16-NEXT:    fcvt h1, s1
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s1, s1, s2
-; CHECKNOFP16-NEXT:    mov h2, v0.h[4]
-; CHECKNOFP16-NEXT:    fcvt h1, s1
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s1, s1, s2
-; CHECKNOFP16-NEXT:    mov h2, v0.h[5]
-; CHECKNOFP16-NEXT:    fcvt h1, s1
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s1, s1, s2
-; CHECKNOFP16-NEXT:    mov h2, v0.h[6]
-; CHECKNOFP16-NEXT:    mov h0, v0.h[7]
-; CHECKNOFP16-NEXT:    fcvt h1, s1
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s1, s1, s2
-; CHECKNOFP16-NEXT:    fcvt h1, s1
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s0, s1, s0
-; CHECKNOFP16-NEXT:    fcvt h0, s0
-; CHECKNOFP16-NEXT:    ret
+; CHECK-GI-FP16-LABEL: add_H:
+; CHECK-GI-FP16:       // %bb.0:
+; CHECK-GI-FP16-NEXT:    faddp v1.8h, v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI2_0
+; CHECK-GI-FP16-NEXT:    faddp v0.8h, v1.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    ldr h1, [x8, :lo12:.LCPI2_0]
+; CHECK-GI-FP16-NEXT:    faddp h0, v0.2h
+; CHECK-GI-FP16-NEXT:    fadd h0, h0, h1
+; CHECK-GI-FP16-NEXT:    ret
   %r = call fast half @llvm.vector.reduce.fadd.f16.v8f16(half -0.0, <8 x half> %bin.rdx)
   ret half %r
 }
 
 define float @add_S(<4 x float> %bin.rdx)  {
-; CHECK-LABEL: add_S:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    faddp v0.4s, v0.4s, v0.4s
-; CHECK-NEXT:    faddp s0, v0.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: add_S:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT:    faddp s0, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: add_S:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    movi v1.2s, #128, lsl #24
+; CHECK-GI-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    ret
   %r = call fast float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %bin.rdx)
   ret float %r
 }
 
 define double @add_D(<2 x double> %bin.rdx)  {
-; CHECK-LABEL: add_D:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    faddp d0, v0.2d
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: add_D:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    faddp d0, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: add_D:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    faddp d0, v0.2d
+; CHECK-GI-NEXT:    mov x8, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT:    fmov d1, x8
+; CHECK-GI-NEXT:    fadd d0, d0, d1
+; CHECK-GI-NEXT:    ret
   %r = call fast double @llvm.vector.reduce.fadd.f64.v2f64(double -0.0, <2 x double> %bin.rdx)
   ret double %r
 }
 
 define half @add_2H(<16 x half> %bin.rdx)  {
-; FULLFP16-LABEL: add_2H:
-; FULLFP16:       // %bb.0:
-; FULLFP16-NEXT:    fadd v0.8h, v0.8h, v1.8h
-; FULLFP16-NEXT:    faddp v1.8h, v0.8h, v0.8h
-; FULLFP16-NEXT:    faddp v0.8h, v1.8h, v0.8h
-; FULLFP16-NEXT:    faddp h0, v0.2h
-; FULLFP16-NEXT:    ret
+; CHECK-SD-NOFP16-LABEL: add_2H:
+; CHECK-SD-NOFP16:       // %bb.0:
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
+; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s3, s2
+; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s5, s3
+; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s4, s2
+; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
+; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[4]
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
+; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s5, s3
+; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[5]
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
+; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s4, s5, s4
+; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
+; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s5, s4
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s1, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
 ;
-; CHECKNOFP16-LABEL: add_2H:
-; CHECKNOFP16:       // %bb.0:
-; CHECKNOFP16-NEXT:    mov h2, v1.h[1]
-; CHECKNOFP16-NEXT:    mov h3, v0.h[1]
-; CHECKNOFP16-NEXT:    fcvt s4, h1
-; CHECKNOFP16-NEXT:    fcvt s5, h0
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fadd s4, s5, s4
-; CHECKNOFP16-NEXT:    mov h5, v0.h[2]
-; CHECKNOFP16-NEXT:    fadd s2, s3, s2
-; CHECKNOFP16-NEXT:    mov h3, v1.h[2]
-; CHECKNOFP16-NEXT:    fcvt h4, s4
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s3, s5, s3
-; CHECKNOFP16-NEXT:    mov h5, v0.h[3]
-; CHECKNOFP16-NEXT:    fadd s2, s4, s2
-; CHECKNOFP16-NEXT:    mov h4, v1.h[3]
-; CHECKNOFP16-NEXT:    fcvt h3, s3
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s4, s5, s4
-; CHECKNOFP16-NEXT:    mov h5, v0.h[4]
-; CHECKNOFP16-NEXT:    fadd s2, s2, s3
-; CHECKNOFP16-NEXT:    mov h3, v1.h[4]
-; CHECKNOFP16-NEXT:    fcvt h4, s4
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s3, s5, s3
-; CHECKNOFP16-NEXT:    mov h5, v0.h[5]
-; CHECKNOFP16-NEXT:    fadd s2, s2, s4
-; CHECKNOFP16-NEXT:    mov h4, v1.h[5]
-; CHECKNOFP16-NEXT:    fcvt h3, s3
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s4, s5, s4
-; CHECKNOFP16-NEXT:    mov h5, v0.h[6]
-; CHECKNOFP16-NEXT:    mov h0, v0.h[7]
-; CHECKNOFP16-NEXT:    fadd s2, s2, s3
-; CHECKNOFP16-NEXT:    fcvt h3, s4
-; CHECKNOFP16-NEXT:    mov h4, v1.h[6]
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    mov h1, v1.h[7]
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s0, s0, s1
-; CHECKNOFP16-NEXT:    fadd s2, s2, s3
-; CHECKNOFP16-NEXT:    fadd s3, s5, s4
-; CHECKNOFP16-NEXT:    fcvt h0, s0
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt h3, s3
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fadd s2, s2, s3
-; CHECKNOFP16-NEXT:    fcvt h1, s2
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s0, s1, s0
-; CHECKNOFP16-NEXT:    fcvt h0, s0
-; CHECKNOFP16-NEXT:    ret
+; CHECK-SD-FP16-LABEL: add_2H:
+; CHECK-SD-FP16:       // %bb.0:
+; CHECK-SD-FP16-NEXT:    fadd v0.8h, v0.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    faddp v1.8h, v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    faddp v0.8h, v1.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    faddp h0, v0.2h
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: add_2H:
+; CHECK-GI-NOFP16:       // %bb.0:
+; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    mov w8, #32768 // =0x8000
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fadd v0.4s, v2.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fadd v1.4s, v3.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fmov s1, w8
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: add_2H:
+; CHECK-GI-FP16:       // %bb.0:
+; CHECK-GI-FP16-NEXT:    fadd v0.8h, v0.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI5_0
+; CHECK-GI-FP16-NEXT:    faddp v1.8h, v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    faddp v0.8h, v1.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    ldr h1, [x8, :lo12:.LCPI5_0]
+; CHECK-GI-FP16-NEXT:    faddp h0, v0.2h
+; CHECK-GI-FP16-NEXT:    fadd h0, h0, h1
+; CHECK-GI-FP16-NEXT:    ret
   %r = call fast half @llvm.vector.reduce.fadd.f16.v16f16(half -0.0, <16 x half> %bin.rdx)
   ret half %r
 }
 
 define float @add_2S(<8 x float> %bin.rdx)  {
-; CHECK-LABEL: add_2S:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    faddp v0.4s, v0.4s, v0.4s
-; CHECK-NEXT:    faddp s0, v0.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: add_2S:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT:    faddp s0, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: add_2S:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    movi v2.2s, #128, lsl #24
+; CHECK-GI-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NEXT:    fadd s0, s0, s2
+; CHECK-GI-NEXT:    ret
   %r = call fast float @llvm.vector.reduce.fadd.f32.v8f32(float -0.0, <8 x float> %bin.rdx)
   ret float %r
 }
 
 define double @add_2D(<4 x double> %bin.rdx)  {
-; CHECK-LABEL: add_2D:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    faddp d0, v0.2d
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: add_2D:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    faddp d0, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: add_2D:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fadd v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    mov x8, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT:    fmov d1, x8
+; CHECK-GI-NEXT:    faddp d0, v0.2d
+; CHECK-GI-NEXT:    fadd d0, d0, d1
+; CHECK-GI-NEXT:    ret
   %r = call fast double @llvm.vector.reduce.fadd.f64.v4f64(double -0.0, <4 x double> %bin.rdx)
   ret double %r
 }
@@ -239,21 +362,39 @@ define float @add_S_init_42(<4 x float> %bin.rdx)  {
 ; The faddp.4s in the loop should not use v0.4s as second operand,
 ; because this introduces an unnecessary cross-iteration dependency.
 define float @fadd_reduction_v4f32_in_loop(ptr %ptr.start) {
-; CHECK-LABEL: fadd_reduction_v4f32_in_loop:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi d0, #0000000000000000
-; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:  .LBB9_1: // %loop
-; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr q1, [x0, x8]
-; CHECK-NEXT:    add x8, x8, #16
-; CHECK-NEXT:    cmp w8, #112
-; CHECK-NEXT:    faddp v1.4s, v1.4s, v1.4s
-; CHECK-NEXT:    faddp s1, v1.2s
-; CHECK-NEXT:    fadd s0, s1, s0
-; CHECK-NEXT:    b.ne .LBB9_1
-; CHECK-NEXT:  // %bb.2: // %exit
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fadd_reduction_v4f32_in_loop:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d0, #0000000000000000
+; CHECK-SD-NEXT:    mov x8, xzr
+; CHECK-SD-NEXT:  .LBB9_1: // %loop
+; CHECK-SD-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NEXT:    ldr q1, [x0, x8]
+; CHECK-SD-NEXT:    add x8, x8, #16
+; CHECK-SD-NEXT:    cmp w8, #112
+; CHECK-SD-NEXT:    faddp v1.4s, v1.4s, v1.4s
+; CHECK-SD-NEXT:    faddp s1, v1.2s
+; CHECK-SD-NEXT:    fadd s0, s1, s0
+; CHECK-SD-NEXT:    b.ne .LBB9_1
+; CHECK-SD-NEXT:  // %bb.2: // %exit
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fadd_reduction_v4f32_in_loop:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi d0, #0000000000000000
+; CHECK-GI-NEXT:    movi v1.2s, #128, lsl #24
+; CHECK-GI-NEXT:    mov x8, xzr
+; CHECK-GI-NEXT:  .LBB9_1: // %loop
+; CHECK-GI-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NEXT:    ldr q2, [x0, x8]
+; CHECK-GI-NEXT:    fadd s0, s1, s0
+; CHECK-GI-NEXT:    add x8, x8, #16
+; CHECK-GI-NEXT:    cmp w8, #112
+; CHECK-GI-NEXT:    faddp v2.4s, v2.4s, v2.4s
+; CHECK-GI-NEXT:    faddp s2, v2.2s
+; CHECK-GI-NEXT:    fadd s0, s2, s0
+; CHECK-GI-NEXT:    b.ne .LBB9_1
+; CHECK-GI-NEXT:  // %bb.2: // %exit
+; CHECK-GI-NEXT:    ret
 entry:
   br label %loop
 
@@ -276,53 +417,102 @@ exit:
 ; The faddp.4h in the loop should not use v0.4h as second operand,
 ; because this introduces an unnecessary cross-iteration dependency.
 define half @fadd_reduction_v4f16_in_loop(ptr %ptr.start) {
-; FULLFP16-LABEL: fadd_reduction_v4f16_in_loop:
-; FULLFP16:       // %bb.0: // %entry
-; FULLFP16-NEXT:    movi d0, #0000000000000000
-; FULLFP16-NEXT:    mov x8, xzr
-; FULLFP16-NEXT:  .LBB10_1: // %loop
-; FULLFP16-NEXT:    // =>This Inner Loop Header: Depth=1
-; FULLFP16-NEXT:    ldr d1, [x0, x8]
-; FULLFP16-NEXT:    add x8, x8, #8
-; FULLFP16-NEXT:    cmp w8, #56
-; FULLFP16-NEXT:    faddp v1.4h, v1.4h, v1.4h
-; FULLFP16-NEXT:    faddp h1, v1.2h
-; FULLFP16-NEXT:    fadd h0, h1, h0
-; FULLFP16-NEXT:    b.ne .LBB10_1
-; FULLFP16-NEXT:  // %bb.2: // %exit
-; FULLFP16-NEXT:    ret
+; CHECK-SD-NOFP16-LABEL: fadd_reduction_v4f16_in_loop:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    movi d0, #0000000000000000
+; CHECK-SD-NOFP16-NEXT:    mov x8, xzr
+; CHECK-SD-NOFP16-NEXT:  .LBB10_1: // %loop
+; CHECK-SD-NOFP16-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NOFP16-NEXT:    ldr d1, [x0, x8]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    add x8, x8, #8
+; CHECK-SD-NOFP16-NEXT:    cmp w8, #56
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s3, s2
+; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s2, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s1, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    b.ne .LBB10_1
+; CHECK-SD-NOFP16-NEXT:  // %bb.2: // %exit
+; CHECK-SD-NOFP16-NEXT:    ret
 ;
-; CHECKNOFP16-LABEL: fadd_reduction_v4f16_in_loop:
-; CHECKNOFP16:       // %bb.0: // %entry
-; CHECKNOFP16-NEXT:    movi d0, #0000000000000000
-; CHECKNOFP16-NEXT:    mov x8, xzr
-; CHECKNOFP16-NEXT:  .LBB10_1: // %loop
-; CHECKNOFP16-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECKNOFP16-NEXT:    ldr d1, [x0, x8]
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    add x8, x8, #8
-; CHECKNOFP16-NEXT:    cmp w8, #56
-; CHECKNOFP16-NEXT:    mov h2, v1.h[1]
-; CHECKNOFP16-NEXT:    fcvt s3, h1
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s2, s3, s2
-; CHECKNOFP16-NEXT:    mov h3, v1.h[2]
-; CHECKNOFP16-NEXT:    mov h1, v1.h[3]
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s2, s2, s3
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s1, s2, s1
-; CHECKNOFP16-NEXT:    fcvt h1, s1
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s0, s1, s0
-; CHECKNOFP16-NEXT:    fcvt h0, s0
-; CHECKNOFP16-NEXT:    b.ne .LBB10_1
-; CHECKNOFP16-NEXT:  // %bb.2: // %exit
-; CHECKNOFP16-NEXT:    ret
+; CHECK-SD-FP16-LABEL: fadd_reduction_v4f16_in_loop:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    movi d0, #0000000000000000
+; CHECK-SD-FP16-NEXT:    mov x8, xzr
+; CHECK-SD-FP16-NEXT:  .LBB10_1: // %loop
+; CHECK-SD-FP16-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-FP16-NEXT:    ldr d1, [x0, x8]
+; CHECK-SD-FP16-NEXT:    add x8, x8, #8
+; CHECK-SD-FP16-NEXT:    cmp w8, #56
+; CHECK-SD-FP16-NEXT:    faddp v1.4h, v1.4h, v1.4h
+; CHECK-SD-FP16-NEXT:    faddp h1, v1.2h
+; CHECK-SD-FP16-NEXT:    fadd h0, h1, h0
+; CHECK-SD-FP16-NEXT:    b.ne .LBB10_1
+; CHECK-SD-FP16-NEXT:  // %bb.2: // %exit
+; CHECK-SD-FP16-NEXT:    ret
+;
+; CHECK-GI-NOFP16-LABEL: fadd_reduction_v4f16_in_loop:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    mov w9, #32768 // =0x8000
+; CHECK-GI-NOFP16-NEXT:    mov x8, xzr
+; CHECK-GI-NOFP16-NEXT:    fmov s0, w9
+; CHECK-GI-NOFP16-NEXT:    mov w9, #0 // =0x0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h0
+; CHECK-GI-NOFP16-NEXT:  .LBB10_1: // %loop
+; CHECK-GI-NOFP16-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NOFP16-NEXT:    ldr d0, [x0, x8]
+; CHECK-GI-NOFP16-NEXT:    fmov s2, w9
+; CHECK-GI-NOFP16-NEXT:    add x8, x8, #8
+; CHECK-GI-NOFP16-NEXT:    cmp w8, #56
+; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-GI-NOFP16-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    fmov w9, s0
+; CHECK-GI-NOFP16-NEXT:    b.ne .LBB10_1
+; CHECK-GI-NOFP16-NEXT:  // %bb.2: // %exit
+; CHECK-GI-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fadd_reduction_v4f16_in_loop:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    movi d0, #0000000000000000
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI10_0
+; CHECK-GI-FP16-NEXT:    ldr h1, [x8, :lo12:.LCPI10_0]
+; CHECK-GI-FP16-NEXT:    mov x8, xzr
+; CHECK-GI-FP16-NEXT:  .LBB10_1: // %loop
+; CHECK-GI-FP16-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-FP16-NEXT:    ldr d2, [x0, x8]
+; CHECK-GI-FP16-NEXT:    fadd h0, h1, h0
+; CHECK-GI-FP16-NEXT:    add x8, x8, #8
+; CHECK-GI-FP16-NEXT:    cmp w8, #56
+; CHECK-GI-FP16-NEXT:    faddp v2.4h, v2.4h, v2.4h
+; CHECK-GI-FP16-NEXT:    faddp h2, v2.2h
+; CHECK-GI-FP16-NEXT:    fadd h0, h2, h0
+; CHECK-GI-FP16-NEXT:    b.ne .LBB10_1
+; CHECK-GI-FP16-NEXT:  // %bb.2: // %exit
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   br label %loop
 
@@ -345,74 +535,126 @@ exit:
 ; The faddp.8h in the loop should not use v0.8h as second operand,
 ; because this introduces an unnecessary cross-iteration dependency.
 define half @fadd_reduction_v8f16_in_loop(ptr %ptr.start) {
-; FULLFP16-LABEL: fadd_reduction_v8f16_in_loop:
-; FULLFP16:       // %bb.0: // %entry
-; FULLFP16-NEXT:    movi d0, #0000000000000000
-; FULLFP16-NEXT:    mov x8, xzr
-; FULLFP16-NEXT:  .LBB11_1: // %loop
-; FULLFP16-NEXT:    // =>This Inner Loop Header: Depth=1
-; FULLFP16-NEXT:    ldr q1, [x0, x8]
-; FULLFP16-NEXT:    add x8, x8, #8
-; FULLFP16-NEXT:    cmp w8, #56
-; FULLFP16-NEXT:    faddp v2.8h, v1.8h, v1.8h
-; FULLFP16-NEXT:    faddp v1.8h, v2.8h, v1.8h
-; FULLFP16-NEXT:    faddp h1, v1.2h
-; FULLFP16-NEXT:    fadd h0, h1, h0
-; FULLFP16-NEXT:    b.ne .LBB11_1
-; FULLFP16-NEXT:  // %bb.2: // %exit
-; FULLFP16-NEXT:    ret
+; CHECK-SD-NOFP16-LABEL: fadd_reduction_v8f16_in_loop:
+; CHECK-SD-NOFP16:       // %bb.0: // %entry
+; CHECK-SD-NOFP16-NEXT:    movi d0, #0000000000000000
+; CHECK-SD-NOFP16-NEXT:    mov x8, xzr
+; CHECK-SD-NOFP16-NEXT:  .LBB11_1: // %loop
+; CHECK-SD-NOFP16-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-NOFP16-NEXT:    ldr q1, [x0, x8]
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    add x8, x8, #8
+; CHECK-SD-NOFP16-NEXT:    cmp w8, #56
+; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s3, s2
+; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
+; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
+; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
+; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
+; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s2, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s1, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    b.ne .LBB11_1
+; CHECK-SD-NOFP16-NEXT:  // %bb.2: // %exit
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fadd_reduction_v8f16_in_loop:
+; CHECK-SD-FP16:       // %bb.0: // %entry
+; CHECK-SD-FP16-NEXT:    movi d0, #0000000000000000
+; CHECK-SD-FP16-NEXT:    mov x8, xzr
+; CHECK-SD-FP16-NEXT:  .LBB11_1: // %loop
+; CHECK-SD-FP16-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-SD-FP16-NEXT:    ldr q1, [x0, x8]
+; CHECK-SD-FP16-NEXT:    add x8, x8, #8
+; CHECK-SD-FP16-NEXT:    cmp w8, #56
+; CHECK-SD-FP16-NEXT:    faddp v2.8h, v1.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    faddp v1.8h, v2.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    faddp h1, v1.2h
+; CHECK-SD-FP16-NEXT:    fadd h0, h1, h0
+; CHECK-SD-FP16-NEXT:    b.ne .LBB11_1
+; CHECK-SD-FP16-NEXT:  // %bb.2: // %exit
+; CHECK-SD-FP16-NEXT:    ret
 ;
-; CHECKNOFP16-LABEL: fadd_reduction_v8f16_in_loop:
-; CHECKNOFP16:       // %bb.0: // %entry
-; CHECKNOFP16-NEXT:    movi d0, #0000000000000000
-; CHECKNOFP16-NEXT:    mov x8, xzr
-; CHECKNOFP16-NEXT:  .LBB11_1: // %loop
-; CHECKNOFP16-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECKNOFP16-NEXT:    ldr q1, [x0, x8]
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    add x8, x8, #8
-; CHECKNOFP16-NEXT:    cmp w8, #56
-; CHECKNOFP16-NEXT:    mov h2, v1.h[1]
-; CHECKNOFP16-NEXT:    fcvt s3, h1
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s2, s3, s2
-; CHECKNOFP16-NEXT:    mov h3, v1.h[2]
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s2, s2, s3
-; CHECKNOFP16-NEXT:    mov h3, v1.h[3]
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s2, s2, s3
-; CHECKNOFP16-NEXT:    mov h3, v1.h[4]
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s2, s2, s3
-; CHECKNOFP16-NEXT:    mov h3, v1.h[5]
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s2, s2, s3
-; CHECKNOFP16-NEXT:    mov h3, v1.h[6]
-; CHECKNOFP16-NEXT:    mov h1, v1.h[7]
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s2, s2, s3
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fadd s1, s2, s1
-; CHECKNOFP16-NEXT:    fcvt h1, s1
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fadd s0, s1, s0
-; CHECKNOFP16-NEXT:    fcvt h0, s0
-; CHECKNOFP16-NEXT:    b.ne .LBB11_1
-; CHECKNOFP16-NEXT:  // %bb.2: // %exit
-; CHECKNOFP16-NEXT:    ret
+; CHECK-GI-NOFP16-LABEL: fadd_reduction_v8f16_in_loop:
+; CHECK-GI-NOFP16:       // %bb.0: // %entry
+; CHECK-GI-NOFP16-NEXT:    mov w9, #32768 // =0x8000
+; CHECK-GI-NOFP16-NEXT:    mov x8, xzr
+; CHECK-GI-NOFP16-NEXT:    fmov s0, w9
+; CHECK-GI-NOFP16-NEXT:    mov w9, #0 // =0x0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h0
+; CHECK-GI-NOFP16-NEXT:  .LBB11_1: // %loop
+; CHECK-GI-NOFP16-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-NOFP16-NEXT:    ldr q0, [x0, x8]
+; CHECK-GI-NOFP16-NEXT:    add x8, x8, #8
+; CHECK-GI-NOFP16-NEXT:    cmp w8, #56
+; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    fadd v0.4s, v2.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fmov s2, w9
+; CHECK-GI-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-GI-NOFP16-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s2
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    fmov w9, s0
+; CHECK-GI-NOFP16-NEXT:    b.ne .LBB11_1
+; CHECK-GI-NOFP16-NEXT:  // %bb.2: // %exit
+; CHECK-GI-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fadd_reduction_v8f16_in_loop:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    movi d0, #0000000000000000
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI11_0
+; CHECK-GI-FP16-NEXT:    ldr h1, [x8, :lo12:.LCPI11_0]
+; CHECK-GI-FP16-NEXT:    mov x8, xzr
+; CHECK-GI-FP16-NEXT:  .LBB11_1: // %loop
+; CHECK-GI-FP16-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-GI-FP16-NEXT:    ldr q2, [x0, x8]
+; CHECK-GI-FP16-NEXT:    fadd h0, h1, h0
+; CHECK-GI-FP16-NEXT:    add x8, x8, #8
+; CHECK-GI-FP16-NEXT:    cmp w8, #56
+; CHECK-GI-FP16-NEXT:    faddp v3.8h, v2.8h, v2.8h
+; CHECK-GI-FP16-NEXT:    faddp v2.8h, v3.8h, v2.8h
+; CHECK-GI-FP16-NEXT:    faddp h2, v2.2h
+; CHECK-GI-FP16-NEXT:    fadd h0, h2, h0
+; CHECK-GI-FP16-NEXT:    b.ne .LBB11_1
+; CHECK-GI-FP16-NEXT:  // %bb.2: // %exit
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   br label %loop
 
@@ -434,91 +676,136 @@ exit:
 
 
 define half @fadd_reduct_reassoc_v8f16(<8 x half> %a, <8 x half> %b) {
-; FULLFP16-LABEL: fadd_reduct_reassoc_v8f16:
-; FULLFP16:       // %bb.0:
-; FULLFP16-NEXT:    fadd v0.8h, v0.8h, v1.8h
-; FULLFP16-NEXT:    faddp v1.8h, v0.8h, v0.8h
-; FULLFP16-NEXT:    faddp v0.8h, v1.8h, v0.8h
-; FULLFP16-NEXT:    faddp h0, v0.2h
-; FULLFP16-NEXT:    ret
+; CHECK-SD-NOFP16-LABEL: fadd_reduct_reassoc_v8f16:
+; CHECK-SD-NOFP16:       // %bb.0:
+; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
+; CHECK-SD-NOFP16-NEXT:    mov h3, v1.h[1]
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s4, s2
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s5, s3
+; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[2]
+; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[2]
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
+; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[3]
+; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[3]
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
+; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[4]
+; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[4]
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
+; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[5]
+; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[5]
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
+; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[6]
+; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
+; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
+; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s4
+; CHECK-SD-NOFP16-NEXT:    fadd s3, s3, s5
+; CHECK-SD-NOFP16-NEXT:    fcvt h2, s2
+; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
+; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s2, s0
+; CHECK-SD-NOFP16-NEXT:    fadd s1, s3, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-SD-NOFP16-NEXT:    ret
+;
+; CHECK-SD-FP16-LABEL: fadd_reduct_reassoc_v8f16:
+; CHECK-SD-FP16:       // %bb.0:
+; CHECK-SD-FP16-NEXT:    fadd v0.8h, v0.8h, v1.8h
+; CHECK-SD-FP16-NEXT:    faddp v1.8h, v0.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    faddp v0.8h, v1.8h, v0.8h
+; CHECK-SD-FP16-NEXT:    faddp h0, v0.2h
+; CHECK-SD-FP16-NEXT:    ret
 ;
-; CHECKNOFP16-LABEL: fadd_reduct_reassoc_v8f16:
-; CHECKNOFP16:       // %bb.0:
-; CHECKNOFP16-NEXT:    mov h2, v0.h[1]
-; CHECKNOFP16-NEXT:    mov h3, v1.h[1]
-; CHECKNOFP16-NEXT:    fcvt s4, h0
-; CHECKNOFP16-NEXT:    fcvt s5, h1
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fadd s2, s4, s2
-; CHECKNOFP16-NEXT:    fadd s3, s5, s3
-; CHECKNOFP16-NEXT:    mov h4, v0.h[2]
-; CHECKNOFP16-NEXT:    mov h5, v1.h[2]
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt h3, s3
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fadd s2, s2, s4
-; CHECKNOFP16-NEXT:    fadd s3, s3, s5
-; CHECKNOFP16-NEXT:    mov h4, v0.h[3]
-; CHECKNOFP16-NEXT:    mov h5, v1.h[3]
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt h3, s3
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fadd s2, s2, s4
-; CHECKNOFP16-NEXT:    fadd s3, s3, s5
-; CHECKNOFP16-NEXT:    mov h4, v0.h[4]
-; CHECKNOFP16-NEXT:    mov h5, v1.h[4]
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt h3, s3
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fadd s2, s2, s4
-; CHECKNOFP16-NEXT:    fadd s3, s3, s5
-; CHECKNOFP16-NEXT:    mov h4, v0.h[5]
-; CHECKNOFP16-NEXT:    mov h5, v1.h[5]
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt h3, s3
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fadd s2, s2, s4
-; CHECKNOFP16-NEXT:    fadd s3, s3, s5
-; CHECKNOFP16-NEXT:    mov h4, v0.h[6]
-; CHECKNOFP16-NEXT:    mov h5, v1.h[6]
-; CHECKNOFP16-NEXT:    mov h1, v1.h[7]
-; CHECKNOFP16-NEXT:    mov h0, v0.h[7]
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt h3, s3
-; CHECKNOFP16-NEXT:    fcvt s4, h4
-; CHECKNOFP16-NEXT:    fcvt s5, h5
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fadd s2, s2, s4
-; CHECKNOFP16-NEXT:    fadd s3, s3, s5
-; CHECKNOFP16-NEXT:    fcvt h2, s2
-; CHECKNOFP16-NEXT:    fcvt h3, s3
-; CHECKNOFP16-NEXT:    fcvt s2, h2
-; CHECKNOFP16-NEXT:    fcvt s3, h3
-; CHECKNOFP16-NEXT:    fadd s0, s2, s0
-; CHECKNOFP16-NEXT:    fadd s1, s3, s1
-; CHECKNOFP16-NEXT:    fcvt h0, s0
-; CHECKNOFP16-NEXT:    fcvt h1, s1
-; CHECKNOFP16-NEXT:    fcvt s1, h1
-; CHECKNOFP16-NEXT:    fcvt s0, h0
-; CHECKNOFP16-NEXT:    fadd s0, s0, s1
-; CHECKNOFP16-NEXT:    fcvt h0, s0
-; CHECKNOFP16-NEXT:    ret
+; CHECK-GI-NOFP16-LABEL: fadd_reduct_reassoc_v8f16:
+; CHECK-GI-NOFP16:       // %bb.0:
+; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT:    mov w8, #32768 // =0x8000
+; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT:    fadd v0.4s, v2.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    fmov s2, w8
+; CHECK-GI-NOFP16-NEXT:    fadd v1.4s, v3.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    fcvt s3, h2
+; CHECK-GI-NOFP16-NEXT:    fcvt s2, h2
+; CHECK-GI-NOFP16-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-GI-NOFP16-NEXT:    faddp v1.4s, v1.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NOFP16-NEXT:    faddp s1, v1.2s
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s3
+; CHECK-GI-NOFP16-NEXT:    fadd s1, s1, s2
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    fcvt h1, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
+; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: fadd_reduct_reassoc_v8f16:
+; CHECK-GI-FP16:       // %bb.0:
+; CHECK-GI-FP16-NEXT:    faddp v2.8h, v0.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    faddp v3.8h, v1.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI12_0
+; CHECK-GI-FP16-NEXT:    faddp v0.8h, v2.8h, v0.8h
+; CHECK-GI-FP16-NEXT:    faddp v1.8h, v3.8h, v1.8h
+; CHECK-GI-FP16-NEXT:    ldr h2, [x8, :lo12:.LCPI12_0]
+; CHECK-GI-FP16-NEXT:    faddp h0, v0.2h
+; CHECK-GI-FP16-NEXT:    faddp h1, v1.2h
+; CHECK-GI-FP16-NEXT:    fadd h0, h0, h2
+; CHECK-GI-FP16-NEXT:    fadd h1, h1, h2
+; CHECK-GI-FP16-NEXT:    fadd h0, h0, h1
+; CHECK-GI-FP16-NEXT:    ret
   %r1 = call fast half @llvm.vector.reduce.fadd.f16.v8f16(half -0.0, <8 x half> %a)
   %r2 = call fast half @llvm.vector.reduce.fadd.f16.v8f16(half -0.0, <8 x half> %b)
   %r = fadd fast half %r1, %r2
@@ -526,14 +813,28 @@ define half @fadd_reduct_reassoc_v8f16(<8 x half> %a, <8 x half> %b) {
 }
 
 define float @fadd_reduct_reassoc_v8f32(<8 x float> %a, <8 x float> %b) {
-; CHECK-LABEL: fadd_reduct_reassoc_v8f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd v2.4s, v2.4s, v3.4s
-; CHECK-NEXT:    fadd v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    fadd v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    faddp v0.4s, v0.4s, v0.4s
-; CHECK-NEXT:    faddp s0, v0.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fadd_reduct_reassoc_v8f32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd v2.4s, v2.4s, v3.4s
+; CHECK-SD-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    fadd v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT:    faddp s0, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fadd_reduct_reassoc_v8f32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    fadd v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    movi v2.2s, #128, lsl #24
+; CHECK-GI-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    faddp v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NEXT:    faddp s1, v1.2s
+; CHECK-GI-NEXT:    fadd s0, s0, s2
+; CHECK-GI-NEXT:    fadd s1, s1, s2
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    ret
   %r1 = call fast float @llvm.vector.reduce.fadd.f32.v8f32(float -0.0, <8 x float> %a)
   %r2 = call fast float @llvm.vector.reduce.fadd.f32.v8f32(float -0.0, <8 x float> %b)
   %r = fadd fast float %r1, %r2
@@ -541,12 +842,24 @@ define float @fadd_reduct_reassoc_v8f32(<8 x float> %a, <8 x float> %b) {
 }
 
 define float @fadd_reduct_reassoc_v4f32(<4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: fadd_reduct_reassoc_v4f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    faddp v0.4s, v0.4s, v0.4s
-; CHECK-NEXT:    faddp s0, v0.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fadd_reduct_reassoc_v4f32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT:    faddp s0, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fadd_reduct_reassoc_v4f32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    faddp v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT:    movi v2.2s, #128, lsl #24
+; CHECK-GI-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NEXT:    faddp s1, v1.2s
+; CHECK-GI-NEXT:    fadd s0, s0, s2
+; CHECK-GI-NEXT:    fadd s1, s1, s2
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    ret
   %r1 = call fast float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %a)
   %r2 = call fast float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %b)
   %r = fadd fast float %r1, %r2
@@ -554,15 +867,27 @@ define float @fadd_reduct_reassoc_v4f32(<4 x float> %a, <4 x float> %b) {
 }
 
 define float @fadd_reduct_reassoc_v4f32_init(float %i, <4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: fadd_reduct_reassoc_v4f32_init:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    faddp v1.4s, v1.4s, v1.4s
-; CHECK-NEXT:    faddp v2.4s, v2.4s, v2.4s
-; CHECK-NEXT:    faddp s1, v1.2s
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    faddp s1, v2.2s
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fadd_reduct_reassoc_v4f32_init:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    faddp v1.4s, v1.4s, v1.4s
+; CHECK-SD-NEXT:    faddp v2.4s, v2.4s, v2.4s
+; CHECK-SD-NEXT:    faddp s1, v1.2s
+; CHECK-SD-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NEXT:    faddp s1, v2.2s
+; CHECK-SD-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fadd_reduct_reassoc_v4f32_init:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    faddp v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT:    faddp v2.4s, v2.4s, v2.4s
+; CHECK-GI-NEXT:    movi v3.2s, #128, lsl #24
+; CHECK-GI-NEXT:    faddp s1, v1.2s
+; CHECK-GI-NEXT:    faddp s2, v2.2s
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    fadd s1, s2, s3
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    ret
   %r1 = call fast float @llvm.vector.reduce.fadd.f32.v4f32(float %i, <4 x float> %a)
   %r2 = call fast float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %b)
   %r = fadd fast float %r1, %r2
@@ -570,13 +895,26 @@ define float @fadd_reduct_reassoc_v4f32_init(float %i, <4 x float> %a, <4 x floa
 }
 
 define float @fadd_reduct_reassoc_v4v8f32(<4 x float> %a, <8 x float> %b) {
-; CHECK-LABEL: fadd_reduct_reassoc_v4v8f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    fadd v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    faddp v0.4s, v0.4s, v0.4s
-; CHECK-NEXT:    faddp s0, v0.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fadd_reduct_reassoc_v4v8f32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT:    faddp s0, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fadd_reduct_reassoc_v4v8f32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fadd v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    movi v2.2s, #128, lsl #24
+; CHECK-GI-NEXT:    faddp v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NEXT:    faddp s1, v1.2s
+; CHECK-GI-NEXT:    fadd s0, s0, s2
+; CHECK-GI-NEXT:    fadd s1, s1, s2
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    ret
   %r1 = call fast float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %a)
   %r2 = call fast float @llvm.vector.reduce.fadd.f32.v8f32(float -0.0, <8 x float> %b)
   %r = fadd fast float %r1, %r2
@@ -584,13 +922,26 @@ define float @fadd_reduct_reassoc_v4v8f32(<4 x float> %a, <8 x float> %b) {
 }
 
 define double @fadd_reduct_reassoc_v4f64(<4 x double> %a, <4 x double> %b) {
-; CHECK-LABEL: fadd_reduct_reassoc_v4f64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd v2.2d, v2.2d, v3.2d
-; CHECK-NEXT:    fadd v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    fadd v0.2d, v0.2d, v2.2d
-; CHECK-NEXT:    faddp d0, v0.2d
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fadd_reduct_reassoc_v4f64:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd v2.2d, v2.2d, v3.2d
+; CHECK-SD-NEXT:    fadd v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    fadd v0.2d, v0.2d, v2.2d
+; CHECK-SD-NEXT:    faddp d0, v0.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fadd_reduct_reassoc_v4f64:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fadd v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    fadd v1.2d, v2.2d, v3.2d
+; CHECK-GI-NEXT:    mov x8, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT:    fmov d2, x8
+; CHECK-GI-NEXT:    faddp d0, v0.2d
+; CHECK-GI-NEXT:    faddp d1, v1.2d
+; CHECK-GI-NEXT:    fadd d0, d0, d2
+; CHECK-GI-NEXT:    fadd d1, d1, d2
+; CHECK-GI-NEXT:    fadd d0, d0, d1
+; CHECK-GI-NEXT:    ret
   %r1 = call fast double @llvm.vector.reduce.fadd.f64.v4f64(double -0.0, <4 x double> %a)
   %r2 = call fast double @llvm.vector.reduce.fadd.f64.v4f64(double -0.0, <4 x double> %b)
   %r = fadd fast double %r1, %r2
@@ -598,15 +949,28 @@ define double @fadd_reduct_reassoc_v4f64(<4 x double> %a, <4 x double> %b) {
 }
 
 define float @fadd_reduct_reassoc_v4f32_extrause(<4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: fadd_reduct_reassoc_v4f32_extrause:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    faddp v0.4s, v0.4s, v0.4s
-; CHECK-NEXT:    faddp v1.4s, v1.4s, v1.4s
-; CHECK-NEXT:    faddp s0, v0.2s
-; CHECK-NEXT:    faddp s1, v1.2s
-; CHECK-NEXT:    fadd s1, s0, s1
-; CHECK-NEXT:    fmul s0, s1, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fadd_reduct_reassoc_v4f32_extrause:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT:    faddp v1.4s, v1.4s, v1.4s
+; CHECK-SD-NEXT:    faddp s0, v0.2s
+; CHECK-SD-NEXT:    faddp s1, v1.2s
+; CHECK-SD-NEXT:    fadd s1, s0, s1
+; CHECK-SD-NEXT:    fmul s0, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fadd_reduct_reassoc_v4f32_extrause:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    faddp v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT:    movi v2.2s, #128, lsl #24
+; CHECK-GI-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NEXT:    faddp s1, v1.2s
+; CHECK-GI-NEXT:    fadd s0, s0, s2
+; CHECK-GI-NEXT:    fadd s1, s1, s2
+; CHECK-GI-NEXT:    fadd s1, s0, s1
+; CHECK-GI-NEXT:    fmul s0, s1, s0
+; CHECK-GI-NEXT:    ret
   %r1 = call fast float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %a)
   %r2 = call fast float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %b)
   %r = fadd fast float %r1, %r2