[GlobalIsel] Combine ADDE

[tschuett]

Clang has them as builtins (__builtin_addc). The middle end has no intrinsics for them. They are used in legalization operations.

AArch64: ADCS Add with carry and set flags

On Neoverse V2, they run at half the throughput of basic arithmetic and have a limited set of pipelines.

[llvmbot]

@llvm/pr-subscribers-llvm-globalisel
@llvm/pr-subscribers-backend-amdgpu

@llvm/pr-subscribers-backend-aarch64

Author: Thorsten Schütt (tschuett)

Changes

Clang has them as builtins (__builtin_addc). The middle end has no intrinsics for them. They are used in legalization operations.

AArch64: ADCS Add with carry and set flags

On Neoverse V2, they run at half the throughput of basic arithmetic and have a limited set of pipelines.

---

Patch is 192.73 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/82413.diff

8 Files Affected:

• (modified) llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h (+8-2)
• (modified) llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h (+17)
• (modified) llvm/include/llvm/Target/GlobalISel/Combine.td (+7-1)
• (modified) llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp (+209)
• (added) llvm/test/CodeGen/AArch64/GlobalISel/combine-adde.mir (+300)
• (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/addo.ll (+36-12)
• (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll (+1043-702)
• (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll (+715-260)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 23728636498ba0..abc2ebdfd878c2 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -810,12 +810,15 @@ class CombinerHelper {
   /// Combine selects.
   bool matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo);
 
-  /// Combine ands,
+  /// Combine ands.
   bool matchAnd(MachineInstr &MI, BuildFnTy &MatchInfo);
 
-  /// Combine ors,
+  /// Combine ors.
   bool matchOr(MachineInstr &MI, BuildFnTy &MatchInfo);
 
+  /// Combine addes.
+  bool matchAddCarryInOut(MachineInstr &MI, BuildFnTy &MatchInfo);
+
 private:
   /// Checks for legality of an indexed variant of \p LdSt.
   bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;
@@ -919,6 +922,7 @@ class CombinerHelper {
   bool isZeroOrZeroSplat(Register Src, bool AllowUndefs);
   bool isConstantSplatVector(Register Src, int64_t SplatValue,
                              bool AllowUndefs);
+  bool isConstantOrConstantVectorI(Register Src);
 
   std::optional<APInt> getConstantOrConstantSplatVector(Register Src);
 
@@ -930,6 +934,8 @@ class CombinerHelper {
 
   // Simplify (cmp cc0 x, y) (&& or ||) (cmp cc1 x, y) -> cmp cc2 x, y.
   bool tryFoldLogicOfFCmps(GLogicalBinOp *Logic, BuildFnTy &MatchInfo);
+
+  bool isZExtOrTruncLegal(LLT ToTy, LLT FromTy) const;
 };
 } // namespace llvm
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index f5a6528d10a973..e46d2d1aac0e86 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -359,6 +359,8 @@ class GBinOpCarryOut : public GenericMachineInstr {
   Register getCarryOutReg() const { return getReg(1); }
   MachineOperand &getLHS() { return getOperand(2); }
   MachineOperand &getRHS() { return getOperand(3); }
+  Register getLHSReg() { return getOperand(2).getReg(); }
+  Register getRHSReg() { return getOperand(3).getReg(); }
 
   static bool classof(const MachineInstr *MI) {
     switch (MI->getOpcode()) {
@@ -448,6 +450,21 @@ class GAddSubCarryInOut : public GAddSubCarryOut {
   }
 };
 
+/// Represents overflowing add operations that also consume a carry-in.
+/// G_UADDE, G_SADDE
+class GAddCarryInOut : public GAddSubCarryInOut {
+public:
+  static bool classof(const MachineInstr *MI) {
+    switch (MI->getOpcode()) {
+    case TargetOpcode::G_UADDE:
+    case TargetOpcode::G_SADDE:
+      return true;
+    default:
+      return false;
+    }
+  }
+};
+
 /// Represents a call to an intrinsic.
 class GIntrinsic final : public GenericMachineInstr {
 public:
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 7eadb718f16415..3a82bc14885beb 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -1253,6 +1253,12 @@ def match_ors : GICombineRule<
         [{ return Helper.matchOr(*${root}, ${matchinfo}); }]),
   (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
 
+def match_addes : GICombineRule<
+  (defs root:$root, build_fn_matchinfo:$matchinfo),
+  (match (wip_match_opcode G_SADDE, G_UADDE):$root,
+        [{ return Helper.matchAddCarryInOut(*${root}, ${matchinfo}); }]),
+  (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
+
 // Combines concat operations
 def concat_matchinfo : GIDefMatchData<"SmallVector<Register>">;
 def combine_concat_vector : GICombineRule<
@@ -1335,7 +1341,7 @@ def all_combines : GICombineGroup<[trivial_combines, insert_vec_elt_combines,
     and_or_disjoint_mask, fma_combines, fold_binop_into_select,
     sub_add_reg, select_to_minmax, redundant_binop_in_equality,
     fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors, 
-    combine_concat_vector]>;
+    combine_concat_vector, match_addes]>;
 
 // A combine group used to for prelegalizer combiners at -O0. The combines in
 // this group have been selected based on experiments to balance code size and
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 779ec49f4d13a7..2cfc7387ed976d 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -6342,6 +6342,23 @@ CombinerHelper::getConstantOrConstantSplatVector(Register Src) {
   return Value;
 }
 
+bool CombinerHelper::isConstantOrConstantVectorI(Register Src) {
+  auto IConstant = getIConstantVRegValWithLookThrough(Src, MRI);
+  if (IConstant)
+    return true;
+  GBuildVector *BuildVector = getOpcodeDef<GBuildVector>(Src, MRI);
+  if (!BuildVector)
+    return false;
+  unsigned NumSources = BuildVector->getNumSources();
+  for (unsigned I = 0; I < NumSources; ++I) {
+    std::optional<ValueAndVReg> IConstant =
+        getIConstantVRegValWithLookThrough(BuildVector->getSourceReg(I), MRI);
+    if (!IConstant)
+      return false;
+  }
+  return true; // FIXME: G_SPLAT_VECTOR
+}
+
 // TODO: use knownbits to determine zeros
 bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select,
                                               BuildFnTy &MatchInfo) {
@@ -6906,3 +6923,195 @@ bool CombinerHelper::matchOr(MachineInstr &MI, BuildFnTy &MatchInfo) {
 
   return false;
 }
+
+bool CombinerHelper::isZExtOrTruncLegal(LLT ToTy, LLT FromTy) const {
+  // Copy.
+  if (ToTy == FromTy)
+    return true;
+
+  if (isLegalOrBeforeLegalizer({TargetOpcode::G_ZEXT, {ToTy, FromTy}}))
+    return true;
+
+  if (isLegalOrBeforeLegalizer({TargetOpcode::G_TRUNC, {ToTy, FromTy}}))
+    return true;
+
+  return false;
+}
+
+bool CombinerHelper::matchAddCarryInOut(MachineInstr &MI,
+                                        BuildFnTy &MatchInfo) {
+  GAddCarryInOut *Add = cast<GAddCarryInOut>(&MI);
+
+  // adde has no flags.
+  Register Dst = Add->getDstReg();
+  Register Carry = Add->getCarryOutReg();
+  Register CarryIn = Add->getCarryInReg();
+  Register LHS = Add->getLHSReg();
+  Register RHS = Add->getRHSReg();
+  bool IsSigned = Add->isSigned();
+  LLT DstTy = MRI.getType(Dst);
+  LLT CarryTy = MRI.getType(Carry);
+  LLT OperandTy = MRI.getType(LHS);
+  LLT CarryInTy = MRI.getType(CarryIn);
+
+  // FIXME: handle undef
+
+  // fold sadde, if the carry is dead -> add(add(LHS, RHS),
+  // zextOrTrunc(CarryIn)), undef.
+  if (MRI.use_nodbg_empty(Carry) && IsSigned && MRI.hasOneNonDBGUse(Dst) &&
+      isLegalOrBeforeLegalizer({TargetOpcode::G_ADD, {DstTy}}) &&
+      isZExtOrTruncLegal(DstTy, CarryInTy)) {
+    MatchInfo = [=](MachineIRBuilder &B) {
+      auto A = B.buildAdd(DstTy, LHS, RHS);
+      Register AReg = A.getReg(0);
+      auto ZextCarryIn = B.buildZExtOrTrunc(DstTy, CarryIn);
+      Register ZextCarryInReg = ZextCarryIn.getReg(0);
+      B.buildAdd(Dst, AReg, ZextCarryInReg);
+      B.buildUndef(Carry);
+    };
+    return true;
+  }
+
+  // We want do fold the [u|s]adde.
+  if (!MRI.hasOneNonDBGUse(Dst) || !MRI.hasOneNonDBGUse(Carry))
+    return false;
+
+  // The parameters of the adde must be integer-like.
+  std::optional<APInt> MaybeLHS = getConstantOrConstantSplatVector(LHS);
+  std::optional<APInt> MaybeRHS = getConstantOrConstantSplatVector(RHS);
+  std::optional<APInt> MaybeCarryIn = getConstantOrConstantSplatVector(CarryIn);
+
+  // fold adde(c, c, c) -> c, carry
+  if (MaybeLHS && MaybeRHS && MaybeCarryIn &&
+      isConstantLegalOrBeforeLegalizer(DstTy) &&
+      isConstantLegalOrBeforeLegalizer(CarryTy)) {
+    // They must all have the same bitwidth. Otherwise APInt might
+    // assert. Prelegalization, they may have widely different bitwidths.
+    unsigned BitWidth =
+        std::max(std::max(MaybeLHS->getBitWidth(), MaybeRHS->getBitWidth()),
+                 MaybeCarryIn->getBitWidth());
+    if (IsSigned) {
+      APInt LHS = MaybeLHS->sext(BitWidth);
+      APInt RHS = MaybeRHS->sext(BitWidth);
+      APInt CarryIn = MaybeCarryIn->zext(BitWidth);
+      bool FirstOverflowed = false;
+      bool SecondOverflowed = false;
+      APInt Result =
+          LHS.sadd_ov(RHS, FirstOverflowed).sadd_ov(CarryIn, SecondOverflowed);
+      MatchInfo = [=](MachineIRBuilder &B) {
+        B.buildConstant(Dst, Result);
+        B.buildConstant(Carry, FirstOverflowed | SecondOverflowed);
+      };
+      return true;
+    } else if (!IsSigned) {
+      APInt LHS = MaybeLHS->zext(BitWidth);
+      APInt RHS = MaybeRHS->zext(BitWidth);
+      APInt CarryIn = MaybeCarryIn->zext(BitWidth);
+      bool FirstOverflowed = false;
+      bool SecondOverflowed = false;
+      APInt Result =
+          LHS.uadd_ov(RHS, FirstOverflowed).uadd_ov(CarryIn, SecondOverflowed);
+      MatchInfo = [=](MachineIRBuilder &B) {
+        B.buildConstant(Dst, Result);
+        B.buildConstant(Carry, FirstOverflowed | SecondOverflowed);
+      };
+      return true;
+    }
+  }
+
+  // canonicalize constant to RHS.
+  if (isConstantOrConstantVectorI(LHS) && !isConstantOrConstantVectorI(RHS)) {
+    if (IsSigned) {
+      MatchInfo = [=](MachineIRBuilder &B) {
+        B.buildSAdde(Dst, Carry, RHS, LHS, CarryIn);
+      };
+      return true;
+    } else {
+      MatchInfo = [=](MachineIRBuilder &B) {
+        B.buildUAdde(Dst, Carry, RHS, LHS, CarryIn);
+      };
+      return true;
+    }
+  }
+
+  // fold adde(LHS, RHS, 0) -> addo(LHS, RHS)
+  if (MaybeCarryIn && *MaybeCarryIn == 0) {
+    if (IsSigned && isLegalOrBeforeLegalizer(
+                        {TargetOpcode::G_SADDO, {DstTy, CarryTy, OperandTy}})) {
+      MatchInfo = [=](MachineIRBuilder &B) {
+        B.buildSAddo(Dst, Carry, LHS, RHS);
+      };
+      return true;
+    } else if (!IsSigned &&
+               isLegalOrBeforeLegalizer(
+                   {TargetOpcode::G_UADDO, {DstTy, CarryTy, OperandTy}}))
+      MatchInfo = [=](MachineIRBuilder &B) {
+        B.buildUAddo(Dst, Carry, LHS, RHS);
+      };
+    return true;
+  }
+
+  // fold adde(LHS, 0, Carry) -> addo(LHS, Carry)
+  if (MaybeRHS && *MaybeRHS == 0) {
+    if (IsSigned &&
+        isLegalOrBeforeLegalizer(
+            {TargetOpcode::G_SADDO, {DstTy, CarryTy, OperandTy}}) &&
+        isZExtOrTruncLegal(OperandTy, CarryInTy)) {
+      MatchInfo = [=](MachineIRBuilder &B) {
+        auto ZextCarryIn = B.buildZExtOrTrunc(OperandTy, CarryIn);
+        Register ZextCarryInReg = ZextCarryIn.getReg(0);
+        B.buildSAddo(Dst, Carry, LHS, ZextCarryInReg);
+      };
+      return true;
+    } else if (!IsSigned &&
+               isLegalOrBeforeLegalizer(
+                   {TargetOpcode::G_UADDO, {DstTy, CarryTy, OperandTy}}) &&
+               isZExtOrTruncLegal(OperandTy, CarryInTy)) {
+      MatchInfo = [=](MachineIRBuilder &B) {
+        auto ZextCarryIn = B.buildZExtOrTrunc(OperandTy, CarryIn);
+        Register ZextCarryInReg = ZextCarryIn.getReg(0);
+        B.buildUAddo(Dst, Carry, LHS, ZextCarryInReg);
+      };
+      return true;
+    }
+  }
+
+  // We lower to 2*addo + 1*or.
+  if (IsSigned &&
+      isLegalOrBeforeLegalizer(
+          {TargetOpcode::G_SADDO, {DstTy, CarryTy, OperandTy}}) &&
+      isLegalOrBeforeLegalizer({TargetOpcode::G_OR, {DstTy}}) &&
+      isZExtOrTruncLegal(OperandTy, CarryInTy)) {
+    MatchInfo = [=](MachineIRBuilder &B) {
+      auto First = B.buildSAddo(DstTy, CarryTy, LHS, RHS);
+      Register FirstResult = First.getReg(0);
+      Register FirstCarry = First.getReg(1);
+      auto ZextCarryIn = B.buildZExtOrTrunc(OperandTy, CarryIn);
+      auto Second = B.buildSAddo(DstTy, CarryTy, FirstResult, ZextCarryIn);
+      Register Result = Second.getReg(0);
+      Register SecondCarry = Second.getReg(1);
+      B.buildCopy(Dst, Result);
+      B.buildOr(Carry, FirstCarry, SecondCarry);
+    };
+    return true;
+  } else if (!IsSigned &&
+             isLegalOrBeforeLegalizer(
+                 {TargetOpcode::G_UADDO, {DstTy, CarryTy, OperandTy}}) &&
+             isLegalOrBeforeLegalizer({TargetOpcode::G_OR, {DstTy}}) &&
+             isZExtOrTruncLegal(OperandTy, CarryInTy)) {
+    MatchInfo = [=](MachineIRBuilder &B) {
+      auto First = B.buildUAddo(DstTy, CarryTy, LHS, RHS);
+      Register FirstResult = First.getReg(0);
+      Register FirstCarry = First.getReg(1);
+      auto ZextCarryIn = B.buildZExtOrTrunc(OperandTy, CarryIn);
+      auto Second = B.buildUAddo(DstTy, CarryTy, FirstResult, ZextCarryIn);
+      Register Result = Second.getReg(0);
+      Register SecondCarry = Second.getReg(1);
+      B.buildCopy(Dst, Result);
+      B.buildOr(Carry, FirstCarry, SecondCarry);
+    };
+    return true;
+  }
+
+  return false;
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-adde.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-adde.mir
new file mode 100644
index 00000000000000..61c7f56f4b2605
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-adde.mir
@@ -0,0 +1,300 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -debugify-and-strip-all-safe -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs -mtriple aarch64-unknown-unknown %s -o - | FileCheck %s
+---
+# add, _ = sadde(_, _, In)
+name:            carryout_unused
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: carryout_unused
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x3
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x4
+    ; CHECK-NEXT: %carry_in:_(s1) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY]], [[COPY]]
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT %carry_in(s1)
+    ; CHECK-NEXT: %add:_(s64) = G_ADD [[ADD]], [[ZEXT]]
+    ; CHECK-NEXT: $x0 = COPY %add(s64)
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = COPY $x2
+    %3:_(s64) = COPY $x3
+    %4:_(s64) = COPY $x4
+    %lhs:_(s64) = COPY %3
+    %rhs:_(s64) = COPY %3
+    %carry_in:_(s1) = G_TRUNC %4
+    %add:_(s64), %carry_out:_(s1) = G_SADDE %lhs, %rhs, %carry_in
+    $x0 = COPY %add
+...
+---
+# add, _ = uadde(_, _, In)
+name:            carryout_unused_unsigned
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: carryout_unused_unsigned
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x3
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x4
+    ; CHECK-NEXT: %carry_in:_(s1) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-NEXT: %add:_(s64), %carry_out:_(s1) = G_UADDE [[COPY]], [[COPY]], %carry_in
+    ; CHECK-NEXT: $x0 = COPY %add(s64)
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = COPY $x2
+    %3:_(s64) = COPY $x3
+    %4:_(s64) = COPY $x4
+    %lhs:_(s64) = COPY %3
+    %rhs:_(s64) = COPY %3
+    %carry_in:_(s1) = G_TRUNC %4
+    %add:_(s64), %carry_out:_(s1) = G_UADDE %lhs, %rhs, %carry_in
+    $x0 = COPY %add
+...
+---
+# add, multi_c = sadde(L, R, In)
+name:            multi_use_unsigned
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: multi_use_unsigned
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x3
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x4
+    ; CHECK-NEXT: %carry_in:_(s1) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-NEXT: %add:_(s64), %carry_out:_(s1) = G_UADDE [[COPY]], [[COPY]], %carry_in
+    ; CHECK-NEXT: %carry_out_ext:_(s64) = G_ANYEXT %carry_out(s1)
+    ; CHECK-NEXT: %carry_out_ext2:_(s64) = G_ANYEXT %carry_out(s1)
+    ; CHECK-NEXT: $x0 = COPY %add(s64)
+    ; CHECK-NEXT: $x1 = COPY %carry_out_ext(s64)
+    ; CHECK-NEXT: $x2 = COPY %carry_out_ext2(s64)
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = COPY $x2
+    %3:_(s64) = COPY $x3
+    %4:_(s64) = COPY $x4
+    %lhs:_(s64) = COPY %3
+    %rhs:_(s64) = COPY %3
+    %carry_in:_(s1) = G_TRUNC %4
+    %add:_(s64), %carry_out:_(s1) = G_UADDE %lhs, %rhs, %carry_in
+    %carry_out_ext:_(s64) = G_ANYEXT %carry_out
+    %carry_out_ext2:_(s64) = G_ANYEXT %carry_out
+    $x0 = COPY %add
+    $x1 = COPY %carry_out_ext
+    $x2 = COPY %carry_out_ext2
+...
+---
+# add, c = sadde(L, R, In)
+name:            constant_fold_signed
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: constant_fold_signed
+    ; CHECK: %add:_(s64) = G_CONSTANT i64 29
+    ; CHECK-NEXT: %carry_out_ext:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: $x0 = COPY %add(s64)
+    ; CHECK-NEXT: $x1 = COPY %carry_out_ext(s64)
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = COPY $x2
+    %3:_(s64) = COPY $x3
+    %4:_(s64) = G_CONSTANT i64 1
+    %lhs:_(s64) = G_CONSTANT i64 11
+    %rhs:_(s64) = G_CONSTANT i64 17
+    %carry_in:_(s1) = G_CONSTANT i1 1
+    %add:_(s64), %carry_out:_(s1) = G_SADDE %lhs, %rhs, %carry_in
+    %carry_out_ext:_(s64) = G_ANYEXT %carry_out
+    $x0 = COPY %add
+    $x1 = COPY %carry_out_ext
+...
+---
+# add, c = uadde(L, R, In)
+name:            constant_fold_unsigned
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: constant_fold_unsigned
+    ; CHECK: %add:_(s64) = G_CONSTANT i64 27
+    ; CHECK-NEXT: %carry_out_ext:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: $x0 = COPY %add(s64)
+    ; CHECK-NEXT: $x1 = COPY %carry_out_ext(s64)
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = COPY $x2
+    %3:_(s64) = COPY $x3
+    %4:_(s64) = G_CONSTANT i64 1
+    %lhs:_(s64) = G_CONSTANT i64 19
+    %rhs:_(s64) = G_CONSTANT i64 7
+    %carry_in:_(s1) = G_CONSTANT i1 1
+    %add:_(s64), %carry_out:_(s1) = G_UADDE %lhs, %rhs, %carry_in
+    %carry_out_ext:_(s64) = G_ANYEXT %carry_out
+    $x0 = COPY %add
+    $x1 = COPY %carry_out_ext
+...
+---
+# add, c = uadde(L, R, In)
+name:            canonicalize_to_rhs_plus_lower
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: canonicalize_to_rhs_plus_lower
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x3
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x4
+    ; CHECK-NEXT: %lhs:_(s64) = G_CONSTANT i64 19
+    ; CHECK-NEXT: %carry_in:_(s1) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-NEXT: [[UADDO:%[0-9]+]]:_(s64), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[COPY]], %lhs
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT %carry_in(s1)
+    ; CHECK-NEXT: [[UADDO2:%[0-9]+]]:_(s64), [[UADDO3:%[0-9]+]]:_(s1) = G_UADDO [[UADDO]], [[ZEXT]]
+    ; CHECK-NEXT: %carry_out:_(s1) = G_OR [[UADDO1]], [[UADDO3]]
+    ; CHECK-NEXT: %carry_out_ext:_(s64) = G_ANYEXT %carry_out(s1)
+    ; CHECK-NEXT: $x0 = COPY [[UADDO2]](s64)
+    ; CHECK-NEXT: $x1 = COPY %carry_out_ext(s64)
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = COPY $x2
+    %3:_(s64) = COPY $x3
+    %4:_(s64) = COPY $x4
+    %lhs:_(s64) = G_CONSTANT i64 19
+    %rhs:_(s64) = COPY %3
+    %carry_in:_(s1) = G_TRUNC %4
+    %add:_(s64), %carry_out:_(s1) = G_UADDE %lhs, %rhs, %carry_in
+    %carry_out_ext:_(s64) = G_ANYEXT %carry_out
+    $x0 = COPY %add
+    $x1 = COPY %carry_out_ext
+...
+---
+# add, c = sadde(L, R, 0)
+name:            fold_to_addo_l_r
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: fold_to_addo_l_r
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x3
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x4
+    ; CHECK-NEXT: %add:_(s64), %carry_out:_(s1) = G_SADDO [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: %carry_out_ext:_(s64) = G_ANYEXT %carry_out(s1)
+    ; CHECK-NEXT: $x0 = COPY %add(s64)
+    ; CHECK-NEXT: $x1 = COPY %carry_out_ext(s64)
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = COPY $x2
+    %3:_(s64) = COPY $x3
+    %4:_(s64) = COPY $x4
+    %lhs:_(s64) = COPY %3
+    %rhs:_(s64) = COPY %4
+    %carry_in:_(s1) = G_CONSTANT i1 0
+    %add:_(s64), %carry_out:_(s1) = G_SADDE %lhs, %rhs, %carry_in
+    %carry_out_ext:_(s64) = G_ANYEXT %carry_out
+    $x0 = COPY %add
+    $x1 = COPY %carry_out_ext
+...
+---
+# add, c = sadde(L, 0, CarryIn)
+name:            fold_to_addo_l_carryin
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: fold_to_addo_l_carryin
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x3
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x4
+    ; CHECK-NEXT: %carry_in:_(s1) = G_TRUNC [[COPY1]](s64)
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT %carry_in(s1)
+    ; CHECK-NEXT: %add:_(s64), %carry_out:_(s1) = G_SADDO [[COPY]], [[ZEXT]]
+    ; CHECK-NEXT: %carry_out_ext:_(s64) = G_ANYEXT %carry_out(s1)
+    ; CHECK-NEXT: $x0 = COPY %add(s64)
+    ; CHECK-NEXT: $x1 = COPY %carry_out_ext(s64)
+    %0:_(s64) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = COPY $x2
+    %3:_(s64) = COPY $x3
+    %4:_(s64) = COPY $x4
+    %lhs:_(s64) = COPY %3
+    %rhs:_(s64) ...
[truncated]