diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 7ceca4a9ba936..76a9ef96b2b04 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -2705,12 +2705,25 @@ bool isKnownNonZero(const Value *V, const APInt &DemandedElts, unsigned Depth,
             Q.DL.getTypeSizeInBits(I->getType()).getFixedValue())
       return isKnownNonZero(I->getOperand(0), Depth, Q);
     break;
-  case Instruction::Sub:
+  case Instruction::Sub: {
     if (auto *C = dyn_cast<Constant>(I->getOperand(0)))
       if (C->isNullValue() &&
           isKnownNonZero(I->getOperand(1), DemandedElts, Depth, Q))
         return true;
-    break;
+
+    KnownBits XKnown =
+        computeKnownBits(I->getOperand(0), DemandedElts, Depth, Q);
+    if (!XKnown.isUnknown()) {
+      KnownBits YKnown =
+          computeKnownBits(I->getOperand(1), DemandedElts, Depth, Q);
+      // If X != Y then X - Y is non zero.
+      std::optional<bool> ne = KnownBits::ne(XKnown, YKnown);
+      // If we are unable to compute if X != Y, we won't be able to do anything
+      // computing the knownbits of the sub expression so just return here.
+      return ne && *ne;
+    }
+    return false;
+  }
   case Instruction::Or:
     // X | Y != 0 if X != 0 or Y != 0.
     return isKnownNonZero(I->getOperand(0), DemandedElts, Depth, Q) ||
diff --git a/llvm/test/Analysis/ValueTracking/known-non-zero.ll b/llvm/test/Analysis/ValueTracking/known-non-zero.ll
index d48d9366d4744..a0a6f2e61b1fe 100644
--- a/llvm/test/Analysis/ValueTracking/known-non-zero.ll
+++ b/llvm/test/Analysis/ValueTracking/known-non-zero.ll
@@ -442,12 +442,7 @@ define i1 @shl_nonzero_and_shift_out_zeros_fail(i32 %ccnt, i32 %y) {
 
 define i1 @sub_nonzero_ops_ne(i8 %xx, i8 %yy, i8 %z) {
 ; CHECK-LABEL: @sub_nonzero_ops_ne(
-; CHECK-NEXT:    [[X:%.*]] = and i8 [[XX:%.*]], -65
-; CHECK-NEXT:    [[Y:%.*]] = or i8 [[YY:%.*]], 64
-; CHECK-NEXT:    [[S:%.*]] = sub i8 [[X]], [[Y]]
-; CHECK-NEXT:    [[EXP:%.*]] = or i8 [[Z:%.*]], [[S]]
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[EXP]], 0
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 false
 ;
   %x = and i8 %xx, 191
   %y = or i8 %yy, 64