diff --git a/crates/ruff_linter/resources/test/fixtures/refurb/FURB101.py b/crates/ruff_linter/resources/test/fixtures/refurb/FURB101.py
index 2f0276a09a826..31b1ccd341e24 100644
--- a/crates/ruff_linter/resources/test/fixtures/refurb/FURB101.py
+++ b/crates/ruff_linter/resources/test/fixtures/refurb/FURB101.py
@@ -28,10 +28,6 @@ def bar(x):
 with open("file.txt", errors="ignore") as f:
     x = f.read()
 
-# FURB101
-with open("file.txt", errors="ignore", mode="rb") as f:
-    x = f.read()
-
 # FURB101
 with open("file.txt", mode="r") as f:  # noqa: FURB120
     x = f.read()
@@ -60,6 +56,11 @@ def bar(x):
 
 # Non-errors.
 
+# Path.read_bytes does not support any kwargs
+with open("file.txt", errors="ignore", mode="rb") as f:
+    x = f.read()
+
+
 f2 = open("file2.txt")
 with open("file.txt") as f:
     x = f2.read()
diff --git a/crates/ruff_linter/src/rules/refurb/rules/read_whole_file.rs b/crates/ruff_linter/src/rules/refurb/rules/read_whole_file.rs
index 518d701c79038..009bdf1c54d4f 100644
--- a/crates/ruff_linter/src/rules/refurb/rules/read_whole_file.rs
+++ b/crates/ruff_linter/src/rules/refurb/rules/read_whole_file.rs
@@ -166,6 +166,10 @@ fn find_file_open<'a>(
     // keyword mode should override that.
     let mode = kw_mode.unwrap_or(pos_mode);
 
+    if matches!(mode, ReadMode::Bytes) && !keywords.is_empty() {
+        return None;
+    }
+
     // Now we need to find what is this variable bound to...
     let scope = semantic.current_scope();
     let bindings: Vec<BindingId> = scope.get_all(var.id.as_str()).collect();
diff --git a/crates/ruff_linter/src/rules/refurb/snapshots/ruff_linter__rules__refurb__tests__FURB101_FURB101.py.snap b/crates/ruff_linter/src/rules/refurb/snapshots/ruff_linter__rules__refurb__tests__FURB101_FURB101.py.snap
index 6b3595549841c..b3db05568d87f 100644
--- a/crates/ruff_linter/src/rules/refurb/snapshots/ruff_linter__rules__refurb__tests__FURB101_FURB101.py.snap
+++ b/crates/ruff_linter/src/rules/refurb/snapshots/ruff_linter__rules__refurb__tests__FURB101_FURB101.py.snap
@@ -41,56 +41,46 @@ FURB101.py:28:6: FURB101 `open` and `read` should be replaced by `Path("file.txt
 29 |     x = f.read()
    |
 
-FURB101.py:32:6: FURB101 `open` and `read` should be replaced by `Path("file.txt").read_bytes(errors="ignore")`
+FURB101.py:32:6: FURB101 `open` and `read` should be replaced by `Path("file.txt").read_text()`
    |
 31 | # FURB101
-32 | with open("file.txt", errors="ignore", mode="rb") as f:
-   |      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ FURB101
+32 | with open("file.txt", mode="r") as f:  # noqa: FURB120
+   |      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ FURB101
 33 |     x = f.read()
    |
 
-FURB101.py:36:6: FURB101 `open` and `read` should be replaced by `Path("file.txt").read_text()`
+FURB101.py:36:6: FURB101 `open` and `read` should be replaced by `Path(foo()).read_bytes()`
    |
 35 | # FURB101
-36 | with open("file.txt", mode="r") as f:  # noqa: FURB120
-   |      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ FURB101
-37 |     x = f.read()
-   |
-
-FURB101.py:40:6: FURB101 `open` and `read` should be replaced by `Path(foo()).read_bytes()`
-   |
-39 | # FURB101
-40 | with open(foo(), "rb") as f:
+36 | with open(foo(), "rb") as f:
    |      ^^^^^^^^^^^^^^^^^^^^^^ FURB101
-41 |     # The body of `with` is non-trivial, but the recommendation holds.
-42 |     bar("pre")
+37 |     # The body of `with` is non-trivial, but the recommendation holds.
+38 |     bar("pre")
    |
 
-FURB101.py:48:6: FURB101 `open` and `read` should be replaced by `Path("a.txt").read_text()`
+FURB101.py:44:6: FURB101 `open` and `read` should be replaced by `Path("a.txt").read_text()`
    |
-47 | # FURB101
-48 | with open("a.txt") as a, open("b.txt", "rb") as b:
+43 | # FURB101
+44 | with open("a.txt") as a, open("b.txt", "rb") as b:
    |      ^^^^^^^^^^^^^^^^^^ FURB101
-49 |     x = a.read()
-50 |     y = b.read()
+45 |     x = a.read()
+46 |     y = b.read()
    |
 
-FURB101.py:48:26: FURB101 `open` and `read` should be replaced by `Path("b.txt").read_bytes()`
+FURB101.py:44:26: FURB101 `open` and `read` should be replaced by `Path("b.txt").read_bytes()`
    |
-47 | # FURB101
-48 | with open("a.txt") as a, open("b.txt", "rb") as b:
+43 | # FURB101
+44 | with open("a.txt") as a, open("b.txt", "rb") as b:
    |                          ^^^^^^^^^^^^^^^^^^^^^^^^ FURB101
-49 |     x = a.read()
-50 |     y = b.read()
+45 |     x = a.read()
+46 |     y = b.read()
    |
 
-FURB101.py:53:18: FURB101 `open` and `read` should be replaced by `Path("file.txt").read_text()`
+FURB101.py:49:18: FURB101 `open` and `read` should be replaced by `Path("file.txt").read_text()`
    |
-52 | # FURB101
-53 | with foo() as a, open("file.txt") as b, foo() as c:
+48 | # FURB101
+49 | with foo() as a, open("file.txt") as b, foo() as c:
    |                  ^^^^^^^^^^^^^^^^^^^^^ FURB101
-54 |     # We have other things in here, multiple with items, but
-55 |     # the user reads the whole file and that bit they can replace.
+50 |     # We have other things in here, multiple with items, but
+51 |     # the user reads the whole file and that bit they can replace.
    |
-
-