diff --git a/src/librustc_mir/borrow_check/mod.rs b/src/librustc_mir/borrow_check/mod.rs
index afdc9f5c02ae1..bb089cb4952d5 100644
--- a/src/librustc_mir/borrow_check/mod.rs
+++ b/src/librustc_mir/borrow_check/mod.rs
@@ -1070,7 +1070,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 
                 (Reservation(WriteKind::MutableBorrow(bk)), BorrowKind::Shallow)
                 | (Reservation(WriteKind::MutableBorrow(bk)), BorrowKind::Shared) if {
-                    tcx.migrate_borrowck()
+                    tcx.migrate_borrowck() && this.borrow_set.location_map.get(&location).is_some()
                 } => {
                     let bi = this.borrow_set.location_map[&location];
                     debug!(
diff --git a/src/test/ui/issues/issue-61623.rs b/src/test/ui/issues/issue-61623.rs
new file mode 100644
index 0000000000000..e5b8747bd8047
--- /dev/null
+++ b/src/test/ui/issues/issue-61623.rs
@@ -0,0 +1,11 @@
+fn f1<'a>(_: &'a mut ()) {}
+
+fn f2<P>(_: P, _: ()) {}
+
+fn f3<'a>(x: &'a ((), &'a mut ())) {
+    f2(|| x.0, f1(x.1))
+//~^ ERROR cannot borrow `*x.1` as mutable, as it is behind a `&` reference
+//~| ERROR cannot borrow `*x.1` as mutable because it is also borrowed as immutable
+}
+
+fn main() {}
diff --git a/src/test/ui/issues/issue-61623.stderr b/src/test/ui/issues/issue-61623.stderr
new file mode 100644
index 0000000000000..883a1c441d6bb
--- /dev/null
+++ b/src/test/ui/issues/issue-61623.stderr
@@ -0,0 +1,22 @@
+error[E0596]: cannot borrow `*x.1` as mutable, as it is behind a `&` reference
+  --> $DIR/issue-61623.rs:6:19
+   |
+LL | fn f3<'a>(x: &'a ((), &'a mut ())) {
+   |              -------------------- help: consider changing this to be a mutable reference: `&'a mut ((), &'a mut ())`
+LL |     f2(|| x.0, f1(x.1))
+   |                   ^^^ `x` is a `&` reference, so the data it refers to cannot be borrowed as mutable
+
+error[E0502]: cannot borrow `*x.1` as mutable because it is also borrowed as immutable
+  --> $DIR/issue-61623.rs:6:19
+   |
+LL |     f2(|| x.0, f1(x.1))
+   |     -- -- -       ^^^ mutable borrow occurs here
+   |     |  |  |
+   |     |  |  first borrow occurs due to use of `x` in closure
+   |     |  immutable borrow occurs here
+   |     immutable borrow later used by call
+
+error: aborting due to 2 previous errors
+
+Some errors have detailed explanations: E0502, E0596.
+For more information about an error, try `rustc --explain E0502`.