diff --git a/src/core/query.ts b/src/core/query.ts
index 04534373df..8e589de4cc 100644
--- a/src/core/query.ts
+++ b/src/core/query.ts
@@ -363,6 +363,8 @@ export class Query<
         // Silently cancel current fetch if the user wants to cancel refetches
         this.cancel({ silent: true })
       } else if (this.promise) {
+        // make sure that retries that were potentially cancelled due to unmounts can continue
+        this.retryer?.continueRetry()
         // Return current promise if we are already fetching
         return this.promise
       }
diff --git a/src/core/retryer.ts b/src/core/retryer.ts
index 82e4d53792..2d8575b072 100644
--- a/src/core/retryer.ts
+++ b/src/core/retryer.ts
@@ -52,6 +52,7 @@ export function isCancelledError(value: any): value is CancelledError {
 export class Retryer<TData = unknown, TError = unknown> {
   cancel: (options?: CancelOptions) => void
   cancelRetry: () => void
+  continueRetry: () => void
   continue: () => void
   failureCount: number
   isPaused: boolean
@@ -72,6 +73,9 @@ export class Retryer<TData = unknown, TError = unknown> {
     this.cancelRetry = () => {
       cancelRetry = true
     }
+    this.continueRetry = () => {
+      cancelRetry = false
+    }
     this.continue = () => continueFn?.()
     this.failureCount = 0
     this.isPaused = false
diff --git a/src/reactjs/tests/useQuery.test.tsx b/src/reactjs/tests/useQuery.test.tsx
index 94d059f2c8..63f7a4fb24 100644
--- a/src/reactjs/tests/useQuery.test.tsx
+++ b/src/reactjs/tests/useQuery.test.tsx
@@ -2711,6 +2711,111 @@ describe('useQuery', () => {
     consoleMock.mockRestore()
   })
 
+  it('should continue retries when observers unmount and remount while waiting for a retry (#3031)', async () => {
+    const key = queryKey()
+    const consoleMock = mockConsoleError()
+    let count = 0
+
+    function Page() {
+      const result = useQuery(
+        key,
+        async () => {
+          count++
+          await sleep(10)
+          return Promise.reject('some error')
+        },
+        {
+          retry: 2,
+          retryDelay: 100,
+        }
+      )
+
+      return (
+        <div>
+          <div>error: {result.error ?? 'null'}</div>
+          <div>failureCount: {result.failureCount}</div>
+        </div>
+      )
+    }
+
+    function App() {
+      const [show, toggle] = React.useReducer(x => !x, true)
+
+      return (
+        <div>
+          <button onClick={toggle}>{show ? 'hide' : 'show'}</button>
+          {show && <Page />}
+        </div>
+      )
+    }
+
+    const rendered = renderWithClient(queryClient, <App />)
+
+    await waitFor(() => rendered.getByText('failureCount: 1'))
+    rendered.getByRole('button', { name: /hide/i }).click()
+    rendered.getByRole('button', { name: /show/i }).click()
+    await waitFor(() => rendered.getByText('error: some error'))
+
+    expect(count).toBe(3)
+
+    consoleMock.mockRestore()
+  })
+
+  it('should restart when observers unmount and remount while waiting for a retry when query was cancelled in between (#3031)', async () => {
+    const key = queryKey()
+    const consoleMock = mockConsoleError()
+    let count = 0
+
+    function Page() {
+      const result = useQuery(
+        key,
+        async () => {
+          count++
+          await sleep(10)
+          return Promise.reject('some error')
+        },
+        {
+          retry: 2,
+          retryDelay: 100,
+        }
+      )
+
+      return (
+        <div>
+          <div>error: {result.error ?? 'null'}</div>
+          <div>failureCount: {result.failureCount}</div>
+        </div>
+      )
+    }
+
+    function App() {
+      const [show, toggle] = React.useReducer(x => !x, true)
+
+      return (
+        <div>
+          <button onClick={toggle}>{show ? 'hide' : 'show'}</button>
+          <button onClick={() => queryClient.cancelQueries({ queryKey: key })}>
+            cancel
+          </button>
+          {show && <Page />}
+        </div>
+      )
+    }
+
+    const rendered = renderWithClient(queryClient, <App />)
+
+    await waitFor(() => rendered.getByText('failureCount: 1'))
+    rendered.getByRole('button', { name: /hide/i }).click()
+    rendered.getByRole('button', { name: /cancel/i }).click()
+    rendered.getByRole('button', { name: /show/i }).click()
+    await waitFor(() => rendered.getByText('error: some error'))
+
+    // initial fetch (1), which will be cancelled, followed by new mount(2) + 2 retries = 4
+    expect(count).toBe(4)
+
+    consoleMock.mockRestore()
+  })
+
   it('should always fetch if refetchOnMount is set to always', async () => {
     const key = queryKey()
     const states: UseQueryResult<string>[] = []