diff --git a/changelog/5206.bugfix.rst b/changelog/5206.bugfix.rst
new file mode 100644
index 00000000000..94df9c6b4b5
--- /dev/null
+++ b/changelog/5206.bugfix.rst
@@ -0,0 +1 @@
+Fix ``--nf`` to not forget about known nodeids with partial test selection.
diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py
index 496931e0fcc..34186c8455b 100755
--- a/src/_pytest/cacheprovider.py
+++ b/src/_pytest/cacheprovider.py
@@ -264,8 +264,8 @@ def __init__(self, config):
         self.cached_nodeids = config.cache.get("cache/nodeids", [])
 
     def pytest_collection_modifyitems(self, session, config, items):
+        new_items = OrderedDict()
         if self.active:
-            new_items = OrderedDict()
             other_items = OrderedDict()
             for item in items:
                 if item.nodeid not in self.cached_nodeids:
@@ -276,7 +276,11 @@ def pytest_collection_modifyitems(self, session, config, items):
             items[:] = self._get_increasing_order(
                 new_items.values()
             ) + self._get_increasing_order(other_items.values())
-        self.cached_nodeids = [x.nodeid for x in items if isinstance(x, pytest.Item)]
+        else:
+            for item in items:
+                if item.nodeid not in self.cached_nodeids:
+                    new_items[item.nodeid] = item
+        self.cached_nodeids.extend(new_items)
 
     def _get_increasing_order(self, items):
         return sorted(items, key=lambda item: item.fspath.mtime(), reverse=True)
diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py
index e2fd5a4ca56..81f605e23ac 100644
--- a/testing/test_cacheprovider.py
+++ b/testing/test_cacheprovider.py
@@ -982,8 +982,13 @@ def test_1(num): assert num
         )
         testdir.tmpdir.join("test_1/test_1.py").setmtime(1)
 
-        result = testdir.runpytest("-v", "--nf")
+        # Running only a subset does not forget about existing ones.
+        result = testdir.runpytest("-v", "--nf", "test_2/test_2.py")
+        result.stdout.fnmatch_lines(
+            ["*test_2/test_2.py::test_1[1*", "*test_2/test_2.py::test_1[2*"]
+        )
 
+        result = testdir.runpytest("-v", "--nf")
         result.stdout.fnmatch_lines(
             [
                 "*test_1/test_1.py::test_1[3*",