diff --git a/pandas/core/frame.py b/pandas/core/frame.py
index fcd822988de20..b22f93d488de2 100644
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -198,6 +198,7 @@
     nargsort,
 )
 
+from pandas.errors import InvalidIndexError
 from pandas.io.common import get_handle
 from pandas.io.formats import (
     console,
@@ -3872,7 +3873,7 @@ def _set_value(
             #  or ValueError
             series._mgr.setitem_inplace(loc, value)
 
-        except (KeyError, TypeError, ValueError):
+        except (KeyError, TypeError, ValueError, InvalidIndexError):
             # set using a non-recursive method & reset the cache
             if takeable:
                 self.iloc[index, col] = value