diff --git a/src/pytest_mock/plugin.py b/src/pytest_mock/plugin.py
index fd039cf..5ee170e 100644
--- a/src/pytest_mock/plugin.py
+++ b/src/pytest_mock/plugin.py
@@ -147,6 +147,7 @@ def _start_patch(self, mock_func, *args, **kwargs):
             module, registering the patch to stop it later and returns the
             mock object resulting from the mock call.
             """
+            self._enforce_no_with_context(inspect.stack())
             p = mock_func(*args, **kwargs)
             mocked = p.start()
             self._patches.append(p)
@@ -154,14 +155,9 @@ def _start_patch(self, mock_func, *args, **kwargs):
                 self._mocks.append(mocked)
             return mocked
 
-        def object(self, *args, **kwargs):
-            """API to mock.patch.object"""
-            self._enforce_no_with_context(inspect.stack())
-            return self._start_patch(self.mock_module.patch.object, *args, **kwargs)
-
         def _enforce_no_with_context(self, stack):
             """raises a ValueError if mocker is used in a with context"""
-            caller = stack[1]
+            caller = stack[2]
             frame = caller[0]
             info = inspect.getframeinfo(frame)
             code_context = " ".join(info.code_context).strip()
@@ -172,6 +168,10 @@ def _enforce_no_with_context(self, stack):
                     "https://github.com/pytest-dev/pytest-mock#note-about-usage-as-context-manager"
                 )
 
+        def object(self, *args, **kwargs):
+            """API to mock.patch.object"""
+            return self._start_patch(self.mock_module.patch.object, *args, **kwargs)
+
         def multiple(self, *args, **kwargs):
             """API to mock.patch.multiple"""
             return self._start_patch(self.mock_module.patch.multiple, *args, **kwargs)