Permalink
Browse files

Added dynamic patcher that patches files loaded after setup

- added modules_to_reload instead of special_names and use it for tempfile
- removed tempfile patch
- see #248
  • Loading branch information...
mrbean-bremen committed Aug 4, 2017
1 parent adef9d9 commit 639c66ce7c5f1765398691b80db70790d090a765
Showing with 178 additions and 79 deletions.
  1. +1 −1 .travis.yml
  2. +1 −2 CHANGES.md
  3. +2 −0 all_tests.py
  4. +75 −0 dynamic_patch_test.py
  5. +0 −16 fake_filesystem_unittest_test.py
  6. +98 −60 pyfakefs/fake_filesystem_unittest.py
  7. +1 −0 requirements.txt
View
@@ -24,7 +24,7 @@ python:
- "3.6"
- "3.7-dev"
- "pypy"
- "pypy3"
# - "pypy3"
- "pypy-5.3.1"
install:
View
@@ -4,8 +4,7 @@ The release versions are PyPi releases.
## Version 3.3 (as yet unreleased)
#### New Features
* Added possibility to patch modules that import file system modules under another name,
e.g. `import os as '_os` ([#231](../../issues/231)).
* Added possibility to reload modules to ensure correct patching ([#248](../../issues/248)).
* Added support for `dir_fd` argument in several `os` functions ([#206](../../issues/206)).
* Added support for open file descriptor as path argument
in `os.utime`, `os.chmod`, `os.chdir`, `os.chown`, `os.listdir`, `os.stat` and `os.lstat`
View
@@ -19,6 +19,7 @@
import unittest
import sys
import dynamic_patch_test
import fake_filesystem_glob_test
import fake_filesystem_shutil_test
import fake_filesystem_test
@@ -46,6 +47,7 @@ def suite(self): # pylint: disable-msg=C6409
loader.loadTestsFromModule(fake_filesystem_unittest_test),
loader.loadTestsFromModule(example_test),
loader.loadTestsFromModule(mox3_stubout_test),
loader.loadTestsFromModule(dynamic_patch_test),
])
if sys.version_info >= (3, 4):
self.addTests([
View
@@ -0,0 +1,75 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Tests for patching modules loaded after `setupPyfakefs()`.
"""
import sys
if sys.version_info < (2, 7):
import unittest2 as unittest
else:
import unittest
from pyfakefs import fake_filesystem_unittest
class TestPyfakefsUnittestBase(fake_filesystem_unittest.TestCase):
def setUp(self):
"""Set up the fake file system"""
self.setUpPyfakefs()
class DynamicImportPatchTest(TestPyfakefsUnittestBase):
def testOsPatch(self):
import os
os.mkdir('test')
self.assertTrue(self.fs.Exists('test'))
self.assertTrue(os.path.exists('test'))
def testOsImportAsPatch(self):
import os as _os
_os.mkdir('test')
self.assertTrue(self.fs.Exists('test'))
self.assertTrue(_os.path.exists('test'))
def testOsPathPatch(self):
import os.path
os.mkdir('test')
self.assertTrue(self.fs.Exists('test'))
self.assertTrue(os.path.exists('test'))
@unittest.skipIf(sys.version_info < (3, 3), 'disk_usage new in Python 3.3')
def testShutilPatch(self):
import shutil
self.fs.SetDiskUsage(100)
self.assertEqual(100, shutil.disk_usage('/').total)
@unittest.skipIf(sys.version_info < (3, 4), 'pathlib new in Python 3.4')
def testPathlibPatch(self):
import pathlib
file_path = 'test.txt'
path = pathlib.Path(file_path)
with path.open('w') as f:
f.write('test')
self.assertTrue(self.fs.Exists(file_path))
file_object = self.fs.GetObject(file_path)
self.assertEqual('test', file_object.contents)
if __name__ == "__main__":
unittest.main()
@@ -126,22 +126,6 @@ def test_fakepathlib(self):
self.assertTrue(self.fs.Exists('/fake_file.txt'))
class TestImportAsOtherName(fake_filesystem_unittest.TestCase):
def __init__(self, methodName='RunTest'):
special_names = {'import_as_example': {'os': '_os'}}
super(TestImportAsOtherName, self).__init__(methodName,
special_names=special_names)
def setUp(self):
self.setUpPyfakefs()
def testFileExists(self):
file_path = '/foo/bar/baz'
self.fs.CreateFile(file_path)
self.assertTrue(self.fs.Exists(file_path))
self.assertTrue(check_if_exists(file_path))
sys.path.append(os.path.join(os.path.dirname(__file__), 'fixtures'))
import module_with_attributes
@@ -36,14 +36,28 @@
pyfakefs by simply changing their base class from `:py:class`unittest.TestCase`
to `:py:class`pyfakefs.fake_filesystem_unittest.TestCase`.
"""
import os
import sys
import doctest
import importlib
import inspect
import shutil
import sys
import tempfile
import shutil
try:
from importlib.machinery import ModuleSpec
except ImportError:
ModuleSpec = object
try:
# python >= 3.4
from importlib import reload
except ImportError:
try:
# python 3.0 - 3.3
from imp import reload
except ImportError:
# python 2 - reload is built-in
pass
from pyfakefs import fake_filesystem
from pyfakefs import fake_filesystem_shutil
@@ -65,7 +79,7 @@
def load_doctests(loader, tests, ignore, module,
additional_skip_names=None,
patch_path=True, special_names=None): # pylint: disable=unused-argument
patch_path=True): # pylint: disable=unused-argument
"""Load the doctest tests for the specified module into unittest.
Args:
loader, tests, ignore : arguments passed in from `load_tests()`
@@ -76,7 +90,7 @@ def load_doctests(loader, tests, ignore, module,
File `example_test.py` in the pyfakefs release provides a usage example.
"""
_patcher = Patcher(additional_skip_names=additional_skip_names,
patch_path=patch_path, special_names=special_names)
patch_path=patch_path)
globs = _patcher.replaceGlobs(vars(module))
tests.addTests(doctest.DocTestSuite(module,
globs=globs,
@@ -91,7 +105,8 @@ class TestCase(unittest.TestCase):
"""
def __init__(self, methodName='runTest', additional_skip_names=None,
patch_path=True, special_names=None):
patch_path=True, modules_to_reload=None,
use_dynamic_patch=True):
"""Creates the test class instance and the stubber used to stub out
file system related modules.
@@ -106,11 +121,11 @@ def __init__(self, methodName='runTest', additional_skip_names=None,
from my_module import path
Irrespective of patch_path, module 'os.path' is still correctly faked
if imported the usual way using `import os` or `import os.path`.
special_names: A dictionary with module names as key and a dictionary as
value, where the key is the original name of the module to be patched,
and the value is the name as it is imported.
This allows to patch modules where some of the file system modules are
imported as another name (e.g. `import os as _os`).
modules_to_reload: A list of modules that need to be reloaded
to be patched dynamically; may be needed if the module
imports file system modules under an alias
Note: this is done independently of `use_dynamic_patch'
use_dynamic_patch: If `True`, dynamic patching after setup is used
If you specify arguments `additional_skip_names` or `patch_path` here
and you have DocTests, consider also specifying the same arguments to
@@ -122,18 +137,14 @@ class MyTestCase(fake_filesystem_unittest.TestCase):
def __init__(self, methodName='runTest'):
super(MyTestCase, self).__init__(
methodName=methodName, additional_skip_names=['posixpath'])
class AnotherTestCase(fake_filesystem_unittest.TestCase):
def __init__(self, methodName='runTest'):
# allow patching a module that imports `os` as `my_os`
special_names = {'amodule': {'os': 'my_os'}}
super(MyTestCase, self).__init__(
methodName=methodName, special_names=special_names)
"""
super(TestCase, self).__init__(methodName)
self._stubber = Patcher(additional_skip_names=additional_skip_names,
patch_path=patch_path, special_names=special_names)
patch_path=patch_path)
self._modules_to_reload = [tempfile]
if modules_to_reload is not None:
self._modules_to_reload.extend(modules_to_reload)
self._use_dynamic_patch = use_dynamic_patch
@property
def fs(self):
@@ -189,6 +200,15 @@ def setUpPyfakefs(self):
"""
self._stubber.setUp()
self.addCleanup(self._stubber.tearDown)
dyn_patcher = DynamicPatcher(self._stubber)
sys.meta_path.insert(0, dyn_patcher)
for module in self._modules_to_reload:
if module.__name__ in sys.modules:
reload(module)
if self._use_dynamic_patch:
self.addCleanup(lambda: sys.meta_path.pop(0))
else:
sys.meta_path.pop(0)
def tearDownPyfakefs(self):
"""This method is deprecated and exists only for backward compatibility.
@@ -226,13 +246,10 @@ class Patcher(object):
if HAS_PATHLIB:
SKIPNAMES.add('pathlib')
def __init__(self, additional_skip_names=None, patch_path=True, special_names=None):
def __init__(self, additional_skip_names=None, patch_path=True):
"""For a description of the arguments, see TestCase.__init__"""
self._skipNames = self.SKIPNAMES.copy()
self._special_names = special_names or {}
self._special_names['tempfile'] = {'os': '_os', 'io': '_io'}
if additional_skip_names is not None:
self._skipNames.update(additional_skip_names)
self._patchPath = patch_path
@@ -307,20 +324,6 @@ def _findModules(self):
self._shutil_modules.add((module, 'shutil'))
if inspect.ismodule(module.__dict__.get('io')):
self._io_modules.add((module, 'io'))
if '__name__' in module.__dict__ and module.__name__ in self._special_names:
module_names = self._special_names[module.__name__]
if 'os' in module_names:
if inspect.ismodule(module.__dict__.get(module_names['os'])):
self._os_modules.add((module, module_names['os']))
if self._patchPath and 'path' in module_names:
if inspect.ismodule(module.__dict__.get(module_names['path'])):
self._path_modules.add((module, module_names['path']))
if self.HAS_PATHLIB and 'pathlib' in module_names:
if inspect.ismodule(module.__dict__.get(module_names['pathlib'])):
self._pathlib_modules.add((module, module_names['pathlib']))
if 'io' in module_names:
if inspect.ismodule(module.__dict__.get(module_names['io'])):
self._io_modules.add((module, module_names['io']))
def _refresh(self):
"""Renew the fake file system and set the _isStale flag to `False`."""
@@ -337,28 +340,8 @@ def _refresh(self):
self.fake_open = fake_filesystem.FakeFileOpen(self.fs)
self.fake_io = fake_filesystem.FakeIoModule(self.fs)
if not self.IS_WINDOWS and 'tempfile' in sys.modules:
self._patch_tempfile()
self._isStale = False
def _patch_tempfile(self):
"""Hack to work around cached `os` functions in `tempfile`.
Shall be replaced by a more generic mechanism.
"""
if 'unlink' in tempfile._TemporaryFileWrapper.__dict__:
# Python 2.6 to 3.2: unlink is a class method of _TemporaryFileWrapper
tempfile._TemporaryFileWrapper.unlink = self.fake_os.unlink
# Python 3.0 to 3.2 (and PyPy3 based on Python 3.2):
# `TemporaryDirectory._rmtree` is used instead of `shutil.rmtree`
# which uses several cached os functions - replace it with `shutil.rmtree`
if 'TemporaryDirectory' in tempfile.__dict__:
tempfile.TemporaryDirectory._rmtree = lambda o, path: shutil.rmtree(path)
else:
# Python > 3.2 - unlink is a default parameter of _TemporaryFileCloser
tempfile._TemporaryFileCloser.close.__defaults__ = (self.fake_os.unlink,)
def setUp(self, doctester=None):
"""Bind the file-related modules to the :py:mod:`pyfakefs` fake
modules real ones. Also bind the fake `file()` and `open()` functions.
@@ -373,7 +356,6 @@ def setUp(self, doctester=None):
# file() was eliminated in Python3
self._stubs.SmartSet(builtins, 'file', self.fake_open)
self._stubs.SmartSet(builtins, 'open', self.fake_open)
for module, attr in self._os_modules:
self._stubs.SmartSet(module, attr, self.fake_os)
for module, attr in self._path_modules:
@@ -411,3 +393,59 @@ def tearDown(self, doctester=None):
"""Clear the fake filesystem bindings created by `setUp()`."""
self._isStale = True
self._stubs.SmartUnsetAll()
class DynamicPatcher(object):
"""A file loader that replaces file system related modules by their
fake implementation if they are loaded after calling `setupPyFakefs()`.
Implements the protocol needed for import hooks.
"""
def __init__(self, patcher):
self._patcher = patcher
self._patching = False
self.modules = {
'os': self._patcher.fake_os,
'os.path': self._patcher.fake_path,
'io': self._patcher.fake_io,
'shutil': self._patcher.fake_shutil
}
if sys.version_info >= (3, 4):
self.modules['pathlib'] = fake_pathlib.FakePathlibModule
# remove all modules that have to be patched from `sys.modules`,
# otherwise the find_... methods will not be called
for module in self.modules:
if self.needs_patch(module) and module in sys.modules:
del sys.modules[module]
def needs_patch(self, name):
"""Check if the module with the given name shall be replaced."""
if self._patching or name not in self.modules:
return False
if (name in sys.modules and
type(sys.modules[name]) == self.modules[name]):
return False
return True
def find_spec(self, fullname, path, target=None):
"""Module finder for Python 3."""
if self.needs_patch(fullname):
return ModuleSpec(fullname, self)
def find_module(self, fullname, path=None):
"""Module finder for Python 2."""
if self.needs_patch(fullname):
return self
def load_module(self, fullname):
"""Replaces the module by its fake implementation."""
# prevent re-entry via the finder
self._patching = True
importlib.import_module(fullname)
self._patching = False
# preserve the original module (currently not used)
sys.modules['original_' + fullname] = sys.modules[fullname]
# replace with fake implementation
sys.modules[fullname] = self.modules[fullname]
return self.modules[fullname]
View
@@ -1,2 +1,3 @@
unittest2; python_version < '2.7'
importlib; python_version < '2.7'
pytest>=2.8.6

0 comments on commit 639c66c

Please sign in to comment.