diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 52026f6a2bce38..88b3511a8d3dcd 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -90,6 +90,16 @@ always available. A string containing the copyright pertaining to the Python interpreter. +.. index:: single: __clearcache__ + +.. function:: clear_caches() + + Clear all caches. Call the ``__clearcache__()`` function for all + imported modules if it is defined. + + ... versionadded: 3.8 + + .. function:: _clear_type_cache() Clear the internal type cache. The type cache is used to speed up attribute diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index accc30649f24ef..3408adc8ed2537 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -311,7 +311,7 @@ defines. It returns a sorted list of strings:: '_clear_type_cache', '_current_frames', '_debugmallocstats', '_getframe', '_home', '_mercurial', '_xoptions', 'abiflags', 'api_version', 'argv', 'base_exec_prefix', 'base_prefix', 'builtin_module_names', 'byteorder', - 'call_tracing', 'callstats', 'copyright', 'displayhook', + 'call_tracing', 'callstats', 'clear_caches', 'copyright', 'displayhook', 'dont_write_bytecode', 'exc_info', 'excepthook', 'exec_prefix', 'executable', 'exit', 'flags', 'float_info', 'float_repr_style', 'getcheckinterval', 'getdefaultencoding', 'getdlopenflags', diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index f0423c376fcdd4..d29694b8750459 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -346,6 +346,12 @@ and manipulating normal distributions of a random variable. >>> temperature_feb.samples(3) # Generate random samples [7.672102882379219, 12.000027119750287, 4.647488369766392] +sys +--- + +Added the :func:`~sys.clear_caches` function. It clears all caches by +calling the ``__clearcache__()`` function for every module if it is defined. +(Contributed by Serhit Storchaka in :issue:`36485`.) tarfile ------- diff --git a/Lib/_strptime.py b/Lib/_strptime.py index f4f3c0b80c1d05..14a0e4f62fde56 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -268,6 +268,7 @@ def compile(self, format): _TimeRE_cache = TimeRE() _CACHE_MAX_SIZE = 5 # Max number of regexes stored in _regex_cache _regex_cache = {} +__clearcache__ = _regex_cache.clear def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon): """Calculate the Julian day based on the year, week of the year, and day of diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 4107db3e3972d7..19feaa4e166aa9 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -267,6 +267,9 @@ def _reset_cache(): POINTER(c_char).from_param = c_char_p.from_param _pointer_type_cache[None] = c_void_p + +__clearcache__ = _reset_cache + def create_unicode_buffer(init, size=None): """create_unicode_buffer(aString) -> character array create_unicode_buffer(anInteger) -> character array diff --git a/Lib/distutils/dir_util.py b/Lib/distutils/dir_util.py index d5cd8e3e24f46a..b8b8a282fb47c6 100644 --- a/Lib/distutils/dir_util.py +++ b/Lib/distutils/dir_util.py @@ -10,6 +10,7 @@ # cache for by mkpath() -- in addition to cheapening redundant calls, # eliminates redundant "creating /foo/bar/baz" messages in dry-run mode _path_created = {} +__clearcache__ = _path_created.clear # I don't use os.makedirs because a) it's new to Python 1.5.2, and # b) it blows up if the directory already exists (I want to silently diff --git a/Lib/doctest.py b/Lib/doctest.py index 79d91a040c2eee..3564e1ab57b1bb 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1853,6 +1853,10 @@ def report_failure(self, out, test, example, got): # class, updated by testmod. master = None +def __clearcache__(): + global master + master = None + def testmod(m=None, name=None, globs=None, verbose=None, report=True, optionflags=0, extraglobs=None, raise_on_error=False, exclude_empty=False): diff --git a/Lib/filecmp.py b/Lib/filecmp.py index e5ad8397e4c539..b32c7773fec212 100644 --- a/Lib/filecmp.py +++ b/Lib/filecmp.py @@ -26,6 +26,8 @@ def clear_cache(): """Clear the filecmp cache.""" _cache.clear() +__clearcache__ = clear_cache + def cmp(f1, f2, shallow=True): """Compare two files. diff --git a/Lib/linecache.py b/Lib/linecache.py index 3afcce1f0a1456..291c4708b2cf84 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -33,6 +33,7 @@ def clearcache(): global cache cache = {} +__clearcache__ = clearcache def getlines(filename, module_globals=None): """Get the lines for a Python source file from the cache. diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 8861b75362dbd7..04814e29491841 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -548,6 +548,7 @@ def _default_mime_types(): } +__clearcache__ = _default_mime_types _default_mime_types() diff --git a/Lib/re.py b/Lib/re.py index 68d62dc2a93b05..b395a5cc9b13df 100644 --- a/Lib/re.py +++ b/Lib/re.py @@ -238,6 +238,8 @@ def purge(): _cache.clear() _compile_repl.cache_clear() +__clearcache__ = purge + def template(pattern, flags=0): "Compile a template pattern, returning a Pattern object" return _compile(pattern, flags|T) diff --git a/Lib/struct.py b/Lib/struct.py index d6bba588636498..998f16a042d7a2 100644 --- a/Lib/struct.py +++ b/Lib/struct.py @@ -11,5 +11,4 @@ ] from _struct import * -from _struct import _clearcache from _struct import __doc__ diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 6724488fcfb088..3a8ce9595b31ef 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -136,7 +136,8 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): zipimport._zip_directory_cache.update(zdc) # clear type cache - sys._clear_type_cache() + if hasattr(sys, '_clear_type_cache'): + sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] @@ -160,98 +161,13 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): def clear_caches(): import gc - # Clear the warnings registry, so they can be displayed again - for mod in sys.modules.values(): - if hasattr(mod, '__warningregistry__'): - del mod.__warningregistry__ - # Flush standard output, so that buffered data is sent to the OS and # associated Python objects are reclaimed. for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): if stream is not None: stream.flush() - # Clear assorted module caches. - # Don't worry about resetting the cache if the module is not loaded - try: - distutils_dir_util = sys.modules['distutils.dir_util'] - except KeyError: - pass - else: - distutils_dir_util._path_created.clear() - re.purge() - - try: - _strptime = sys.modules['_strptime'] - except KeyError: - pass - else: - _strptime._regex_cache.clear() - - try: - urllib_parse = sys.modules['urllib.parse'] - except KeyError: - pass - else: - urllib_parse.clear_cache() - - try: - urllib_request = sys.modules['urllib.request'] - except KeyError: - pass - else: - urllib_request.urlcleanup() - - try: - linecache = sys.modules['linecache'] - except KeyError: - pass - else: - linecache.clearcache() - - try: - mimetypes = sys.modules['mimetypes'] - except KeyError: - pass - else: - mimetypes._default_mime_types() - - try: - filecmp = sys.modules['filecmp'] - except KeyError: - pass - else: - filecmp._cache.clear() - - try: - struct = sys.modules['struct'] - except KeyError: - pass - else: - struct._clearcache() - - try: - doctest = sys.modules['doctest'] - except KeyError: - pass - else: - doctest.master = None - - try: - ctypes = sys.modules['ctypes'] - except KeyError: - pass - else: - ctypes._reset_cache() - - try: - typing = sys.modules['typing'] - except KeyError: - pass - else: - for f in typing._cleanups: - f() - + sys.clear_caches() gc.collect() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 4bd54af3629c88..5dc6d3398272b6 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -15,6 +15,8 @@ # strings to intern in test_intern() numruns = 0 +cache = [] +__clearcache__ = cache.clear class SysModuleTest(unittest.TestCase): @@ -556,6 +558,11 @@ def test_sys_getwindowsversion_no_instantiation(self): test.support.get_attribute(sys, "getwindowsversion") self.assert_raise_on_new_sys_type(sys.getwindowsversion()) + def test_clear_caches(self): + cache.append(True) + sys.clear_caches() + self.assertEqual(cache, []) + @test.support.cpython_only def test_clear_type_cache(self): sys._clear_type_cache() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 0d66ebbd18456e..ae943552066a0c 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -45,8 +45,7 @@ def assertNotIsSubclass(self, cls, class_or_tuple, msg=None): raise self.failureException(message) def clear_caches(self): - for f in typing._cleanups: - f() + typing.__clearcache__() class Employee: diff --git a/Lib/typing.py b/Lib/typing.py index 530d4633fe4c22..78f663eb5c7a29 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -233,6 +233,10 @@ def _remove_dups_flatten(parameters): _cleanups = [] +def __clearcache__(): + for f in _cleanups: + f() + def _tp_cache(func): """Internal wrapper caching __getitem__ of generic types with a fallback to diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 8b6c9b10609152..adab39aca47e80 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -86,6 +86,7 @@ def clear_cache(): _parse_cache.clear() _safe_quoters.clear() +__clearcache__ = clear_cache # Helpers for bytes handling # For 3.2, we deliberately require applications that diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index df2ff06f0fc9a2..a85776003b7535 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -302,6 +302,8 @@ def urlcleanup(): if _opener: _opener = None +__clearcache__ = urlcleanup + # copied from cookielib.py _cut_port_re = re.compile(r":\d+$", re.ASCII) def request_host(request): diff --git a/Lib/warnings.py b/Lib/warnings.py index 00f740ca3a95ba..981dd1a7bbd104 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -538,6 +538,13 @@ def _filters_mutated(): _warnings_defaults = False +def __clearcache__(): + # Clear the warnings registry, so they can be displayed again + for mod in list(sys.modules.values()): + if hasattr(mod, '__warningregistry__'): + del mod.__warningregistry__ + + # Module initialization _processoptions(sys.warnoptions) if not _warnings_defaults: diff --git a/Lib/xml/etree/ElementPath.py b/Lib/xml/etree/ElementPath.py index ef32917b14d41e..7eb975707f11d5 100644 --- a/Lib/xml/etree/ElementPath.py +++ b/Lib/xml/etree/ElementPath.py @@ -251,6 +251,7 @@ def select(context, result): } _cache = {} +__clearcache__ = _cache.clear class _SelectorContext: parent_map = None diff --git a/Misc/NEWS.d/next/Library/2019-03-30-14-01-43.bpo-36485.OM96Zf.rst b/Misc/NEWS.d/next/Library/2019-03-30-14-01-43.bpo-36485.OM96Zf.rst new file mode 100644 index 00000000000000..99a0c4526f8823 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-03-30-14-01-43.bpo-36485.OM96Zf.rst @@ -0,0 +1 @@ +Added the :func:`sys.clear_caches` function. diff --git a/Modules/_struct.c b/Modules/_struct.c index 90839b2ead7508..5e88151a051dd9 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2112,14 +2112,14 @@ cache_struct_converter(PyObject *fmt, PyStructObject **ptr) } /*[clinic input] -_clearcache +__clearcache__ Clear the internal cache. [clinic start generated code]*/ static PyObject * -_clearcache_impl(PyObject *module) -/*[clinic end generated code: output=ce4fb8a7bf7cb523 input=463eaae04bab3211]*/ +__clearcache___impl(PyObject *module) +/*[clinic end generated code: output=7d1757711e960187 input=d8c1789f69c336ae]*/ { Py_CLEAR(cache); Py_RETURN_NONE; @@ -2264,7 +2264,7 @@ iter_unpack_impl(PyObject *module, PyStructObject *s_object, } static struct PyMethodDef module_functions[] = { - _CLEARCACHE_METHODDEF + __CLEARCACHE___METHODDEF CALCSIZE_METHODDEF ITER_UNPACK_METHODDEF {"pack", (PyCFunction)(void(*)(void))pack, METH_FASTCALL, pack_doc}, diff --git a/Modules/clinic/_struct.c.h b/Modules/clinic/_struct.c.h index 908c44266c4ec6..c438627bef21c6 100644 --- a/Modules/clinic/_struct.c.h +++ b/Modules/clinic/_struct.c.h @@ -167,22 +167,22 @@ PyDoc_STRVAR(Struct_iter_unpack__doc__, #define STRUCT_ITER_UNPACK_METHODDEF \ {"iter_unpack", (PyCFunction)Struct_iter_unpack, METH_O, Struct_iter_unpack__doc__}, -PyDoc_STRVAR(_clearcache__doc__, -"_clearcache($module, /)\n" +PyDoc_STRVAR(__clearcache____doc__, +"__clearcache__($module, /)\n" "--\n" "\n" "Clear the internal cache."); -#define _CLEARCACHE_METHODDEF \ - {"_clearcache", (PyCFunction)_clearcache, METH_NOARGS, _clearcache__doc__}, +#define __CLEARCACHE___METHODDEF \ + {"__clearcache__", (PyCFunction)__clearcache__, METH_NOARGS, __clearcache____doc__}, static PyObject * -_clearcache_impl(PyObject *module); +__clearcache___impl(PyObject *module); static PyObject * -_clearcache(PyObject *module, PyObject *Py_UNUSED(ignored)) +__clearcache__(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return _clearcache_impl(module); + return __clearcache___impl(module); } PyDoc_STRVAR(calcsize__doc__, @@ -386,4 +386,4 @@ iter_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -/*[clinic end generated code: output=b642e1002d25ebdd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c92e2f149a1703cb input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index c70b721983c1c4..b8a938e649dd40 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -971,6 +971,26 @@ sys__debugmallocstats(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys__debugmallocstats_impl(module); } +PyDoc_STRVAR(sys_clear_caches__doc__, +"clear_caches($module, /)\n" +"--\n" +"\n" +"Clear all caches.\n" +"\n" +"Call the __clearcache__() function in every imported module."); + +#define SYS_CLEAR_CACHES_METHODDEF \ + {"clear_caches", (PyCFunction)sys_clear_caches, METH_NOARGS, sys_clear_caches__doc__}, + +static PyObject * +sys_clear_caches_impl(PyObject *module); + +static PyObject * +sys_clear_caches(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return sys_clear_caches_impl(module); +} + PyDoc_STRVAR(sys__clear_type_cache__doc__, "_clear_type_cache($module, /)\n" "--\n" @@ -1060,4 +1080,4 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=3ba4c194d00f1866 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3fc46ed6688d455c input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 3de94e8468bee1..5882d411f39a14 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1596,6 +1596,41 @@ extern PyObject *_Py_GetDXProfile(PyObject *, PyObject *); #endif +/*[clinic input] +sys.clear_caches + +Clear all caches. + +Call the __clearcache__() function in every imported module. +[clinic start generated code]*/ + +static PyObject * +sys_clear_caches_impl(PyObject *module) +/*[clinic end generated code: output=7bbac9db7bbef6c3 input=9e739bbc710495ff]*/ +{ + _Py_IDENTIFIER(__clearcache__); + PyObject *modules = PyMapping_Values(PyImport_GetModuleDict()); + for (Py_ssize_t i = PyList_GET_SIZE(modules); i-- > 0;) { + PyObject *mod, *func, *res; + mod = PyList_GET_ITEM(modules, i); + if (_PyObject_LookupAttrId(mod, &PyId___clearcache__, &func) < 0) { + Py_DECREF(modules); + return NULL; + } + if (func) { + res = _PyObject_CallNoArg(func); + Py_DECREF(func); + if (res == NULL) { + Py_DECREF(modules); + return NULL; + } + Py_DECREF(res); + } + } + Py_DECREF(modules); + Py_RETURN_NONE; +} + /*[clinic input] sys._clear_type_cache @@ -1644,6 +1679,7 @@ static PyMethodDef sys_methods[] = { {"breakpointhook", (PyCFunction)(void(*)(void))sys_breakpointhook, METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc}, SYS_CALLSTATS_METHODDEF + SYS_CLEAR_CACHES_METHODDEF SYS__CLEAR_TYPE_CACHE_METHODDEF SYS__CURRENT_FRAMES_METHODDEF SYS_DISPLAYHOOK_METHODDEF