bpo-33499: Add PYTHONPYCACHEPREFIX env var for alt bytecode cache location.

Doc/library/compileall.rst

@@ -109,6 +109,11 @@ There is no command-line option to control the optimization level used by the
 :func:`compile` function, because the Python interpreter itself already
 provides the option: :program:`python -O -m compileall`.
 
+Similarly, the :func:`compile` function respects the :attr:`sys.pycache_prefix`
+setting. The generated bytecode cache will only be useful if :func:`compile` is
+run with the same :attr:`sys.pycache_prefix` (if any) that will be used at
+runtime.
+
 Public functions
 ----------------
 

Doc/library/sys.rst

@@ -209,6 +209,26 @@ always available.
    yourself to control bytecode file generation.
 
 
+.. data:: pycache_prefix
+
+   If this is set (not ``None``), Python will write bytecode-cache ``.pyc``
+   files to (and read them from) a parallel directory tree rooted at this
+   directory, rather than from ``__pycache__`` directories in the source code
+   tree. Any ``__pycache__`` directories in the source code tree will be ignored
+   and new `.pyc` files written within the pycache prefix. Thus if you use
+   :mod:`compileall` as a pre-build step, you must ensure you run it with the
+   same pycache prefix (if any) that you will use at runtime.
+
+   A relative path is interpreted relative to the current working directory.
+
+   This value is initially set based on the value of the :option:`-X`
+   ``pycache_prefix=PATH`` command-line option or the
+   :envvar:`PYTHONPYCACHEPREFIX` environment variable (command-line takes
+   precedence). If neither are set, it is ``None``.
+
+   .. versionadded:: 3.8
+
+
 .. function:: excepthook(type, value, traceback)
 
    This function prints out a given traceback and exception to ``sys.stderr``.

Doc/using/cmdline.rst

@@ -442,6 +442,9 @@ Miscellaneous options
      the default locale-aware mode. ``-X utf8=0`` explicitly disables UTF-8
      mode (even when it would otherwise activate automatically).
      See :envvar:`PYTHONUTF8` for more details.
+   * ``-X pycache_prefix=PATH`` enables writing ``.pyc`` files to a parallel
+     tree rooted at the given directory instead of to the code tree. See also
+     :envvar:`PYTHONPYCACHEPREFIX`.
 
    It also allows passing arbitrary values and retrieving them through the
    :data:`sys._xoptions` dictionary.
@@ -461,6 +464,9 @@ Miscellaneous options
    .. versionadded:: 3.7
       The ``-X importtime``, ``-X dev`` and ``-X utf8`` options.
 
+   .. versionadded:: 3.8
+      The ``-X pycache_prefix`` option.
+
 
 Options you shouldn't use
 ~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -587,6 +593,16 @@ conflict.
    specifying the :option:`-B` option.
 
 
+.. envvar:: PYTHONPYCACHEPREFIX
+
+   If this is set, Python will write ``.pyc`` files in a mirror directory tree
+   at this path, instead of in ``__pycache__`` directories within the source
+   tree. This is equivalent to specifying the :option:`-X`
+   ``pycache_prefix=PATH`` option.
+
+   .. versionadded:: 3.8
+
+
 .. envvar:: PYTHONHASHSEED
 
    If this variable is not set or set to ``random``, a random value is used

Include/pystate.h

@@ -1,4 +1,3 @@
-
 /* Thread and interpreter state structures and their interfaces */
 
 
@@ -44,6 +43,7 @@ typedef struct {
     int coerce_c_locale;    /* PYTHONCOERCECLOCALE, -1 means unknown */
     int coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */
     int utf8_mode;          /* PYTHONUTF8, -X utf8; -1 means unknown */
+    wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */
 
     wchar_t *program_name;  /* Program name, see also Py_GetProgramName() */
     int argc;               /* Number of command line arguments,
@@ -101,6 +101,7 @@ typedef struct {
     PyObject *warnoptions;         /* sys.warnoptions list, can be NULL */
     PyObject *xoptions;            /* sys._xoptions dict, can be NULL */
     PyObject *module_search_path;  /* sys.path list */
+    PyObject *pycache_prefix;      /* sys.pycache_prefix str, can be NULL */
 } _PyMainInterpreterConfig;
 
 #define _PyMainInterpreterConfig_INIT \

Lib/importlib/_bootstrap_external.py

@@ -102,6 +102,15 @@ def _path_isdir(path):
     return _path_is_mode_type(path, 0o040000)
 
 
+def _path_isabs(path):
+    """Replacement for os.path.isabs.
+
+    Considers a Windows drive-relative path (no drive, but starts with slash) to
+    still be "absolute".
+    """
+    return path.startswith(path_separators) or path[1:3] in _pathseps_with_colon
+
+
 def _write_atomic(path, data, mode=0o666):
     """Best-effort function to write data to a path atomically.
     Be prepared to handle a FileExistsError if concurrent writing of the
@@ -312,7 +321,33 @@ def cache_from_source(path, debug_override=None, *, optimization=None):
         if not optimization.isalnum():
             raise ValueError('{!r} is not alphanumeric'.format(optimization))
         almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization)
-    return _path_join(head, _PYCACHE, almost_filename + BYTECODE_SUFFIXES[0])
+    filename = almost_filename + BYTECODE_SUFFIXES[0]
+    if sys.pycache_prefix is not None:
+        # We need an absolute path to the py file to avoid the possibility of
+        # collisions within sys.pycache_prefix, if someone has two different
+        # `foo/bar.py` on their system and they import both of them using the
+        # same sys.pycache_prefix. Let's say sys.pycache_prefix is
+        # `C:\Bytecode`; the idea here is that if we get `Foo\Bar`, we first
+        # make it absolute (`C:\Somewhere\Foo\Bar`), then make it root-relative
+        # (`Somewhere\Foo\Bar`), so we end up placing the bytecode file in an
+        # unambiguous `C:\Bytecode\Somewhere\Foo\Bar\`.
+        if not _path_isabs(head):
+            head = _path_join(_os.getcwd(), head)
+
+        # Strip initial drive from a Windows path. We know we have an absolute
+        # path here, so the second part of the check rules out a POSIX path that
+        # happens to contain a colon at the second character.
+        if head[1] == ':' and head[0] not in path_separators:
+            head = head[2:]
+
+        # Strip initial path separator from `head` to complete the conversion
+        # back to a root-relative path before joining.
+        return _path_join(
+            sys.pycache_prefix,
+            head.lstrip(path_separators),
+            filename,
+        )
+    return _path_join(head, _PYCACHE, filename)
 
 
 def source_from_cache(path):
@@ -328,23 +363,29 @@ def source_from_cache(path):
         raise NotImplementedError('sys.implementation.cache_tag is None')
     path = _os.fspath(path)
     head, pycache_filename = _path_split(path)
-    head, pycache = _path_split(head)
-    if pycache != _PYCACHE:
-        raise ValueError('{} not bottom-level directory in '
-                         '{!r}'.format(_PYCACHE, path))
+    found_in_pycache_prefix = False
+    if sys.pycache_prefix is not None:
+        stripped_path = sys.pycache_prefix.rstrip(path_separators)
+        if head.startswith(stripped_path + path_sep):
+            head = head[len(stripped_path):]
+            found_in_pycache_prefix = True
+    if not found_in_pycache_prefix:
+        head, pycache = _path_split(head)
+        if pycache != _PYCACHE:
+            raise ValueError(f'{_PYCACHE} not bottom-level directory in '
+                             f'{path!r}')
     dot_count = pycache_filename.count('.')
     if dot_count not in {2, 3}:
-        raise ValueError('expected only 2 or 3 dots in '
-                         '{!r}'.format(pycache_filename))
+        raise ValueError(f'expected only 2 or 3 dots in {pycache_filename!r}')
     elif dot_count == 3:
         optimization = pycache_filename.rsplit('.', 2)[-2]
         if not optimization.startswith(_OPT):
             raise ValueError("optimization portion of filename does not start "
-                             "with {!r}".format(_OPT))
+                             f"with {_OPT!r}")
         opt_level = optimization[len(_OPT):]
         if not opt_level.isalnum():
-            raise ValueError("optimization level {!r} is not an alphanumeric "
-                             "value".format(optimization))
+            raise ValueError(f"optimization level {optimization!r} is not an "
+                             "alphanumeric value")
     base_filename = pycache_filename.partition('.')[0]
     return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
 
@@ -1533,6 +1574,7 @@ def _setup(_bootstrap_module):
     setattr(self_module, '_os', os_module)
     setattr(self_module, 'path_sep', path_sep)
     setattr(self_module, 'path_separators', ''.join(path_separators))
+    setattr(self_module, '_pathseps_with_colon', {f':{s}' for s in path_separators})
 
     # Directly load the _thread module (needed during bootstrap).
     thread_module = _bootstrap._builtin_from_name('_thread')

Lib/test/test_cmd_line.py

@@ -519,6 +519,32 @@ def test_sys_flags_set(self):
             with self.subTest(envar_value=value):
                 assert_python_ok('-c', code, **env_vars)
 
+    def test_set_pycache_prefix(self):
+        # sys.pycache_prefix can be set from either -X pycache_prefix or
+        # PYTHONPYCACHEPREFIX env var, with the former taking precedence.
+        NO_VALUE = object()  # `-X pycache_prefix` with no `=PATH`
+        cases = [
+            # (PYTHONPYCACHEPREFIX, -X pycache_prefix, sys.pycache_prefix)
+            (None, None, None),
+            ('foo', None, 'foo'),
+            (None, 'bar', 'bar'),
+            ('foo', 'bar', 'bar'),
+            ('foo', '', None),
+            ('foo', NO_VALUE, None),
+        ]
+        for envval, opt, expected in cases:
+            exp_clause = "is None" if expected is None else f'== "{expected}"'
+            code = f"import sys; sys.exit(not sys.pycache_prefix {exp_clause})"
+            args = ['-c', code]
+            env = {} if envval is None else {'PYTHONPYCACHEPREFIX': envval}
+            if opt is NO_VALUE:
+                args[:0] = ['-X', 'pycache_prefix']
+            elif opt is not None:
+                args[:0] = ['-X', f'pycache_prefix={opt}']
+            with self.subTest(envval=envval, opt=opt):
+                with support.temp_cwd():
+                    assert_python_ok(*args, **env)
+
     def run_xdev(self, *args, check_exitcode=True, xdev=True):
         env = dict(os.environ)
         env.pop('PYTHONWARNINGS', None)

Lib/test/test_importlib/test_util.py

@@ -1,9 +1,10 @@
-from . import util
+from . import util
 abc = util.import_importlib('importlib.abc')
 init = util.import_importlib('importlib')
 machinery = util.import_importlib('importlib.machinery')
 importlib_util = util.import_importlib('importlib.util')
 
+import contextlib
 import importlib.util
 import os
 import pathlib
@@ -12,6 +13,7 @@
 from test import support
 import types
 import unittest
+import unittest.mock
 import warnings
 
 
@@ -557,8 +559,8 @@ class PEP3147Tests:
 
     tag = sys.implementation.cache_tag
 
-    @unittest.skipUnless(sys.implementation.cache_tag is not None,
-                         'requires sys.implementation.cache_tag not be None')
+    @unittest.skipIf(sys.implementation.cache_tag is None,
+                     'requires sys.implementation.cache_tag not be None')
     def test_cache_from_source(self):
         # Given the path to a .py file, return the path to its PEP 3147
         # defined .pyc file (i.e. under __pycache__).
@@ -678,18 +680,17 @@ def test_sep_altsep_and_sep_cache_from_source(self):
             self.util.cache_from_source('\\foo\\bar\\baz/qux.py', optimization=''),
             '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
 
-    @unittest.skipUnless(sys.implementation.cache_tag is not None,
-                         'requires sys.implementation.cache_tag not be None')
+    @unittest.skipIf(sys.implementation.cache_tag is None,
+                     'requires sys.implementation.cache_tag not be None')
     def test_source_from_cache_path_like_arg(self):
         path = pathlib.PurePath('foo', 'bar', 'baz', 'qux.py')
         expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
                               'qux.{}.pyc'.format(self.tag))
         self.assertEqual(self.util.cache_from_source(path, optimization=''),
                          expect)
 
-    @unittest.skipUnless(sys.implementation.cache_tag is not None,
-                         'requires sys.implementation.cache_tag to not be '
-                         'None')
+    @unittest.skipIf(sys.implementation.cache_tag is None,
+                     'requires sys.implementation.cache_tag to not be None')
     def test_source_from_cache(self):
         # Given the path to a PEP 3147 defined .pyc file, return the path to
         # its source.  This tests the good path.
@@ -749,15 +750,87 @@ def test_source_from_cache_missing_optimization(self):
         with self.assertRaises(ValueError):
             self.util.source_from_cache(path)
 
-    @unittest.skipUnless(sys.implementation.cache_tag is not None,
-                         'requires sys.implementation.cache_tag to not be '
-                         'None')
+    @unittest.skipIf(sys.implementation.cache_tag is None,
+                     'requires sys.implementation.cache_tag to not be None')
     def test_source_from_cache_path_like_arg(self):
         path = pathlib.PurePath('foo', 'bar', 'baz', '__pycache__',
                                 'qux.{}.pyc'.format(self.tag))
         expect = os.path.join('foo', 'bar', 'baz', 'qux.py')
         self.assertEqual(self.util.source_from_cache(path), expect)
 
+    @unittest.skipIf(sys.implementation.cache_tag is None,
+                     'requires sys.implementation.cache_tag to not be None')
+    def test_cache_from_source_respects_pycache_prefix(self):
+        # If pycache_prefix is set, cache_from_source will return a bytecode
+        # path inside that directory (in a subdirectory mirroring the .py file's
+        # path) rather than in a __pycache__ dir next to the py file.
+        pycache_prefixes = [
+            os.path.join(os.path.sep, 'tmp', 'bytecode'),
+            os.path.join(os.path.sep, 'tmp', '\u2603'),  # non-ASCII in path!
+            os.path.join(os.path.sep, 'tmp', 'trailing-slash') + os.path.sep,
+        ]
+        drive = ''
+        if os.name == 'nt':
+            drive = 'C:'
+            pycache_prefixes = [
+                f'{drive}{prefix}' for prefix in pycache_prefixes]
+            pycache_prefixes += [r'\\?\C:\foo', r'\\localhost\c$\bar']
+        for pycache_prefix in pycache_prefixes:
+            with self.subTest(path=pycache_prefix):
+                path = drive + os.path.join(
+                    os.path.sep, 'foo', 'bar', 'baz', 'qux.py')
+                expect = os.path.join(
+                    pycache_prefix, 'foo', 'bar', 'baz',
+                    'qux.{}.pyc'.format(self.tag))
+                with util.temporary_pycache_prefix(pycache_prefix):
+                    self.assertEqual(
+                        self.util.cache_from_source(path, optimization=''),
+                        expect)
+
+    @unittest.skipIf(sys.implementation.cache_tag is None,
+                     'requires sys.implementation.cache_tag to not be None')
+    def test_cache_from_source_respects_pycache_prefix_relative(self):
+        # If the .py path we are given is relative, we will resolve to an
+        # absolute path before prefixing with pycache_prefix, to avoid any
+        # possible ambiguity.
+        pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode')
+        path = os.path.join('foo', 'bar', 'baz', 'qux.py')
+        root = os.path.splitdrive(os.getcwd())[0] + os.path.sep
+        expect = os.path.join(
+            pycache_prefix,
+            os.path.relpath(os.getcwd(), root),
+            'foo', 'bar', 'baz', f'qux.{self.tag}.pyc')
+        with util.temporary_pycache_prefix(pycache_prefix):
+            self.assertEqual(
+                self.util.cache_from_source(path, optimization=''),
+                expect)
+
+    @unittest.skipIf(sys.implementation.cache_tag is None,
+                     'requires sys.implementation.cache_tag to not be None')
+    def test_source_from_cache_inside_pycache_prefix(self):
+        # If pycache_prefix is set and the cache path we get is inside it,
+        # we return an absolute path to the py file based on the remainder of
+        # the path within pycache_prefix.
+        pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode')
+        path = os.path.join(pycache_prefix, 'foo', 'bar', 'baz',
+                            f'qux.{self.tag}.pyc')
+        expect = os.path.join(os.path.sep, 'foo', 'bar', 'baz', 'qux.py')
+        with util.temporary_pycache_prefix(pycache_prefix):
+            self.assertEqual(self.util.source_from_cache(path), expect)
+
+    @unittest.skipIf(sys.implementation.cache_tag is None,
+                     'requires sys.implementation.cache_tag to not be None')
+    def test_source_from_cache_outside_pycache_prefix(self):
+        # If pycache_prefix is set but the cache path we get is not inside
+        # it, just ignore it and handle the cache path according to the default
+        # behavior.
+        pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode')
+        path = os.path.join('foo', 'bar', 'baz', '__pycache__',
+                            f'qux.{self.tag}.pyc')
+        expect = os.path.join('foo', 'bar', 'baz', 'qux.py')
+        with util.temporary_pycache_prefix(pycache_prefix):
+            self.assertEqual(self.util.source_from_cache(path), expect)
+
 
 (Frozen_PEP3147Tests,
  Source_PEP3147Tests

Lib/test/test_importlib/util.py

@@ -319,6 +319,17 @@ def ensure_bytecode_path(bytecode_path):
             raise
 
 
+@contextlib.contextmanager
+def temporary_pycache_prefix(prefix):
+    """Adjust and restore sys.pycache_prefix."""
+    _orig_prefix = sys.pycache_prefix
+    sys.pycache_prefix = prefix
+    try:
+        yield
+    finally:
+        sys.pycache_prefix = _orig_prefix
+
+
 @contextlib.contextmanager
 def create_modules(*names):
     """Temporarily create each named module with an attribute (named 'attr')

Misc/NEWS.d/next/Core and Builtins/2018-05-15-10-48-47.bpo-33499.uBEc06.rst

@@ -0,0 +1,3 @@
+Add :envvar:`PYTHONPYCACHEPREFIX` environment variable and :option:`-X`
+``pycache_prefix`` command-line option to set an alternate root directory for
+writing module bytecode cache files.