diff --git a/.travis.yml b/.travis.yml index c207bd72da227c1..4e8be5e03502321 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ matrix: # (Updating the version is fine as long as no warnings are raised by doing so.) - python -m pip install sphinx~=1.6.1 blurb script: - - make check suspicious html SPHINXOPTS="-q -W -j4" + - make check suspicious html SPHINXBUILD="sphinx-build" SPHINXOPTS="-q -W -j4" - os: linux language: c compiler: gcc diff --git a/Doc/Makefile b/Doc/Makefile index 307d1e0e7de10b0..69e7e2eecea2042 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -17,7 +17,7 @@ ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_elements.papersize=$(PA .PHONY: help build html htmlhelp latex text changes linkcheck \ suspicious coverage doctest pydoc-topics htmlview clean dist check serve \ - autobuild-dev autobuild-stable venv + autobuild-dev autobuild-stable help: @echo "Please use \`make ' where is one of" @@ -39,7 +39,7 @@ help: @echo " check to run a check for frequent markup errors" @echo " serve to serve the documentation on the localhost (8000)" -build: +build: venv -mkdir -p build # Look first for a Misc/NEWS file (building from a source release tarball # or old repo) and use that, otherwise look for a Misc/NEWS.d directory @@ -122,9 +122,11 @@ clean: -rm -rf build/* $(VENVDIR)/* venv: - $(PYTHON) -m venv $(VENVDIR) - $(VENVDIR)/bin/python3 -m pip install -U Sphinx blurb - @echo "The venv has been created in the $(VENVDIR) directory" + @if [ "$(SPHINXBUILD)" == "PATH=$(VENVDIR)/bin:$$PATH sphinx-build" ]; then \ + $(PYTHON) -m venv $(VENVDIR); \ + echo "A virtual environment for Docs has been made in the $(VENVDIR) directory"; \ + $(VENVDIR)/bin/python3 -m pip install Sphinx blurb; \ + fi dist: rm -rf dist diff --git a/Doc/README.rst b/Doc/README.rst index a29d1f3a708a436..c0a8d89a8c7925b 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -21,21 +21,16 @@ tree but are maintained separately and are available from * `Sphinx `_ * `blurb `_ -The easiest way to install these tools is to create a virtual environment and -install the tools into there. +You could manually create a virtual environment and install them, but there is +a ``Makefile`` already set up to do this for you, as long as you have a working +Python 3 interpreter available. Using make ---------- -To get started on UNIX, you can create a virtual environment with the command :: - - make venv - -That will install all the tools necessary to build the documentation. Assuming -the virtual environment was created in the ``env`` directory (the default; -configurable with the VENVDIR variable), you can run the following command to -build the HTML output files:: +A Makefile has been prepared so that (on Unix), after you change into the +``Doc/`` directory you can simply run :: make html @@ -44,8 +39,17 @@ look for instances of sphinxbuild and blurb installed on your process PATH (configurable with the SPHINXBUILD and BLURB variables). On Windows, we try to emulate the Makefile as closely as possible with a -``make.bat`` file. If you need to specify the Python interpreter to use, -set the PYTHON environment variable instead. +``make.bat`` file. + +To use a Python interpreter that's not called ``python3``, use the standard +way to set Makefile variables, using e.g. :: + + make html PYTHON=python + +On Windows, set the PYTHON environment variable instead. + +To use a specific sphinx-build (something other than ``sphinx-build``), set +the SPHINXBUILD variable. Available make targets are: @@ -104,11 +108,14 @@ Available make targets are: Without make ------------ -First, install the tool dependencies from PyPI. - -Then, from the ``Doc`` directory, run :: +Install the Sphinx package and its dependencies from PyPI. In this situation, +you'll have to create a virtual environment manually, and install Sphinx into +it. Change into the ``Doc`` directory and run :: - sphinx-build -b . build/ + $ python3 -m venv venv + $ source venv/bin/activate + (venv) $ pip install Sphinx + (venv) $ sphinx-build -b . build/ where ```` is one of html, text, latex, or htmlhelp (for explanations see the make targets above). diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index dc1939db17e4c30..2f77bb359d24e55 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -7,6 +7,213 @@ Initialization, Finalization, and Threads ***************************************** +.. _pre-init-safe: + +Before Python Initialization +============================ + +In an application embedding Python, the :c:func:`Py_Initialize` function must +be called before using any other Python/C API functions; with the exception of +a few functions and the :ref:`global configuration variables +`. + +The following functions can be safely called before Python is initialized: + +* Configuration functions: + + * :c:func:`PyImport_AppendInittab` + * :c:func:`PyImport_ExtendInittab` + * :c:func:`PyInitFrozenExtensions` + * :c:func:`PyMem_SetAllocator` + * :c:func:`PyMem_SetupDebugHooks` + * :c:func:`PyObject_SetArenaAllocator` + * :c:func:`Py_SetPath` + * :c:func:`Py_SetProgramName` + * :c:func:`Py_SetPythonHome` + * :c:func:`Py_SetStandardStreamEncoding` + +* Informative functions: + + * :c:func:`PyMem_GetAllocator` + * :c:func:`PyObject_GetArenaAllocator` + * :c:func:`Py_GetBuildInfo` + * :c:func:`Py_GetCompiler` + * :c:func:`Py_GetCopyright` + * :c:func:`Py_GetPlatform` + * :c:func:`Py_GetProgramName` + * :c:func:`Py_GetVersion` + +* Utilities: + + * :c:func:`Py_DecodeLocale` + +* Memory allocators: + + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawFree` + +.. note:: + + The following functions **should not be called** before + :c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, + :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix` and + :c:func:`Py_GetProgramFullPath` and :c:func:`Py_GetPythonHome`. + + +.. _global-conf-vars: + +Global configuration variables +============================== + +Python has variables for the global configuration to control different features +and options. By default, these flags are controlled by :ref:`command line +options `. + +When a flag is set by an option, the value of the flag is the number of times +that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` +to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. + +.. c:var:: Py_BytesWarningFlag + + Issue a warning when comparing :class:`bytes` or :class:`bytearray` with + :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater + or equal to ``2``. + + Set by the :option:`-b` option. + +.. c:var:: Py_DebugFlag + + Turn on parser debugging output (for expert only, depending on compilation + options). + + Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment + variable. + +.. c:var:: Py_DontWriteBytecodeFlag + + If set to non-zero, Python won't try to write ``.pyc`` files on the + import of source modules. + + Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` + environment variable. + +.. c:var:: Py_FrozenFlag + + Suppress error messages when calculating the module search path in + :c:func:`Py_GetPath`. + + Private flag used by ``_freeze_importlib`` and ``frozenmain`` programs. + +.. c:var:: Py_HashRandomizationFlag + + Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to + a non-empty string. + + If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment + variable to initialize the secret hash seed. + +.. c:var:: Py_IgnoreEnvironmentFlag + + Ignore all :envvar:`PYTHON*` environment variables, e.g. + :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. + + Set by the :option:`-E` and :option:`-I` options. + +.. c:var:: Py_InspectFlag + + When a script is passed as first argument or the :option:`-c` option is used, + enter interactive mode after executing the script or the command, even when + :data:`sys.stdin` does not appear to be a terminal. + + Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment + variable. + +.. c:var:: Py_InteractiveFlag + + Set by the :option:`-i` option. + +.. c:var:: Py_IsolatedFlag + + Run Python in isolated mode. In isolated mode :data:`sys.path` contains + neither the script's directory nor the user's site-packages directory. + + Set by the :option:`-I` option. + + .. versionadded:: 3.4 + +.. c:var:: Py_LegacyWindowsFSEncodingFlag + + If the flag is non-zero, use the ``mbcs`` encoding instead of the UTF-8 + encoding for the filesystem encoding. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + variable is set to a non-empty string. + + See :pep:`529` for more details. + + Availability: Windows. + +.. c:var:: Py_LegacyWindowsStdioFlag + + If the flag is non-zero, use :class:`io.FileIO` instead of + :class:`WindowsConsoleIO` for :mod:`sys` standard streams. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment + variable is set to a non-empty string. + + See :pep:`528` for more details. + + Availability: Windows. + +.. c:var:: Py_NoSiteFlag + + Disable the import of the module :mod:`site` and the site-dependent + manipulations of :data:`sys.path` that it entails. Also disable these + manipulations if :mod:`site` is explicitly imported later (call + :func:`site.main` if you want them to be triggered). + + Set by the :option:`-S` option. + +.. c:var:: Py_NoUserSiteDirectory + + Don't add the :data:`user site-packages directory ` to + :data:`sys.path`. + + Set by the :option:`-s` and :option:`-I` options, and the + :envvar:`PYTHONNOUSERSITE` environment variable. + +.. c:var:: Py_OptimizeFlag + + Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment + variable. + +.. c:var:: Py_QuietFlag + + Don't display the copyright and version messages even in interactive mode. + + Set by the :option:`-q` option. + + .. versionadded:: 3.2 + +.. c:var:: Py_UnbufferedStdioFlag + + Force the stdout and stderr streams to be unbuffered. + + Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` + environment variable. + +.. c:var:: Py_VerboseFlag + + Print a message each time a module is initialized, showing the place + (filename or built-in module) from which it is loaded. If greater or equal + to ``2``, print a message for each file that is checked for when + searching for a module. Also provides information on module cleanup at exit. + + Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment + variable. + Initializing and finalizing the interpreter =========================================== @@ -27,9 +234,11 @@ Initializing and finalizing the interpreter single: PySys_SetArgvEx() single: Py_FinalizeEx() - Initialize the Python interpreter. In an application embedding Python, this - should be called before using any other Python/C API functions; with the - exception of :c:func:`Py_SetProgramName`, :c:func:`Py_SetPythonHome` and :c:func:`Py_SetPath`. This initializes + Initialize the Python interpreter. In an application embedding Python, + this should be called before using any other Python/C API functions; see + :ref:`Before Python Initialization ` for the few exceptions. + + This initializes the table of loaded modules (``sys.modules``), and creates the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes the module search path (``sys.path``). It does not set ``sys.argv``; use diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index 7cde1a0701eb8fd..ced8837d37363f1 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -285,6 +285,10 @@ the full reference. See the :func:`setup` function for a list of keyword arguments accepted by the Distribution constructor. :func:`setup` creates a Distribution instance. + .. versionchanged:: 3.7 + :class:`~distutils.core.Distribution` now raises a :exc:`TypeError` if + ``classifiers``, ``keywords`` and ``platforms`` fields are not specified + as a list. .. class:: Command diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst index 38e0202e4ac1f6f..542ad544843fdea 100644 --- a/Doc/distutils/setupscript.rst +++ b/Doc/distutils/setupscript.rst @@ -581,17 +581,19 @@ This information includes: | | description of the | | | | | package | | | +----------------------+---------------------------+-----------------+--------+ -| ``long_description`` | longer description of the | long string | \(5) | +| ``long_description`` | longer description of the | long string | \(4) | | | package | | | +----------------------+---------------------------+-----------------+--------+ -| ``download_url`` | location where the | URL | \(4) | +| ``download_url`` | location where the | URL | | | | package may be downloaded | | | +----------------------+---------------------------+-----------------+--------+ -| ``classifiers`` | a list of classifiers | list of strings | \(4) | +| ``classifiers`` | a list of classifiers | list of strings | (6)(7) | +----------------------+---------------------------+-----------------+--------+ -| ``platforms`` | a list of platforms | list of strings | | +| ``platforms`` | a list of platforms | list of strings | (6)(8) | +----------------------+---------------------------+-----------------+--------+ -| ``license`` | license for the package | short string | \(6) | +| ``keywords`` | a list of keywords | list of strings | (6)(8) | ++----------------------+---------------------------+-----------------+--------+ +| ``license`` | license for the package | short string | \(5) | +----------------------+---------------------------+-----------------+--------+ Notes: @@ -607,22 +609,30 @@ Notes: provided, distutils lists it as the author in :file:`PKG-INFO`. (4) - These fields should not be used if your package is to be compatible with Python - versions prior to 2.2.3 or 2.3. The list is available from the `PyPI website - `_. - -(5) The ``long_description`` field is used by PyPI when you are :ref:`registering ` a package, to :ref:`build its home page `. -(6) +(5) The ``license`` field is a text indicating the license covering the package where the license is not a selection from the "License" Trove classifiers. See the ``Classifier`` field. Notice that there's a ``licence`` distribution option which is deprecated but still acts as an alias for ``license``. +(6) + This field must be a list. + +(7) + The valid classifiers are listed on + `PyPI `_. + +(8) + To preserve backward compatibility, this field also accepts a string. If + you pass a comma-separated string ``'foo, bar'``, it will be converted to + ``['foo', 'bar']``, Otherwise, it will be converted to a list of one + string. + 'short string' A single line of text, not more than 200 characters. @@ -650,7 +660,7 @@ information is sometimes used to indicate sub-releases. These are 1.0.1a2 the second alpha release of the first patch version of 1.0 -``classifiers`` are specified in a Python list:: +``classifiers`` must be specified in a list:: setup(..., classifiers=[ @@ -671,6 +681,11 @@ information is sometimes used to indicate sub-releases. These are ], ) +.. versionchanged:: 3.7 + :class:`~distutils.core.setup` now raises a :exc:`TypeError` if + ``classifiers``, ``keywords`` and ``platforms`` fields are not specified + as a list. + .. _debug-setup-script: Debugging the setup script diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 7c273533aba5b53..ea1c29a397ed86e 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -40,7 +40,7 @@ A Simple Example Let's create an extension module called ``spam`` (the favorite food of Monty Python fans...) and let's say we want to create a Python interface to the C -library function :c:func:`system`. [#]_ This function takes a null-terminated +library function :c:func:`system` [#]_. This function takes a null-terminated character string as argument and returns an integer. We want this function to be callable from Python as follows:: @@ -917,7 +917,7 @@ It is also possible to :dfn:`borrow` [#]_ a reference to an object. The borrower of a reference should not call :c:func:`Py_DECREF`. The borrower must not hold on to the object longer than the owner from which it was borrowed. Using a borrowed reference after the owner has disposed of it risks using freed -memory and should be avoided completely. [#]_ +memory and should be avoided completely [#]_. The advantage of borrowing over owning a reference is that you don't need to take care of disposing of the reference on all possible paths through the code @@ -1088,7 +1088,7 @@ checking. The C function calling mechanism guarantees that the argument list passed to C functions (``args`` in the examples) is never *NULL* --- in fact it guarantees -that it is always a tuple. [#]_ +that it is always a tuple [#]_. It is a severe error to ever let a *NULL* pointer "escape" to the Python user. diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 0e36ba0aec07ae1..62fbdb87a530007 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -659,7 +659,7 @@ Fortunately, Python's cyclic-garbage collector will eventually figure out that the list is garbage and free it. In the second version of the :class:`Noddy` example, we allowed any kind of -object to be stored in the :attr:`first` or :attr:`last` attributes. [#]_ This +object to be stored in the :attr:`first` or :attr:`last` attributes [#]_. This means that :class:`Noddy` objects can participate in cycles:: >>> import noddy2 diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index e8466ee5423c686..fa8c6939408100a 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -844,7 +844,7 @@ backreferences in a RE. For example, the following RE detects doubled words in a string. :: - >>> p = re.compile(r'(\b\w+)\s+\1') + >>> p = re.compile(r'\b(\w+)\s+\1\b') >>> p.search('Paris in the the spring').group() 'the the' @@ -943,9 +943,9 @@ number of the group. There's naturally a variant that uses the group name instead of the number. This is another Python extension: ``(?P=name)`` indicates that the contents of the group called *name* should again be matched at the current point. The regular expression for finding doubled words, -``(\b\w+)\s+\1`` can also be written as ``(?P\b\w+)\s+(?P=word)``:: +``\b(\w+)\s+\1\b`` can also be written as ``\b(?P\w+)\s+(?P=word)\b``:: - >>> p = re.compile(r'(?P\b\w+)\s+(?P=word)') + >>> p = re.compile(r'\b(?P\w+)\s+(?P=word)\b') >>> p.search('Paris in the the spring').group() 'the the' diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index b2ad87b5d020572..b9735de2078d18a 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -216,9 +216,9 @@ The fix is to call the :func:`ensure_future` function or the Detect exceptions never consumed -------------------------------- -Python usually calls :func:`sys.displayhook` on unhandled exceptions. If +Python usually calls :func:`sys.excepthook` on unhandled exceptions. If :meth:`Future.set_exception` is called, but the exception is never consumed, -:func:`sys.displayhook` is not called. Instead, :ref:`a log is emitted +:func:`sys.excepthook` is not called. Instead, :ref:`a log is emitted ` when the future is deleted by the garbage collector, with the traceback where the exception was raised. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index cda829694a32774..4b0d8c048ae7d9a 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -618,6 +618,25 @@ added elements by appending to the right and popping to the left:: d.append(elem) yield s / n +A `round-robin scheduler +`_ can be implemented with +input iterators stored in a :class:`deque`. Values are yielded from the active +iterator in position zero. If that iterator is exhausted, it can be removed +with :meth:`~deque.popleft`; otherwise, it can be cycled back to the end with +the :meth:`~deque.rotate` method:: + + def roundrobin(*iterables): + "roundrobin('ABC', 'D', 'EF') --> A D E B F C" + iterators = deque(map(iter, iterables)) + while iterators: + try: + while True: + yield next(iterators[0]) + iterators.rotate(-1) + except StopIteration: + # Remove an exhausted iterator. + iterators.popleft() + The :meth:`rotate` method provides a way to implement :class:`deque` slicing and deletion. For example, a pure Python implementation of ``del d[n]`` relies on the :meth:`rotate` method to position elements to be popped:: diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 19793693b7ba68c..48ca0da6b95f3ab 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -137,6 +137,28 @@ Functions and classes provided: ``page.close()`` will be called when the :keyword:`with` block is exited. +.. _simplifying-support-for-single-optional-context-managers: + +.. function:: nullcontext(enter_result=None) + + Return a context manager that returns enter_result from ``__enter__``, but + otherwise does nothing. It is intended to be used as a stand-in for an + optional context manager, for example:: + + def process_file(file_or_path): + if isinstance(file_or_path, str): + # If string, open file + cm = open(file_or_path) + else: + # Caller is responsible for closing file + cm = nullcontext(file_or_path) + + with cm as file: + # Perform processing on the file + + .. versionadded:: 3.7 + + .. function:: suppress(*exceptions) Return a context manager that suppresses any of the specified exceptions @@ -433,24 +455,6 @@ statements to manage arbitrary resources that don't natively support the context management protocol. -Simplifying support for single optional context managers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In the specific case of a single optional context manager, :class:`ExitStack` -instances can be used as a "do nothing" context manager, allowing a context -manager to easily be omitted without affecting the overall structure of -the source code:: - - def debug_trace(details): - if __debug__: - return TraceContext(details) - # Don't do anything special with the context in release mode - return ExitStack() - - with debug_trace(): - # Suite is traced in debug mode, but runs normally otherwise - - Catching exceptions from ``__enter__`` methods ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 64aa3ac7c8aefdb..3d29ac49b9191a1 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -20,8 +20,10 @@ the Unix :program:`ftp` program and other FTP clients. A :class:`~netrc.netrc` instance or subclass instance encapsulates data from a netrc file. The initialization argument, if present, specifies the file to parse. If - no argument is given, the file :file:`.netrc` in the user's home directory will - be read. Parse errors will raise :exc:`NetrcParseError` with diagnostic + no argument is given, the file :file:`.netrc` in the user's home directory -- + as determined by :func:`os.path.expanduser` -- will be read. Otherwise, + a :exc:`FileNotFoundError` exception will be raised. + Parse errors will raise :exc:`NetrcParseError` with diagnostic information including the file name, line number, and terminating token. If no argument is specified on a POSIX system, the presence of passwords in the :file:`.netrc` file will raise a :exc:`NetrcParseError` if the file @@ -32,6 +34,10 @@ the Unix :program:`ftp` program and other FTP clients. .. versionchanged:: 3.4 Added the POSIX permission check. + .. versionchanged:: 3.7 + :func:`os.path.expanduser` is used to find the location of the + :file:`.netrc` file when *file* is not passed as argument. + .. exception:: NetrcParseError @@ -82,4 +88,3 @@ Instances of :class:`~netrc.netrc` have public instance variables: punctuation is allowed in passwords, however, note that whitespace and non-printable characters are not allowed in passwords. This is a limitation of the way the .netrc file is parsed and may be removed in the future. - diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 4d4a6161057cc5c..6094a7b871454ed 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -69,7 +69,7 @@ Scheduler Objects Schedule a new event. The *time* argument should be a numeric type compatible with the return value of the *timefunc* function passed to the constructor. Events scheduled for the same *time* will be executed in the order of their - *priority*. + *priority*. A lower number represents a higher priority. Executing the event means executing ``action(*argument, **kwargs)``. *argument* is a sequence holding the positional arguments for *action*. diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 9b3bdd5489d4a90..45bb65ff0715e33 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -429,6 +429,10 @@ Certificate handling Matching of IP addresses, when present in the subjectAltName field of the certificate, is now supported. + .. versionchanged:: 3.7 + Allow wildcard when it is the leftmost and the only character + in that segment. + .. function:: cert_time_to_seconds(cert_time) Return the time in seconds since the Epoch, given the ``cert_time`` diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 9883d8bbe865960..d28a5d548431ef3 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -146,6 +146,8 @@ See :pep:`484` for more details. ``Derived`` is expected. This is useful when you want to prevent logic errors with minimal runtime cost. +.. versionadded:: 3.5.2 + Callable -------- @@ -494,6 +496,8 @@ The module defines the following classes, functions and decorators: ``Type[Any]`` is equivalent to ``Type`` which in turn is equivalent to ``type``, which is the root of Python's metaclass hierarchy. + .. versionadded:: 3.5.2 + .. class:: Iterable(Generic[T_co]) A generic version of :class:`collections.abc.Iterable`. @@ -674,6 +678,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.defaultdict`. + .. versionadded:: 3.5.2 + .. class:: Counter(collections.Counter, Dict[T, int]) A generic version of :class:`collections.Counter`. @@ -762,6 +768,8 @@ The module defines the following classes, functions and decorators: def add_unicode_checkmark(text: Text) -> Text: return text + u' \u2713' + .. versionadded:: 3.5.2 + .. class:: io Wrapper namespace for I/O stream types. @@ -847,6 +855,8 @@ The module defines the following classes, functions and decorators: UserId = NewType('UserId', int) first_user = UserId(1) + .. versionadded:: 3.5.2 + .. function:: cast(typ, val) Cast a value to a type. @@ -1054,3 +1064,5 @@ The module defines the following classes, functions and decorators: "forward reference", to hide the ``expensive_mod`` reference from the interpreter runtime. Type annotations for local variables are not evaluated, so the second annotation does not need to be enclosed in quotes. + + .. versionadded:: 3.5.2 diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index e52f140029c57ab..4755488d91db061 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -219,6 +219,22 @@ Command-line options Stop the test run on the first error or failure. +.. cmdoption:: -k + + Only run test methods and classes that match the pattern or substring. + This option may be used multiple times, in which case all test cases that + match of the given patterns are included. + + Patterns that contain a wildcard character (``*``) are matched against the + test name using :meth:`fnmatch.fnmatchcase`; otherwise simple case-sensitive + substring matching is used. + + Patterns are matched against the fully qualified test method name as + imported by the test loader. + + For example, ``-k foo`` matches ``foo_tests.SomeTest.test_something``, + ``bar_tests.SomeTest.test_foo``, but not ``bar_tests.FooTest.test_something``. + .. cmdoption:: --locals Show local variables in tracebacks. @@ -229,6 +245,9 @@ Command-line options .. versionadded:: 3.5 The command-line option ``--locals``. +.. versionadded:: 3.7 + The command-line option ``-k``. + The command line can also be used for test discovery, for running all of the tests in a project or just a subset. @@ -1745,6 +1764,21 @@ Loading and running tests This affects all the :meth:`loadTestsFrom\*` methods. + .. attribute:: testNamePatterns + + List of Unix shell-style wildcard test name patterns that test methods + have to match to be included in test suites (see ``-v`` option). + + If this attribute is not ``None`` (the default), all test methods to be + included in test suites must match one of the patterns in this list. + Note that matches are always performed using :meth:`fnmatch.fnmatchcase`, + so unlike patterns passed to the ``-v`` option, simple substring patterns + will have to be converted using ``*`` wildcards. + + This affects all the :meth:`loadTestsFrom\*` methods. + + .. versionadded:: 3.7 + .. class:: TestResult diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 7d31932f9656e49..e3b90e673caaf0c 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -69,10 +69,10 @@ structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. .. method:: request_rate(useragent) Returns the contents of the ``Request-rate`` parameter from - ``robots.txt`` in the form of a :func:`~collections.namedtuple` - ``(requests, seconds)``. If there is no such parameter or it doesn't - apply to the *useragent* specified or the ``robots.txt`` entry for this - parameter has invalid syntax, return ``None``. + ``robots.txt`` as a :term:`named tuple` ``RequestRate(requests, seconds)``. + If there is no such parameter or it doesn't apply to the *useragent* + specified or the ``robots.txt`` entry for this parameter has invalid + syntax, return ``None``. .. versionadded:: 3.6 diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 2fa894a533f3a81..7176d819425094e 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -212,6 +212,13 @@ to each other are automatically concatenated. :: >>> 'Py' 'thon' 'Python' +This feature is particularly useful when you want to break long strings:: + + >>> text = ('Put several strings within parentheses ' + ... 'to have them joined together.') + >>> text + 'Put several strings within parentheses to have them joined together.' + This only works with two literals though, not with variables or expressions:: >>> prefix = 'Py' @@ -227,13 +234,6 @@ If you want to concatenate variables or a variable and a literal, use ``+``:: >>> prefix + 'thon' 'Python' -This feature is particularly useful when you want to break long strings:: - - >>> text = ('Put several strings within parentheses ' - ... 'to have them joined together.') - >>> text - 'Put several strings within parentheses to have them joined together.' - Strings can be *indexed* (subscripted), with the first character having index 0. There is no separate character type; a character is simply a string of size one:: diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index fc557eebe4ac3e9..d022e2cd2a0ace9 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -212,7 +212,7 @@ Miscellaneous options .. cmdoption:: -d - Turn on parser debugging output (for wizards only, depending on compilation + Turn on parser debugging output (for expert only, depending on compilation options). See also :envvar:`PYTHONDEBUG`. diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 71e8358a420b614..514c3c293c080c5 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -298,6 +298,12 @@ README.rst is now included in the list of distutils standard READMEs and therefore included in source distributions. (Contributed by Ryan Gonzalez in :issue:`11913`.) +:class:`distutils.core.setup` now raises a :exc:`TypeError` if +``classifiers``, ``keywords`` and ``platforms`` fields are not specified +as a list. However, to minimize backwards incompatibility concerns, +``keywords`` and ``platforms`` fields still accept a comma separated string. +(Contributed by Berker Peksag in :issue:`19610`.) + http.client ----------- diff --git a/Include/internal/mem.h b/Include/internal/mem.h index 471cdf45df27668..a731e30e6af7d2d 100644 --- a/Include/internal/mem.h +++ b/Include/internal/mem.h @@ -7,54 +7,6 @@ extern "C" { #include "objimpl.h" #include "pymem.h" -#ifdef WITH_PYMALLOC -#include "internal/pymalloc.h" -#endif - -/* Low-level memory runtime state */ - -struct _pymem_runtime_state { - struct _allocator_runtime_state { - PyMemAllocatorEx mem; - PyMemAllocatorEx obj; - PyMemAllocatorEx raw; - } allocators; -#ifdef WITH_PYMALLOC - /* Array of objects used to track chunks of memory (arenas). */ - struct arena_object* arenas; - /* The head of the singly-linked, NULL-terminated list of available - arena_objects. */ - struct arena_object* unused_arena_objects; - /* The head of the doubly-linked, NULL-terminated at each end, - list of arena_objects associated with arenas that have pools - available. */ - struct arena_object* usable_arenas; - /* Number of slots currently allocated in the `arenas` vector. */ - unsigned int maxarenas; - /* Number of arenas allocated that haven't been free()'d. */ - size_t narenas_currently_allocated; - /* High water mark (max value ever seen) for - * narenas_currently_allocated. */ - size_t narenas_highwater; - /* Total number of times malloc() called to allocate an arena. */ - size_t ntimes_arena_allocated; - poolp usedpools[MAX_POOLS]; - Py_ssize_t num_allocated_blocks; -#endif /* WITH_PYMALLOC */ - size_t serialno; /* incremented on each debug {m,re}alloc */ -}; - -PyAPI_FUNC(void) _PyMem_Initialize(struct _pymem_runtime_state *); - - -/* High-level memory runtime state */ - -struct _pyobj_runtime_state { - PyObjectArenaAllocator allocator_arenas; -}; - -PyAPI_FUNC(void) _PyObject_Initialize(struct _pyobj_runtime_state *); - /* GC runtime state */ diff --git a/Include/internal/pymalloc.h b/Include/internal/pymalloc.h deleted file mode 100644 index 723d9e7e671a8ee..000000000000000 --- a/Include/internal/pymalloc.h +++ /dev/null @@ -1,443 +0,0 @@ - -/* An object allocator for Python. - - Here is an introduction to the layers of the Python memory architecture, - showing where the object allocator is actually used (layer +2), It is - called for every object allocation and deallocation (PyObject_New/Del), - unless the object-specific allocators implement a proprietary allocation - scheme (ex.: ints use a simple free list). This is also the place where - the cyclic garbage collector operates selectively on container objects. - - - Object-specific allocators - _____ ______ ______ ________ - [ int ] [ dict ] [ list ] ... [ string ] Python core | -+3 | <----- Object-specific memory -----> | <-- Non-object memory --> | - _______________________________ | | - [ Python's object allocator ] | | -+2 | ####### Object memory ####### | <------ Internal buffers ------> | - ______________________________________________________________ | - [ Python's raw memory allocator (PyMem_ API) ] | -+1 | <----- Python memory (under PyMem manager's control) ------> | | - __________________________________________________________________ - [ Underlying general-purpose allocator (ex: C library malloc) ] - 0 | <------ Virtual memory allocated for the python process -------> | - - ========================================================================= - _______________________________________________________________________ - [ OS-specific Virtual Memory Manager (VMM) ] --1 | <--- Kernel dynamic storage allocation & management (page-based) ---> | - __________________________________ __________________________________ - [ ] [ ] --2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> | - -*/ -/*==========================================================================*/ - -/* A fast, special-purpose memory allocator for small blocks, to be used - on top of a general-purpose malloc -- heavily based on previous art. */ - -/* Vladimir Marangozov -- August 2000 */ - -/* - * "Memory management is where the rubber meets the road -- if we do the wrong - * thing at any level, the results will not be good. And if we don't make the - * levels work well together, we are in serious trouble." (1) - * - * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles, - * "Dynamic Storage Allocation: A Survey and Critical Review", - * in Proc. 1995 Int'l. Workshop on Memory Management, September 1995. - */ - -#ifndef Py_INTERNAL_PYMALLOC_H -#define Py_INTERNAL_PYMALLOC_H - -/* #undef WITH_MEMORY_LIMITS */ /* disable mem limit checks */ - -/*==========================================================================*/ - -/* - * Allocation strategy abstract: - * - * For small requests, the allocator sub-allocates blocks of memory. - * Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the - * system's allocator. - * - * Small requests are grouped in size classes spaced 8 bytes apart, due - * to the required valid alignment of the returned address. Requests of - * a particular size are serviced from memory pools of 4K (one VMM page). - * Pools are fragmented on demand and contain free lists of blocks of one - * particular size class. In other words, there is a fixed-size allocator - * for each size class. Free pools are shared by the different allocators - * thus minimizing the space reserved for a particular size class. - * - * This allocation strategy is a variant of what is known as "simple - * segregated storage based on array of free lists". The main drawback of - * simple segregated storage is that we might end up with lot of reserved - * memory for the different free lists, which degenerate in time. To avoid - * this, we partition each free list in pools and we share dynamically the - * reserved space between all free lists. This technique is quite efficient - * for memory intensive programs which allocate mainly small-sized blocks. - * - * For small requests we have the following table: - * - * Request in bytes Size of allocated block Size class idx - * ---------------------------------------------------------------- - * 1-8 8 0 - * 9-16 16 1 - * 17-24 24 2 - * 25-32 32 3 - * 33-40 40 4 - * 41-48 48 5 - * 49-56 56 6 - * 57-64 64 7 - * 65-72 72 8 - * ... ... ... - * 497-504 504 62 - * 505-512 512 63 - * - * 0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying - * allocator. - */ - -/*==========================================================================*/ - -/* - * -- Main tunable settings section -- - */ - -/* - * Alignment of addresses returned to the user. 8-bytes alignment works - * on most current architectures (with 32-bit or 64-bit address busses). - * The alignment value is also used for grouping small requests in size - * classes spaced ALIGNMENT bytes apart. - * - * You shouldn't change this unless you know what you are doing. - */ -#define ALIGNMENT 8 /* must be 2^N */ -#define ALIGNMENT_SHIFT 3 - -/* Return the number of bytes in size class I, as a uint. */ -#define INDEX2SIZE(I) (((unsigned int)(I) + 1) << ALIGNMENT_SHIFT) - -/* - * Max size threshold below which malloc requests are considered to be - * small enough in order to use preallocated memory pools. You can tune - * this value according to your application behaviour and memory needs. - * - * Note: a size threshold of 512 guarantees that newly created dictionaries - * will be allocated from preallocated memory pools on 64-bit. - * - * The following invariants must hold: - * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512 - * 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT - * - * Although not required, for better performance and space efficiency, - * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2. - */ -#define SMALL_REQUEST_THRESHOLD 512 -#define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT) - -#if NB_SMALL_SIZE_CLASSES > 64 -#error "NB_SMALL_SIZE_CLASSES should be less than 64" -#endif /* NB_SMALL_SIZE_CLASSES > 64 */ - -/* - * The system's VMM page size can be obtained on most unices with a - * getpagesize() call or deduced from various header files. To make - * things simpler, we assume that it is 4K, which is OK for most systems. - * It is probably better if this is the native page size, but it doesn't - * have to be. In theory, if SYSTEM_PAGE_SIZE is larger than the native page - * size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation - * violation fault. 4K is apparently OK for all the platforms that python - * currently targets. - */ -#define SYSTEM_PAGE_SIZE (4 * 1024) -#define SYSTEM_PAGE_SIZE_MASK (SYSTEM_PAGE_SIZE - 1) - -/* - * Maximum amount of memory managed by the allocator for small requests. - */ -#ifdef WITH_MEMORY_LIMITS -#ifndef SMALL_MEMORY_LIMIT -#define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MiB -- more? */ -#endif -#endif - -/* - * The allocator sub-allocates blocks of memory (called arenas) aligned - * on a page boundary. This is a reserved virtual address space for the - * current process (obtained through a malloc()/mmap() call). In no way this - * means that the memory arenas will be used entirely. A malloc() is - * usually an address range reservation for bytes, unless all pages within - * this space are referenced subsequently. So malloc'ing big blocks and not - * using them does not mean "wasting memory". It's an addressable range - * wastage... - * - * Arenas are allocated with mmap() on systems supporting anonymous memory - * mappings to reduce heap fragmentation. - */ -#define ARENA_SIZE (256 << 10) /* 256 KiB */ - -#ifdef WITH_MEMORY_LIMITS -#define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE) -#endif - -/* - * Size of the pools used for small blocks. Should be a power of 2, - * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k. - */ -#define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */ -#define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK - -/* - * -- End of tunable settings section -- - */ - -/*==========================================================================*/ - -/* - * Locking - * - * To reduce lock contention, it would probably be better to refine the - * crude function locking with per size class locking. I'm not positive - * however, whether it's worth switching to such locking policy because - * of the performance penalty it might introduce. - * - * The following macros describe the simplest (should also be the fastest) - * lock object on a particular platform and the init/fini/lock/unlock - * operations on it. The locks defined here are not expected to be recursive - * because it is assumed that they will always be called in the order: - * INIT, [LOCK, UNLOCK]*, FINI. - */ - -/* - * Python's threads are serialized, so object malloc locking is disabled. - */ -#define SIMPLELOCK_DECL(lock) /* simple lock declaration */ -#define SIMPLELOCK_INIT(lock) /* allocate (if needed) and initialize */ -#define SIMPLELOCK_FINI(lock) /* free/destroy an existing lock */ -#define SIMPLELOCK_LOCK(lock) /* acquire released lock */ -#define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */ - -/* When you say memory, my mind reasons in terms of (pointers to) blocks */ -typedef uint8_t pyblock; - -/* Pool for small blocks. */ -struct pool_header { - union { pyblock *_padding; - unsigned int count; } ref; /* number of allocated blocks */ - pyblock *freeblock; /* pool's free list head */ - struct pool_header *nextpool; /* next pool of this size class */ - struct pool_header *prevpool; /* previous pool "" */ - unsigned int arenaindex; /* index into arenas of base adr */ - unsigned int szidx; /* block size class index */ - unsigned int nextoffset; /* bytes to virgin block */ - unsigned int maxnextoffset; /* largest valid nextoffset */ -}; - -typedef struct pool_header *poolp; - -/* Record keeping for arenas. */ -struct arena_object { - /* The address of the arena, as returned by malloc. Note that 0 - * will never be returned by a successful malloc, and is used - * here to mark an arena_object that doesn't correspond to an - * allocated arena. - */ - uintptr_t address; - - /* Pool-aligned pointer to the next pool to be carved off. */ - pyblock* pool_address; - - /* The number of available pools in the arena: free pools + never- - * allocated pools. - */ - unsigned int nfreepools; - - /* The total number of pools in the arena, whether or not available. */ - unsigned int ntotalpools; - - /* Singly-linked list of available pools. */ - struct pool_header* freepools; - - /* Whenever this arena_object is not associated with an allocated - * arena, the nextarena member is used to link all unassociated - * arena_objects in the singly-linked `unused_arena_objects` list. - * The prevarena member is unused in this case. - * - * When this arena_object is associated with an allocated arena - * with at least one available pool, both members are used in the - * doubly-linked `usable_arenas` list, which is maintained in - * increasing order of `nfreepools` values. - * - * Else this arena_object is associated with an allocated arena - * all of whose pools are in use. `nextarena` and `prevarena` - * are both meaningless in this case. - */ - struct arena_object* nextarena; - struct arena_object* prevarena; -}; - -#define POOL_OVERHEAD _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT) - -#define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */ - -/* Round pointer P down to the closest pool-aligned address <= P, as a poolp */ -#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE)) - -/* Return total number of blocks in pool of size index I, as a uint. */ -#define NUMBLOCKS(I) \ - ((unsigned int)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I)) - -/*==========================================================================*/ - -/* - * This malloc lock - */ -SIMPLELOCK_DECL(_malloc_lock) -#define LOCK() SIMPLELOCK_LOCK(_malloc_lock) -#define UNLOCK() SIMPLELOCK_UNLOCK(_malloc_lock) -#define LOCK_INIT() SIMPLELOCK_INIT(_malloc_lock) -#define LOCK_FINI() SIMPLELOCK_FINI(_malloc_lock) - -/* - * Pool table -- headed, circular, doubly-linked lists of partially used pools. - -This is involved. For an index i, usedpools[i+i] is the header for a list of -all partially used pools holding small blocks with "size class idx" i. So -usedpools[0] corresponds to blocks of size 8, usedpools[2] to blocks of size -16, and so on: index 2*i <-> blocks of size (i+1)<freeblock points to -the start of a singly-linked list of free blocks within the pool. When a -block is freed, it's inserted at the front of its pool's freeblock list. Note -that the available blocks in a pool are *not* linked all together when a pool -is initialized. Instead only "the first two" (lowest addresses) blocks are -set up, returning the first such block, and setting pool->freeblock to a -one-block list holding the second such block. This is consistent with that -pymalloc strives at all levels (arena, pool, and block) never to touch a piece -of memory until it's actually needed. - -So long as a pool is in the used state, we're certain there *is* a block -available for allocating, and pool->freeblock is not NULL. If pool->freeblock -points to the end of the free list before we've carved the entire pool into -blocks, that means we simply haven't yet gotten to one of the higher-address -blocks. The offset from the pool_header to the start of "the next" virgin -block is stored in the pool_header nextoffset member, and the largest value -of nextoffset that makes sense is stored in the maxnextoffset member when a -pool is initialized. All the blocks in a pool have been passed out at least -once when and only when nextoffset > maxnextoffset. - - -Major obscurity: While the usedpools vector is declared to have poolp -entries, it doesn't really. It really contains two pointers per (conceptual) -poolp entry, the nextpool and prevpool members of a pool_header. The -excruciating initialization code below fools C so that - - usedpool[i+i] - -"acts like" a genuine poolp, but only so long as you only reference its -nextpool and prevpool members. The "- 2*sizeof(block *)" gibberish is -compensating for that a pool_header's nextpool and prevpool members -immediately follow a pool_header's first two members: - - union { block *_padding; - uint count; } ref; - block *freeblock; - -each of which consume sizeof(block *) bytes. So what usedpools[i+i] really -contains is a fudged-up pointer p such that *if* C believes it's a poolp -pointer, then p->nextpool and p->prevpool are both p (meaning that the headed -circular list is empty). - -It's unclear why the usedpools setup is so convoluted. It could be to -minimize the amount of cache required to hold this heavily-referenced table -(which only *needs* the two interpool pointer members of a pool_header). OTOH, -referencing code has to remember to "double the index" and doing so isn't -free, usedpools[0] isn't a strictly legal pointer, and we're crucially relying -on that C doesn't insert any padding anywhere in a pool_header at or before -the prevpool member. -**************************************************************************** */ - -#define MAX_POOLS (2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8) - -/*========================================================================== -Arena management. - -`arenas` is a vector of arena_objects. It contains maxarenas entries, some of -which may not be currently used (== they're arena_objects that aren't -currently associated with an allocated arena). Note that arenas proper are -separately malloc'ed. - -Prior to Python 2.5, arenas were never free()'ed. Starting with Python 2.5, -we do try to free() arenas, and use some mild heuristic strategies to increase -the likelihood that arenas eventually can be freed. - -unused_arena_objects - - This is a singly-linked list of the arena_objects that are currently not - being used (no arena is associated with them). Objects are taken off the - head of the list in new_arena(), and are pushed on the head of the list in - PyObject_Free() when the arena is empty. Key invariant: an arena_object - is on this list if and only if its .address member is 0. - -usable_arenas - - This is a doubly-linked list of the arena_objects associated with arenas - that have pools available. These pools are either waiting to be reused, - or have not been used before. The list is sorted to have the most- - allocated arenas first (ascending order based on the nfreepools member). - This means that the next allocation will come from a heavily used arena, - which gives the nearly empty arenas a chance to be returned to the system. - In my unscientific tests this dramatically improved the number of arenas - that could be freed. - -Note that an arena_object associated with an arena all of whose pools are -currently in use isn't on either list. -*/ - -/* How many arena_objects do we initially allocate? - * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4 MiB before growing the - * `arenas` vector. - */ -#define INITIAL_ARENA_OBJECTS 16 - -#endif /* Py_INTERNAL_PYMALLOC_H */ diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 67b4a516a767c96..7056e105ff7dcb4 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -64,9 +64,7 @@ typedef struct pyruntimestate { int nexitfuncs; void (*pyexitfunc)(void); - struct _pyobj_runtime_state obj; struct _gc_runtime_state gc; - struct _pymem_runtime_state mem; struct _warnings_runtime_state warnings; struct _ceval_runtime_state ceval; struct _gilstate_runtime_state gilstate; diff --git a/Include/pydebug.h b/Include/pydebug.h index d3b95966aa604e5..bd4aafe3b49f83a 100644 --- a/Include/pydebug.h +++ b/Include/pydebug.h @@ -15,7 +15,6 @@ PyAPI_DATA(int) Py_InspectFlag; PyAPI_DATA(int) Py_OptimizeFlag; PyAPI_DATA(int) Py_NoSiteFlag; PyAPI_DATA(int) Py_BytesWarningFlag; -PyAPI_DATA(int) Py_UseClassExceptionsFlag; PyAPI_DATA(int) Py_FrozenFlag; PyAPI_DATA(int) Py_IgnoreEnvironmentFlag; PyAPI_DATA(int) Py_DontWriteBytecodeFlag; diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index a75b77cc73372c5..d32c98b69856884 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -7,19 +7,7 @@ extern "C" { #endif -PyAPI_FUNC(void) Py_SetProgramName(wchar_t *); -PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); - -PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *); -PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); - #ifndef Py_LIMITED_API -/* Only used by applications that embed the interpreter and need to - * override the standard encoding determination mechanism - */ -PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding, - const char *errors); - typedef struct { const char *prefix; const char *msg; @@ -42,13 +30,34 @@ typedef struct { Don't abort() the process on such error. */ #define _Py_INIT_USER_ERR(MSG) \ (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1} +#define _Py_INIT_NO_MEMORY() _Py_INIT_USER_ERR("memory allocation failed") #define _Py_INIT_FAILED(err) \ (err.msg != NULL) +#endif + + +PyAPI_FUNC(void) Py_SetProgramName(wchar_t *); +PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); + +PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *); +PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); + +#ifndef Py_LIMITED_API +/* Only used by applications that embed the interpreter and need to + * override the standard encoding determination mechanism + */ +PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding, + const char *errors); + /* PEP 432 Multi-phase initialization API (Private while provisional!) */ PyAPI_FUNC(_PyInitError) _Py_InitializeCore(const _PyCoreConfig *); PyAPI_FUNC(int) _Py_IsCoreInitialized(void); -PyAPI_FUNC(_PyInitError) _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *); + +PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *); +PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_ReadEnv(_PyMainInterpreterConfig *); +PyAPI_FUNC(void) _PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *); + PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *); #endif @@ -93,6 +102,11 @@ PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetPath(void); +#ifdef Py_BUILD_CORE +PyAPI_FUNC(_PyInitError) _PyPathConfig_Init( + const _PyMainInterpreterConfig *main_config); +PyAPI_FUNC(void) _PyPathConfig_Fini(void); +#endif PyAPI_FUNC(void) Py_SetPath(const wchar_t *); #ifdef MS_WINDOWS int _Py_CheckPython3(); diff --git a/Include/pymem.h b/Include/pymem.h index 928851a3d70e459..57a34cf90783b6c 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -105,8 +105,14 @@ PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size); PyAPI_FUNC(void) PyMem_Free(void *ptr); #ifndef Py_LIMITED_API +/* strdup() using PyMem_RawMalloc() */ PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str); + +/* strdup() using PyMem_Malloc() */ PyAPI_FUNC(char *) _PyMem_Strdup(const char *str); + +/* wcsdup() using PyMem_RawMalloc() */ +PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const wchar_t *str); #endif /* Macros. */ diff --git a/Include/pyport.h b/Include/pyport.h index 0e82543ac783551..f2e247a374e063c 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -784,7 +784,9 @@ extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; #endif /* Py_BUILD_CORE */ #ifdef __ANDROID__ -#include +/* The Android langinfo.h header is not used. */ +#undef HAVE_LANGINFO_H +#undef CODESET #endif /* Maximum value of the Windows DWORD type */ diff --git a/Include/pystate.h b/Include/pystate.h index a3840c96cb02ed4..60d001c4926c20d 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -39,7 +39,8 @@ typedef struct { } _PyCoreConfig; #define _PyCoreConfig_INIT \ - {.ignore_environment = 0, \ + (_PyCoreConfig){\ + .ignore_environment = 0, \ .use_hash_seed = -1, \ .hash_seed = 0, \ ._disable_importlib = 0, \ @@ -59,9 +60,19 @@ typedef struct { */ typedef struct { int install_signal_handlers; + /* PYTHONPATH environment variable */ + wchar_t *module_search_path_env; + /* PYTHONHOME environment variable, see also Py_SetPythonHome(). */ + wchar_t *home; + /* Program name, see also Py_GetProgramName() */ + wchar_t *program_name; } _PyMainInterpreterConfig; -#define _PyMainInterpreterConfig_INIT {-1} +#define _PyMainInterpreterConfig_INIT \ + (_PyMainInterpreterConfig){\ + .install_signal_handlers = -1, \ + .module_search_path_env = NULL, \ + .home = NULL} typedef struct _is { diff --git a/Include/warnings.h b/Include/warnings.h index a3f83ff6967e406..25f715e3a8bb05d 100644 --- a/Include/warnings.h +++ b/Include/warnings.h @@ -7,6 +7,9 @@ extern "C" { #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) _PyWarnings_Init(void); #endif +#ifdef Py_BUILD_CORE +PyAPI_FUNC(PyObject*) _PyWarnings_InitWithConfig(const _PyCoreConfig *config); +#endif PyAPI_FUNC(int) PyErr_WarnEx( PyObject *category, diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 962cedab490eb22..c1f8a84617fce43 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -5,7 +5,7 @@ from collections import deque from functools import wraps -__all__ = ["asynccontextmanager", "contextmanager", "closing", +__all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", "AbstractContextManager", "ContextDecorator", "ExitStack", "redirect_stdout", "redirect_stderr", "suppress"] @@ -469,3 +469,24 @@ def _fix_exception_context(new_exc, old_exc): exc_details[1].__context__ = fixed_ctx raise return received_exc and suppressed_exc + + +class nullcontext(AbstractContextManager): + """Context manager that does no additional processing. + + Used as a stand-in for a normal context manager, when a particular + block of code is only sometimes used with a normal context manager: + + cm = optional_cm if condition else nullcontext() + with cm: + # Perform operation, using optional_cm if condition is True + """ + + def __init__(self, enter_result=None): + self.enter_result = enter_result + + def __enter__(self): + return self.enter_result + + def __exit__(self, *excinfo): + pass diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py index 62a24516cfafe59..78c29ede6c2c4e3 100644 --- a/Lib/distutils/dist.py +++ b/Lib/distutils/dist.py @@ -1188,12 +1188,38 @@ def get_long_description(self): def get_keywords(self): return self.keywords or [] + def set_keywords(self, value): + # If 'keywords' is a string, it will be converted to a list + # by Distribution.finalize_options(). To maintain backwards + # compatibility, do not raise an exception if 'keywords' is + # a string. + if not isinstance(value, (list, str)): + msg = "'keywords' should be a 'list', not %r" + raise TypeError(msg % type(value).__name__) + self.keywords = value + def get_platforms(self): return self.platforms or ["UNKNOWN"] + def set_platforms(self, value): + # If 'platforms' is a string, it will be converted to a list + # by Distribution.finalize_options(). To maintain backwards + # compatibility, do not raise an exception if 'platforms' is + # a string. + if not isinstance(value, (list, str)): + msg = "'platforms' should be a 'list', not %r" + raise TypeError(msg % type(value).__name__) + self.platforms = value + def get_classifiers(self): return self.classifiers or [] + def set_classifiers(self, value): + if not isinstance(value, list): + msg = "'classifiers' should be a 'list', not %r" + raise TypeError(msg % type(value).__name__) + self.classifiers = value + def get_download_url(self): return self.download_url or "UNKNOWN" diff --git a/Lib/distutils/tests/test_dist.py b/Lib/distutils/tests/test_dist.py index 1f104cef675d9a6..50b456ec9471c7f 100644 --- a/Lib/distutils/tests/test_dist.py +++ b/Lib/distutils/tests/test_dist.py @@ -195,6 +195,13 @@ def test_finalize_options(self): self.assertEqual(dist.metadata.platforms, ['one', 'two']) self.assertEqual(dist.metadata.keywords, ['one', 'two']) + attrs = {'keywords': 'foo bar', + 'platforms': 'foo bar'} + dist = Distribution(attrs=attrs) + dist.finalize_options() + self.assertEqual(dist.metadata.platforms, ['foo bar']) + self.assertEqual(dist.metadata.keywords, ['foo bar']) + def test_get_command_packages(self): dist = Distribution() self.assertEqual(dist.command_packages, None) @@ -338,9 +345,46 @@ def test_classifier(self): attrs = {'name': 'Boa', 'version': '3.0', 'classifiers': ['Programming Language :: Python :: 3']} dist = Distribution(attrs) + self.assertEqual(dist.get_classifiers(), + ['Programming Language :: Python :: 3']) meta = self.format_metadata(dist) self.assertIn('Metadata-Version: 1.1', meta) + def test_classifier_invalid_type(self): + attrs = {'name': 'Boa', 'version': '3.0', + 'classifiers': ('Programming Language :: Python :: 3',)} + msg = "'classifiers' should be a 'list', not 'tuple'" + with self.assertRaises(TypeError, msg=msg): + Distribution(attrs) + + def test_keywords(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'keywords': ['spam', 'eggs', 'life of brian']} + dist = Distribution(attrs) + self.assertEqual(dist.get_keywords(), + ['spam', 'eggs', 'life of brian']) + + def test_keywords_invalid_type(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'keywords': ('spam', 'eggs', 'life of brian')} + msg = "'keywords' should be a 'list', not 'tuple'" + with self.assertRaises(TypeError, msg=msg): + Distribution(attrs) + + def test_platforms(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'platforms': ['GNU/Linux', 'Some Evil Platform']} + dist = Distribution(attrs) + self.assertEqual(dist.get_platforms(), + ['GNU/Linux', 'Some Evil Platform']) + + def test_platforms_invalid_types(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'platforms': ('GNU/Linux', 'Some Evil Platform')} + msg = "'platforms' should be a 'list', not 'tuple'" + with self.assertRaises(TypeError, msg=msg): + Distribution(attrs) + def test_download_url(self): attrs = {'name': 'Boa', 'version': '3.0', 'download_url': 'http://example.org/boa'} diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 79eaeb7eb45bf48..447dafcc515e6c5 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -79,9 +79,6 @@ def __init__(self, master, path, *, _htest=False, _utest=False): creating ModuleBrowserTreeItem as the rootnode for the tree and subsequently in the children. """ - global file_open - if not (_htest or _utest): - file_open = pyshell.flist.open self.master = master self.path = path self._htest = _htest @@ -95,9 +92,13 @@ def close(self, event=None): def init(self): "Create browser tkinter widgets, including the tree." + global file_open root = self.master - # reset pyclbr + flist = (pyshell.flist if not (self._htest or self._utest) + else pyshell.PyShellFileList(root)) + file_open = flist.open pyclbr._modules.clear() + # create top self.top = top = ListedToplevel(root) top.protocol("WM_DELETE_WINDOW", self.close) @@ -107,6 +108,7 @@ def init(self): (root.winfo_rootx(), root.winfo_rooty() + 200)) self.settitle() top.focus_set() + # create scrolled canvas theme = idleConf.CurrentTheme() background = idleConf.GetHighlight(theme, 'normal')['background'] @@ -236,8 +238,6 @@ class Nested_in_func(TreeNode): def nested_in_class(): pass def closure(): class Nested_in_closure: pass - global file_open - file_open = pyshell.PyShellFileList(parent).open ModuleBrowser(parent, file, _htest=True) if __name__ == "__main__": diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 68450c921f2fad3..b51c45c97e50f2a 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -668,7 +668,7 @@ def open_module_browser(self, event=None): def open_path_browser(self, event=None): from idlelib import pathbrowser - pathbrowser.PathBrowser(self.flist) + pathbrowser.PathBrowser(self.root) return "break" def open_turtle_demo(self, event = None): diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index 59e03c5aab3c9e2..34eb332c1df434b 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -4,17 +4,19 @@ (Higher, because should exclude 3 lines that .coveragerc won't exclude.) """ +from collections import deque import os.path -import unittest import pyclbr +from tkinter import Tk -from idlelib import browser, filelist -from idlelib.tree import TreeNode from test.support import requires +import unittest from unittest import mock -from tkinter import Tk from idlelib.idle_test.mock_idle import Func -from collections import deque + +from idlelib import browser +from idlelib import filelist +from idlelib.tree import TreeNode class ModuleBrowserTest(unittest.TestCase): @@ -29,6 +31,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.mb.close() + cls.root.update_idletasks() cls.root.destroy() del cls.root, cls.mb @@ -38,6 +41,7 @@ def test_init(self): eq(mb.path, __file__) eq(pyclbr._modules, {}) self.assertIsInstance(mb.node, TreeNode) + self.assertIsNotNone(browser.file_open) def test_settitle(self): mb = self.mb @@ -151,10 +155,9 @@ def test_getsublist(self): self.assertEqual(sub0.name, 'f0') self.assertEqual(sub1.name, 'C0(base)') - - def test_ondoubleclick(self): + @mock.patch('idlelib.browser.file_open') + def test_ondoubleclick(self, fopen): mbt = self.mbt - fopen = browser.file_open = mock.Mock() with mock.patch('os.path.exists', return_value=False): mbt.OnDoubleClick() @@ -165,8 +168,6 @@ def test_ondoubleclick(self): fopen.assert_called() fopen.called_with(fname) - del browser.file_open - class ChildBrowserTreeItemTest(unittest.TestCase): @@ -212,14 +213,13 @@ def test_getsublist(self): eq(self.cbt_F1.GetSubList(), []) - def test_ondoubleclick(self): - fopen = browser.file_open = mock.Mock() + @mock.patch('idlelib.browser.file_open') + def test_ondoubleclick(self, fopen): goto = fopen.return_value.gotoline = mock.Mock() self.cbt_F1.OnDoubleClick() fopen.assert_called() goto.assert_called() goto.assert_called_with(self.cbt_F1.obj.lineno) - del browser.file_open # Failure test would have to raise OSError or AttributeError. diff --git a/Lib/idlelib/idle_test/test_pathbrowser.py b/Lib/idlelib/idle_test/test_pathbrowser.py index 813cbcc63167ccb..74b716a3199327a 100644 --- a/Lib/idlelib/idle_test/test_pathbrowser.py +++ b/Lib/idlelib/idle_test/test_pathbrowser.py @@ -1,11 +1,68 @@ +""" Test idlelib.pathbrowser. +""" + + +import os.path +import pyclbr # for _modules +import sys # for sys.path +from tkinter import Tk + +from test.support import requires import unittest -import os -import sys -import idlelib +from idlelib.idle_test.mock_idle import Func + +import idlelib # for __file__ +from idlelib import browser from idlelib import pathbrowser +from idlelib.tree import TreeNode + class PathBrowserTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.pb = pathbrowser.PathBrowser(cls.root, _utest=True) + + @classmethod + def tearDownClass(cls): + cls.pb.close() + cls.root.update_idletasks() + cls.root.destroy() + del cls.root, cls.pb + + def test_init(self): + pb = self.pb + eq = self.assertEqual + eq(pb.master, self.root) + eq(pyclbr._modules, {}) + self.assertIsInstance(pb.node, TreeNode) + self.assertIsNotNone(browser.file_open) + + def test_settitle(self): + pb = self.pb + self.assertEqual(pb.top.title(), 'Path Browser') + self.assertEqual(pb.top.iconname(), 'Path Browser') + + def test_rootnode(self): + pb = self.pb + rn = pb.rootnode() + self.assertIsInstance(rn, pathbrowser.PathBrowserTreeItem) + + def test_close(self): + pb = self.pb + pb.top.destroy = Func() + pb.node.destroy = Func() + pb.close() + self.assertTrue(pb.top.destroy.called) + self.assertTrue(pb.node.destroy.called) + del pb.top.destroy, pb.node.destroy + + +class DirBrowserTreeItemTest(unittest.TestCase): + def test_DirBrowserTreeItem(self): # Issue16226 - make sure that getting a sublist works d = pathbrowser.DirBrowserTreeItem('') @@ -16,6 +73,9 @@ def test_DirBrowserTreeItem(self): self.assertEqual(d.ispackagedir(dir), True) self.assertEqual(d.ispackagedir(dir + '/Icons'), False) + +class PathBrowserTreeItemTest(unittest.TestCase): + def test_PathBrowserTreeItem(self): p = pathbrowser.PathBrowserTreeItem() self.assertEqual(p.GetText(), 'sys.path') @@ -23,5 +83,6 @@ def test_PathBrowserTreeItem(self): self.assertEqual(len(sub), len(sys.path)) self.assertEqual(type(sub[0]), pathbrowser.DirBrowserTreeItem) + if __name__ == '__main__': unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/pathbrowser.py b/Lib/idlelib/pathbrowser.py index c0aa2a1590916cb..c61ae0f4fb96cc8 100644 --- a/Lib/idlelib/pathbrowser.py +++ b/Lib/idlelib/pathbrowser.py @@ -9,13 +9,14 @@ class PathBrowser(ModuleBrowser): - def __init__(self, flist, *, _htest=False, _utest=False): + def __init__(self, master, *, _htest=False, _utest=False): """ _htest - bool, change box location when running htest """ + self.master = master self._htest = _htest self._utest = _utest - self.init(flist) + self.init() def settitle(self): "Set window titles." @@ -100,8 +101,7 @@ def listmodules(self, allnames): def _path_browser(parent): # htest # - flist = PyShellFileList(parent) - PathBrowser(flist, _htest=True) + PathBrowser(parent, _htest=True) parent.mainloop() if __name__ == "__main__": diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 6d0511ebfe90cf9..a5660099af75145 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -76,7 +76,8 @@ >>> def encode_complex(obj): ... if isinstance(obj, complex): ... return [obj.real, obj.imag] - ... raise TypeError(repr(obj) + " is not JSON serializable") + ... raise TypeError(f'Object of type {obj.__class__.__name__} ' + ... f'is not JSON serializable') ... >>> json.dumps(2 + 1j, default=encode_complex) '[2.0, 1.0]' @@ -344,8 +345,8 @@ def loads(s, *, encoding=None, cls=None, object_hook=None, parse_float=None, s, 0) else: if not isinstance(s, (bytes, bytearray)): - raise TypeError('the JSON object must be str, bytes or bytearray, ' - 'not {!r}'.format(s.__class__.__name__)) + raise TypeError(f'the JSON object must be str, bytes or bytearray, ' + f'not {s.__class__.__name__}') s = s.decode(detect_encoding(s), 'surrogatepass') if (cls is None and object_hook is None and diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 41a497c5da01603..fb083ed61bb1f8b 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -176,8 +176,8 @@ def default(self, o): return JSONEncoder.default(self, o) """ - raise TypeError("Object of type '%s' is not JSON serializable" % - o.__class__.__name__) + raise TypeError(f'Object of type {o.__class__.__name__} ' + f'is not JSON serializable') def encode(self, o): """Return a JSON string representation of a Python data structure. @@ -373,7 +373,8 @@ def _iterencode_dict(dct, _current_indent_level): elif _skipkeys: continue else: - raise TypeError("key " + repr(key) + " is not a string") + raise TypeError(f'keys must be str, int, float, bool or None, ' + f'not {key.__class__.__name__}') if first: first = False else: diff --git a/Lib/netrc.py b/Lib/netrc.py index baf8f1d99d9d305..f0ae48cfed9e67e 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -23,10 +23,7 @@ class netrc: def __init__(self, file=None): default_netrc = file is None if file is None: - try: - file = os.path.join(os.environ['HOME'], ".netrc") - except KeyError: - raise OSError("Could not find .netrc: $HOME is not set") from None + file = os.path.join(os.path.expanduser("~"), ".netrc") self.hosts = {} self.macros = {} with open(file) as fp: diff --git a/Lib/ssl.py b/Lib/ssl.py index 75caae0c440566c..fa83606e7cd5a57 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -221,7 +221,7 @@ class CertificateError(ValueError): pass -def _dnsname_match(dn, hostname, max_wildcards=1): +def _dnsname_match(dn, hostname): """Matching according to RFC 6125, section 6.4.3 http://tools.ietf.org/html/rfc6125#section-6.4.3 @@ -233,7 +233,12 @@ def _dnsname_match(dn, hostname, max_wildcards=1): leftmost, *remainder = dn.split(r'.') wildcards = leftmost.count('*') - if wildcards > max_wildcards: + if wildcards == 1 and len(leftmost) > 1: + # Only match wildcard in leftmost segment. + raise CertificateError( + "wildcard can only be present in the leftmost segment: " + repr(dn)) + + if wildcards > 1: # Issue #17980: avoid denials of service by refusing more # than one wildcard per fragment. A survey of established # policy among SSL implementations showed it to be a diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9871a28dbf2d2b3..ce01c8ce586d659 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -257,12 +257,12 @@ def _list_cases(self, suite): if isinstance(test, unittest.TestSuite): self._list_cases(test) elif isinstance(test, unittest.TestCase): - if support._match_test(test): + if support.match_test(test): print(test.id()) def list_cases(self): support.verbose = False - support.match_tests = self.ns.match_tests + support.set_match_tests(self.ns.match_tests) for test in self.selected: abstest = get_abs_module(self.ns, test) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index dbd463435c781bd..12bf422c902dc1c 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -102,7 +102,7 @@ def runtest(ns, test): if use_timeout: faulthandler.dump_traceback_later(ns.timeout, exit=True) try: - support.match_tests = ns.match_tests + support.set_match_tests(ns.match_tests) # reset the environment_altered flag to detect if a test altered # the environment support.environment_altered = False diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 527cf7fbf95328c..e8648964d1e29e1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -88,6 +88,7 @@ "requires_IEEE_754", "skip_unless_xattr", "requires_zlib", "anticipate_failure", "load_package_tests", "detect_api_mismatch", "check__all__", "requires_android_level", "requires_multiprocessing_queue", + "skip_unless_bind_unix_socket", # sys "is_jython", "is_android", "check_impl_detail", "unix_shell", "setswitchinterval", @@ -278,7 +279,6 @@ def get_attribute(obj, name): # small sizes, to make sure they work.) real_max_memuse = 0 failfast = False -match_tests = None # _original_stdout is meant to hold stdout at the time regrtest began. # This may be "the real" stdout, or IDLE's emulation of stdout, or whatever. @@ -1900,21 +1900,67 @@ def _run_suite(suite): raise TestFailed(err) -def _match_test(test): - global match_tests +# By default, don't filter tests +_match_test_func = None +_match_test_patterns = None - if match_tests is None: + +def match_test(test): + # Function used by support.run_unittest() and regrtest --list-cases + if _match_test_func is None: return True - test_id = test.id() + else: + return _match_test_func(test.id()) + + +def _is_full_match_test(pattern): + # If a pattern contains at least one dot, it's considered + # as a full test identifier. + # Example: 'test.test_os.FileTests.test_access'. + # + # Reject patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, reject 'test_access*'. + return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) - for match_test in match_tests: - if fnmatch.fnmatchcase(test_id, match_test): - return True - for name in test_id.split("."): - if fnmatch.fnmatchcase(name, match_test): +def set_match_tests(patterns): + global _match_test_func, _match_test_patterns + + if patterns == _match_test_patterns: + # No change: no need to recompile patterns. + return + + if not patterns: + func = None + # set_match_tests(None) behaves as set_match_tests(()) + patterns = () + elif all(map(_is_full_match_test, patterns)): + # Simple case: all patterns are full test identifier. + # The test.bisect utility only uses such full test identifiers. + func = set(patterns).__contains__ + else: + regex = '|'.join(map(fnmatch.translate, patterns)) + # The search *is* case sensitive on purpose: + # don't use flags=re.IGNORECASE + regex_match = re.compile(regex).match + + def match_test_regex(test_id): + if regex_match(test_id): + # The regex matchs the whole identifier like + # 'test.test_os.FileTests.test_access' return True - return False + else: + # Try to match parts of the test identifier. + # For example, split 'test.test_os.FileTests.test_access' + # into: 'test', 'test_os', 'FileTests' and 'test_access'. + return any(map(regex_match, test_id.split("."))) + + func = match_test_regex + + # Create a copy since patterns can be mutable and so modified later + _match_test_patterns = tuple(patterns) + _match_test_func = func + def run_unittest(*classes): @@ -1931,7 +1977,7 @@ def run_unittest(*classes): suite.addTest(cls) else: suite.addTest(unittest.makeSuite(cls)) - _filter_suite(suite, _match_test) + _filter_suite(suite, match_test) _run_suite(suite) #======================================================================= @@ -2387,6 +2433,28 @@ def skip_unless_xattr(test): msg = "no non-broken extended attribute support" return test if ok else unittest.skip(msg)(test) +_bind_nix_socket_error = None +def skip_unless_bind_unix_socket(test): + """Decorator for tests requiring a functional bind() for unix sockets.""" + if not hasattr(socket, 'AF_UNIX'): + return unittest.skip('No UNIX Sockets')(test) + global _bind_nix_socket_error + if _bind_nix_socket_error is None: + path = TESTFN + "can_bind_unix_socket" + with socket.socket(socket.AF_UNIX) as sock: + try: + sock.bind(path) + _bind_nix_socket_error = False + except OSError as e: + _bind_nix_socket_error = e + finally: + unlink(path) + if _bind_nix_socket_error: + msg = 'Requires a functional unix bind(): %s' % _bind_nix_socket_error + return unittest.skip(msg)(test) + else: + return test + def fs_is_case_insensitive(directory): """Detects if the file system for the specified directory is case-insensitive.""" diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 1a8bc13429648f4..a6e4ecf7958df73 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -470,7 +470,7 @@ def test_sock_client_ops(self): sock = socket.socket() self._basetest_sock_recv_into(httpd, sock) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_unix_sock_client_ops(self): with test_utils.run_test_unix_server() as httpd: sock = socket.socket(socket.AF_UNIX) @@ -606,7 +606,7 @@ def test_create_connection(self): lambda: MyProto(loop=self.loop), *httpd.address) self._basetest_create_connection(conn_fut) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_create_unix_connection(self): # Issue #20682: On Mac OS X Tiger, getsockname() returns a # zero-length address for UNIX socket. @@ -736,8 +736,8 @@ def test_create_ssl_connection(self): self._test_create_ssl_connection(httpd, create_connection, peername=httpd.address) + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_ssl_unix_connection(self): # Issue #20682: On Mac OS X Tiger, getsockname() returns a # zero-length address for UNIX socket. @@ -961,7 +961,7 @@ def _make_unix_server(self, factory, **kwargs): return server, path - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_create_unix_server(self): proto = MyProto(loop=self.loop) server, path = self._make_unix_server(lambda: proto) @@ -1053,8 +1053,8 @@ def test_create_server_ssl(self): # stop serving server.close() + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_unix_server_ssl(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( @@ -1113,8 +1113,8 @@ def test_create_server_ssl_verify_failed(self): self.assertIsNone(proto.transport) server.close() + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_unix_server_ssl_verify_failed(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( @@ -1171,8 +1171,8 @@ def test_create_server_ssl_match_failed(self): proto.transport.close() server.close() + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_unix_server_ssl_verified(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( @@ -2155,6 +2155,10 @@ def tearDown(self): super().tearDown() def test_get_event_loop_new_process(self): + # Issue bpo-32126: The multiprocessing module used by + # ProcessPoolExecutor is not functional when the + # multiprocessing.synchronize module cannot be imported. + support.import_module('multiprocessing.synchronize') async def main(): pool = concurrent.futures.ProcessPoolExecutor() result = await self.loop.run_in_executor( diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 6d16d2007967d31..a1e5bd7fab6c8e1 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -9,6 +9,7 @@ import threading import unittest from unittest import mock +from test import support try: import ssl except ImportError: @@ -57,7 +58,7 @@ def test_open_connection(self): loop=self.loop) self._basetest_open_connection(conn_fut) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_open_unix_connection(self): with test_utils.run_test_unix_server() as httpd: conn_fut = asyncio.open_unix_connection(httpd.address, @@ -86,8 +87,8 @@ def test_open_connection_no_loop_ssl(self): self._basetest_open_connection_no_loop_ssl(conn_fut) + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_open_unix_connection_no_loop_ssl(self): with test_utils.run_test_unix_server(use_ssl=True) as httpd: conn_fut = asyncio.open_unix_connection( @@ -113,7 +114,7 @@ def test_open_connection_error(self): loop=self.loop) self._basetest_open_connection_error(conn_fut) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_open_unix_connection_error(self): with test_utils.run_test_unix_server() as httpd: conn_fut = asyncio.open_unix_connection(httpd.address, @@ -634,7 +635,7 @@ def client(addr): server.stop() self.assertEqual(msg, b"hello world!\n") - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_start_unix_server(self): class MyServer: diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index fe758ba5e135293..04284fa57d1962d 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -13,6 +13,7 @@ import threading import unittest from unittest import mock +from test import support if sys.platform == 'win32': raise unittest.SkipTest('UNIX only') @@ -239,6 +240,7 @@ def setUp(self): self.loop = asyncio.SelectorEventLoop() self.set_event_loop(self.loop) + @support.skip_unless_bind_unix_socket def test_create_unix_server_existing_path_sock(self): with test_utils.unix_socket_path() as path: sock = socket.socket(socket.AF_UNIX) @@ -251,6 +253,7 @@ def test_create_unix_server_existing_path_sock(self): srv.close() self.loop.run_until_complete(srv.wait_closed()) + @support.skip_unless_bind_unix_socket def test_create_unix_server_pathlib(self): with test_utils.unix_socket_path() as path: path = pathlib.Path(path) @@ -308,6 +311,7 @@ def test_create_unix_server_path_dgram(self): @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), 'no socket.SOCK_NONBLOCK (linux only)') + @support.skip_unless_bind_unix_socket def test_create_unix_server_path_stream_bittype(self): sock = socket.socket( socket.AF_UNIX, socket.SOCK_STREAM | socket.SOCK_NONBLOCK) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index bb5b2a3b9f0d734..2fe0feca5a3ca44 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -593,6 +593,16 @@ def test_forced_io_encoding(self): self.maxDiff = None self.assertEqual(out.strip(), expected_output) + def test_pre_initialization_api(self): + """ + Checks the few parts of the C-API that work before the runtine + is initialized (via Py_Initialize()). + """ + env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) + out, err = self.run_embedded_interpreter("pre_initialization_api", env=env) + self.assertEqual(out, '') + self.assertEqual(err, '') + class SkipitemTest(unittest.TestCase): diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 64b6578ff94eb30..1a5e6edad9b28f2 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -252,6 +252,16 @@ def close(self): 1 / 0 self.assertEqual(state, [1]) + +class NullcontextTestCase(unittest.TestCase): + def test_nullcontext(self): + class C: + pass + c = C() + with nullcontext(c) as c_in: + self.assertIs(c_in, c) + + class FileContextTestCase(unittest.TestCase): def testWithOpen(self): diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index 791052102140b76..eb9064edea9115d 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -93,12 +93,15 @@ def test_failures(self): def test_non_string_keys_dict(self): data = {'a' : 1, (1, 2) : 2} + with self.assertRaisesRegex(TypeError, + 'keys must be str, int, float, bool or None, not tuple'): + self.dumps(data) - #This is for c encoder - self.assertRaises(TypeError, self.dumps, data) - - #This is for python encoder - self.assertRaises(TypeError, self.dumps, data, indent=True) + def test_not_serializable(self): + import sys + with self.assertRaisesRegex(TypeError, + 'Object of type module is not JSON serializable'): + self.dumps(sys) def test_truncated_input(self): test_cases = [ diff --git a/Lib/test/test_msilib.py b/Lib/test/test_msilib.py index f656f7234e2d333..6d89ca4b77321eb 100644 --- a/Lib/test/test_msilib.py +++ b/Lib/test/test_msilib.py @@ -1,7 +1,65 @@ """ Test suite for the code in msilib """ +import os.path import unittest -from test.support import import_module +from test.support import TESTFN, import_module, unlink msilib = import_module('msilib') +import msilib.schema + + +def init_database(): + path = TESTFN + '.msi' + db = msilib.init_database( + path, + msilib.schema, + 'Python Tests', + 'product_code', + '1.0', + 'PSF', + ) + return db, path + + +class MsiDatabaseTestCase(unittest.TestCase): + + def test_view_fetch_returns_none(self): + db, db_path = init_database() + properties = [] + view = db.OpenView('SELECT Property, Value FROM Property') + view.Execute(None) + while True: + record = view.Fetch() + if record is None: + break + properties.append(record.GetString(1)) + view.Close() + db.Close() + self.assertEqual( + properties, + [ + 'ProductName', 'ProductCode', 'ProductVersion', + 'Manufacturer', 'ProductLanguage', + ] + ) + self.addCleanup(unlink, db_path) + + def test_database_open_failed(self): + with self.assertRaises(msilib.MSIError) as cm: + msilib.OpenDatabase('non-existent.msi', msilib.MSIDBOPEN_READONLY) + self.assertEqual(str(cm.exception), 'open failed') + + def test_database_create_failed(self): + db_path = os.path.join(TESTFN, 'test.msi') + with self.assertRaises(msilib.MSIError) as cm: + msilib.OpenDatabase(db_path, msilib.MSIDBOPEN_CREATE) + self.assertEqual(str(cm.exception), 'create failed') + + def test_get_property_vt_empty(self): + db, db_path = init_database() + summary = db.GetSummaryInformation(0) + self.assertIsNone(summary.GetProperty(msilib.PID_SECURITY)) + db.Close() + self.addCleanup(unlink, db_path) + class Test_make_id(unittest.TestCase): #http://msdn.microsoft.com/en-us/library/aa369212(v=vs.85).aspx diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index ca6f27d03c3afd2..f59e5371acad3ba 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,4 +1,5 @@ import netrc, os, unittest, sys, tempfile, textwrap +from unittest import mock from test import support @@ -126,8 +127,44 @@ def test_security(self): os.chmod(fn, 0o622) self.assertRaises(netrc.NetrcParseError, netrc.netrc) -def test_main(): - support.run_unittest(NetrcTestCase) + def test_file_not_found_in_home(self): + d = support.TESTFN + os.mkdir(d) + self.addCleanup(support.rmtree, d) + with support.EnvironmentVarGuard() as environ: + environ.set('HOME', d) + self.assertRaises(FileNotFoundError, netrc.netrc) + + def test_file_not_found_explicit(self): + self.assertRaises(FileNotFoundError, netrc.netrc, + file='unlikely_netrc') + + def test_home_not_set(self): + fake_home = support.TESTFN + os.mkdir(fake_home) + self.addCleanup(support.rmtree, fake_home) + fake_netrc_path = os.path.join(fake_home, '.netrc') + with open(fake_netrc_path, 'w') as f: + f.write('machine foo.domain.com login bar password pass') + os.chmod(fake_netrc_path, 0o600) + + orig_expanduser = os.path.expanduser + called = [] + + def fake_expanduser(s): + called.append(s) + with support.EnvironmentVarGuard() as environ: + environ.set('HOME', fake_home) + result = orig_expanduser(s) + return result + + with support.swap_attr(os.path, 'expanduser', fake_expanduser): + nrc = netrc.netrc() + login, account, password = nrc.authenticators('foo.domain.com') + self.assertEqual(login, 'bar') + + self.assertTrue(called) + if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py index bb780bfa004b578..8c1032b986bf61f 100644 --- a/Lib/test/test_nntplib.py +++ b/Lib/test/test_nntplib.py @@ -165,6 +165,7 @@ def check_article_resp(self, resp, article, art_num=None): # XXX this could exceptionally happen... self.assertNotIn(article.lines[-1], (b".", b".\n", b".\r\n")) + @unittest.skipIf(True, "FIXME: see bpo-32128") def test_article_head_body(self): resp, count, first, last, name = self.server.group(self.GROUP_NAME) # Try to find an available article diff --git a/Lib/test/test_pwd.py b/Lib/test/test_pwd.py index ac9cff789eea28f..c13a7c9294f983d 100644 --- a/Lib/test/test_pwd.py +++ b/Lib/test/test_pwd.py @@ -4,19 +4,11 @@ pwd = support.import_module('pwd') -def _getpwall(): - # Android does not have getpwall. - if hasattr(pwd, 'getpwall'): - return pwd.getpwall() - elif hasattr(pwd, 'getpwuid'): - return [pwd.getpwuid(0)] - else: - return [] - +@unittest.skipUnless(hasattr(pwd, 'getpwall'), 'Does not have getpwall()') class PwdTest(unittest.TestCase): def test_values(self): - entries = _getpwall() + entries = pwd.getpwall() for e in entries: self.assertEqual(len(e), 7) @@ -42,7 +34,7 @@ def test_values(self): # and check afterwards (done in test_values_extended) def test_values_extended(self): - entries = _getpwall() + entries = pwd.getpwall() entriesbyname = {} entriesbyuid = {} @@ -66,13 +58,12 @@ def test_errors(self): self.assertRaises(TypeError, pwd.getpwuid, 3.14) self.assertRaises(TypeError, pwd.getpwnam) self.assertRaises(TypeError, pwd.getpwnam, 42) - if hasattr(pwd, 'getpwall'): - self.assertRaises(TypeError, pwd.getpwall, 42) + self.assertRaises(TypeError, pwd.getpwall, 42) # try to get some errors bynames = {} byuids = {} - for (n, p, u, g, gecos, d, s) in _getpwall(): + for (n, p, u, g, gecos, d, s) in pwd.getpwall(): bynames[n] = u byuids[u] = n @@ -106,17 +97,13 @@ def test_errors(self): # loop, say), pwd.getpwuid() might still be able to find data for that # uid. Using sys.maxint may provoke the same problems, but hopefully # it will be a more repeatable failure. - # Android accepts a very large span of uids including sys.maxsize and - # -1; it raises KeyError with 1 or 2 for example. fakeuid = sys.maxsize self.assertNotIn(fakeuid, byuids) - if not support.is_android: - self.assertRaises(KeyError, pwd.getpwuid, fakeuid) + self.assertRaises(KeyError, pwd.getpwuid, fakeuid) # -1 shouldn't be a valid uid because it has a special meaning in many # uid-related functions - if not support.is_android: - self.assertRaises(KeyError, pwd.getpwuid, -1) + self.assertRaises(KeyError, pwd.getpwuid, -1) # should be out of uid_t range self.assertRaises(KeyError, pwd.getpwuid, 2**128) self.assertRaises(KeyError, pwd.getpwuid, -2**128) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 5c1a571f1b6d702..75198b70ad4ff5f 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -3,7 +3,6 @@ import threading import unittest import urllib.robotparser -from collections import namedtuple from test import support from http.server import BaseHTTPRequestHandler, HTTPServer @@ -87,6 +86,10 @@ def test_request_rate(self): self.parser.crawl_delay(agent), self.crawl_delay ) if self.request_rate: + self.assertIsInstance( + self.parser.request_rate(agent), + urllib.robotparser.RequestRate + ) self.assertEqual( self.parser.request_rate(agent).requests, self.request_rate.requests @@ -108,7 +111,7 @@ class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase): Disallow: /%7ejoe/index.html """ agent = 'figtree' - request_rate = namedtuple('req_rate', 'requests seconds')(9, 30) + request_rate = urllib.robotparser.RequestRate(9, 30) crawl_delay = 3 good = [('figtree', '/foo.html')] bad = ['/tmp', '/tmp.html', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', @@ -237,7 +240,7 @@ class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase): Request-rate: 3/15 Disallow: /cyberworld/map/ """ - request_rate = namedtuple('req_rate', 'requests seconds')(3, 15) + request_rate = urllib.robotparser.RequestRate(3, 15) crawl_delay = 1 good = ['/', '/test.html'] bad = ['/cyberworld/map/index.html'] diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index aa2429ac9827aef..c65290b945f6c98 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -512,10 +512,11 @@ def fail(cert, hostname): fail(cert, 'Xa.com') fail(cert, '.a.com') - # only match one left-most wildcard + # only match wildcards when they are the only thing + # in left-most segment cert = {'subject': ((('commonName', 'f*.com'),),)} - ok(cert, 'foo.com') - ok(cert, 'f.com') + fail(cert, 'foo.com') + fail(cert, 'f.com') fail(cert, 'bar.com') fail(cert, 'foo.a.com') fail(cert, 'bar.foo.com') @@ -552,8 +553,8 @@ def fail(cert, hostname): # are supported. idna = 'www*.pythön.org'.encode("idna").decode("ascii") cert = {'subject': ((('commonName', idna),),)} - ok(cert, 'www.pythön.org'.encode("idna").decode("ascii")) - ok(cert, 'www1.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'www.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'www1.pythön.org'.encode("idna").decode("ascii")) fail(cert, 'ftp.pythön.org'.encode("idna").decode("ascii")) fail(cert, 'pythön.org'.encode("idna").decode("ascii")) @@ -637,7 +638,7 @@ def fail(cert, hostname): # Issue #17980: avoid denials of service by refusing more than one # wildcard per fragment. cert = {'subject': ((('commonName', 'a*b.com'),),)} - ok(cert, 'axxb.com') + fail(cert, 'axxb.com') cert = {'subject': ((('commonName', 'a*b.co*'),),)} fail(cert, 'axxb.com') cert = {'subject': ((('commonName', 'a*b*.com'),),)} diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 4a577efbeb9ccb2..e06f7b8e9952b8d 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -483,6 +483,64 @@ def test_optim_args_from_interpreter_flags(self): with self.subTest(opts=opts): self.check_options(opts, 'optim_args_from_interpreter_flags') + def test_match_test(self): + class Test: + def __init__(self, test_id): + self.test_id = test_id + + def id(self): + return self.test_id + + test_access = Test('test.test_os.FileTests.test_access') + test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') + + with support.swap_attr(support, '_match_test_func', None): + # match all + support.set_match_tests([]) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match all using None + support.set_match_tests(None) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match the full test identifier + support.set_match_tests([test_access.id()]) + self.assertTrue(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # match the module name + support.set_match_tests(['test_os']) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Test '*' pattern + support.set_match_tests(['test_*']) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Test case sensitivity + support.set_match_tests(['filetests']) + self.assertFalse(support.match_test(test_access)) + support.set_match_tests(['FileTests']) + self.assertTrue(support.match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + support.set_match_tests(['*test_os.*.test_*']) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Multiple patterns + support.set_match_tests([test_access.id(), test_chdir.id()]) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + support.set_match_tests(['test_access', 'DONTMATCH']) + self.assertTrue(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 083c2aa8aab54d6..00a69158ec3ad42 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -512,60 +512,73 @@ def test_find_mac(self): self.assertEqual(mac, 0x1234567890ab) - def check_node(self, node, requires=None, network=False): + def check_node(self, node, requires=None, *, random=False): if requires and node is None: self.skipTest('requires ' + requires) hex = '%012x' % node if support.verbose >= 2: print(hex, end=' ') - if network: - # 47 bit will never be set in IEEE 802 addresses obtained - # from network cards. - self.assertFalse(node & 0x010000000000, hex) + # The MAC address will be universally administered (i.e. the second + # least significant bit of the first octet must be unset) for any + # physical interface, such as an ethernet port or wireless adapter. + # There are some cases where this won't be the case. Randomly + # generated MACs may not be universally administered, but they must + # have their multicast bit set, though this is tested in the + # `test_random_getnode()` method specifically. Another case is the + # Travis-CI case, which apparently only has locally administered MAC + # addresses. + if not random and not os.getenv('TRAVIS'): + self.assertFalse(node & (1 << 41), '%012x' % node) self.assertTrue(0 < node < (1 << 48), "%s is not an RFC 4122 node ID" % hex) @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(os.getenv('TRAVIS'), + 'Travis-CI has no universally administered MAC addresses') def test_ifconfig_getnode(self): node = self.uuid._ifconfig_getnode() - self.check_node(node, 'ifconfig', True) + self.check_node(node, 'ifconfig') @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipIf(os.getenv('TRAVIS'), + 'Travis-CI has no universally administered MAC addresses') def test_ip_getnode(self): node = self.uuid._ip_getnode() - self.check_node(node, 'ip', True) + self.check_node(node, 'ip') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_arp_getnode(self): node = self.uuid._arp_getnode() - self.check_node(node, 'arp', True) + self.check_node(node, 'arp') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_lanscan_getnode(self): node = self.uuid._lanscan_getnode() - self.check_node(node, 'lanscan', True) + self.check_node(node, 'lanscan') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_netstat_getnode(self): node = self.uuid._netstat_getnode() - self.check_node(node, 'netstat', True) + self.check_node(node, 'netstat') @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_ipconfig_getnode(self): node = self.uuid._ipconfig_getnode() - self.check_node(node, 'ipconfig', True) + self.check_node(node, 'ipconfig') @unittest.skipUnless(importable('win32wnet'), 'requires win32wnet') @unittest.skipUnless(importable('netbios'), 'requires netbios') def test_netbios_getnode(self): node = self.uuid._netbios_getnode() - self.check_node(node, network=True) + self.check_node(node) def test_random_getnode(self): node = self.uuid._random_getnode() - # Least significant bit of first octet must be set. - self.assertTrue(node & 0x010000000000, '%012x' % node) - self.check_node(node) + # The multicast bit, i.e. the least significant bit of first octet, + # must be set for randomly generated MAC addresses. See RFC 4122, + # $4.1.6. + self.assertTrue(node & (1 << 40), '%012x' % node) + self.check_node(node, random=True) @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_unix_getnode(self): diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index e860debc0f39204..eb03b4ab87707a1 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -8,7 +8,7 @@ import functools import warnings -from fnmatch import fnmatch +from fnmatch import fnmatch, fnmatchcase from . import case, suite, util @@ -70,6 +70,7 @@ class TestLoader(object): """ testMethodPrefix = 'test' sortTestMethodsUsing = staticmethod(util.three_way_cmp) + testNamePatterns = None suiteClass = suite.TestSuite _top_level_dir = None @@ -222,11 +223,15 @@ def loadTestsFromNames(self, names, module=None): def getTestCaseNames(self, testCaseClass): """Return a sorted sequence of method names found within testCaseClass """ - def isTestMethod(attrname, testCaseClass=testCaseClass, - prefix=self.testMethodPrefix): - return attrname.startswith(prefix) and \ - callable(getattr(testCaseClass, attrname)) - testFnNames = list(filter(isTestMethod, dir(testCaseClass))) + def shouldIncludeMethod(attrname): + testFunc = getattr(testCaseClass, attrname) + isTestMethod = attrname.startswith(self.testMethodPrefix) and callable(testFunc) + if not isTestMethod: + return False + fullName = '%s.%s' % (testCaseClass.__module__, testFunc.__qualname__) + return self.testNamePatterns is None or \ + any(fnmatchcase(fullName, pattern) for pattern in self.testNamePatterns) + testFnNames = list(filter(shouldIncludeMethod, dir(testCaseClass))) if self.sortTestMethodsUsing: testFnNames.sort(key=functools.cmp_to_key(self.sortTestMethodsUsing)) return testFnNames @@ -486,16 +491,17 @@ def _find_test_path(self, full_path, pattern, namespace=False): defaultTestLoader = TestLoader() -def _makeLoader(prefix, sortUsing, suiteClass=None): +def _makeLoader(prefix, sortUsing, suiteClass=None, testNamePatterns=None): loader = TestLoader() loader.sortTestMethodsUsing = sortUsing loader.testMethodPrefix = prefix + loader.testNamePatterns = testNamePatterns if suiteClass: loader.suiteClass = suiteClass return loader -def getTestCaseNames(testCaseClass, prefix, sortUsing=util.three_way_cmp): - return _makeLoader(prefix, sortUsing).getTestCaseNames(testCaseClass) +def getTestCaseNames(testCaseClass, prefix, sortUsing=util.three_way_cmp, testNamePatterns=None): + return _makeLoader(prefix, sortUsing, testNamePatterns=testNamePatterns).getTestCaseNames(testCaseClass) def makeSuite(testCaseClass, prefix='test', sortUsing=util.three_way_cmp, suiteClass=suite.TestSuite): diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 807604f08dfd141..e62469aa2a170f2 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -46,6 +46,12 @@ def _convert_names(names): return [_convert_name(name) for name in names] +def _convert_select_pattern(pattern): + if not '*' in pattern: + pattern = '*%s*' % pattern + return pattern + + class TestProgram(object): """A command-line program that runs a set of tests; this is primarily for making test modules conveniently executable. @@ -53,7 +59,7 @@ class TestProgram(object): # defaults for testing module=None verbosity = 1 - failfast = catchbreak = buffer = progName = warnings = None + failfast = catchbreak = buffer = progName = warnings = testNamePatterns = None _discovery_parser = None def __init__(self, module='__main__', defaultTest=None, argv=None, @@ -140,8 +146,13 @@ def parseArgs(self, argv): self.testNames = list(self.defaultTest) self.createTests() - def createTests(self): - if self.testNames is None: + def createTests(self, from_discovery=False, Loader=None): + if self.testNamePatterns: + self.testLoader.testNamePatterns = self.testNamePatterns + if from_discovery: + loader = self.testLoader if Loader is None else Loader() + self.test = loader.discover(self.start, self.pattern, self.top) + elif self.testNames is None: self.test = self.testLoader.loadTestsFromModule(self.module) else: self.test = self.testLoader.loadTestsFromNames(self.testNames, @@ -179,6 +190,11 @@ def _getParentArgParser(self): action='store_true', help='Buffer stdout and stderr during tests') self.buffer = False + if self.testNamePatterns is None: + parser.add_argument('-k', dest='testNamePatterns', + action='append', type=_convert_select_pattern, + help='Only run tests which match the given substring') + self.testNamePatterns = [] return parser @@ -225,8 +241,7 @@ def _do_discovery(self, argv, Loader=None): self._initArgParsers() self._discovery_parser.parse_args(argv, self) - loader = self.testLoader if Loader is None else Loader() - self.test = loader.discover(self.start, self.pattern, self.top) + self.createTests(from_discovery=True, Loader=Loader) def runTests(self): if self.catchbreak: diff --git a/Lib/unittest/test/test_loader.py b/Lib/unittest/test/test_loader.py index 1131a755eaa3f2b..15b01863f514545 100644 --- a/Lib/unittest/test/test_loader.py +++ b/Lib/unittest/test/test_loader.py @@ -1226,6 +1226,33 @@ def test_3(self): pass names = ['test_1', 'test_2', 'test_3'] self.assertEqual(loader.getTestCaseNames(TestC), names) + # "Return a sorted sequence of method names found within testCaseClass" + # + # If TestLoader.testNamePatterns is set, only tests that match one of these + # patterns should be included. + def test_getTestCaseNames__testNamePatterns(self): + class MyTest(unittest.TestCase): + def test_1(self): pass + def test_2(self): pass + def foobar(self): pass + + loader = unittest.TestLoader() + + loader.testNamePatterns = [] + self.assertEqual(loader.getTestCaseNames(MyTest), []) + + loader.testNamePatterns = ['*1'] + self.assertEqual(loader.getTestCaseNames(MyTest), ['test_1']) + + loader.testNamePatterns = ['*1', '*2'] + self.assertEqual(loader.getTestCaseNames(MyTest), ['test_1', 'test_2']) + + loader.testNamePatterns = ['*My*'] + self.assertEqual(loader.getTestCaseNames(MyTest), ['test_1', 'test_2']) + + loader.testNamePatterns = ['*my*'] + self.assertEqual(loader.getTestCaseNames(MyTest), []) + ################################################################ ### /Tests for TestLoader.getTestCaseNames() diff --git a/Lib/unittest/test/test_program.py b/Lib/unittest/test/test_program.py index 1cfc17959e074aa..4a62ae1b11306ec 100644 --- a/Lib/unittest/test/test_program.py +++ b/Lib/unittest/test/test_program.py @@ -2,6 +2,7 @@ import os import sys +import subprocess from test import support import unittest import unittest.test @@ -409,6 +410,33 @@ def testParseArgsAbsolutePathsThatCannotBeConverted(self): # for invalid filenames should we raise a useful error rather than # leaving the current error message (import of filename fails) in place? + def testParseArgsSelectedTestNames(self): + program = self.program + argv = ['progname', '-k', 'foo', '-k', 'bar', '-k', '*pat*'] + + program.createTests = lambda: None + program.parseArgs(argv) + + self.assertEqual(program.testNamePatterns, ['*foo*', '*bar*', '*pat*']) + + def testSelectedTestNamesFunctionalTest(self): + def run_unittest(args): + p = subprocess.Popen([sys.executable, '-m', 'unittest'] + args, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, cwd=os.path.dirname(__file__)) + with p: + _, stderr = p.communicate() + return stderr.decode() + + t = '_test_warnings' + self.assertIn('Ran 7 tests', run_unittest([t])) + self.assertIn('Ran 7 tests', run_unittest(['-k', 'TestWarnings', t])) + self.assertIn('Ran 7 tests', run_unittest(['discover', '-p', '*_test*', '-k', 'TestWarnings'])) + self.assertIn('Ran 2 tests', run_unittest(['-k', 'f', t])) + self.assertIn('Ran 7 tests', run_unittest(['-k', 't', t])) + self.assertIn('Ran 3 tests', run_unittest(['-k', '*t', t])) + self.assertIn('Ran 7 tests', run_unittest(['-k', '*test_warnings.*Warning*', t])) + self.assertIn('Ran 1 test', run_unittest(['-k', '*test_warnings.*warning*', t])) + if __name__ == '__main__': unittest.main() diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 9dab4c1c3a8880c..daac29c68dc36d2 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -16,6 +16,9 @@ __all__ = ["RobotFileParser"] +RequestRate = collections.namedtuple("RequestRate", "requests seconds") + + class RobotFileParser: """ This class provides a set of methods to read, parse and answer questions about a single robots.txt file. @@ -136,11 +139,7 @@ def parse(self, lines): # check if all values are sane if (len(numbers) == 2 and numbers[0].strip().isdigit() and numbers[1].strip().isdigit()): - req_rate = collections.namedtuple('req_rate', - 'requests seconds') - entry.req_rate = req_rate - entry.req_rate.requests = int(numbers[0]) - entry.req_rate.seconds = int(numbers[1]) + entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) state = 2 if state == 2: self._add_entry(entry) diff --git a/Lib/uuid.py b/Lib/uuid.py index 020c6e73c863d47..9e7c672f7a0a6f4 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -342,11 +342,29 @@ def _popen(command, *args): env=env) return proc +# For MAC (a.k.a. IEEE 802, or EUI-48) addresses, the second least significant +# bit of the first octet signifies whether the MAC address is universally (0) +# or locally (1) administered. Network cards from hardware manufacturers will +# always be universally administered to guarantee global uniqueness of the MAC +# address, but any particular machine may have other interfaces which are +# locally administered. An example of the latter is the bridge interface to +# the Touch Bar on MacBook Pros. +# +# This bit works out to be the 42nd bit counting from 1 being the least +# significant, or 1<<41. We'll skip over any locally administered MAC +# addresses, as it makes no sense to use those in UUID calculation. +# +# See https://en.wikipedia.org/wiki/MAC_address#Universal_vs._local + +def _is_universal(mac): + return not (mac & (1 << 41)) + def _find_mac(command, args, hw_identifiers, get_index): + first_local_mac = None try: proc = _popen(command, *args.split()) if not proc: - return + return None with proc: for line in proc.stdout: words = line.lower().rstrip().split() @@ -355,8 +373,9 @@ def _find_mac(command, args, hw_identifiers, get_index): try: word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) - if mac: + if _is_universal(mac): return mac + first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by # VPNs, do not have a colon-delimited MAC address @@ -366,6 +385,7 @@ def _find_mac(command, args, hw_identifiers, get_index): pass except OSError: pass + return first_local_mac or None def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" @@ -375,6 +395,7 @@ def _ifconfig_getnode(): mac = _find_mac('ifconfig', args, keywords, lambda i: i+1) if mac: return mac + return None def _ip_getnode(): """Get the hardware address on Unix by running ip.""" @@ -382,6 +403,7 @@ def _ip_getnode(): mac = _find_mac('ip', 'link list', [b'link/ether'], lambda i: i+1) if mac: return mac + return None def _arp_getnode(): """Get the hardware address on Unix by running arp.""" @@ -404,8 +426,10 @@ def _arp_getnode(): # This works on Linux, FreeBSD and NetBSD mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) + # Return None instead of 0. if mac: return mac + return None def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" @@ -415,32 +439,36 @@ def _lanscan_getnode(): def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" # This might work on AIX, Tru64 UNIX. + first_local_mac = None try: proc = _popen('netstat', '-ia') if not proc: - return + return None with proc: words = proc.stdout.readline().rstrip().split() try: i = words.index(b'Address') except ValueError: - return + return None for line in proc.stdout: try: words = line.rstrip().split() word = words[i] if len(word) == 17 and word.count(b':') == 5: mac = int(word.replace(b':', b''), 16) - if mac: + if _is_universal(mac): return mac + first_local_mac = first_local_mac or mac except (ValueError, IndexError): pass except OSError: pass + return first_local_mac or None def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" import os, re + first_local_mac = None dirs = ['', r'c:\windows\system32', r'c:\winnt\system32'] try: import ctypes @@ -458,18 +486,23 @@ def _ipconfig_getnode(): for line in pipe: value = line.split(':')[-1].strip().lower() if re.match('([0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): - return int(value.replace('-', ''), 16) + mac = int(value.replace('-', ''), 16) + if _is_universal(mac): + return mac + first_local_mac = first_local_mac or mac + return first_local_mac or None def _netbios_getnode(): """Get the hardware address on Windows using NetBIOS calls. See http://support.microsoft.com/kb/118623 for details.""" import win32wnet, netbios + first_local_mac = None ncb = netbios.NCB() ncb.Command = netbios.NCBENUM ncb.Buffer = adapters = netbios.LANA_ENUM() adapters._pack() if win32wnet.Netbios(ncb) != 0: - return + return None adapters._unpack() for i in range(adapters.length): ncb.Reset() @@ -488,7 +521,11 @@ def _netbios_getnode(): bytes = status.adapter_address[:6] if len(bytes) != 6: continue - return int.from_bytes(bytes, 'big') + mac = int.from_bytes(bytes, 'big') + if _is_universal(mac): + return mac + first_local_mac = first_local_mac or mac + return first_local_mac or None _generate_time_safe = _UuidCreate = None @@ -601,9 +638,19 @@ def _windll_getnode(): return UUID(bytes=bytes_(_buffer.raw)).node def _random_getnode(): - """Get a random node ID, with eighth bit set as suggested by RFC 4122.""" + """Get a random node ID.""" + # RFC 4122, $4.1.6 says "For systems with no IEEE address, a randomly or + # pseudo-randomly generated value may be used; see Section 4.5. The + # multicast bit must be set in such addresses, in order that they will + # never conflict with addresses obtained from network cards." + # + # The "multicast bit" of a MAC address is defined to be "the least + # significant bit of the first octet". This works out to be the 41st bit + # counting from 1 being the least significant bit, or 1<<40. + # + # See https://en.wikipedia.org/wiki/MAC_address#Unicast_vs._multicast import random - return random.getrandbits(48) | 0x010000000000 + return random.getrandbits(48) | (1 << 40) _node = None @@ -626,13 +673,14 @@ def getnode(): getters = [_unix_getnode, _ifconfig_getnode, _ip_getnode, _arp_getnode, _lanscan_getnode, _netstat_getnode] - for getter in getters + [_random_getnode]: + for getter in getters: try: _node = getter() except: continue if _node is not None: return _node + return _random_getnode() _last_timestamp = None diff --git a/Lib/warnings.py b/Lib/warnings.py index b2605f84aec0f53..5b62569c977350d 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -128,7 +128,6 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ - import re assert action in ("error", "ignore", "always", "default", "module", "once"), "invalid action: %r" % (action,) assert isinstance(message, str), "message must be a string" @@ -137,8 +136,20 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, assert isinstance(module, str), "module must be a string" assert isinstance(lineno, int) and lineno >= 0, \ "lineno must be an int >= 0" - _add_filter(action, re.compile(message, re.I), category, - re.compile(module), lineno, append=append) + + if message or module: + import re + + if message: + message = re.compile(message, re.I) + else: + message = None + if module: + module = re.compile(module) + else: + module = None + + _add_filter(action, message, category, module, lineno, append=append) def simplefilter(action, category=Warning, lineno=0, append=False): """Insert a simple entry into the list of warnings filters (at the front). diff --git a/Makefile.pre.in b/Makefile.pre.in index 566afba2538ac05..d196d5f838e888e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1015,7 +1015,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/ceval.h \ $(srcdir)/Include/internal/gil.h \ $(srcdir)/Include/internal/mem.h \ - $(srcdir)/Include/internal/pymalloc.h \ $(srcdir)/Include/internal/pystate.h \ $(srcdir)/Include/internal/warnings.h \ $(DTRACE_HEADERS) diff --git a/Misc/ACKS b/Misc/ACKS index 05113903f06de7a..54d8d62b633f706 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1031,6 +1031,7 @@ Bill van Melle Lucas Prado Melo Ezio Melotti Doug Mennella +Dimitri Merejkowsky Brian Merrell Alexis Métaireau Luke Mewburn @@ -1466,6 +1467,7 @@ Nathan Paul Simons Guilherme Simões Adam Simpkins Ravi Sinha +Mandeep Singh Janne Sinkkonen Ng Pheng Siong Yann Sionneau diff --git a/Misc/NEWS.d/3.5.2rc1.rst b/Misc/NEWS.d/3.5.2rc1.rst index c33dcb2309f636d..caed84a06f78d53 100644 --- a/Misc/NEWS.d/3.5.2rc1.rst +++ b/Misc/NEWS.d/3.5.2rc1.rst @@ -420,7 +420,7 @@ patch by ingrid. .. section: Library A new version of typing.py provides several new classes and features: -@overload outside stubs, Reversible, DefaultDict, Text, ContextManager, +@overload outside stubs, DefaultDict, Text, ContextManager, Type[], NewType(), TYPE_CHECKING, and numerous bug fixes (note that some of the new features are not yet implemented in mypy or other static analyzers). Also classes for PEP 492 (Awaitable, AsyncIterable, AsyncIterator) have been diff --git a/Misc/NEWS.d/next/Build/2017-11-18-11-19-28.bpo-32059.a0Hxgp.rst b/Misc/NEWS.d/next/Build/2017-11-18-11-19-28.bpo-32059.a0Hxgp.rst new file mode 100644 index 000000000000000..a071bd8b3250483 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-18-11-19-28.bpo-32059.a0Hxgp.rst @@ -0,0 +1,2 @@ +``detect_modules()`` in ``setup.py`` now also searches the sysroot paths +when cross-compiling. diff --git a/Misc/NEWS.d/next/Build/2017-11-21-16-56-24.bpo-29040.14lCSr.rst b/Misc/NEWS.d/next/Build/2017-11-21-16-56-24.bpo-29040.14lCSr.rst new file mode 100644 index 000000000000000..60f05db8d4f0438 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-21-16-56-24.bpo-29040.14lCSr.rst @@ -0,0 +1,2 @@ +Support building Android with Unified Headers. The first NDK release to +support Unified Headers is android-ndk-r14. diff --git a/Misc/NEWS.d/next/Build/2017-11-21-17-12-24.bpo-28762.R6uu8w.rst b/Misc/NEWS.d/next/Build/2017-11-21-17-12-24.bpo-28762.R6uu8w.rst new file mode 100644 index 000000000000000..0c57e33c0a5221b --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-21-17-12-24.bpo-28762.R6uu8w.rst @@ -0,0 +1 @@ +Revert the last commit, the F_LOCK macro is defined by Android Unified Headers. diff --git a/Misc/NEWS.d/next/Build/2017-11-21-17-27-59.bpo-28538.DsNBS7.rst b/Misc/NEWS.d/next/Build/2017-11-21-17-27-59.bpo-28538.DsNBS7.rst new file mode 100644 index 000000000000000..db435b008f724c6 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-21-17-27-59.bpo-28538.DsNBS7.rst @@ -0,0 +1,2 @@ +Revert the previous changes, the if_nameindex structure is defined by +Unified Headers. diff --git a/Misc/NEWS.d/next/C API/2017-11-24-21-25-43.bpo-32125.K8zWgn.rst b/Misc/NEWS.d/next/C API/2017-11-24-21-25-43.bpo-32125.K8zWgn.rst new file mode 100644 index 000000000000000..d71c66415d3e00c --- /dev/null +++ b/Misc/NEWS.d/next/C API/2017-11-24-21-25-43.bpo-32125.K8zWgn.rst @@ -0,0 +1,2 @@ +The ``Py_UseClassExceptionsFlag`` flag has been removed. It was deprecated +and wasn't used anymore since Python 2.0. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-24-01-13-58.bpo-32096.CQTHXJ.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-24-01-13-58.bpo-32096.CQTHXJ.rst new file mode 100644 index 000000000000000..d2a770b9375fa0b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-24-01-13-58.bpo-32096.CQTHXJ.rst @@ -0,0 +1,4 @@ +Revert memory allocator changes in the C API: move structures back from +_PyRuntime to Objects/obmalloc.c. The memory allocators are once again initialized +statically, and so PyMem_RawMalloc() and Py_DecodeLocale() can be +called before _PyRuntime_Initialize(). diff --git a/Misc/NEWS.d/next/IDLE/2017-11-21-08-26-08.bpo-32100.P43qx2.rst b/Misc/NEWS.d/next/IDLE/2017-11-21-08-26-08.bpo-32100.P43qx2.rst new file mode 100644 index 000000000000000..c5ee6736a845655 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-11-21-08-26-08.bpo-32100.P43qx2.rst @@ -0,0 +1,2 @@ +IDLE: Fix old and new bugs in pathbrowser; improve tests. +Patch mostly by Cheryl Sabella. diff --git a/Misc/NEWS.d/next/Library/2017-11-22-12-54-46.bpo-28684.NLiDKZ.rst b/Misc/NEWS.d/next/Library/2017-11-22-12-54-46.bpo-28684.NLiDKZ.rst new file mode 100644 index 000000000000000..9d8e4da822f3773 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-22-12-54-46.bpo-28684.NLiDKZ.rst @@ -0,0 +1,5 @@ +The new test.support.skip_unless_bind_unix_socket() decorator is used here to +skip asyncio tests that fail because the platform lacks a functional bind() +function for unix domain sockets (as it is the case for non root users on the +recent Android versions that run now SELinux in enforcing mode). + diff --git a/Misc/NEWS.d/next/Library/2017-11-22-17-21-01.bpo-10049.ttsBqb.rst b/Misc/NEWS.d/next/Library/2017-11-22-17-21-01.bpo-10049.ttsBqb.rst new file mode 100644 index 000000000000000..b6153c235d0cc48 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-22-17-21-01.bpo-10049.ttsBqb.rst @@ -0,0 +1,3 @@ +Added *nullcontext* no-op context manager to contextlib. This provides a +simpler and faster alternative to ExitStack() when handling optional context +managers. diff --git a/Misc/NEWS.d/next/Library/2017-11-22-19-52-17.bpo-32071.4WNhUH.rst b/Misc/NEWS.d/next/Library/2017-11-22-19-52-17.bpo-32071.4WNhUH.rst new file mode 100644 index 000000000000000..2f0f54041cbf6af --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-22-19-52-17.bpo-32071.4WNhUH.rst @@ -0,0 +1,2 @@ +Added the ``-k`` command-line option to ``python -m unittest`` to run only +tests that match the given pattern(s). diff --git a/Misc/NEWS.d/next/Library/2017-11-23-16-15-55.bpo-19610.Dlca2P.rst b/Misc/NEWS.d/next/Library/2017-11-23-16-15-55.bpo-19610.Dlca2P.rst new file mode 100644 index 000000000000000..5ea87a45722c0ef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-23-16-15-55.bpo-19610.Dlca2P.rst @@ -0,0 +1,5 @@ +``setup()`` now raises :exc:`TypeError` for invalid types. + +The ``distutils.dist.Distribution`` class now explicitly raises an exception +when ``classifiers``, ``keywords`` and ``platforms`` fields are not +specified as a list. diff --git a/Misc/NEWS.d/next/Library/2017-11-23-21-47-36.bpo-12382.xWT9k0.rst b/Misc/NEWS.d/next/Library/2017-11-23-21-47-36.bpo-12382.xWT9k0.rst new file mode 100644 index 000000000000000..d9b54259cc6ce30 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-23-21-47-36.bpo-12382.xWT9k0.rst @@ -0,0 +1,2 @@ +:func:`msilib.OpenDatabase` now raises a better exception message when it +couldn't open or create an MSI file. Initial patch by William Tisäter. diff --git a/Misc/NEWS.d/next/Library/2017-11-23-22-12-11.bpo-31325.8jAUxN.rst b/Misc/NEWS.d/next/Library/2017-11-23-22-12-11.bpo-31325.8jAUxN.rst new file mode 100644 index 000000000000000..89a193c9ef5b8b7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-23-22-12-11.bpo-31325.8jAUxN.rst @@ -0,0 +1,5 @@ +Fix wrong usage of :func:`collections.namedtuple` in +the :meth:`RobotFileParser.parse() ` +method. + +Initial patch by Robin Wellner. diff --git a/Misc/NEWS.d/next/Library/2017-11-24-11-50-41.bpo-28334.3gGGlt.rst b/Misc/NEWS.d/next/Library/2017-11-24-11-50-41.bpo-28334.3gGGlt.rst new file mode 100644 index 000000000000000..074036b1d31e369 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-24-11-50-41.bpo-28334.3gGGlt.rst @@ -0,0 +1,3 @@ +Use :func:`os.path.expanduser` to find the ``~/.netrc`` file in +:class:`netrc.netrc`. If it does not exist, :exc:`FileNotFoundError` +is raised. Patch by Dimitri Merejkowsky. diff --git a/Misc/NEWS.d/next/Library/2017-11-24-14-07-55.bpo-12239.Nj3A0x.rst b/Misc/NEWS.d/next/Library/2017-11-24-14-07-55.bpo-12239.Nj3A0x.rst new file mode 100644 index 000000000000000..20d94779e4453aa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-24-14-07-55.bpo-12239.Nj3A0x.rst @@ -0,0 +1,2 @@ +Make :meth:`msilib.SummaryInformation.GetProperty` return ``None`` when the +value of property is ``VT_EMPTY``. Initial patch by Mark Mc Mahon. diff --git a/Misc/NEWS.d/next/Library/2017-11-26-17-00-52.bpo-23033.YGXRWT.rst b/Misc/NEWS.d/next/Library/2017-11-26-17-00-52.bpo-23033.YGXRWT.rst new file mode 100644 index 000000000000000..cecc10aebb99124 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-26-17-00-52.bpo-23033.YGXRWT.rst @@ -0,0 +1,3 @@ +Wildcard is now supported in hostname when it is one and only character in +the left most segment of hostname in second argument of +:meth:`ssl.match_hostname`. Patch by Mandeep Singh. diff --git a/Misc/NEWS.d/next/Tests/2017-11-24-18-15-12.bpo-32126.PLmNLn.rst b/Misc/NEWS.d/next/Tests/2017-11-24-18-15-12.bpo-32126.PLmNLn.rst new file mode 100644 index 000000000000000..b5ba9d574e35b49 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-11-24-18-15-12.bpo-32126.PLmNLn.rst @@ -0,0 +1,2 @@ +Skip test_get_event_loop_new_process in test.test_asyncio.test_events when +sem_open() is not functional. diff --git a/Misc/NEWS.d/next/Windows/2017-11-19-09-46-27.bpo-1102.NY-g1F.rst b/Misc/NEWS.d/next/Windows/2017-11-19-09-46-27.bpo-1102.NY-g1F.rst new file mode 100644 index 000000000000000..6a6618e96925585 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2017-11-19-09-46-27.bpo-1102.NY-g1F.rst @@ -0,0 +1,4 @@ +Return ``None`` when ``View.Fetch()`` returns ``ERROR_NO_MORE_ITEMS`` +instead of raising ``MSIError``. + +Initial patch by Anthony Tuininga. diff --git a/Misc/python.man b/Misc/python.man index 9f71d69dfaf260e..b4110295536606f 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -124,7 +124,7 @@ This terminates the option list (following options are passed as arguments to the command). .TP .B \-d -Turn on parser debugging output (for wizards only, depending on +Turn on parser debugging output (for expert only, depending on compilation options). .TP .B \-E diff --git a/Modules/_json.c b/Modules/_json.c index 13218a6ecce587b..5a9464e34fb7f33 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1650,8 +1650,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyAccu *acc, continue; } else { - /* TODO: include repr of key */ - PyErr_SetString(PyExc_TypeError, "keys must be a string"); + PyErr_Format(PyExc_TypeError, + "keys must be str, int, float, bool or None, " + "not %.100s", key->ob_type->tp_name); goto bail; } diff --git a/Modules/getpath.c b/Modules/getpath.c index dd3387a9d77f98c..02d626cea73b38e 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -7,7 +7,7 @@ #include #ifdef __APPLE__ -#include +# include #endif /* Search in some common locations for the associated Python libraries. @@ -97,7 +97,7 @@ */ #ifdef __cplusplus - extern "C" { +extern "C" { #endif @@ -109,13 +109,43 @@ #define LANDMARK L"os.py" #endif -static wchar_t prefix[MAXPATHLEN+1]; -static wchar_t exec_prefix[MAXPATHLEN+1]; -static wchar_t progpath[MAXPATHLEN+1]; -static wchar_t *module_search_path = NULL; +#define DECODE_LOCALE_ERR(NAME, LEN) \ + ((LEN) == (size_t)-2) \ + ? _Py_INIT_USER_ERR("cannot decode " #NAME) \ + : _Py_INIT_NO_MEMORY() + + +typedef struct { + wchar_t *prefix; + wchar_t *exec_prefix; + wchar_t *program_name; + wchar_t *module_search_path; +} PyPathConfig; + +typedef struct { + wchar_t *path_env; /* PATH environment variable */ + wchar_t *home; /* PYTHONHOME environment variable */ + wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ + + wchar_t *program_name; /* Program name */ + wchar_t *pythonpath; /* PYTHONPATH define */ + wchar_t *prefix; /* PREFIX define */ + wchar_t *exec_prefix; /* EXEC_PREFIX define */ + + wchar_t *lib_python; /* "lib/pythonX.Y" */ + wchar_t argv0_path[MAXPATHLEN+1]; + wchar_t zip_path[MAXPATHLEN+1]; /* ".../lib/pythonXY.zip" */ + + int prefix_found; /* found platform independent libraries? */ + int exec_prefix_found; /* found the platform dependent libraries? */ +} PyCalculatePath; + +static const wchar_t delimiter[2] = {DELIM, '\0'}; +static const wchar_t separator[2] = {SEP, '\0'}; +static PyPathConfig path_config = {.module_search_path = NULL}; -/* Get file status. Encode the path to the locale encoding. */ +/* Get file status. Encode the path to the locale encoding. */ static int _Py_wstat(const wchar_t* path, struct stat *buf) { @@ -131,6 +161,7 @@ _Py_wstat(const wchar_t* path, struct stat *buf) return err; } + static void reduce(wchar_t *dir) { @@ -140,14 +171,17 @@ reduce(wchar_t *dir) dir[i] = '\0'; } + static int isfile(wchar_t *filename) /* Is file, not directory */ { struct stat buf; - if (_Py_wstat(filename, &buf) != 0) + if (_Py_wstat(filename, &buf) != 0) { return 0; - if (!S_ISREG(buf.st_mode)) + } + if (!S_ISREG(buf.st_mode)) { return 0; + } return 1; } @@ -155,41 +189,50 @@ isfile(wchar_t *filename) /* Is file, not directory */ static int ismodule(wchar_t *filename) /* Is module -- check for .pyc too */ { - if (isfile(filename)) + if (isfile(filename)) { return 1; + } /* Check for the compiled version of prefix. */ if (wcslen(filename) < MAXPATHLEN) { wcscat(filename, L"c"); - if (isfile(filename)) + if (isfile(filename)) { return 1; + } } return 0; } +/* Is executable file */ static int -isxfile(wchar_t *filename) /* Is executable file */ +isxfile(wchar_t *filename) { struct stat buf; - if (_Py_wstat(filename, &buf) != 0) + if (_Py_wstat(filename, &buf) != 0) { return 0; - if (!S_ISREG(buf.st_mode)) + } + if (!S_ISREG(buf.st_mode)) { return 0; - if ((buf.st_mode & 0111) == 0) + } + if ((buf.st_mode & 0111) == 0) { return 0; + } return 1; } +/* Is directory */ static int -isdir(wchar_t *filename) /* Is directory */ +isdir(wchar_t *filename) { struct stat buf; - if (_Py_wstat(filename, &buf) != 0) + if (_Py_wstat(filename, &buf) != 0) { return 0; - if (!S_ISDIR(buf.st_mode)) + } + if (!S_ISDIR(buf.st_mode)) { return 0; + } return 1; } @@ -207,58 +250,67 @@ static void joinpath(wchar_t *buffer, wchar_t *stuff) { size_t n, k; - if (stuff[0] == SEP) + if (stuff[0] == SEP) { n = 0; + } else { n = wcslen(buffer); - if (n > 0 && buffer[n-1] != SEP && n < MAXPATHLEN) + if (n > 0 && buffer[n-1] != SEP && n < MAXPATHLEN) { buffer[n++] = SEP; + } } - if (n > MAXPATHLEN) + if (n > MAXPATHLEN) { Py_FatalError("buffer overflow in getpath.c's joinpath()"); + } k = wcslen(stuff); - if (n + k > MAXPATHLEN) + if (n + k > MAXPATHLEN) { k = MAXPATHLEN - n; + } wcsncpy(buffer+n, stuff, k); buffer[n+k] = '\0'; } + /* copy_absolute requires that path be allocated at least MAXPATHLEN + 1 bytes and that p be no more than MAXPATHLEN bytes. */ static void copy_absolute(wchar_t *path, wchar_t *p, size_t pathlen) { - if (p[0] == SEP) + if (p[0] == SEP) { wcscpy(path, p); + } else { if (!_Py_wgetcwd(path, pathlen)) { /* unable to get the current directory */ wcscpy(path, p); return; } - if (p[0] == '.' && p[1] == SEP) + if (p[0] == '.' && p[1] == SEP) { p += 2; + } joinpath(path, p); } } + /* absolutize() requires that path be allocated at least MAXPATHLEN+1 bytes. */ static void absolutize(wchar_t *path) { wchar_t buffer[MAXPATHLEN+1]; - if (path[0] == SEP) + if (path[0] == SEP) { return; + } copy_absolute(buffer, path, MAXPATHLEN+1); wcscpy(path, buffer); } + /* search for a prefix value in an environment file. If found, copy it to the provided buffer, which is expected to be no more than MAXPATHLEN bytes long. */ - static int find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) { @@ -272,15 +324,18 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) PyObject * decoded; int n; - if (p == NULL) + if (p == NULL) { break; + } n = strlen(p); if (p[n - 1] != '\n') { /* line has overflowed - bail */ break; } - if (p[0] == '#') /* Comment - skip */ + if (p[0] == '#') { + /* Comment - skip */ continue; + } decoded = PyUnicode_DecodeUTF8(buffer, n, "surrogateescape"); if (decoded != NULL) { Py_ssize_t k; @@ -307,92 +362,136 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) return result; } + /* search_for_prefix requires that argv0_path be no more than MAXPATHLEN bytes long. */ static int -search_for_prefix(wchar_t *argv0_path, wchar_t *home, wchar_t *_prefix, - wchar_t *lib_python) +search_for_prefix(PyCalculatePath *calculate, wchar_t *prefix) { size_t n; wchar_t *vpath; /* If PYTHONHOME is set, we believe it unconditionally */ - if (home) { - wchar_t *delim; - wcsncpy(prefix, home, MAXPATHLEN); + if (calculate->home) { + wcsncpy(prefix, calculate->home, MAXPATHLEN); prefix[MAXPATHLEN] = L'\0'; - delim = wcschr(prefix, DELIM); - if (delim) + wchar_t *delim = wcschr(prefix, DELIM); + if (delim) { *delim = L'\0'; - joinpath(prefix, lib_python); + } + joinpath(prefix, calculate->lib_python); joinpath(prefix, LANDMARK); return 1; } /* Check to see if argv[0] is in the build directory */ - wcsncpy(prefix, argv0_path, MAXPATHLEN); + wcsncpy(prefix, calculate->argv0_path, MAXPATHLEN); prefix[MAXPATHLEN] = L'\0'; joinpath(prefix, L"Modules/Setup"); if (isfile(prefix)) { /* Check VPATH to see if argv0_path is in the build directory. */ vpath = Py_DecodeLocale(VPATH, NULL); if (vpath != NULL) { - wcsncpy(prefix, argv0_path, MAXPATHLEN); + wcsncpy(prefix, calculate->argv0_path, MAXPATHLEN); prefix[MAXPATHLEN] = L'\0'; joinpath(prefix, vpath); PyMem_RawFree(vpath); joinpath(prefix, L"Lib"); joinpath(prefix, LANDMARK); - if (ismodule(prefix)) + if (ismodule(prefix)) { return -1; + } } } /* Search from argv0_path, until root is found */ - copy_absolute(prefix, argv0_path, MAXPATHLEN+1); + copy_absolute(prefix, calculate->argv0_path, MAXPATHLEN+1); do { n = wcslen(prefix); - joinpath(prefix, lib_python); + joinpath(prefix, calculate->lib_python); joinpath(prefix, LANDMARK); - if (ismodule(prefix)) + if (ismodule(prefix)) { return 1; + } prefix[n] = L'\0'; reduce(prefix); } while (prefix[0]); /* Look at configure's PREFIX */ - wcsncpy(prefix, _prefix, MAXPATHLEN); + wcsncpy(prefix, calculate->prefix, MAXPATHLEN); prefix[MAXPATHLEN] = L'\0'; - joinpath(prefix, lib_python); + joinpath(prefix, calculate->lib_python); joinpath(prefix, LANDMARK); - if (ismodule(prefix)) + if (ismodule(prefix)) { return 1; + } /* Fail */ return 0; } +static void +calculate_prefix(PyCalculatePath *calculate, wchar_t *prefix) +{ + calculate->prefix_found = search_for_prefix(calculate, prefix); + if (!calculate->prefix_found) { + if (!Py_FrozenFlag) { + fprintf(stderr, + "Could not find platform independent libraries \n"); + } + wcsncpy(prefix, calculate->prefix, MAXPATHLEN); + joinpath(prefix, calculate->lib_python); + } + else { + reduce(prefix); + } +} + + +static void +calculate_reduce_prefix(PyCalculatePath *calculate, wchar_t *prefix) +{ + /* Reduce prefix and exec_prefix to their essence, + * e.g. /usr/local/lib/python1.5 is reduced to /usr/local. + * If we're loading relative to the build directory, + * return the compiled-in defaults instead. + */ + if (calculate->prefix_found > 0) { + reduce(prefix); + reduce(prefix); + /* The prefix is the root directory, but reduce() chopped + * off the "/". */ + if (!prefix[0]) { + wcscpy(prefix, separator); + } + } + else { + wcsncpy(prefix, calculate->prefix, MAXPATHLEN); + } +} + + /* search_for_exec_prefix requires that argv0_path be no more than MAXPATHLEN bytes long. */ static int -search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, - wchar_t *_exec_prefix, wchar_t *lib_python) +search_for_exec_prefix(PyCalculatePath *calculate, wchar_t *exec_prefix) { size_t n; /* If PYTHONHOME is set, we believe it unconditionally */ - if (home) { - wchar_t *delim; - delim = wcschr(home, DELIM); - if (delim) + if (calculate->home) { + wchar_t *delim = wcschr(calculate->home, DELIM); + if (delim) { wcsncpy(exec_prefix, delim+1, MAXPATHLEN); - else - wcsncpy(exec_prefix, home, MAXPATHLEN); + } + else { + wcsncpy(exec_prefix, calculate->home, MAXPATHLEN); + } exec_prefix[MAXPATHLEN] = L'\0'; - joinpath(exec_prefix, lib_python); + joinpath(exec_prefix, calculate->lib_python); joinpath(exec_prefix, L"lib-dynload"); return 1; } @@ -400,13 +499,14 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, /* Check to see if argv[0] is in the build directory. "pybuilddir.txt" is written by setup.py and contains the relative path to the location of shared library modules. */ - wcsncpy(exec_prefix, argv0_path, MAXPATHLEN); + wcsncpy(exec_prefix, calculate->argv0_path, MAXPATHLEN); exec_prefix[MAXPATHLEN] = L'\0'; joinpath(exec_prefix, L"pybuilddir.txt"); if (isfile(exec_prefix)) { FILE *f = _Py_wfopen(exec_prefix, L"rb"); - if (f == NULL) + if (f == NULL) { errno = 0; + } else { char buf[MAXPATHLEN+1]; PyObject *decoded; @@ -422,7 +522,7 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, Py_DECREF(decoded); if (k >= 0) { rel_builddir_path[k] = L'\0'; - wcsncpy(exec_prefix, argv0_path, MAXPATHLEN); + wcsncpy(exec_prefix, calculate->argv0_path, MAXPATHLEN); exec_prefix[MAXPATHLEN] = L'\0'; joinpath(exec_prefix, rel_builddir_path); return -1; @@ -432,54 +532,71 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, } /* Search from argv0_path, until root is found */ - copy_absolute(exec_prefix, argv0_path, MAXPATHLEN+1); + copy_absolute(exec_prefix, calculate->argv0_path, MAXPATHLEN+1); do { n = wcslen(exec_prefix); - joinpath(exec_prefix, lib_python); + joinpath(exec_prefix, calculate->lib_python); joinpath(exec_prefix, L"lib-dynload"); - if (isdir(exec_prefix)) + if (isdir(exec_prefix)) { return 1; + } exec_prefix[n] = L'\0'; reduce(exec_prefix); } while (exec_prefix[0]); /* Look at configure's EXEC_PREFIX */ - wcsncpy(exec_prefix, _exec_prefix, MAXPATHLEN); + wcsncpy(exec_prefix, calculate->exec_prefix, MAXPATHLEN); exec_prefix[MAXPATHLEN] = L'\0'; - joinpath(exec_prefix, lib_python); + joinpath(exec_prefix, calculate->lib_python); joinpath(exec_prefix, L"lib-dynload"); - if (isdir(exec_prefix)) + if (isdir(exec_prefix)) { return 1; + } /* Fail */ return 0; } + static void -calculate_path(void) +calculate_exec_prefix(PyCalculatePath *calculate, wchar_t *exec_prefix) { - extern wchar_t *Py_GetProgramName(void); - - static const wchar_t delimiter[2] = {DELIM, '\0'}; - static const wchar_t separator[2] = {SEP, '\0'}; - char *_rtpypath = Py_GETENV("PYTHONPATH"); /* XXX use wide version on Windows */ - wchar_t *rtpypath = NULL; - wchar_t *home = Py_GetPythonHome(); - char *_path = getenv("PATH"); - wchar_t *path_buffer = NULL; - wchar_t *path = NULL; - wchar_t *prog = Py_GetProgramName(); - wchar_t argv0_path[MAXPATHLEN+1]; - wchar_t zip_path[MAXPATHLEN+1]; - int pfound, efound; /* 1 if found; -1 if found build directory */ - wchar_t *buf; - size_t bufsz; - size_t prefixsz; - wchar_t *defpath; -#ifdef WITH_NEXT_FRAMEWORK - NSModule pythonModule; - const char* modPath; -#endif + calculate->exec_prefix_found = search_for_exec_prefix(calculate, exec_prefix); + if (!calculate->exec_prefix_found) { + if (!Py_FrozenFlag) { + fprintf(stderr, + "Could not find platform dependent libraries \n"); + } + wcsncpy(exec_prefix, calculate->exec_prefix, MAXPATHLEN); + joinpath(exec_prefix, L"lib/lib-dynload"); + } + /* If we found EXEC_PREFIX do *not* reduce it! (Yet.) */ +} + + +static void +calculate_reduce_exec_prefix(PyCalculatePath *calculate, wchar_t *exec_prefix) +{ + if (calculate->exec_prefix_found > 0) { + reduce(exec_prefix); + reduce(exec_prefix); + reduce(exec_prefix); + if (!exec_prefix[0]) { + wcscpy(exec_prefix, separator); + } + } + else { + wcsncpy(exec_prefix, calculate->exec_prefix, MAXPATHLEN); + } +} + + +static _PyInitError +calculate_program_name(PyCalculatePath *calculate, PyPathConfig *config) +{ + wchar_t program_name[MAXPATHLEN+1]; + memset(program_name, 0, sizeof(program_name)); + #ifdef __APPLE__ #if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4 uint32_t nsexeclength = MAXPATHLEN; @@ -488,32 +605,15 @@ calculate_path(void) #endif char execpath[MAXPATHLEN+1]; #endif - wchar_t *_pythonpath, *_prefix, *_exec_prefix; - wchar_t *lib_python; - - _pythonpath = Py_DecodeLocale(PYTHONPATH, NULL); - _prefix = Py_DecodeLocale(PREFIX, NULL); - _exec_prefix = Py_DecodeLocale(EXEC_PREFIX, NULL); - lib_python = Py_DecodeLocale("lib/python" VERSION, NULL); - - if (!_pythonpath || !_prefix || !_exec_prefix || !lib_python) { - Py_FatalError( - "Unable to decode path variables in getpath.c: " - "memory error"); - } - - if (_path) { - path_buffer = Py_DecodeLocale(_path, NULL); - path = path_buffer; - } /* If there is no slash in the argv0 path, then we have to * assume python is on the user's $PATH, since there's no * other way to find a directory to start the search from. If * $PATH isn't exported, you lose. */ - if (wcschr(prog, SEP)) - wcsncpy(progpath, prog, MAXPATHLEN); + if (wcschr(calculate->program_name, SEP)) { + wcsncpy(program_name, calculate->program_name, MAXPATHLEN); + } #ifdef __APPLE__ /* On Mac OS X, if a script uses an interpreter of the form * "#!/opt/python2.3/bin/python", the kernel only passes "python" @@ -525,48 +625,69 @@ calculate_path(void) * will fail if a relative path was used. but in that case, * absolutize() should help us out below */ - else if(0 == _NSGetExecutablePath(execpath, &nsexeclength) && execpath[0] == SEP) { - size_t r = mbstowcs(progpath, execpath, MAXPATHLEN+1); + else if(0 == _NSGetExecutablePath(execpath, &nsexeclength) && + execpath[0] == SEP) + { + size_t r = mbstowcs(program_name, execpath, MAXPATHLEN+1); if (r == (size_t)-1 || r > MAXPATHLEN) { /* Could not convert execpath, or it's too long. */ - progpath[0] = '\0'; + program_name[0] = '\0'; } } #endif /* __APPLE__ */ - else if (path) { + else if (calculate->path_env) { + wchar_t *path = calculate->path_env; while (1) { wchar_t *delim = wcschr(path, DELIM); if (delim) { size_t len = delim - path; - if (len > MAXPATHLEN) + if (len > MAXPATHLEN) { len = MAXPATHLEN; - wcsncpy(progpath, path, len); - *(progpath + len) = '\0'; + } + wcsncpy(program_name, path, len); + program_name[len] = '\0'; + } + else { + wcsncpy(program_name, path, MAXPATHLEN); } - else - wcsncpy(progpath, path, MAXPATHLEN); - joinpath(progpath, prog); - if (isxfile(progpath)) + joinpath(program_name, calculate->program_name); + if (isxfile(program_name)) { break; + } if (!delim) { - progpath[0] = L'\0'; + program_name[0] = L'\0'; break; } path = delim + 1; } } - else - progpath[0] = '\0'; - PyMem_RawFree(path_buffer); - if (progpath[0] != SEP && progpath[0] != '\0') - absolutize(progpath); - wcsncpy(argv0_path, progpath, MAXPATHLEN); - argv0_path[MAXPATHLEN] = '\0'; + else { + program_name[0] = '\0'; + } + if (program_name[0] != SEP && program_name[0] != '\0') { + absolutize(program_name); + } + + config->program_name = _PyMem_RawWcsdup(program_name); + if (config->program_name == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); +} + + +static _PyInitError +calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_name) +{ + wcsncpy(calculate->argv0_path, program_name, MAXPATHLEN); + calculate->argv0_path[MAXPATHLEN] = '\0'; #ifdef WITH_NEXT_FRAMEWORK + NSModule pythonModule; + /* On Mac OS X we have a special case if we're running from a framework. ** This is because the python home should be set relative to the library, ** which is in the framework, not relative to the executable, which may @@ -574,7 +695,7 @@ calculate_path(void) */ pythonModule = NSModuleForSymbol(NSLookupAndBindSymbol("_Py_Initialize")); /* Use dylib functions to find out where the framework was loaded from */ - modPath = NSLibraryNameForModule(pythonModule); + const char* modPath = NSLibraryNameForModule(pythonModule); if (modPath != NULL) { /* We're in a framework. */ /* See if we might be in the build directory. The framework in the @@ -584,147 +705,141 @@ calculate_path(void) ** be running the interpreter in the build directory, so we use the ** build-directory-specific logic to find Lib and such. */ - wchar_t* wbuf = Py_DecodeLocale(modPath, NULL); + size_t len; + wchar_t* wbuf = Py_DecodeLocale(modPath, &len); if (wbuf == NULL) { - Py_FatalError("Cannot decode framework location"); + return DECODE_LOCALE_ERR("framework location", len); } - wcsncpy(argv0_path, wbuf, MAXPATHLEN); - reduce(argv0_path); - joinpath(argv0_path, lib_python); - joinpath(argv0_path, LANDMARK); - if (!ismodule(argv0_path)) { + wcsncpy(calculate->argv0_path, wbuf, MAXPATHLEN); + reduce(calculate->argv0_path); + joinpath(calculate->argv0_path, calculate->lib_python); + joinpath(calculate->argv0_path, LANDMARK); + if (!ismodule(calculate->argv0_path)) { /* We are in the build directory so use the name of the executable - we know that the absolute path is passed */ - wcsncpy(argv0_path, progpath, MAXPATHLEN); + wcsncpy(calculate->argv0_path, program_name, MAXPATHLEN); } else { - /* Use the location of the library as the progpath */ - wcsncpy(argv0_path, wbuf, MAXPATHLEN); + /* Use the location of the library as the program_name */ + wcsncpy(calculate->argv0_path, wbuf, MAXPATHLEN); } PyMem_RawFree(wbuf); } #endif #if HAVE_READLINK - { - wchar_t tmpbuffer[MAXPATHLEN+1]; - int linklen = _Py_wreadlink(progpath, tmpbuffer, MAXPATHLEN); - while (linklen != -1) { - if (tmpbuffer[0] == SEP) - /* tmpbuffer should never be longer than MAXPATHLEN, - but extra check does not hurt */ - wcsncpy(argv0_path, tmpbuffer, MAXPATHLEN); - else { - /* Interpret relative to progpath */ - reduce(argv0_path); - joinpath(argv0_path, tmpbuffer); - } - linklen = _Py_wreadlink(argv0_path, tmpbuffer, MAXPATHLEN); + wchar_t tmpbuffer[MAXPATHLEN+1]; + int linklen = _Py_wreadlink(program_name, tmpbuffer, MAXPATHLEN); + while (linklen != -1) { + if (tmpbuffer[0] == SEP) { + /* tmpbuffer should never be longer than MAXPATHLEN, + but extra check does not hurt */ + wcsncpy(calculate->argv0_path, tmpbuffer, MAXPATHLEN); + } + else { + /* Interpret relative to program_name */ + reduce(calculate->argv0_path); + joinpath(calculate->argv0_path, tmpbuffer); } + linklen = _Py_wreadlink(calculate->argv0_path, tmpbuffer, MAXPATHLEN); } #endif /* HAVE_READLINK */ - reduce(argv0_path); + reduce(calculate->argv0_path); /* At this point, argv0_path is guaranteed to be less than - MAXPATHLEN bytes long. - */ + MAXPATHLEN bytes long. */ + return _Py_INIT_OK(); +} - /* Search for an environment configuration file, first in the - executable's directory and then in the parent directory. - If found, open it for use when searching for prefixes. - */ - { - wchar_t tmpbuffer[MAXPATHLEN+1]; - wchar_t *env_cfg = L"pyvenv.cfg"; - FILE * env_file = NULL; +/* Search for an "pyvenv.cfg" environment configuration file, first in the + executable's directory and then in the parent directory. + If found, open it for use when searching for prefixes. +*/ +static void +calculate_read_pyenv(PyCalculatePath *calculate) +{ + wchar_t tmpbuffer[MAXPATHLEN+1]; + wchar_t *env_cfg = L"pyvenv.cfg"; + FILE *env_file; - wcscpy(tmpbuffer, argv0_path); + wcscpy(tmpbuffer, calculate->argv0_path); + joinpath(tmpbuffer, env_cfg); + env_file = _Py_wfopen(tmpbuffer, L"r"); + if (env_file == NULL) { + errno = 0; + + reduce(tmpbuffer); + reduce(tmpbuffer); joinpath(tmpbuffer, env_cfg); + env_file = _Py_wfopen(tmpbuffer, L"r"); if (env_file == NULL) { errno = 0; - reduce(tmpbuffer); - reduce(tmpbuffer); - joinpath(tmpbuffer, env_cfg); - env_file = _Py_wfopen(tmpbuffer, L"r"); - if (env_file == NULL) { - errno = 0; - } - } - if (env_file != NULL) { - /* Look for a 'home' variable and set argv0_path to it, if found */ - if (find_env_config_value(env_file, L"home", tmpbuffer)) { - wcscpy(argv0_path, tmpbuffer); - } - fclose(env_file); - env_file = NULL; } } - pfound = search_for_prefix(argv0_path, home, _prefix, lib_python); - if (!pfound) { - if (!Py_FrozenFlag) - fprintf(stderr, - "Could not find platform independent libraries \n"); - wcsncpy(prefix, _prefix, MAXPATHLEN); - joinpath(prefix, lib_python); + if (env_file == NULL) { + return; } - else - reduce(prefix); - wcsncpy(zip_path, prefix, MAXPATHLEN); - zip_path[MAXPATHLEN] = L'\0'; - if (pfound > 0) { /* Use the reduced prefix returned by Py_GetPrefix() */ - reduce(zip_path); - reduce(zip_path); - } - else - wcsncpy(zip_path, _prefix, MAXPATHLEN); - joinpath(zip_path, L"lib/python00.zip"); - bufsz = wcslen(zip_path); /* Replace "00" with version */ - zip_path[bufsz - 6] = VERSION[0]; - zip_path[bufsz - 5] = VERSION[2]; - - efound = search_for_exec_prefix(argv0_path, home, - _exec_prefix, lib_python); - if (!efound) { - if (!Py_FrozenFlag) - fprintf(stderr, - "Could not find platform dependent libraries \n"); - wcsncpy(exec_prefix, _exec_prefix, MAXPATHLEN); - joinpath(exec_prefix, L"lib/lib-dynload"); + /* Look for a 'home' variable and set argv0_path to it, if found */ + if (find_env_config_value(env_file, L"home", tmpbuffer)) { + wcscpy(calculate->argv0_path, tmpbuffer); } - /* If we found EXEC_PREFIX do *not* reduce it! (Yet.) */ + fclose(env_file); +} - if ((!pfound || !efound) && !Py_FrozenFlag) - fprintf(stderr, - "Consider setting $PYTHONHOME to [:]\n"); - /* Calculate size of return buffer. - */ - bufsz = 0; +static void +calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix) +{ + wcsncpy(calculate->zip_path, prefix, MAXPATHLEN); + calculate->zip_path[MAXPATHLEN] = L'\0'; - if (_rtpypath && _rtpypath[0] != '\0') { - size_t rtpypath_len; - rtpypath = Py_DecodeLocale(_rtpypath, &rtpypath_len); - if (rtpypath != NULL) - bufsz += rtpypath_len + 1; + if (calculate->prefix_found > 0) { + /* Use the reduced prefix returned by Py_GetPrefix() */ + reduce(calculate->zip_path); + reduce(calculate->zip_path); + } + else { + wcsncpy(calculate->zip_path, calculate->prefix, MAXPATHLEN); + } + joinpath(calculate->zip_path, L"lib/python00.zip"); + + /* Replace "00" with version */ + size_t bufsz = wcslen(calculate->zip_path); + calculate->zip_path[bufsz - 6] = VERSION[0]; + calculate->zip_path[bufsz - 5] = VERSION[2]; +} + + +static _PyInitError +calculate_module_search_path(PyCalculatePath *calculate, + const wchar_t *prefix, const wchar_t *exec_prefix, + PyPathConfig *config) +{ + /* Calculate size of return buffer */ + size_t bufsz = 0; + if (calculate->module_search_path_env != NULL) { + bufsz += wcslen(calculate->module_search_path_env) + 1; } - defpath = _pythonpath; - prefixsz = wcslen(prefix) + 1; + wchar_t *defpath = calculate->pythonpath; + size_t prefixsz = wcslen(prefix) + 1; while (1) { wchar_t *delim = wcschr(defpath, DELIM); - if (defpath[0] != SEP) + if (defpath[0] != SEP) { /* Paths are relative to prefix */ bufsz += prefixsz; + } - if (delim) + if (delim) { bufsz += delim - defpath + 1; + } else { bufsz += wcslen(defpath) + 1; break; @@ -732,38 +847,39 @@ calculate_path(void) defpath = delim + 1; } - bufsz += wcslen(zip_path) + 1; + bufsz += wcslen(calculate->zip_path) + 1; bufsz += wcslen(exec_prefix) + 1; - buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); + /* Allocate the buffer */ + wchar_t *buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); if (buf == NULL) { - Py_FatalError( - "Not enough memory for dynamic PYTHONPATH"); + return _Py_INIT_NO_MEMORY(); } + buf[0] = '\0'; /* Run-time value of $PYTHONPATH goes first */ - if (rtpypath) { - wcscpy(buf, rtpypath); + if (calculate->module_search_path_env) { + wcscpy(buf, calculate->module_search_path_env); wcscat(buf, delimiter); } - else - buf[0] = '\0'; /* Next is the default zip path */ - wcscat(buf, zip_path); + wcscat(buf, calculate->zip_path); wcscat(buf, delimiter); /* Next goes merge of compile-time $PYTHONPATH with * dynamically located prefix. */ - defpath = _pythonpath; + defpath = calculate->pythonpath; while (1) { wchar_t *delim = wcschr(defpath, DELIM); if (defpath[0] != SEP) { wcscat(buf, prefix); if (prefixsz >= 2 && prefix[prefixsz - 2] != SEP && - defpath[0] != (delim ? DELIM : L'\0')) { /* not empty */ + defpath[0] != (delim ? DELIM : L'\0')) + { + /* not empty */ wcscat(buf, separator); } } @@ -772,7 +888,7 @@ calculate_path(void) size_t len = delim - defpath + 1; size_t end = wcslen(buf) + len; wcsncat(buf, defpath, len); - *(buf + end) = '\0'; + buf[end] = '\0'; } else { wcscat(buf, defpath); @@ -785,40 +901,190 @@ calculate_path(void) /* Finally, on goes the directory for dynamic-load modules */ wcscat(buf, exec_prefix); - /* And publish the results */ - module_search_path = buf; + config->module_search_path = buf; + return _Py_INIT_OK(); +} - /* Reduce prefix and exec_prefix to their essence, - * e.g. /usr/local/lib/python1.5 is reduced to /usr/local. - * If we're loading relative to the build directory, - * return the compiled-in defaults instead. - */ - if (pfound > 0) { - reduce(prefix); - reduce(prefix); - /* The prefix is the root directory, but reduce() chopped - * off the "/". */ - if (!prefix[0]) - wcscpy(prefix, separator); + +static _PyInitError +calculate_init(PyCalculatePath *calculate, + const _PyMainInterpreterConfig *main_config) +{ + calculate->home = main_config->home; + calculate->module_search_path_env = main_config->module_search_path_env; + calculate->program_name = main_config->program_name; + + size_t len; + char *path = getenv("PATH"); + if (path) { + calculate->path_env = Py_DecodeLocale(path, &len); + if (!calculate->path_env) { + return DECODE_LOCALE_ERR("PATH environment variable", len); + } } - else - wcsncpy(prefix, _prefix, MAXPATHLEN); - if (efound > 0) { - reduce(exec_prefix); - reduce(exec_prefix); - reduce(exec_prefix); - if (!exec_prefix[0]) - wcscpy(exec_prefix, separator); + calculate->pythonpath = Py_DecodeLocale(PYTHONPATH, &len); + if (!calculate->pythonpath) { + return DECODE_LOCALE_ERR("PYTHONPATH define", len); + } + calculate->prefix = Py_DecodeLocale(PREFIX, &len); + if (!calculate->prefix) { + return DECODE_LOCALE_ERR("PREFIX define", len); } - else - wcsncpy(exec_prefix, _exec_prefix, MAXPATHLEN); + calculate->exec_prefix = Py_DecodeLocale(EXEC_PREFIX, &len); + if (!calculate->prefix) { + return DECODE_LOCALE_ERR("EXEC_PREFIX define", len); + } + calculate->lib_python = Py_DecodeLocale("lib/python" VERSION, &len); + if (!calculate->lib_python) { + return DECODE_LOCALE_ERR("EXEC_PREFIX define", len); + } + return _Py_INIT_OK(); +} + - PyMem_RawFree(_pythonpath); - PyMem_RawFree(_prefix); - PyMem_RawFree(_exec_prefix); - PyMem_RawFree(lib_python); - PyMem_RawFree(rtpypath); +static void +calculate_free(PyCalculatePath *calculate) +{ + PyMem_RawFree(calculate->pythonpath); + PyMem_RawFree(calculate->prefix); + PyMem_RawFree(calculate->exec_prefix); + PyMem_RawFree(calculate->lib_python); + PyMem_RawFree(calculate->path_env); +} + + +static _PyInitError +calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config) +{ + _PyInitError err = calculate_program_name(calculate, config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = calculate_argv0_path(calculate, config->program_name); + if (_Py_INIT_FAILED(err)) { + return err; + } + + calculate_read_pyenv(calculate); + + wchar_t prefix[MAXPATHLEN+1]; + memset(prefix, 0, sizeof(prefix)); + calculate_prefix(calculate, prefix); + + calculate_zip_path(calculate, prefix); + + wchar_t exec_prefix[MAXPATHLEN+1]; + memset(exec_prefix, 0, sizeof(exec_prefix)); + calculate_exec_prefix(calculate, exec_prefix); + + if ((!calculate->prefix_found || !calculate->exec_prefix_found) && + !Py_FrozenFlag) + { + fprintf(stderr, + "Consider setting $PYTHONHOME to [:]\n"); + } + + err = calculate_module_search_path(calculate, prefix, exec_prefix, + config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + calculate_reduce_prefix(calculate, prefix); + + config->prefix = _PyMem_RawWcsdup(prefix); + if (config->prefix == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + calculate_reduce_exec_prefix(calculate, exec_prefix); + + config->exec_prefix = _PyMem_RawWcsdup(exec_prefix); + if (config->exec_prefix == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + return _Py_INIT_OK(); +} + + +static void +pathconfig_clear(PyPathConfig *config) +{ +#define CLEAR(ATTR) \ + do { \ + PyMem_RawFree(ATTR); \ + ATTR = NULL; \ + } while (0) + + CLEAR(config->prefix); + CLEAR(config->exec_prefix); + CLEAR(config->program_name); + CLEAR(config->module_search_path); +#undef CLEAR +} + + +/* Initialize paths for Py_GetPath(), Py_GetPrefix(), Py_GetExecPrefix() + and Py_GetProgramFullPath() */ +_PyInitError +_PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) +{ + if (path_config.module_search_path) { + /* Already initialized */ + return _Py_INIT_OK(); + } + + PyCalculatePath calculate; + memset(&calculate, 0, sizeof(calculate)); + + _PyInitError err = calculate_init(&calculate, main_config); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + PyPathConfig new_path_config; + memset(&new_path_config, 0, sizeof(new_path_config)); + + err = calculate_path_impl(&calculate, &new_path_config); + if (_Py_INIT_FAILED(err)) { + pathconfig_clear(&new_path_config); + goto done; + } + + path_config = new_path_config; + err = _Py_INIT_OK(); + +done: + calculate_free(&calculate); + return err; +} + + +static void +calculate_path(void) +{ + _PyInitError err; + _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; + + err = _PyMainInterpreterConfig_ReadEnv(&config); + if (!_Py_INIT_FAILED(err)) { + err = _PyPathConfig_Init(&config); + } + _PyMainInterpreterConfig_Clear(&config); + + if (_Py_INIT_FAILED(err)) { + _Py_FatalInitError(err); + } +} + + +void +_PyPathConfig_Fini(void) +{ + pathconfig_clear(&path_config); } @@ -826,54 +1092,61 @@ calculate_path(void) void Py_SetPath(const wchar_t *path) { - if (module_search_path != NULL) { - PyMem_RawFree(module_search_path); - module_search_path = NULL; - } - if (path != NULL) { - extern wchar_t *Py_GetProgramName(void); - wchar_t *prog = Py_GetProgramName(); - wcsncpy(progpath, prog, MAXPATHLEN); - exec_prefix[0] = prefix[0] = L'\0'; - module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t)); - if (module_search_path != NULL) - wcscpy(module_search_path, path); + if (path == NULL) { + pathconfig_clear(&path_config); + return; } + + PyPathConfig new_config; + new_config.program_name = _PyMem_RawWcsdup(Py_GetProgramName()); + new_config.exec_prefix = _PyMem_RawWcsdup(L""); + new_config.prefix = _PyMem_RawWcsdup(L""); + new_config.module_search_path = _PyMem_RawWcsdup(path); + + pathconfig_clear(&path_config); + path_config = new_config; } + wchar_t * Py_GetPath(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(); - return module_search_path; + } + return path_config.module_search_path; } + wchar_t * Py_GetPrefix(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(); - return prefix; + } + return path_config.prefix; } + wchar_t * Py_GetExecPrefix(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(); - return exec_prefix; + } + return path_config.exec_prefix; } + wchar_t * Py_GetProgramFullPath(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(); - return progpath; + } + return path_config.program_name; } - #ifdef __cplusplus } #endif diff --git a/Modules/main.c b/Modules/main.c index e5e4f33f51c072c..1b1d8041ef114ff 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -36,6 +36,22 @@ extern "C" { #endif +#define DECODE_LOCALE_ERR(NAME, LEN) \ + (((LEN) == -2) \ + ? _Py_INIT_USER_ERR("cannot decode " #NAME) \ + : _Py_INIT_NO_MEMORY()) + + +#define SET_DECODE_ERROR(NAME, LEN) \ + do { \ + if ((LEN) == (size_t)-2) { \ + pymain->err = _Py_INIT_USER_ERR("cannot decode " #NAME); \ + } \ + else { \ + pymain->err = _Py_INIT_NO_MEMORY(); \ + } \ + } while (0) + /* For Py_GetArgcArgv(); set by main() */ static wchar_t **orig_argv; static int orig_argc; @@ -389,6 +405,7 @@ typedef struct { /* non-zero is stdin is a TTY or if -i option is used */ int stdin_is_interactive; _PyCoreConfig core_config; + _PyMainInterpreterConfig config; _Py_CommandLineDetails cmdline; PyObject *main_importer_path; /* non-zero if filename, command (-c) or module (-m) is set @@ -408,6 +425,7 @@ typedef struct { {.status = 0, \ .cf = {.cf_flags = 0}, \ .core_config = _PyCoreConfig_INIT, \ + .config = _PyMainInterpreterConfig_INIT, \ .main_importer_path = NULL, \ .run_code = -1, \ .program_name = NULL, \ @@ -415,9 +433,6 @@ typedef struct { .env_warning_options = {0, NULL}} -#define INIT_NO_MEMORY() _Py_INIT_ERR("memory allocation failed") - - static void pymain_optlist_clear(_Py_OptList *list) { @@ -441,6 +456,8 @@ pymain_free_impl(_PyMain *pymain) Py_CLEAR(pymain->main_importer_path); PyMem_RawFree(pymain->program_name); + _PyMainInterpreterConfig_Clear(&pymain->config); + #ifdef __INSURE__ /* Insure++ is a memory analysis tool that aids in discovering * memory leaks and other memory problems. On Python exit, the @@ -502,15 +519,13 @@ pymain_run_main_from_importer(_PyMain *pymain) static wchar_t* -pymain_strdup(_PyMain *pymain, wchar_t *str) +pymain_wstrdup(_PyMain *pymain, wchar_t *str) { - size_t len = wcslen(str) + 1; /* +1 for NUL character */ - wchar_t *str2 = PyMem_RawMalloc(sizeof(wchar_t) * len); + wchar_t *str2 = _PyMem_RawWcsdup(str); if (str2 == NULL) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return NULL; } - memcpy(str2, str, len * sizeof(wchar_t)); return str2; } @@ -518,7 +533,7 @@ pymain_strdup(_PyMain *pymain, wchar_t *str) static int pymain_optlist_append(_PyMain *pymain, _Py_OptList *list, wchar_t *str) { - wchar_t *str2 = pymain_strdup(pymain, str); + wchar_t *str2 = pymain_wstrdup(pymain, str); if (str2 == NULL) { return -1; } @@ -527,7 +542,7 @@ pymain_optlist_append(_PyMain *pymain, _Py_OptList *list, wchar_t *str) wchar_t **options2 = (wchar_t **)PyMem_RawRealloc(list->options, size); if (options2 == NULL) { PyMem_RawFree(str2); - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } options2[list->len] = str2; @@ -560,7 +575,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain) size_t len = wcslen(_PyOS_optarg) + 1 + 1; wchar_t *command = PyMem_RawMalloc(sizeof(wchar_t) * len); if (command == NULL) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } memcpy(command, _PyOS_optarg, len * sizeof(wchar_t)); @@ -706,7 +721,7 @@ pymain_add_xoptions(_PyMain *pymain) for (size_t i=0; i < options->len; i++) { wchar_t *option = options->options[i]; if (_PySys_AddXOptionWithError(option) < 0) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } } @@ -737,11 +752,11 @@ pymain_add_warnings_options(_PyMain *pymain) PySys_ResetWarnOptions(); if (pymain_add_warnings_optlist(&pymain->env_warning_options) < 0) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } if (pymain_add_warnings_optlist(&pymain->cmdline.warning_options) < 0) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } return 0; @@ -762,14 +777,12 @@ pymain_warnings_envvar(_PyMain *pymain) wchar_t *wp; if ((wp = _wgetenv(L"PYTHONWARNINGS")) && *wp != L'\0') { - wchar_t *buf, *warning, *context = NULL; + wchar_t *warning, *context = NULL; - buf = (wchar_t *)PyMem_RawMalloc((wcslen(wp) + 1) * sizeof(wchar_t)); + wchar_t *buf = pymain_wstrdup(pymain, wp); if (buf == NULL) { - pymain->err = INIT_NO_MEMORY(); return -1; } - wcscpy(buf, wp); for (warning = wcstok_s(buf, L",", &context); warning != NULL; warning = wcstok_s(NULL, L",", &context)) { @@ -792,7 +805,7 @@ pymain_warnings_envvar(_PyMain *pymain) C89 wcstok */ buf = (char *)PyMem_RawMalloc(strlen(p) + 1); if (buf == NULL) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } strcpy(buf, p); @@ -802,15 +815,8 @@ pymain_warnings_envvar(_PyMain *pymain) size_t len; wchar_t *warning = Py_DecodeLocale(p, &len); if (warning == NULL) { - if (len == (size_t)-2) { - pymain->err = _Py_INIT_ERR("failed to decode " - "PYTHONWARNINGS"); - return -1; - } - else { - pymain->err = INIT_NO_MEMORY(); - return -1; - } + SET_DECODE_ERROR("PYTHONWARNINGS environment variable", len); + return -1; } if (pymain_optlist_append(pymain, &pymain->env_warning_options, warning) < 0) { @@ -894,7 +900,7 @@ pymain_get_program_name(_PyMain *pymain) buffer = PyMem_RawMalloc(len * sizeof(wchar_t)); if (buffer == NULL) { - pymain->err = INIT_NO_MEMORY(); + pymain->err = _Py_INIT_NO_MEMORY(); return -1; } @@ -911,15 +917,8 @@ pymain_get_program_name(_PyMain *pymain) size_t len; wchar_t* wbuf = Py_DecodeLocale(pyvenv_launcher, &len); if (wbuf == NULL) { - if (len == (size_t)-2) { - pymain->err = _Py_INIT_ERR("failed to decode " - "__PYVENV_LAUNCHER__"); - return -1; - } - else { - pymain->err = INIT_NO_MEMORY(); - return -1; - } + SET_DECODE_ERROR("__PYVENV_LAUNCHER__", len); + return -1; } pymain->program_name = wbuf; } @@ -929,7 +928,7 @@ pymain_get_program_name(_PyMain *pymain) if (pymain->program_name == NULL) { /* Use argv[0] by default */ - pymain->program_name = pymain_strdup(pymain, pymain->argv[0]); + pymain->program_name = pymain_wstrdup(pymain, pymain->argv[0]); if (pymain->program_name == NULL) { return -1; } @@ -950,20 +949,16 @@ pymain_get_program_name(_PyMain *pymain) static int pymain_init_main_interpreter(_PyMain *pymain) { - _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; _PyInitError err; - /* TODO: Moar config options! */ - config.install_signal_handlers = 1; - /* TODO: Print any exceptions raised by these operations */ - err = _Py_ReadMainInterpreterConfig(&config); + err = _PyMainInterpreterConfig_Read(&pymain->config); if (_Py_INIT_FAILED(err)) { pymain->err = err; return -1; } - err = _Py_InitializeMainInterpreter(&config); + err = _Py_InitializeMainInterpreter(&pymain->config); if (_Py_INIT_FAILED(err)) { pymain->err = err; return -1; @@ -1362,6 +1357,114 @@ pymain_set_flags_from_env(_PyMain *pymain) } +static int +config_get_env_var_dup(wchar_t **dest, wchar_t *wname, char *name) +{ + if (Py_IgnoreEnvironmentFlag) { + *dest = NULL; + return 0; + } + +#ifdef MS_WINDOWS + wchar_t *var = _wgetenv(wname); + if (!var || var[0] == '\0') { + *dest = NULL; + return 0; + } + + wchar_t *copy = _PyMem_RawWcsdup(var); + if (copy == NULL) { + return -1; + } + + *dest = copy; +#else + char *var = getenv(name); + if (!var || var[0] == '\0') { + *dest = NULL; + return 0; + } + + size_t len; + wchar_t *wvar = Py_DecodeLocale(var, &len); + if (!wvar) { + if (len == (size_t)-2) { + return -2; + } + else { + return -1; + } + } + *dest = wvar; +#endif + return 0; +} + + +static _PyInitError +config_init_pythonpath(_PyMainInterpreterConfig *config) +{ + wchar_t *path; + int res = config_get_env_var_dup(&path, L"PYTHONPATH", "PYTHONPATH"); + if (res < 0) { + return DECODE_LOCALE_ERR("PYTHONHOME", res); + } + config->module_search_path_env = path; + return _Py_INIT_OK(); +} + + +static _PyInitError +config_init_pythonhome(_PyMainInterpreterConfig *config) +{ + wchar_t *home; + + home = Py_GetPythonHome(); + if (home) { + /* Py_SetPythonHome() has been called before Py_Main(), + use its value */ + config->home = _PyMem_RawWcsdup(home); + if (config->home == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); + } + + int res = config_get_env_var_dup(&home, L"PYTHONHOME", "PYTHONHOME"); + if (res < 0) { + return DECODE_LOCALE_ERR("PYTHONHOME", res); + } + config->home = home; + return _Py_INIT_OK(); +} + + +_PyInitError +_PyMainInterpreterConfig_ReadEnv(_PyMainInterpreterConfig *config) +{ + _PyInitError err = config_init_pythonhome(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = config_init_pythonpath(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + /* FIXME: _PyMainInterpreterConfig_Read() has the same code. Remove it + here? See also pymain_get_program_name() and pymain_parse_envvars(). */ + config->program_name = _PyMem_RawWcsdup(Py_GetProgramName()); + if (config->program_name == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + return _Py_INIT_OK(); +} + + + + static int pymain_parse_envvars(_PyMain *pymain) { @@ -1384,6 +1487,21 @@ pymain_parse_envvars(_PyMain *pymain) } core_config->allocator = Py_GETENV("PYTHONMALLOC"); + /* FIXME: move pymain_get_program_name() code into + _PyMainInterpreterConfig_ReadEnv(). + Problem: _PyMainInterpreterConfig_ReadEnv() doesn't have access + to argv[0]. */ + Py_SetProgramName(pymain->program_name); + /* Don't free program_name here: the argument to Py_SetProgramName + must remain valid until Py_FinalizeEx is called. The string is freed + by pymain_free(). */ + + _PyInitError err = _PyMainInterpreterConfig_ReadEnv(&pymain->config); + if (_Py_INIT_FAILED(pymain->err)) { + pymain->err = err; + return -1; + } + /* -X options */ if (pymain_get_xoption(pymain, L"showrefcount")) { core_config->show_ref_count = 1; @@ -1467,11 +1585,6 @@ pymain_init_python(_PyMain *pymain) return -1; } - Py_SetProgramName(pymain->program_name); - /* Don't free program_name here: the argument to Py_SetProgramName - must remain valid until Py_FinalizeEx is called. The string is freed - by pymain_free(). */ - if (pymain_add_xoptions(pymain)) { return -1; } @@ -1522,6 +1635,8 @@ pymain_init(_PyMain *pymain) } pymain->core_config._disable_importlib = 0; + /* TODO: Moar config options! */ + pymain->config.install_signal_handlers = 1; orig_argc = pymain->argc; /* For Py_GetArgcArgv() */ orig_argv = pymain->argv; diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 7c6973ec035f10b..9bd97986300f3d0 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1,6 +1,4 @@ #include "Python.h" -#include "internal/mem.h" -#include "internal/pystate.h" #include @@ -180,24 +178,39 @@ static struct { #define PYDBG_FUNCS \ _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree +static PyMemAllocatorEx _PyMem_Raw = { +#ifdef Py_DEBUG + &_PyMem_Debug.raw, PYRAWDBG_FUNCS +#else + NULL, PYRAW_FUNCS +#endif + }; -#define _PyMem_Raw _PyRuntime.mem.allocators.raw - -#define _PyMem _PyRuntime.mem.allocators.mem +static PyMemAllocatorEx _PyMem = { +#ifdef Py_DEBUG + &_PyMem_Debug.mem, PYDBG_FUNCS +#else + NULL, PYMEM_FUNCS +#endif + }; -#define _PyObject _PyRuntime.mem.allocators.obj +static PyMemAllocatorEx _PyObject = { +#ifdef Py_DEBUG + &_PyMem_Debug.obj, PYDBG_FUNCS +#else + NULL, PYOBJ_FUNCS +#endif + }; void _PyMem_GetDefaultRawAllocator(PyMemAllocatorEx *alloc_p) { - PyMemAllocatorEx pymem_raw = { #ifdef Py_DEBUG - &_PyMem_Debug.raw, PYRAWDBG_FUNCS + PyMemAllocatorEx alloc = {&_PyMem_Debug.raw, PYDBG_FUNCS}; #else - NULL, PYRAW_FUNCS + PyMemAllocatorEx alloc = {NULL, PYRAW_FUNCS}; #endif - }; - *alloc_p = pymem_raw; + *alloc_p = alloc; } int @@ -259,62 +272,22 @@ _PyMem_SetupAllocators(const char *opt) return 0; } +#undef PYRAW_FUNCS +#undef PYMEM_FUNCS +#undef PYOBJ_FUNCS +#undef PYRAWDBG_FUNCS +#undef PYDBG_FUNCS -void -_PyObject_Initialize(struct _pyobj_runtime_state *state) -{ - PyObjectArenaAllocator _PyObject_Arena = {NULL, +static PyObjectArenaAllocator _PyObject_Arena = {NULL, #ifdef MS_WINDOWS - _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree + _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree #elif defined(ARENAS_USE_MMAP) - _PyObject_ArenaMmap, _PyObject_ArenaMunmap -#else - _PyObject_ArenaMalloc, _PyObject_ArenaFree -#endif - }; - - state->allocator_arenas = _PyObject_Arena; -} - - -void -_PyMem_Initialize(struct _pymem_runtime_state *state) -{ - PyMemAllocatorEx pymem = { -#ifdef Py_DEBUG - &_PyMem_Debug.mem, PYDBG_FUNCS + _PyObject_ArenaMmap, _PyObject_ArenaMunmap #else - NULL, PYMEM_FUNCS -#endif - }; - PyMemAllocatorEx pyobject = { -#ifdef Py_DEBUG - &_PyMem_Debug.obj, PYDBG_FUNCS -#else - NULL, PYOBJ_FUNCS + _PyObject_ArenaMalloc, _PyObject_ArenaFree #endif }; - _PyMem_GetDefaultRawAllocator(&state->allocators.raw); - state->allocators.mem = pymem; - state->allocators.obj = pyobject; - -#ifdef WITH_PYMALLOC - Py_BUILD_ASSERT(NB_SMALL_SIZE_CLASSES == 64); - - for (int i = 0; i < 8; i++) { - for (int j = 0; j < 8; j++) { - int x = i * 8 + j; - poolp *addr = &(state->usedpools[2*(x)]); - poolp val = (poolp)((uint8_t *)addr - 2*sizeof(pyblock *)); - state->usedpools[x * 2] = val; - state->usedpools[x * 2 + 1] = val; - }; - }; -#endif /* WITH_PYMALLOC */ -} - - #ifdef WITH_PYMALLOC static int _PyMem_DebugEnabled(void) @@ -401,13 +374,13 @@ PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) { - *allocator = _PyRuntime.obj.allocator_arenas; + *allocator = _PyObject_Arena; } void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) { - _PyRuntime.obj.allocator_arenas = *allocator; + _PyObject_Arena = *allocator; } void * @@ -442,8 +415,7 @@ PyMem_RawRealloc(void *ptr, size_t new_size) return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size); } -void -PyMem_RawFree(void *ptr) +void PyMem_RawFree(void *ptr) { _PyMem_Raw.free(_PyMem_Raw.ctx, ptr); } @@ -483,6 +455,24 @@ PyMem_Free(void *ptr) } +wchar_t* +_PyMem_RawWcsdup(const wchar_t *str) +{ + size_t len = wcslen(str); + if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) { + return NULL; + } + + size_t size = (len + 1) * sizeof(wchar_t); + wchar_t *str2 = PyMem_RawMalloc(size); + if (str2 == NULL) { + return NULL; + } + + memcpy(str2, str, size); + return str2; +} + char * _PyMem_RawStrdup(const char *str) { @@ -563,10 +553,497 @@ static int running_on_valgrind = -1; #endif +/* An object allocator for Python. + + Here is an introduction to the layers of the Python memory architecture, + showing where the object allocator is actually used (layer +2), It is + called for every object allocation and deallocation (PyObject_New/Del), + unless the object-specific allocators implement a proprietary allocation + scheme (ex.: ints use a simple free list). This is also the place where + the cyclic garbage collector operates selectively on container objects. + + + Object-specific allocators + _____ ______ ______ ________ + [ int ] [ dict ] [ list ] ... [ string ] Python core | ++3 | <----- Object-specific memory -----> | <-- Non-object memory --> | + _______________________________ | | + [ Python's object allocator ] | | ++2 | ####### Object memory ####### | <------ Internal buffers ------> | + ______________________________________________________________ | + [ Python's raw memory allocator (PyMem_ API) ] | ++1 | <----- Python memory (under PyMem manager's control) ------> | | + __________________________________________________________________ + [ Underlying general-purpose allocator (ex: C library malloc) ] + 0 | <------ Virtual memory allocated for the python process -------> | + + ========================================================================= + _______________________________________________________________________ + [ OS-specific Virtual Memory Manager (VMM) ] +-1 | <--- Kernel dynamic storage allocation & management (page-based) ---> | + __________________________________ __________________________________ + [ ] [ ] +-2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> | + +*/ +/*==========================================================================*/ + +/* A fast, special-purpose memory allocator for small blocks, to be used + on top of a general-purpose malloc -- heavily based on previous art. */ + +/* Vladimir Marangozov -- August 2000 */ + +/* + * "Memory management is where the rubber meets the road -- if we do the wrong + * thing at any level, the results will not be good. And if we don't make the + * levels work well together, we are in serious trouble." (1) + * + * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles, + * "Dynamic Storage Allocation: A Survey and Critical Review", + * in Proc. 1995 Int'l. Workshop on Memory Management, September 1995. + */ + +/* #undef WITH_MEMORY_LIMITS */ /* disable mem limit checks */ + +/*==========================================================================*/ + +/* + * Allocation strategy abstract: + * + * For small requests, the allocator sub-allocates blocks of memory. + * Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the + * system's allocator. + * + * Small requests are grouped in size classes spaced 8 bytes apart, due + * to the required valid alignment of the returned address. Requests of + * a particular size are serviced from memory pools of 4K (one VMM page). + * Pools are fragmented on demand and contain free lists of blocks of one + * particular size class. In other words, there is a fixed-size allocator + * for each size class. Free pools are shared by the different allocators + * thus minimizing the space reserved for a particular size class. + * + * This allocation strategy is a variant of what is known as "simple + * segregated storage based on array of free lists". The main drawback of + * simple segregated storage is that we might end up with lot of reserved + * memory for the different free lists, which degenerate in time. To avoid + * this, we partition each free list in pools and we share dynamically the + * reserved space between all free lists. This technique is quite efficient + * for memory intensive programs which allocate mainly small-sized blocks. + * + * For small requests we have the following table: + * + * Request in bytes Size of allocated block Size class idx + * ---------------------------------------------------------------- + * 1-8 8 0 + * 9-16 16 1 + * 17-24 24 2 + * 25-32 32 3 + * 33-40 40 4 + * 41-48 48 5 + * 49-56 56 6 + * 57-64 64 7 + * 65-72 72 8 + * ... ... ... + * 497-504 504 62 + * 505-512 512 63 + * + * 0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying + * allocator. + */ + +/*==========================================================================*/ + +/* + * -- Main tunable settings section -- + */ + +/* + * Alignment of addresses returned to the user. 8-bytes alignment works + * on most current architectures (with 32-bit or 64-bit address busses). + * The alignment value is also used for grouping small requests in size + * classes spaced ALIGNMENT bytes apart. + * + * You shouldn't change this unless you know what you are doing. + */ +#define ALIGNMENT 8 /* must be 2^N */ +#define ALIGNMENT_SHIFT 3 + +/* Return the number of bytes in size class I, as a uint. */ +#define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT) + +/* + * Max size threshold below which malloc requests are considered to be + * small enough in order to use preallocated memory pools. You can tune + * this value according to your application behaviour and memory needs. + * + * Note: a size threshold of 512 guarantees that newly created dictionaries + * will be allocated from preallocated memory pools on 64-bit. + * + * The following invariants must hold: + * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512 + * 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT + * + * Although not required, for better performance and space efficiency, + * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2. + */ +#define SMALL_REQUEST_THRESHOLD 512 +#define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT) + +/* + * The system's VMM page size can be obtained on most unices with a + * getpagesize() call or deduced from various header files. To make + * things simpler, we assume that it is 4K, which is OK for most systems. + * It is probably better if this is the native page size, but it doesn't + * have to be. In theory, if SYSTEM_PAGE_SIZE is larger than the native page + * size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation + * violation fault. 4K is apparently OK for all the platforms that python + * currently targets. + */ +#define SYSTEM_PAGE_SIZE (4 * 1024) +#define SYSTEM_PAGE_SIZE_MASK (SYSTEM_PAGE_SIZE - 1) + +/* + * Maximum amount of memory managed by the allocator for small requests. + */ +#ifdef WITH_MEMORY_LIMITS +#ifndef SMALL_MEMORY_LIMIT +#define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MB -- more? */ +#endif +#endif + +/* + * The allocator sub-allocates blocks of memory (called arenas) aligned + * on a page boundary. This is a reserved virtual address space for the + * current process (obtained through a malloc()/mmap() call). In no way this + * means that the memory arenas will be used entirely. A malloc() is + * usually an address range reservation for bytes, unless all pages within + * this space are referenced subsequently. So malloc'ing big blocks and not + * using them does not mean "wasting memory". It's an addressable range + * wastage... + * + * Arenas are allocated with mmap() on systems supporting anonymous memory + * mappings to reduce heap fragmentation. + */ +#define ARENA_SIZE (256 << 10) /* 256KB */ + +#ifdef WITH_MEMORY_LIMITS +#define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE) +#endif + +/* + * Size of the pools used for small blocks. Should be a power of 2, + * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k. + */ +#define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */ +#define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK + +/* + * -- End of tunable settings section -- + */ + +/*==========================================================================*/ + +/* + * Locking + * + * To reduce lock contention, it would probably be better to refine the + * crude function locking with per size class locking. I'm not positive + * however, whether it's worth switching to such locking policy because + * of the performance penalty it might introduce. + * + * The following macros describe the simplest (should also be the fastest) + * lock object on a particular platform and the init/fini/lock/unlock + * operations on it. The locks defined here are not expected to be recursive + * because it is assumed that they will always be called in the order: + * INIT, [LOCK, UNLOCK]*, FINI. + */ + +/* + * Python's threads are serialized, so object malloc locking is disabled. + */ +#define SIMPLELOCK_DECL(lock) /* simple lock declaration */ +#define SIMPLELOCK_INIT(lock) /* allocate (if needed) and initialize */ +#define SIMPLELOCK_FINI(lock) /* free/destroy an existing lock */ +#define SIMPLELOCK_LOCK(lock) /* acquire released lock */ +#define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */ + +/* When you say memory, my mind reasons in terms of (pointers to) blocks */ +typedef uint8_t block; + +/* Pool for small blocks. */ +struct pool_header { + union { block *_padding; + uint count; } ref; /* number of allocated blocks */ + block *freeblock; /* pool's free list head */ + struct pool_header *nextpool; /* next pool of this size class */ + struct pool_header *prevpool; /* previous pool "" */ + uint arenaindex; /* index into arenas of base adr */ + uint szidx; /* block size class index */ + uint nextoffset; /* bytes to virgin block */ + uint maxnextoffset; /* largest valid nextoffset */ +}; + +typedef struct pool_header *poolp; + +/* Record keeping for arenas. */ +struct arena_object { + /* The address of the arena, as returned by malloc. Note that 0 + * will never be returned by a successful malloc, and is used + * here to mark an arena_object that doesn't correspond to an + * allocated arena. + */ + uintptr_t address; + + /* Pool-aligned pointer to the next pool to be carved off. */ + block* pool_address; + + /* The number of available pools in the arena: free pools + never- + * allocated pools. + */ + uint nfreepools; + + /* The total number of pools in the arena, whether or not available. */ + uint ntotalpools; + + /* Singly-linked list of available pools. */ + struct pool_header* freepools; + + /* Whenever this arena_object is not associated with an allocated + * arena, the nextarena member is used to link all unassociated + * arena_objects in the singly-linked `unused_arena_objects` list. + * The prevarena member is unused in this case. + * + * When this arena_object is associated with an allocated arena + * with at least one available pool, both members are used in the + * doubly-linked `usable_arenas` list, which is maintained in + * increasing order of `nfreepools` values. + * + * Else this arena_object is associated with an allocated arena + * all of whose pools are in use. `nextarena` and `prevarena` + * are both meaningless in this case. + */ + struct arena_object* nextarena; + struct arena_object* prevarena; +}; + +#define POOL_OVERHEAD _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT) + +#define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */ + +/* Round pointer P down to the closest pool-aligned address <= P, as a poolp */ +#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE)) + +/* Return total number of blocks in pool of size index I, as a uint. */ +#define NUMBLOCKS(I) ((uint)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I)) + +/*==========================================================================*/ + +/* + * This malloc lock + */ +SIMPLELOCK_DECL(_malloc_lock) +#define LOCK() SIMPLELOCK_LOCK(_malloc_lock) +#define UNLOCK() SIMPLELOCK_UNLOCK(_malloc_lock) +#define LOCK_INIT() SIMPLELOCK_INIT(_malloc_lock) +#define LOCK_FINI() SIMPLELOCK_FINI(_malloc_lock) + +/* + * Pool table -- headed, circular, doubly-linked lists of partially used pools. + +This is involved. For an index i, usedpools[i+i] is the header for a list of +all partially used pools holding small blocks with "size class idx" i. So +usedpools[0] corresponds to blocks of size 8, usedpools[2] to blocks of size +16, and so on: index 2*i <-> blocks of size (i+1)<freeblock points to +the start of a singly-linked list of free blocks within the pool. When a +block is freed, it's inserted at the front of its pool's freeblock list. Note +that the available blocks in a pool are *not* linked all together when a pool +is initialized. Instead only "the first two" (lowest addresses) blocks are +set up, returning the first such block, and setting pool->freeblock to a +one-block list holding the second such block. This is consistent with that +pymalloc strives at all levels (arena, pool, and block) never to touch a piece +of memory until it's actually needed. + +So long as a pool is in the used state, we're certain there *is* a block +available for allocating, and pool->freeblock is not NULL. If pool->freeblock +points to the end of the free list before we've carved the entire pool into +blocks, that means we simply haven't yet gotten to one of the higher-address +blocks. The offset from the pool_header to the start of "the next" virgin +block is stored in the pool_header nextoffset member, and the largest value +of nextoffset that makes sense is stored in the maxnextoffset member when a +pool is initialized. All the blocks in a pool have been passed out at least +once when and only when nextoffset > maxnextoffset. + + +Major obscurity: While the usedpools vector is declared to have poolp +entries, it doesn't really. It really contains two pointers per (conceptual) +poolp entry, the nextpool and prevpool members of a pool_header. The +excruciating initialization code below fools C so that + + usedpool[i+i] + +"acts like" a genuine poolp, but only so long as you only reference its +nextpool and prevpool members. The "- 2*sizeof(block *)" gibberish is +compensating for that a pool_header's nextpool and prevpool members +immediately follow a pool_header's first two members: + + union { block *_padding; + uint count; } ref; + block *freeblock; + +each of which consume sizeof(block *) bytes. So what usedpools[i+i] really +contains is a fudged-up pointer p such that *if* C believes it's a poolp +pointer, then p->nextpool and p->prevpool are both p (meaning that the headed +circular list is empty). + +It's unclear why the usedpools setup is so convoluted. It could be to +minimize the amount of cache required to hold this heavily-referenced table +(which only *needs* the two interpool pointer members of a pool_header). OTOH, +referencing code has to remember to "double the index" and doing so isn't +free, usedpools[0] isn't a strictly legal pointer, and we're crucially relying +on that C doesn't insert any padding anywhere in a pool_header at or before +the prevpool member. +**************************************************************************** */ + +#define PTA(x) ((poolp )((uint8_t *)&(usedpools[2*(x)]) - 2*sizeof(block *))) +#define PT(x) PTA(x), PTA(x) + +static poolp usedpools[2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8] = { + PT(0), PT(1), PT(2), PT(3), PT(4), PT(5), PT(6), PT(7) +#if NB_SMALL_SIZE_CLASSES > 8 + , PT(8), PT(9), PT(10), PT(11), PT(12), PT(13), PT(14), PT(15) +#if NB_SMALL_SIZE_CLASSES > 16 + , PT(16), PT(17), PT(18), PT(19), PT(20), PT(21), PT(22), PT(23) +#if NB_SMALL_SIZE_CLASSES > 24 + , PT(24), PT(25), PT(26), PT(27), PT(28), PT(29), PT(30), PT(31) +#if NB_SMALL_SIZE_CLASSES > 32 + , PT(32), PT(33), PT(34), PT(35), PT(36), PT(37), PT(38), PT(39) +#if NB_SMALL_SIZE_CLASSES > 40 + , PT(40), PT(41), PT(42), PT(43), PT(44), PT(45), PT(46), PT(47) +#if NB_SMALL_SIZE_CLASSES > 48 + , PT(48), PT(49), PT(50), PT(51), PT(52), PT(53), PT(54), PT(55) +#if NB_SMALL_SIZE_CLASSES > 56 + , PT(56), PT(57), PT(58), PT(59), PT(60), PT(61), PT(62), PT(63) +#if NB_SMALL_SIZE_CLASSES > 64 +#error "NB_SMALL_SIZE_CLASSES should be less than 64" +#endif /* NB_SMALL_SIZE_CLASSES > 64 */ +#endif /* NB_SMALL_SIZE_CLASSES > 56 */ +#endif /* NB_SMALL_SIZE_CLASSES > 48 */ +#endif /* NB_SMALL_SIZE_CLASSES > 40 */ +#endif /* NB_SMALL_SIZE_CLASSES > 32 */ +#endif /* NB_SMALL_SIZE_CLASSES > 24 */ +#endif /* NB_SMALL_SIZE_CLASSES > 16 */ +#endif /* NB_SMALL_SIZE_CLASSES > 8 */ +}; + +/*========================================================================== +Arena management. + +`arenas` is a vector of arena_objects. It contains maxarenas entries, some of +which may not be currently used (== they're arena_objects that aren't +currently associated with an allocated arena). Note that arenas proper are +separately malloc'ed. + +Prior to Python 2.5, arenas were never free()'ed. Starting with Python 2.5, +we do try to free() arenas, and use some mild heuristic strategies to increase +the likelihood that arenas eventually can be freed. + +unused_arena_objects + + This is a singly-linked list of the arena_objects that are currently not + being used (no arena is associated with them). Objects are taken off the + head of the list in new_arena(), and are pushed on the head of the list in + PyObject_Free() when the arena is empty. Key invariant: an arena_object + is on this list if and only if its .address member is 0. + +usable_arenas + + This is a doubly-linked list of the arena_objects associated with arenas + that have pools available. These pools are either waiting to be reused, + or have not been used before. The list is sorted to have the most- + allocated arenas first (ascending order based on the nfreepools member). + This means that the next allocation will come from a heavily used arena, + which gives the nearly empty arenas a chance to be returned to the system. + In my unscientific tests this dramatically improved the number of arenas + that could be freed. + +Note that an arena_object associated with an arena all of whose pools are +currently in use isn't on either list. +*/ + +/* Array of objects used to track chunks of memory (arenas). */ +static struct arena_object* arenas = NULL; +/* Number of slots currently allocated in the `arenas` vector. */ +static uint maxarenas = 0; + +/* The head of the singly-linked, NULL-terminated list of available + * arena_objects. + */ +static struct arena_object* unused_arena_objects = NULL; + +/* The head of the doubly-linked, NULL-terminated at each end, list of + * arena_objects associated with arenas that have pools available. + */ +static struct arena_object* usable_arenas = NULL; + +/* How many arena_objects do we initially allocate? + * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the + * `arenas` vector. + */ +#define INITIAL_ARENA_OBJECTS 16 + +/* Number of arenas allocated that haven't been free()'d. */ +static size_t narenas_currently_allocated = 0; + +/* Total number of times malloc() called to allocate an arena. */ +static size_t ntimes_arena_allocated = 0; +/* High water mark (max value ever seen) for narenas_currently_allocated. */ +static size_t narenas_highwater = 0; + +static Py_ssize_t _Py_AllocatedBlocks = 0; + Py_ssize_t _Py_GetAllocatedBlocks(void) { - return _PyRuntime.mem.num_allocated_blocks; + return _Py_AllocatedBlocks; } @@ -590,7 +1067,7 @@ new_arena(void) if (debug_stats) _PyObject_DebugMallocStats(stderr); - if (_PyRuntime.mem.unused_arena_objects == NULL) { + if (unused_arena_objects == NULL) { uint i; uint numarenas; size_t nbytes; @@ -598,18 +1075,18 @@ new_arena(void) /* Double the number of arena objects on each allocation. * Note that it's possible for `numarenas` to overflow. */ - numarenas = _PyRuntime.mem.maxarenas ? _PyRuntime.mem.maxarenas << 1 : INITIAL_ARENA_OBJECTS; - if (numarenas <= _PyRuntime.mem.maxarenas) + numarenas = maxarenas ? maxarenas << 1 : INITIAL_ARENA_OBJECTS; + if (numarenas <= maxarenas) return NULL; /* overflow */ #if SIZEOF_SIZE_T <= SIZEOF_INT - if (numarenas > SIZE_MAX / sizeof(*_PyRuntime.mem.arenas)) + if (numarenas > SIZE_MAX / sizeof(*arenas)) return NULL; /* overflow */ #endif - nbytes = numarenas * sizeof(*_PyRuntime.mem.arenas); - arenaobj = (struct arena_object *)PyMem_RawRealloc(_PyRuntime.mem.arenas, nbytes); + nbytes = numarenas * sizeof(*arenas); + arenaobj = (struct arena_object *)PyMem_RawRealloc(arenas, nbytes); if (arenaobj == NULL) return NULL; - _PyRuntime.mem.arenas = arenaobj; + arenas = arenaobj; /* We might need to fix pointers that were copied. However, * new_arena only gets called when all the pages in the @@ -617,45 +1094,45 @@ new_arena(void) * into the old array. Thus, we don't have to worry about * invalid pointers. Just to be sure, some asserts: */ - assert(_PyRuntime.mem.usable_arenas == NULL); - assert(_PyRuntime.mem.unused_arena_objects == NULL); + assert(usable_arenas == NULL); + assert(unused_arena_objects == NULL); /* Put the new arenas on the unused_arena_objects list. */ - for (i = _PyRuntime.mem.maxarenas; i < numarenas; ++i) { - _PyRuntime.mem.arenas[i].address = 0; /* mark as unassociated */ - _PyRuntime.mem.arenas[i].nextarena = i < numarenas - 1 ? - &_PyRuntime.mem.arenas[i+1] : NULL; + for (i = maxarenas; i < numarenas; ++i) { + arenas[i].address = 0; /* mark as unassociated */ + arenas[i].nextarena = i < numarenas - 1 ? + &arenas[i+1] : NULL; } /* Update globals. */ - _PyRuntime.mem.unused_arena_objects = &_PyRuntime.mem.arenas[_PyRuntime.mem.maxarenas]; - _PyRuntime.mem.maxarenas = numarenas; + unused_arena_objects = &arenas[maxarenas]; + maxarenas = numarenas; } /* Take the next available arena object off the head of the list. */ - assert(_PyRuntime.mem.unused_arena_objects != NULL); - arenaobj = _PyRuntime.mem.unused_arena_objects; - _PyRuntime.mem.unused_arena_objects = arenaobj->nextarena; + assert(unused_arena_objects != NULL); + arenaobj = unused_arena_objects; + unused_arena_objects = arenaobj->nextarena; assert(arenaobj->address == 0); - address = _PyRuntime.obj.allocator_arenas.alloc(_PyRuntime.obj.allocator_arenas.ctx, ARENA_SIZE); + address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE); if (address == NULL) { /* The allocation failed: return NULL after putting the * arenaobj back. */ - arenaobj->nextarena = _PyRuntime.mem.unused_arena_objects; - _PyRuntime.mem.unused_arena_objects = arenaobj; + arenaobj->nextarena = unused_arena_objects; + unused_arena_objects = arenaobj; return NULL; } arenaobj->address = (uintptr_t)address; - ++_PyRuntime.mem.narenas_currently_allocated; - ++_PyRuntime.mem.ntimes_arena_allocated; - if (_PyRuntime.mem.narenas_currently_allocated > _PyRuntime.mem.narenas_highwater) - _PyRuntime.mem.narenas_highwater = _PyRuntime.mem.narenas_currently_allocated; + ++narenas_currently_allocated; + ++ntimes_arena_allocated; + if (narenas_currently_allocated > narenas_highwater) + narenas_highwater = narenas_currently_allocated; arenaobj->freepools = NULL; /* pool_address <- first pool-aligned address in the arena nfreepools <- number of whole pools that fit after alignment */ - arenaobj->pool_address = (pyblock*)arenaobj->address; + arenaobj->pool_address = (block*)arenaobj->address; arenaobj->nfreepools = ARENA_SIZE / POOL_SIZE; assert(POOL_SIZE * arenaobj->nfreepools == ARENA_SIZE); excess = (uint)(arenaobj->address & POOL_SIZE_MASK); @@ -753,9 +1230,9 @@ address_in_range(void *p, poolp pool) // the GIL. The following dance forces the compiler to read pool->arenaindex // only once. uint arenaindex = *((volatile uint *)&pool->arenaindex); - return arenaindex < _PyRuntime.mem.maxarenas && - (uintptr_t)p - _PyRuntime.mem.arenas[arenaindex].address < ARENA_SIZE && - _PyRuntime.mem.arenas[arenaindex].address != 0; + return arenaindex < maxarenas && + (uintptr_t)p - arenas[arenaindex].address < ARENA_SIZE && + arenas[arenaindex].address != 0; } @@ -777,7 +1254,7 @@ address_in_range(void *p, poolp pool) static int pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) { - pyblock *bp; + block *bp; poolp pool; poolp next; uint size; @@ -803,7 +1280,7 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) * Most frequent paths first */ size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT; - pool = _PyRuntime.mem.usedpools[size + size]; + pool = usedpools[size + size]; if (pool != pool->nextpool) { /* * There is a used pool for this size class. @@ -812,7 +1289,7 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) ++pool->ref.count; bp = pool->freeblock; assert(bp != NULL); - if ((pool->freeblock = *(pyblock **)bp) != NULL) { + if ((pool->freeblock = *(block **)bp) != NULL) { goto success; } @@ -821,10 +1298,10 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) */ if (pool->nextoffset <= pool->maxnextoffset) { /* There is room for another block. */ - pool->freeblock = (pyblock*)pool + + pool->freeblock = (block*)pool + pool->nextoffset; pool->nextoffset += INDEX2SIZE(size); - *(pyblock **)(pool->freeblock) = NULL; + *(block **)(pool->freeblock) = NULL; goto success; } @@ -839,27 +1316,27 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) /* There isn't a pool of the right size class immediately * available: use a free pool. */ - if (_PyRuntime.mem.usable_arenas == NULL) { + if (usable_arenas == NULL) { /* No arena has a free pool: allocate a new arena. */ #ifdef WITH_MEMORY_LIMITS - if (_PyRuntime.mem.narenas_currently_allocated >= MAX_ARENAS) { + if (narenas_currently_allocated >= MAX_ARENAS) { goto failed; } #endif - _PyRuntime.mem.usable_arenas = new_arena(); - if (_PyRuntime.mem.usable_arenas == NULL) { + usable_arenas = new_arena(); + if (usable_arenas == NULL) { goto failed; } - _PyRuntime.mem.usable_arenas->nextarena = - _PyRuntime.mem.usable_arenas->prevarena = NULL; + usable_arenas->nextarena = + usable_arenas->prevarena = NULL; } - assert(_PyRuntime.mem.usable_arenas->address != 0); + assert(usable_arenas->address != 0); /* Try to get a cached free pool. */ - pool = _PyRuntime.mem.usable_arenas->freepools; + pool = usable_arenas->freepools; if (pool != NULL) { /* Unlink from cached pools. */ - _PyRuntime.mem.usable_arenas->freepools = pool->nextpool; + usable_arenas->freepools = pool->nextpool; /* This arena already had the smallest nfreepools * value, so decreasing nfreepools doesn't change @@ -868,18 +1345,18 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) * become wholly allocated, we need to remove its * arena_object from usable_arenas. */ - --_PyRuntime.mem.usable_arenas->nfreepools; - if (_PyRuntime.mem.usable_arenas->nfreepools == 0) { + --usable_arenas->nfreepools; + if (usable_arenas->nfreepools == 0) { /* Wholly allocated: remove. */ - assert(_PyRuntime.mem.usable_arenas->freepools == NULL); - assert(_PyRuntime.mem.usable_arenas->nextarena == NULL || - _PyRuntime.mem.usable_arenas->nextarena->prevarena == - _PyRuntime.mem.usable_arenas); - - _PyRuntime.mem.usable_arenas = _PyRuntime.mem.usable_arenas->nextarena; - if (_PyRuntime.mem.usable_arenas != NULL) { - _PyRuntime.mem.usable_arenas->prevarena = NULL; - assert(_PyRuntime.mem.usable_arenas->address != 0); + assert(usable_arenas->freepools == NULL); + assert(usable_arenas->nextarena == NULL || + usable_arenas->nextarena->prevarena == + usable_arenas); + + usable_arenas = usable_arenas->nextarena; + if (usable_arenas != NULL) { + usable_arenas->prevarena = NULL; + assert(usable_arenas->address != 0); } } else { @@ -888,15 +1365,15 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) * off all the arena's pools for the first * time. */ - assert(_PyRuntime.mem.usable_arenas->freepools != NULL || - _PyRuntime.mem.usable_arenas->pool_address <= - (pyblock*)_PyRuntime.mem.usable_arenas->address + + assert(usable_arenas->freepools != NULL || + usable_arenas->pool_address <= + (block*)usable_arenas->address + ARENA_SIZE - POOL_SIZE); } init_pool: /* Frontlink to used pools. */ - next = _PyRuntime.mem.usedpools[size + size]; /* == prev */ + next = usedpools[size + size]; /* == prev */ pool->nextpool = next; pool->prevpool = next; next->nextpool = pool; @@ -909,7 +1386,7 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) */ bp = pool->freeblock; assert(bp != NULL); - pool->freeblock = *(pyblock **)bp; + pool->freeblock = *(block **)bp; goto success; } /* @@ -919,35 +1396,35 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) */ pool->szidx = size; size = INDEX2SIZE(size); - bp = (pyblock *)pool + POOL_OVERHEAD; + bp = (block *)pool + POOL_OVERHEAD; pool->nextoffset = POOL_OVERHEAD + (size << 1); pool->maxnextoffset = POOL_SIZE - size; pool->freeblock = bp + size; - *(pyblock **)(pool->freeblock) = NULL; + *(block **)(pool->freeblock) = NULL; goto success; } /* Carve off a new pool. */ - assert(_PyRuntime.mem.usable_arenas->nfreepools > 0); - assert(_PyRuntime.mem.usable_arenas->freepools == NULL); - pool = (poolp)_PyRuntime.mem.usable_arenas->pool_address; - assert((pyblock*)pool <= (pyblock*)_PyRuntime.mem.usable_arenas->address + + assert(usable_arenas->nfreepools > 0); + assert(usable_arenas->freepools == NULL); + pool = (poolp)usable_arenas->pool_address; + assert((block*)pool <= (block*)usable_arenas->address + ARENA_SIZE - POOL_SIZE); - pool->arenaindex = (uint)(_PyRuntime.mem.usable_arenas - _PyRuntime.mem.arenas); - assert(&_PyRuntime.mem.arenas[pool->arenaindex] == _PyRuntime.mem.usable_arenas); + pool->arenaindex = (uint)(usable_arenas - arenas); + assert(&arenas[pool->arenaindex] == usable_arenas); pool->szidx = DUMMY_SIZE_IDX; - _PyRuntime.mem.usable_arenas->pool_address += POOL_SIZE; - --_PyRuntime.mem.usable_arenas->nfreepools; + usable_arenas->pool_address += POOL_SIZE; + --usable_arenas->nfreepools; - if (_PyRuntime.mem.usable_arenas->nfreepools == 0) { - assert(_PyRuntime.mem.usable_arenas->nextarena == NULL || - _PyRuntime.mem.usable_arenas->nextarena->prevarena == - _PyRuntime.mem.usable_arenas); + if (usable_arenas->nfreepools == 0) { + assert(usable_arenas->nextarena == NULL || + usable_arenas->nextarena->prevarena == + usable_arenas); /* Unlink the arena: it is completely allocated. */ - _PyRuntime.mem.usable_arenas = _PyRuntime.mem.usable_arenas->nextarena; - if (_PyRuntime.mem.usable_arenas != NULL) { - _PyRuntime.mem.usable_arenas->prevarena = NULL; - assert(_PyRuntime.mem.usable_arenas->address != 0); + usable_arenas = usable_arenas->nextarena; + if (usable_arenas != NULL) { + usable_arenas->prevarena = NULL; + assert(usable_arenas->address != 0); } } @@ -970,13 +1447,13 @@ _PyObject_Malloc(void *ctx, size_t nbytes) { void* ptr; if (pymalloc_alloc(ctx, &ptr, nbytes)) { - _PyRuntime.mem.num_allocated_blocks++; + _Py_AllocatedBlocks++; return ptr; } ptr = PyMem_RawMalloc(nbytes); if (ptr != NULL) { - _PyRuntime.mem.num_allocated_blocks++; + _Py_AllocatedBlocks++; } return ptr; } @@ -992,13 +1469,13 @@ _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize) if (pymalloc_alloc(ctx, &ptr, nbytes)) { memset(ptr, 0, nbytes); - _PyRuntime.mem.num_allocated_blocks++; + _Py_AllocatedBlocks++; return ptr; } ptr = PyMem_RawCalloc(nelem, elsize); if (ptr != NULL) { - _PyRuntime.mem.num_allocated_blocks++; + _Py_AllocatedBlocks++; } return ptr; } @@ -1011,7 +1488,7 @@ static int pymalloc_free(void *ctx, void *p) { poolp pool; - pyblock *lastfree; + block *lastfree; poolp next, prev; uint size; @@ -1038,8 +1515,8 @@ pymalloc_free(void *ctx, void *p) * list in any case). */ assert(pool->ref.count > 0); /* else it was empty */ - *(pyblock **)p = lastfree = pool->freeblock; - pool->freeblock = (pyblock *)p; + *(block **)p = lastfree = pool->freeblock; + pool->freeblock = (block *)p; if (!lastfree) { /* Pool was full, so doesn't currently live in any list: * link it to the front of the appropriate usedpools[] list. @@ -1050,7 +1527,7 @@ pymalloc_free(void *ctx, void *p) --pool->ref.count; assert(pool->ref.count > 0); /* else the pool is empty */ size = pool->szidx; - next = _PyRuntime.mem.usedpools[size + size]; + next = usedpools[size + size]; prev = next->prevpool; /* insert pool before next: prev <-> pool <-> next */ @@ -1084,7 +1561,7 @@ pymalloc_free(void *ctx, void *p) /* Link the pool to freepools. This is a singly-linked * list, and pool->prevpool isn't used there. */ - ao = &_PyRuntime.mem.arenas[pool->arenaindex]; + ao = &arenas[pool->arenaindex]; pool->nextpool = ao->freepools; ao->freepools = pool; nf = ++ao->nfreepools; @@ -1113,9 +1590,9 @@ pymalloc_free(void *ctx, void *p) * usable_arenas pointer. */ if (ao->prevarena == NULL) { - _PyRuntime.mem.usable_arenas = ao->nextarena; - assert(_PyRuntime.mem.usable_arenas == NULL || - _PyRuntime.mem.usable_arenas->address != 0); + usable_arenas = ao->nextarena; + assert(usable_arenas == NULL || + usable_arenas->address != 0); } else { assert(ao->prevarena->nextarena == ao); @@ -1131,14 +1608,14 @@ pymalloc_free(void *ctx, void *p) /* Record that this arena_object slot is * available to be reused. */ - ao->nextarena = _PyRuntime.mem.unused_arena_objects; - _PyRuntime.mem.unused_arena_objects = ao; + ao->nextarena = unused_arena_objects; + unused_arena_objects = ao; /* Free the entire arena. */ - _PyRuntime.obj.allocator_arenas.free(_PyRuntime.obj.allocator_arenas.ctx, + _PyObject_Arena.free(_PyObject_Arena.ctx, (void *)ao->address, ARENA_SIZE); ao->address = 0; /* mark unassociated */ - --_PyRuntime.mem.narenas_currently_allocated; + --narenas_currently_allocated; goto success; } @@ -1149,12 +1626,12 @@ pymalloc_free(void *ctx, void *p) * ao->nfreepools was 0 before, ao isn't * currently on the usable_arenas list. */ - ao->nextarena = _PyRuntime.mem.usable_arenas; + ao->nextarena = usable_arenas; ao->prevarena = NULL; - if (_PyRuntime.mem.usable_arenas) - _PyRuntime.mem.usable_arenas->prevarena = ao; - _PyRuntime.mem.usable_arenas = ao; - assert(_PyRuntime.mem.usable_arenas->address != 0); + if (usable_arenas) + usable_arenas->prevarena = ao; + usable_arenas = ao; + assert(usable_arenas->address != 0); goto success; } @@ -1183,8 +1660,8 @@ pymalloc_free(void *ctx, void *p) } else { /* ao is at the head of the list */ - assert(_PyRuntime.mem.usable_arenas == ao); - _PyRuntime.mem.usable_arenas = ao->nextarena; + assert(usable_arenas == ao); + usable_arenas = ao->nextarena; } ao->nextarena->prevarena = ao->prevarena; @@ -1209,7 +1686,7 @@ pymalloc_free(void *ctx, void *p) assert(ao->nextarena == NULL || nf <= ao->nextarena->nfreepools); assert(ao->prevarena == NULL || nf > ao->prevarena->nfreepools); assert(ao->nextarena == NULL || ao->nextarena->prevarena == ao); - assert((_PyRuntime.mem.usable_arenas == ao && ao->prevarena == NULL) + assert((usable_arenas == ao && ao->prevarena == NULL) || ao->prevarena->nextarena == ao); goto success; @@ -1228,7 +1705,7 @@ _PyObject_Free(void *ctx, void *p) return; } - _PyRuntime.mem.num_allocated_blocks--; + _Py_AllocatedBlocks--; if (!pymalloc_free(ctx, p)) { /* pymalloc didn't allocate this address */ PyMem_RawFree(p); @@ -1353,13 +1830,15 @@ _Py_GetAllocatedBlocks(void) #define DEADBYTE 0xDB /* dead (newly freed) memory */ #define FORBIDDENBYTE 0xFB /* untouchable bytes at each end of a block */ +static size_t serialno = 0; /* incremented on each debug {m,re}alloc */ + /* serialno is always incremented via calling this routine. The point is * to supply a single place to set a breakpoint. */ static void bumpserialno(void) { - ++_PyRuntime.mem.serialno; + ++serialno; } #define SST SIZEOF_SIZE_T @@ -1466,7 +1945,7 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) /* at tail, write pad (SST bytes) and serialno (SST bytes) */ tail = data + nbytes; memset(tail, FORBIDDENBYTE, SST); - write_size_t(tail + SST, _PyRuntime.mem.serialno); + write_size_t(tail + SST, serialno); return data; } @@ -1526,7 +2005,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */ size_t total; /* 2 * SST + nbytes + 2 * SST */ size_t original_nbytes; - size_t serialno; + size_t block_serialno; #define ERASED_SIZE 64 uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ @@ -1542,7 +2021,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) total = nbytes + 4*SST; tail = data + original_nbytes; - serialno = read_size_t(tail + SST); + block_serialno = read_size_t(tail + SST); /* Mark the header, the trailer, ERASED_SIZE bytes at the begin and ERASED_SIZE bytes at the end as dead and save the copy of erased bytes. */ @@ -1565,7 +2044,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) else { head = r; bumpserialno(); - serialno = _PyRuntime.mem.serialno; + block_serialno = serialno; } write_size_t(head, nbytes); @@ -1575,7 +2054,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) tail = data + nbytes; memset(tail, FORBIDDENBYTE, SST); - write_size_t(tail + SST, serialno); + write_size_t(tail + SST, block_serialno); /* Restore saved bytes. */ if (original_nbytes <= sizeof(save)) { @@ -1923,16 +2402,16 @@ _PyObject_DebugMallocStats(FILE *out) * to march over all the arenas. If we're lucky, most of the memory * will be living in full pools -- would be a shame to miss them. */ - for (i = 0; i < _PyRuntime.mem.maxarenas; ++i) { + for (i = 0; i < maxarenas; ++i) { uint j; - uintptr_t base = _PyRuntime.mem.arenas[i].address; + uintptr_t base = arenas[i].address; /* Skip arenas which are not allocated. */ - if (_PyRuntime.mem.arenas[i].address == (uintptr_t)NULL) + if (arenas[i].address == (uintptr_t)NULL) continue; narenas += 1; - numfreepools += _PyRuntime.mem.arenas[i].nfreepools; + numfreepools += arenas[i].nfreepools; /* round up to pool alignment */ if (base & (uintptr_t)POOL_SIZE_MASK) { @@ -1942,8 +2421,8 @@ _PyObject_DebugMallocStats(FILE *out) } /* visit every pool in the arena */ - assert(base <= (uintptr_t) _PyRuntime.mem.arenas[i].pool_address); - for (j = 0; base < (uintptr_t) _PyRuntime.mem.arenas[i].pool_address; + assert(base <= (uintptr_t) arenas[i].pool_address); + for (j = 0; base < (uintptr_t) arenas[i].pool_address; ++j, base += POOL_SIZE) { poolp p = (poolp)base; const uint sz = p->szidx; @@ -1952,7 +2431,7 @@ _PyObject_DebugMallocStats(FILE *out) if (p->ref.count == 0) { /* currently unused */ #ifdef Py_DEBUG - assert(pool_is_in_list(p, _PyRuntime.mem.arenas[i].freepools)); + assert(pool_is_in_list(p, arenas[i].freepools)); #endif continue; } @@ -1962,11 +2441,11 @@ _PyObject_DebugMallocStats(FILE *out) numfreeblocks[sz] += freeblocks; #ifdef Py_DEBUG if (freeblocks > 0) - assert(pool_is_in_list(p, _PyRuntime.mem.usedpools[sz + sz])); + assert(pool_is_in_list(p, usedpools[sz + sz])); #endif } } - assert(narenas == _PyRuntime.mem.narenas_currently_allocated); + assert(narenas == narenas_currently_allocated); fputc('\n', out); fputs("class size num pools blocks in use avail blocks\n" @@ -1994,10 +2473,10 @@ _PyObject_DebugMallocStats(FILE *out) } fputc('\n', out); if (_PyMem_DebugEnabled()) - (void)printone(out, "# times object malloc called", _PyRuntime.mem.serialno); - (void)printone(out, "# arenas allocated total", _PyRuntime.mem.ntimes_arena_allocated); - (void)printone(out, "# arenas reclaimed", _PyRuntime.mem.ntimes_arena_allocated - narenas); - (void)printone(out, "# arenas highwater mark", _PyRuntime.mem.narenas_highwater); + (void)printone(out, "# times object malloc called", serialno); + (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); + (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); + (void)printone(out, "# arenas highwater mark", narenas_highwater); (void)printone(out, "# arenas allocated current", narenas); PyOS_snprintf(buf, sizeof(buf), diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index f6b2b65d1ffba57..8d4fea8ede15daa 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -2171,19 +2171,6 @@ kind_maxchar_limit(unsigned int kind) } } -static inline Py_UCS4 -align_maxchar(Py_UCS4 maxchar) -{ - if (maxchar <= 127) - return 127; - else if (maxchar <= 255) - return 255; - else if (maxchar <= 65535) - return 65535; - else - return MAX_UNICODE; -} - static PyObject* _PyUnicode_FromUCS1(const Py_UCS1* u, Py_ssize_t size) { diff --git a/PC/_msi.c b/PC/_msi.c index df6c881b4ec44fd..000d81f139f3541 100644 --- a/PC/_msi.c +++ b/PC/_msi.c @@ -315,6 +315,12 @@ msierror(int status) case ERROR_INVALID_PARAMETER: PyErr_SetString(MSIError, "invalid parameter"); return NULL; + case ERROR_OPEN_FAILED: + PyErr_SetString(MSIError, "open failed"); + return NULL; + case ERROR_CREATE_FAILED: + PyErr_SetString(MSIError, "create failed"); + return NULL; default: PyErr_Format(MSIError, "unknown error %x", status); return NULL; @@ -569,6 +575,8 @@ summary_getproperty(msiobj* si, PyObject *args) if (sval != sbuf) free(sval); return result; + case VT_EMPTY: + Py_RETURN_NONE; } PyErr_Format(PyExc_NotImplementedError, "result of type %d", type); return NULL; @@ -723,8 +731,12 @@ view_fetch(msiobj *view, PyObject*args) int status; MSIHANDLE result; - if ((status = MsiViewFetch(view->h, &result)) != ERROR_SUCCESS) + status = MsiViewFetch(view->h, &result); + if (status == ERROR_NO_MORE_ITEMS) { + Py_RETURN_NONE; + } else if (status != ERROR_SUCCESS) { return msierror(status); + } return record_new(result); } diff --git a/PC/getpathp.c b/PC/getpathp.c index 9bbc7bf0b27ba54..fe0226ba5d277cb 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -116,14 +116,34 @@ #define LANDMARK L"lib\\os.py" #endif -static wchar_t prefix[MAXPATHLEN+1]; -static wchar_t progpath[MAXPATHLEN+1]; -static wchar_t dllpath[MAXPATHLEN+1]; -static wchar_t *module_search_path = NULL; +typedef struct { + wchar_t *prefix; + wchar_t *program_name; + wchar_t *dll_path; + wchar_t *module_search_path; +} PyPathConfig; + +typedef struct { + wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ + wchar_t *path_env; /* PATH environment variable */ + wchar_t *home; /* PYTHONHOME environment variable */ + + /* Registry key "Software\Python\PythonCore\PythonPath" */ + wchar_t *machine_path; /* from HKEY_LOCAL_MACHINE */ + wchar_t *user_path; /* from HKEY_CURRENT_USER */ + + wchar_t *program_name; /* Program name */ + wchar_t argv0_path[MAXPATHLEN+1]; + wchar_t zip_path[MAXPATHLEN+1]; +} PyCalculatePath; + + +static PyPathConfig path_config = {.module_search_path = NULL}; +/* determine if "ch" is a separator character */ static int -is_sep(wchar_t ch) /* determine if "ch" is a separator character */ +is_sep(wchar_t ch) { #ifdef ALTSEP return ch == SEP || ch == ALTSEP; @@ -132,28 +152,31 @@ is_sep(wchar_t ch) /* determine if "ch" is a separator character */ #endif } + /* assumes 'dir' null terminated in bounds. Never writes - beyond existing terminator. -*/ + beyond existing terminator. */ static void reduce(wchar_t *dir) { size_t i = wcsnlen_s(dir, MAXPATHLEN+1); - if (i >= MAXPATHLEN+1) + if (i >= MAXPATHLEN+1) { Py_FatalError("buffer overflow in getpathp.c's reduce()"); + } while (i > 0 && !is_sep(dir[i])) --i; dir[i] = '\0'; } + static int change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext) { size_t src_len = wcsnlen_s(src, MAXPATHLEN+1); size_t i = src_len; - if (i >= MAXPATHLEN+1) + if (i >= MAXPATHLEN+1) { Py_FatalError("buffer overflow in getpathp.c's reduce()"); + } while (i > 0 && src[i] != '.' && !is_sep(src[i])) --i; @@ -163,11 +186,13 @@ change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext) return -1; } - if (is_sep(src[i])) + if (is_sep(src[i])) { i = src_len; + } if (wcsncpy_s(dest, MAXPATHLEN+1, src, i) || - wcscat_s(dest, MAXPATHLEN+1, ext)) { + wcscat_s(dest, MAXPATHLEN+1, ext)) + { dest[0] = '\0'; return -1; } @@ -175,22 +200,25 @@ change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext) return 0; } + static int exists(wchar_t *filename) { return GetFileAttributesW(filename) != 0xFFFFFFFF; } -/* Assumes 'filename' MAXPATHLEN+1 bytes long - - may extend 'filename' by one character. -*/ + +/* Is module -- check for .pyc too. + Assumes 'filename' MAXPATHLEN+1 bytes long - + may extend 'filename' by one character. */ static int -ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc too */ +ismodule(wchar_t *filename, int update_filename) { size_t n; - if (exists(filename)) + if (exists(filename)) { return 1; + } /* Check for the compiled version of prefix. */ n = wcsnlen_s(filename, MAXPATHLEN+1); @@ -199,13 +227,15 @@ ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc filename[n] = L'c'; filename[n + 1] = L'\0'; exist = exists(filename); - if (!update_filename) + if (!update_filename) { filename[n] = L'\0'; + } return exist; } return 0; } + /* Add a path component, by appending stuff to buffer. buffer must have at least MAXPATHLEN + 1 bytes allocated, and contain a NUL-terminated string with no more than MAXPATHLEN characters (not counting @@ -217,7 +247,9 @@ ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc */ static int _PathCchCombineEx_Initialized = 0; -typedef HRESULT(__stdcall *PPathCchCombineEx)(PWSTR pszPathOut, size_t cchPathOut, PCWSTR pszPathIn, PCWSTR pszMore, unsigned long dwFlags); +typedef HRESULT(__stdcall *PPathCchCombineEx) (PWSTR pszPathOut, size_t cchPathOut, + PCWSTR pszPathIn, PCWSTR pszMore, + unsigned long dwFlags); static PPathCchCombineEx _PathCchCombineEx; static void @@ -225,28 +257,32 @@ join(wchar_t *buffer, const wchar_t *stuff) { if (_PathCchCombineEx_Initialized == 0) { HMODULE pathapi = LoadLibraryW(L"api-ms-win-core-path-l1-1-0.dll"); - if (pathapi) + if (pathapi) { _PathCchCombineEx = (PPathCchCombineEx)GetProcAddress(pathapi, "PathCchCombineEx"); - else + } + else { _PathCchCombineEx = NULL; + } _PathCchCombineEx_Initialized = 1; } if (_PathCchCombineEx) { - if (FAILED(_PathCchCombineEx(buffer, MAXPATHLEN+1, buffer, stuff, 0))) + if (FAILED(_PathCchCombineEx(buffer, MAXPATHLEN+1, buffer, stuff, 0))) { Py_FatalError("buffer overflow in getpathp.c's join()"); + } } else { - if (!PathCombineW(buffer, buffer, stuff)) + if (!PathCombineW(buffer, buffer, stuff)) { Py_FatalError("buffer overflow in getpathp.c's join()"); + } } } + /* gotlandmark only called by search_for_prefix, which ensures 'prefix' is null terminated in bounds. join() ensures - 'landmark' can not overflow prefix if too long. -*/ + 'landmark' can not overflow prefix if too long. */ static int -gotlandmark(const wchar_t *landmark) +gotlandmark(wchar_t *prefix, const wchar_t *landmark) { int ok; Py_ssize_t n = wcsnlen_s(prefix, MAXPATHLEN); @@ -257,27 +293,29 @@ gotlandmark(const wchar_t *landmark) return ok; } + /* assumes argv0_path is MAXPATHLEN+1 bytes long, already \0 term'd. assumption provided by only caller, calculate_path() */ static int -search_for_prefix(wchar_t *argv0_path, const wchar_t *landmark) +search_for_prefix(wchar_t *prefix, wchar_t *argv0_path, const wchar_t *landmark) { /* Search from argv0_path, until landmark is found */ wcscpy_s(prefix, MAXPATHLEN + 1, argv0_path); do { - if (gotlandmark(landmark)) + if (gotlandmark(prefix, landmark)) { return 1; + } reduce(prefix); } while (prefix[0]); return 0; } + #ifdef Py_ENABLE_SHARED /* a string loaded from the DLL at startup.*/ extern const char *PyWin_DLLVersionString; - /* Load a PYTHONPATH value from the registry. Load from either HKEY_LOCAL_MACHINE or HKEY_CURRENT_USER. @@ -290,7 +328,6 @@ extern const char *PyWin_DLLVersionString; work on Win16, where the buffer sizes werent available in advance. It could be simplied now Win16/Win32s is dead! */ - static wchar_t * getpythonregpath(HKEY keyBase, int skipcore) { @@ -315,7 +352,9 @@ getpythonregpath(HKEY keyBase, int skipcore) sizeof(WCHAR)*(versionLen-1) + sizeof(keySuffix); keyBuf = keyBufPtr = PyMem_RawMalloc(keyBufLen); - if (keyBuf==NULL) goto done; + if (keyBuf==NULL) { + goto done; + } memcpy_s(keyBufPtr, keyBufLen, keyPrefix, sizeof(keyPrefix)-sizeof(WCHAR)); keyBufPtr += Py_ARRAY_LENGTH(keyPrefix) - 1; @@ -329,17 +368,25 @@ getpythonregpath(HKEY keyBase, int skipcore) 0, /* reserved */ KEY_READ, &newKey); - if (rc!=ERROR_SUCCESS) goto done; + if (rc!=ERROR_SUCCESS) { + goto done; + } /* Find out how big our core buffer is, and how many subkeys we have */ rc = RegQueryInfoKey(newKey, NULL, NULL, NULL, &numKeys, NULL, NULL, NULL, NULL, &dataSize, NULL, NULL); - if (rc!=ERROR_SUCCESS) goto done; - if (skipcore) dataSize = 0; /* Only count core ones if we want them! */ + if (rc!=ERROR_SUCCESS) { + goto done; + } + if (skipcore) { + dataSize = 0; /* Only count core ones if we want them! */ + } /* Allocate a temp array of char buffers, so we only need to loop reading the registry once */ ppPaths = PyMem_RawMalloc( sizeof(WCHAR *) * numKeys ); - if (ppPaths==NULL) goto done; + if (ppPaths==NULL) { + goto done; + } memset(ppPaths, 0, sizeof(WCHAR *) * numKeys); /* Loop over all subkeys, allocating a temp sub-buffer. */ for(index=0;indexdll_path = _PyMem_RawWcsdup(dll_path); + if (config->dll_path == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); +} + + +static _PyInitError +get_program_name(PyCalculatePath *calculate, PyPathConfig *config) +{ + wchar_t program_name[MAXPATHLEN+1]; + memset(program_name, 0, sizeof(program_name)); + + if (GetModuleFileNameW(NULL, program_name, MAXPATHLEN)) { + goto done; + } /* If there is no slash in the argv0 path, then we have to * assume python is on the user's $PATH, since there's no @@ -454,12 +524,15 @@ get_progpath(void) * $PATH isn't exported, you lose. */ #ifdef ALTSEP - if (wcschr(prog, SEP) || wcschr(prog, ALTSEP)) + if (wcschr(calculate->program_name, SEP) || wcschr(calculate->program_name, ALTSEP)) #else - if (wcschr(prog, SEP)) + if (wcschr(calculate->program_name, SEP)) #endif - wcsncpy(progpath, prog, MAXPATHLEN); - else if (path) { + { + wcsncpy(program_name, calculate->program_name, MAXPATHLEN); + } + else if (calculate->path_env) { + wchar_t *path = calculate->path_env; while (1) { wchar_t *delim = wcschr(path, DELIM); @@ -467,28 +540,39 @@ get_progpath(void) size_t len = delim - path; /* ensure we can't overwrite buffer */ len = min(MAXPATHLEN,len); - wcsncpy(progpath, path, len); - *(progpath + len) = '\0'; + wcsncpy(program_name, path, len); + program_name[len] = '\0'; + } + else { + wcsncpy(program_name, path, MAXPATHLEN); } - else - wcsncpy(progpath, path, MAXPATHLEN); /* join() is safe for MAXPATHLEN+1 size buffer */ - join(progpath, prog); - if (exists(progpath)) + join(program_name, calculate->program_name); + if (exists(program_name)) { break; + } if (!delim) { - progpath[0] = '\0'; + program_name[0] = '\0'; break; } path = delim + 1; } } - else - progpath[0] = '\0'; + else { + program_name[0] = '\0'; + } + +done: + config->program_name = _PyMem_RawWcsdup(program_name); + if (config->program_name == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); } + static int find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) { @@ -502,15 +586,18 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) PyObject * decoded; size_t n; - if (p == NULL) + if (p == NULL) { break; + } n = strlen(p); if (p[n - 1] != '\n') { /* line has overflowed - bail */ break; } - if (p[0] == '#') /* Comment - skip */ + if (p[0] == '#') { + /* Comment - skip */ continue; + } decoded = PyUnicode_DecodeUTF8(buffer, n, "surrogateescape"); if (decoded != NULL) { Py_ssize_t k; @@ -537,12 +624,15 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) return result; } + static int -read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) +read_pth_file(PyPathConfig *config, wchar_t *prefix, const wchar_t *path, + int *isolated, int *nosite) { FILE *sp_file = _Py_wfopen(path, L"r"); - if (sp_file == NULL) - return -1; + if (sp_file == NULL) { + return 0; + } wcscpy_s(prefix, MAXPATHLEN+1, path); reduce(prefix); @@ -558,10 +648,12 @@ read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) while (!feof(sp_file)) { char line[MAXPATHLEN + 1]; char *p = fgets(line, MAXPATHLEN + 1, sp_file); - if (!p) + if (!p) { break; - if (*p == '\0' || *p == '\r' || *p == '\n' || *p == '#') + } + if (*p == '\0' || *p == '\r' || *p == '\n' || *p == '#') { continue; + } while (*++p) { if (*p == '\r' || *p == '\n') { *p = '\0'; @@ -611,124 +703,142 @@ read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) PyMem_RawFree(wline); } - module_search_path = buf; - fclose(sp_file); - return 0; + config->module_search_path = buf; + return 1; error: PyMem_RawFree(buf); fclose(sp_file); - return -1; + return 0; } static void -calculate_path(void) +calculate_init(PyCalculatePath *calculate, + const _PyMainInterpreterConfig *main_config) { - wchar_t argv0_path[MAXPATHLEN+1]; - wchar_t *buf; - size_t bufsz; - wchar_t *pythonhome = Py_GetPythonHome(); - wchar_t *envpath = NULL; - - int skiphome, skipdefault; - wchar_t *machinepath = NULL; - wchar_t *userpath = NULL; - wchar_t zip_path[MAXPATHLEN+1]; + calculate->home = main_config->home; + calculate->module_search_path_env = main_config->module_search_path_env; + calculate->program_name = main_config->program_name; - if (!Py_IgnoreEnvironmentFlag) { - envpath = _wgetenv(L"PYTHONPATH"); - } + calculate->path_env = _wgetenv(L"PATH"); +} - get_progpath(); - /* progpath guaranteed \0 terminated in MAXPATH+1 bytes. */ - wcscpy_s(argv0_path, MAXPATHLEN+1, progpath); - reduce(argv0_path); - /* Search for a sys.path file */ - { - wchar_t spbuffer[MAXPATHLEN+1]; +static int +get_pth_filename(wchar_t *spbuffer, PyPathConfig *config) +{ + if (config->dll_path[0]) { + if (!change_ext(spbuffer, config->dll_path, L"._pth") && exists(spbuffer)) { + return 1; + } + } + if (config->program_name[0]) { + if (!change_ext(spbuffer, config->program_name, L"._pth") && exists(spbuffer)) { + return 1; + } + } + return 0; +} - if ((dllpath[0] && !change_ext(spbuffer, dllpath, L"._pth") && exists(spbuffer)) || - (progpath[0] && !change_ext(spbuffer, progpath, L"._pth") && exists(spbuffer))) { - if (!read_pth_file(spbuffer, prefix, &Py_IsolatedFlag, &Py_NoSiteFlag)) { - return; - } - } +static int +calculate_pth_file(PyPathConfig *config, wchar_t *prefix) +{ + wchar_t spbuffer[MAXPATHLEN+1]; + + if (!get_pth_filename(spbuffer, config)) { + return 0; } - /* Search for an environment configuration file, first in the - executable's directory and then in the parent directory. - If found, open it for use when searching for prefixes. - */ + return read_pth_file(config, prefix, spbuffer, + &Py_IsolatedFlag, &Py_NoSiteFlag); +} - { - wchar_t envbuffer[MAXPATHLEN+1]; - wchar_t tmpbuffer[MAXPATHLEN+1]; - const wchar_t *env_cfg = L"pyvenv.cfg"; - FILE * env_file = NULL; - wcscpy_s(envbuffer, MAXPATHLEN+1, argv0_path); +/* Search for an environment configuration file, first in the + executable's directory and then in the parent directory. + If found, open it for use when searching for prefixes. +*/ +static void +calculate_pyvenv_file(PyCalculatePath *calculate) +{ + wchar_t envbuffer[MAXPATHLEN+1]; + const wchar_t *env_cfg = L"pyvenv.cfg"; + + wcscpy_s(envbuffer, MAXPATHLEN+1, calculate->argv0_path); + join(envbuffer, env_cfg); + + FILE *env_file = _Py_wfopen(envbuffer, L"r"); + if (env_file == NULL) { + errno = 0; + reduce(envbuffer); + reduce(envbuffer); join(envbuffer, env_cfg); env_file = _Py_wfopen(envbuffer, L"r"); if (env_file == NULL) { errno = 0; - reduce(envbuffer); - reduce(envbuffer); - join(envbuffer, env_cfg); - env_file = _Py_wfopen(envbuffer, L"r"); - if (env_file == NULL) { - errno = 0; - } - } - if (env_file != NULL) { - /* Look for a 'home' variable and set argv0_path to it, if found */ - if (find_env_config_value(env_file, L"home", tmpbuffer)) { - wcscpy_s(argv0_path, MAXPATHLEN+1, tmpbuffer); - } - fclose(env_file); - env_file = NULL; } } - /* Calculate zip archive path from DLL or exe path */ - change_ext(zip_path, dllpath[0] ? dllpath : progpath, L".zip"); + if (env_file == NULL) { + return; + } - if (pythonhome == NULL || *pythonhome == '\0') { - if (zip_path[0] && exists(zip_path)) { - wcscpy_s(prefix, MAXPATHLEN+1, zip_path); - reduce(prefix); - pythonhome = prefix; - } else if (search_for_prefix(argv0_path, LANDMARK)) - pythonhome = prefix; - else - pythonhome = NULL; + /* Look for a 'home' variable and set argv0_path to it, if found */ + wchar_t tmpbuffer[MAXPATHLEN+1]; + if (find_env_config_value(env_file, L"home", tmpbuffer)) { + wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, tmpbuffer); } - else - wcscpy_s(prefix, MAXPATHLEN+1, pythonhome); + fclose(env_file); +} + - if (envpath && *envpath == '\0') - envpath = NULL; +#define INIT_ERR_BUFFER_OVERFLOW() _Py_INIT_ERR("buffer overflow") - skiphome = pythonhome==NULL ? 0 : 1; +static void +calculate_home_prefix(PyCalculatePath *calculate, wchar_t *prefix) +{ + if (calculate->home == NULL || *calculate->home == '\0') { + if (calculate->zip_path[0] && exists(calculate->zip_path)) { + wcscpy_s(prefix, MAXPATHLEN+1, calculate->zip_path); + reduce(prefix); + calculate->home = prefix; + } + else if (search_for_prefix(prefix, calculate->argv0_path, LANDMARK)) { + calculate->home = prefix; + } + else { + calculate->home = NULL; + } + } + else { + wcscpy_s(prefix, MAXPATHLEN+1, calculate->home); + } +} + + +static _PyInitError +calculate_module_search_path(PyCalculatePath *calculate, PyPathConfig *config, wchar_t *prefix) +{ + int skiphome = calculate->home==NULL ? 0 : 1; #ifdef Py_ENABLE_SHARED - machinepath = getpythonregpath(HKEY_LOCAL_MACHINE, skiphome); - userpath = getpythonregpath(HKEY_CURRENT_USER, skiphome); + calculate->machine_path = getpythonregpath(HKEY_LOCAL_MACHINE, skiphome); + calculate->user_path = getpythonregpath(HKEY_CURRENT_USER, skiphome); #endif /* We only use the default relative PYTHONPATH if we haven't anything better to use! */ - skipdefault = envpath!=NULL || pythonhome!=NULL || \ - machinepath!=NULL || userpath!=NULL; + int skipdefault = (calculate->module_search_path_env!=NULL || calculate->home!=NULL || \ + calculate->machine_path!=NULL || calculate->user_path!=NULL); /* We need to construct a path from the following parts. (1) the PYTHONPATH environment variable, if set; (2) for Win32, the zip archive file path; - (3) for Win32, the machinepath and userpath, if set; + (3) for Win32, the machine_path and user_path, if set; (4) the PYTHONPATH config macro, with the leading "." - of each component replaced with pythonhome, if set; + of each component replaced with home, if set; (5) the directory containing the executable (argv0_path). The length calculation calculates #4 first. Extra rules: @@ -737,74 +847,80 @@ calculate_path(void) */ /* Calculate size of return buffer */ - if (pythonhome != NULL) { + size_t bufsz = 0; + if (calculate->home != NULL) { wchar_t *p; bufsz = 1; for (p = PYTHONPATH; *p; p++) { - if (*p == DELIM) + if (*p == DELIM) { bufsz++; /* number of DELIM plus one */ + } } - bufsz *= wcslen(pythonhome); + bufsz *= wcslen(calculate->home); } - else - bufsz = 0; bufsz += wcslen(PYTHONPATH) + 1; - bufsz += wcslen(argv0_path) + 1; - if (userpath) - bufsz += wcslen(userpath) + 1; - if (machinepath) - bufsz += wcslen(machinepath) + 1; - bufsz += wcslen(zip_path) + 1; - if (envpath != NULL) - bufsz += wcslen(envpath) + 1; - - module_search_path = buf = PyMem_RawMalloc(bufsz*sizeof(wchar_t)); + bufsz += wcslen(calculate->argv0_path) + 1; + if (calculate->user_path) { + bufsz += wcslen(calculate->user_path) + 1; + } + if (calculate->machine_path) { + bufsz += wcslen(calculate->machine_path) + 1; + } + bufsz += wcslen(calculate->zip_path) + 1; + if (calculate->module_search_path_env != NULL) { + bufsz += wcslen(calculate->module_search_path_env) + 1; + } + + wchar_t *buf, *start_buf; + buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); if (buf == NULL) { /* We can't exit, so print a warning and limp along */ fprintf(stderr, "Can't malloc dynamic PYTHONPATH.\n"); - if (envpath) { + if (calculate->module_search_path_env) { fprintf(stderr, "Using environment $PYTHONPATH.\n"); - module_search_path = envpath; + config->module_search_path = calculate->module_search_path_env; } else { fprintf(stderr, "Using default static path.\n"); - module_search_path = PYTHONPATH; + config->module_search_path = PYTHONPATH; } - PyMem_RawFree(machinepath); - PyMem_RawFree(userpath); - return; + return _Py_INIT_OK(); } + start_buf = buf; - if (envpath) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), envpath)) - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + if (calculate->module_search_path_env) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->module_search_path_env)) { + return INIT_ERR_BUFFER_OVERFLOW(); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; } - if (zip_path[0]) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), zip_path)) - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + if (calculate->zip_path[0]) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->zip_path)) { + return INIT_ERR_BUFFER_OVERFLOW(); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; } - if (userpath) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), userpath)) - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + if (calculate->user_path) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->user_path)) { + return INIT_ERR_BUFFER_OVERFLOW(); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; - PyMem_RawFree(userpath); } - if (machinepath) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), machinepath)) - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + if (calculate->machine_path) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->machine_path)) { + return INIT_ERR_BUFFER_OVERFLOW(); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; - PyMem_RawFree(machinepath); } - if (pythonhome == NULL) { + if (calculate->home == NULL) { if (!skipdefault) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), PYTHONPATH)) - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + if (wcscpy_s(buf, bufsz - (buf - start_buf), PYTHONPATH)) { + return INIT_ERR_BUFFER_OVERFLOW(); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; } @@ -814,13 +930,16 @@ calculate_path(void) size_t n; for (;;) { q = wcschr(p, DELIM); - if (q == NULL) + if (q == NULL) { n = wcslen(p); - else + } + else { n = q-p; + } if (p[0] == '.' && is_sep(p[1])) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), pythonhome)) - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->home)) { + return INIT_ERR_BUFFER_OVERFLOW(); + } buf = wcschr(buf, L'\0'); p++; n--; @@ -828,17 +947,19 @@ calculate_path(void) wcsncpy(buf, p, n); buf += n; *buf++ = DELIM; - if (q == NULL) + if (q == NULL) { break; + } p = q+1; } } - if (argv0_path) { - wcscpy(buf, argv0_path); + if (calculate->argv0_path) { + wcscpy(buf, calculate->argv0_path); buf = wcschr(buf, L'\0'); *buf++ = DELIM; } *(buf - 1) = L'\0'; + /* Now to pull one last hack/trick. If sys.prefix is empty, then try and find it somewhere on the paths we calculated. We scan backwards, as our general policy @@ -847,7 +968,7 @@ calculate_path(void) on the path, and that our 'prefix' directory is the parent of that. */ - if (*prefix==L'\0') { + if (prefix[0] == L'\0') { wchar_t lookBuf[MAXPATHLEN+1]; wchar_t *look = buf - 1; /* 'buf' is at the end of the buffer */ while (1) { @@ -857,22 +978,165 @@ calculate_path(void) start of the path in question - even if this is one character before the start of the buffer */ - while (look >= module_search_path && *look != DELIM) + while (look >= start_buf && *look != DELIM) look--; nchars = lookEnd-look; wcsncpy(lookBuf, look+1, nchars); lookBuf[nchars] = L'\0'; /* Up one level to the parent */ reduce(lookBuf); - if (search_for_prefix(lookBuf, LANDMARK)) { + if (search_for_prefix(prefix, lookBuf, LANDMARK)) { break; } /* If we are out of paths to search - give up */ - if (look < module_search_path) + if (look < start_buf) { break; + } look--; } } + + config->module_search_path = start_buf; + return _Py_INIT_OK(); +} + + +static _PyInitError +calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config, + const _PyMainInterpreterConfig *main_config) +{ + _PyInitError err; + + err = get_dll_path(calculate, config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = get_program_name(calculate, config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + /* program_name guaranteed \0 terminated in MAXPATH+1 bytes. */ + wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, config->program_name); + reduce(calculate->argv0_path); + + wchar_t prefix[MAXPATHLEN+1]; + memset(prefix, 0, sizeof(prefix)); + + /* Search for a sys.path file */ + if (calculate_pth_file(config, prefix)) { + goto done; + } + + calculate_pyvenv_file(calculate); + + /* Calculate zip archive path from DLL or exe path */ + change_ext(calculate->zip_path, + config->dll_path[0] ? config->dll_path : config->program_name, + L".zip"); + + calculate_home_prefix(calculate, prefix); + + err = calculate_module_search_path(calculate, config, prefix); + if (_Py_INIT_FAILED(err)) { + return err; + } + +done: + config->prefix = _PyMem_RawWcsdup(prefix); + if (config->prefix == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + return _Py_INIT_OK(); +} + + +static void +calculate_free(PyCalculatePath *calculate) +{ + PyMem_RawFree(calculate->machine_path); + PyMem_RawFree(calculate->user_path); +} + + +static void +pathconfig_clear(PyPathConfig *config) +{ +#define CLEAR(ATTR) \ + do { \ + PyMem_RawFree(ATTR); \ + ATTR = NULL; \ + } while (0) + + CLEAR(config->prefix); + CLEAR(config->program_name); + CLEAR(config->dll_path); + CLEAR(config->module_search_path); +#undef CLEAR +} + + +/* Initialize paths for Py_GetPath(), Py_GetPrefix(), Py_GetExecPrefix() + and Py_GetProgramFullPath() */ +_PyInitError +_PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) +{ + if (path_config.module_search_path) { + /* Already initialized */ + return _Py_INIT_OK(); + } + + _PyInitError err; + + PyCalculatePath calculate; + memset(&calculate, 0, sizeof(calculate)); + + calculate_init(&calculate, main_config); + + PyPathConfig new_path_config; + memset(&new_path_config, 0, sizeof(new_path_config)); + + err = calculate_path_impl(&calculate, &new_path_config, main_config); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + path_config = new_path_config; + err = _Py_INIT_OK(); + +done: + if (_Py_INIT_FAILED(err)) { + pathconfig_clear(&new_path_config); + } + calculate_free(&calculate); + return err; +} + + +static void +calculate_path(void) +{ + _PyInitError err; + _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; + + err = _PyMainInterpreterConfig_ReadEnv(&config); + if (!_Py_INIT_FAILED(err)) { + err = _PyPathConfig_Init(&config); + } + _PyMainInterpreterConfig_Clear(&config); + + if (_Py_INIT_FAILED(err)) { + _Py_FatalInitError(err); + } +} + + +void +_PyPathConfig_Fini(void) +{ + pathconfig_clear(&path_config); } @@ -881,51 +1145,62 @@ calculate_path(void) void Py_SetPath(const wchar_t *path) { - if (module_search_path != NULL) { - PyMem_RawFree(module_search_path); - module_search_path = NULL; - } - if (path != NULL) { - extern wchar_t *Py_GetProgramName(void); - wchar_t *prog = Py_GetProgramName(); - wcsncpy(progpath, prog, MAXPATHLEN); - prefix[0] = L'\0'; - module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t)); - if (module_search_path != NULL) - wcscpy(module_search_path, path); + if (path_config.module_search_path != NULL) { + pathconfig_clear(&path_config); + } + + if (path == NULL) { + return; } + + PyPathConfig new_config; + new_config.program_name = _PyMem_RawWcsdup(Py_GetProgramName()); + new_config.prefix = _PyMem_RawWcsdup(L""); + new_config.dll_path = _PyMem_RawWcsdup(L""); + new_config.module_search_path = _PyMem_RawWcsdup(path); + + pathconfig_clear(&path_config); + path_config = new_config; } + wchar_t * Py_GetPath(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(); - return module_search_path; + } + return path_config.module_search_path; } + wchar_t * Py_GetPrefix(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(); - return prefix; + } + return path_config.prefix; } + wchar_t * Py_GetExecPrefix(void) { return Py_GetPrefix(); } + wchar_t * Py_GetProgramFullPath(void) { - if (!module_search_path) + if (!path_config.module_search_path) { calculate_path(); - return progpath; + } + return path_config.program_name; } + /* Load python3.dll before loading any extension module that might refer to it. That way, we can be sure that always the python3.dll corresponding to this python DLL is loaded, not a python3.dll that might be on the path @@ -939,20 +1214,23 @@ _Py_CheckPython3() { wchar_t py3path[MAXPATHLEN+1]; wchar_t *s; - if (python3_checked) + if (python3_checked) { return hPython3 != NULL; + } python3_checked = 1; /* If there is a python3.dll next to the python3y.dll, assume this is a build tree; use that DLL */ - wcscpy(py3path, dllpath); + wcscpy(py3path, path_config.dll_path); s = wcsrchr(py3path, L'\\'); - if (!s) + if (!s) { s = py3path; + } wcscpy(s, L"\\python3.dll"); hPython3 = LoadLibraryExW(py3path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); - if (hPython3 != NULL) + if (hPython3 != NULL) { return 1; + } /* Check sys.prefix\DLLs\python3.dll */ wcscpy(py3path, Py_GetPrefix()); diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 246de78463d95b5..3793cbda8829d84 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -116,7 +116,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index b37e9930e3146c8..1d33c6e2cc2802d 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -141,9 +141,6 @@ Include - - Include - Include @@ -1031,4 +1028,4 @@ Resource Files - \ No newline at end of file + diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c index dbdf13440afe3db..20ef8a71f710bc3 100644 --- a/Parser/pgenmain.c +++ b/Parser/pgenmain.c @@ -60,8 +60,6 @@ main(int argc, char **argv) filename = argv[1]; graminit_h = argv[2]; graminit_c = argv[3]; - _PyObject_Initialize(&_PyRuntime.obj); - _PyMem_Initialize(&_PyRuntime.mem); g = getgrammar(filename); fp = fopen(graminit_c, "w"); if (fp == NULL) { diff --git a/Programs/_testembed.c b/Programs/_testembed.c index e68e68de327b129..21aa76e9de38bf2 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -125,6 +125,29 @@ static int test_forced_io_encoding(void) return 0; } + +/********************************************************* + * Test parts of the C-API that work before initialization + *********************************************************/ + +static int test_pre_initialization_api(void) +{ + /* Leading "./" ensures getpath.c can still find the standard library */ + wchar_t *program = Py_DecodeLocale("./spam", NULL); + if (program == NULL) { + fprintf(stderr, "Fatal error: cannot decode program name\n"); + return 1; + } + Py_SetProgramName(program); + + Py_Initialize(); + Py_Finalize(); + + PyMem_RawFree(program); + return 0; +} + + /* ********************************************************* * List of test cases and the function that implements it. * @@ -146,6 +169,7 @@ struct TestCase static struct TestCase TestCases[] = { { "forced_io_encoding", test_forced_io_encoding }, { "repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters }, + { "pre_initialization_api", test_pre_initialization_api }, { NULL, NULL } }; diff --git a/Python/_warnings.c b/Python/_warnings.c index f2110edc52d1967..086a70d7e68f0a5 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -11,6 +11,10 @@ MODULE_NAME " provides basic warning filtering support.\n" _Py_IDENTIFIER(argv); _Py_IDENTIFIER(stderr); +_Py_IDENTIFIER(ignore); +_Py_IDENTIFIER(error); +_Py_IDENTIFIER(always); +_Py_static_string(PyId_default, "default"); static int check_matched(PyObject *obj, PyObject *arg) @@ -35,15 +39,15 @@ check_matched(PyObject *obj, PyObject *arg) A NULL return value can mean false or an error. */ static PyObject * -get_warnings_attr(const char *attr, int try_import) +get_warnings_attr(_Py_Identifier *attr_id, int try_import) { - static PyObject *warnings_str = NULL; + PyObject *warnings_str; PyObject *warnings_module, *obj; + _Py_IDENTIFIER(warnings); + warnings_str = _PyUnicode_FromId(&PyId_warnings); if (warnings_str == NULL) { - warnings_str = PyUnicode_InternFromString("warnings"); - if (warnings_str == NULL) - return NULL; + return NULL; } /* don't try to import after the start of the Python finallization */ @@ -64,7 +68,7 @@ get_warnings_attr(const char *attr, int try_import) return NULL; } - obj = PyObject_GetAttrString(warnings_module, attr); + obj = _PyObject_GetAttrId(warnings_module, attr_id); Py_DECREF(warnings_module); if (obj == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); @@ -77,8 +81,9 @@ static PyObject * get_once_registry(void) { PyObject *registry; + _Py_IDENTIFIER(onceregistry); - registry = get_warnings_attr("onceregistry", 0); + registry = get_warnings_attr(&PyId_onceregistry, 0); if (registry == NULL) { if (PyErr_Occurred()) return NULL; @@ -102,8 +107,9 @@ static PyObject * get_default_action(void) { PyObject *default_action; + _Py_IDENTIFIER(defaultaction); - default_action = get_warnings_attr("defaultaction", 0); + default_action = get_warnings_attr(&PyId_defaultaction, 0); if (default_action == NULL) { if (PyErr_Occurred()) { return NULL; @@ -132,8 +138,9 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, PyObject *action; Py_ssize_t i; PyObject *warnings_filters; + _Py_IDENTIFIER(filters); - warnings_filters = get_warnings_attr("filters", 0); + warnings_filters = get_warnings_attr(&PyId_filters, 0); if (warnings_filters == NULL) { if (PyErr_Occurred()) return NULL; @@ -389,11 +396,13 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, PyObject *sourceline, PyObject *source) { PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL; + _Py_IDENTIFIER(_showwarnmsg); + _Py_IDENTIFIER(WarningMessage); /* If the source parameter is set, try to get the Python implementation. The Python implementation is able to log the traceback where the source was allocated, whereas the C implementation doesn't. */ - show_fn = get_warnings_attr("_showwarnmsg", source != NULL); + show_fn = get_warnings_attr(&PyId__showwarnmsg, source != NULL); if (show_fn == NULL) { if (PyErr_Occurred()) return -1; @@ -407,7 +416,7 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, goto error; } - warnmsg_cls = get_warnings_attr("WarningMessage", 0); + warnmsg_cls = get_warnings_attr(&PyId_WarningMessage, 0); if (warnmsg_cls == NULL) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_RuntimeError, @@ -1144,60 +1153,22 @@ static PyMethodDef warnings_functions[] = { static PyObject * -create_filter(PyObject *category, const char *action) +create_filter(PyObject *category, _Py_Identifier *id) { - static PyObject *ignore_str = NULL; - static PyObject *error_str = NULL; - static PyObject *default_str = NULL; - static PyObject *always_str = NULL; - PyObject *action_obj = NULL; - - if (!strcmp(action, "ignore")) { - if (ignore_str == NULL) { - ignore_str = PyUnicode_InternFromString("ignore"); - if (ignore_str == NULL) - return NULL; - } - action_obj = ignore_str; - } - else if (!strcmp(action, "error")) { - if (error_str == NULL) { - error_str = PyUnicode_InternFromString("error"); - if (error_str == NULL) - return NULL; - } - action_obj = error_str; - } - else if (!strcmp(action, "default")) { - if (default_str == NULL) { - default_str = PyUnicode_InternFromString("default"); - if (default_str == NULL) - return NULL; - } - action_obj = default_str; - } - else if (!strcmp(action, "always")) { - if (always_str == NULL) { - always_str = PyUnicode_InternFromString("always"); - if (always_str == NULL) - return NULL; - } - action_obj = always_str; - } - else { - Py_FatalError("unknown action"); + PyObject *action_str = _PyUnicode_FromId(id); + if (action_str == NULL) { + return NULL; } /* This assumes the line number is zero for now. */ - return PyTuple_Pack(5, action_obj, Py_None, + return PyTuple_Pack(5, action_str, Py_None, category, Py_None, _PyLong_Zero); } static PyObject * -init_filters(void) +init_filters(const _PyCoreConfig *config) { - PyInterpreterState *interp = PyThreadState_GET()->interp; - int dev_mode = interp->core_config.dev_mode; + int dev_mode = config->dev_mode; Py_ssize_t count = 2; if (dev_mode) { @@ -1209,48 +1180,47 @@ init_filters(void) } #endif PyObject *filters = PyList_New(count); - unsigned int pos = 0; /* Post-incremented in each use. */ - unsigned int x; - const char *bytes_action, *resource_action; - if (filters == NULL) return NULL; + size_t pos = 0; /* Post-incremented in each use. */ #ifndef Py_DEBUG if (!dev_mode) { PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_DeprecationWarning, "ignore")); + create_filter(PyExc_DeprecationWarning, &PyId_ignore)); PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_PendingDeprecationWarning, "ignore")); + create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore)); PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_ImportWarning, "ignore")); + create_filter(PyExc_ImportWarning, &PyId_ignore)); } #endif + _Py_Identifier *bytes_action; if (Py_BytesWarningFlag > 1) - bytes_action = "error"; + bytes_action = &PyId_error; else if (Py_BytesWarningFlag) - bytes_action = "default"; + bytes_action = &PyId_default; else - bytes_action = "ignore"; + bytes_action = &PyId_ignore; PyList_SET_ITEM(filters, pos++, create_filter(PyExc_BytesWarning, bytes_action)); + _Py_Identifier *resource_action; /* resource usage warnings are enabled by default in pydebug mode */ #ifdef Py_DEBUG - resource_action = "always"; + resource_action = &PyId_always; #else - resource_action = (dev_mode ? "always" : "ignore"); + resource_action = (dev_mode ? &PyId_always : &PyId_ignore); #endif PyList_SET_ITEM(filters, pos++, create_filter(PyExc_ResourceWarning, resource_action)); if (dev_mode) { PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_Warning, "default")); + create_filter(PyExc_Warning, &PyId_default)); } - for (x = 0; x < pos; x += 1) { + for (size_t x = 0; x < pos; x++) { if (PyList_GET_ITEM(filters, x) == NULL) { Py_DECREF(filters); return NULL; @@ -1273,8 +1243,8 @@ static struct PyModuleDef warningsmodule = { }; -PyMODINIT_FUNC -_PyWarnings_Init(void) +PyObject* +_PyWarnings_InitWithConfig(const _PyCoreConfig *config) { PyObject *m; @@ -1283,7 +1253,7 @@ _PyWarnings_Init(void) return NULL; if (_PyRuntime.warnings.filters == NULL) { - _PyRuntime.warnings.filters = init_filters(); + _PyRuntime.warnings.filters = init_filters(config); if (_PyRuntime.warnings.filters == NULL) return NULL; } @@ -1314,3 +1284,12 @@ _PyWarnings_Init(void) _PyRuntime.warnings.filters_version = 0; return m; } + + +PyMODINIT_FUNC +_PyWarnings_Init(void) +{ + PyInterpreterState *interp = PyThreadState_GET()->interp; + const _PyCoreConfig *config = &interp->core_config; + return _PyWarnings_InitWithConfig(config); +} diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 36fcf61c33d2d35..b89cbc88d4b90ef 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -48,8 +48,6 @@ _Py_IDENTIFIER(threading); extern "C" { #endif -extern wchar_t *Py_GetPath(void); - extern grammar _PyParser_Grammar; /* From graminit.c */ /* Forward */ @@ -118,7 +116,6 @@ int Py_InspectFlag; /* Needed to determine whether to exit at SystemExit */ int Py_OptimizeFlag = 0; /* Needed by compile.c */ int Py_NoSiteFlag; /* Suppress 'import site' */ int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */ -int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */ int Py_FrozenFlag; /* Needed by getpath.c */ int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */ int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.pyc) */ @@ -769,7 +766,9 @@ _Py_InitializeCore(const _PyCoreConfig *config) } /* Initialize _warnings. */ - _PyWarnings_Init(); + if (_PyWarnings_InitWithConfig(&interp->core_config) == NULL) { + return _Py_INIT_ERR("can't initialize warnings"); + } /* This call sets up builtin and frozen import support */ if (!interp->core_config._disable_importlib) { @@ -797,15 +796,40 @@ _Py_InitializeCore(const _PyCoreConfig *config) */ _PyInitError -_Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *config) +_PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *config) { /* Signal handlers are installed by default */ if (config->install_signal_handlers < 0) { config->install_signal_handlers = 1; } + + if (config->program_name == NULL) { + config->program_name = _PyMem_RawWcsdup(Py_GetProgramName()); + if (config->program_name == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + return _Py_INIT_OK(); } + +void +_PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *config) +{ +#define CLEAR(ATTR) \ + do { \ + PyMem_RawFree(ATTR); \ + ATTR = NULL; \ + } while (0) + + CLEAR(config->module_search_path_env); + CLEAR(config->home); + CLEAR(config->program_name); +#undef CLEAR +} + + /* Update interpreter state based on supplied configuration settings * * After calling this function, most of the restrictions on the interpreter @@ -851,16 +875,22 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) _PyRuntime.initialized = 1; return _Py_INIT_OK(); } + /* TODO: Report exceptions rather than fatal errors below here */ if (_PyTime_Init() < 0) return _Py_INIT_ERR("can't initialize time"); - /* Finish setting up the sys module and import system */ /* GetPath may initialize state that _PySys_EndInit locks in, and so has to be called first. */ - /* TODO: Call Py_GetPath() in Py_ReadConfig, rather than here */ - PySys_SetPath(Py_GetPath()); + err = _PyPathConfig_Init(&interp->config); + if (_Py_INIT_FAILED(err)) { + return err; + } + wchar_t *sys_path = Py_GetPath(); + + /* Finish setting up the sys module and import system */ + PySys_SetPath(sys_path); if (_PySys_EndInit(interp->sysdict) < 0) return _Py_INIT_ERR("can't finish initializing sys"); @@ -880,7 +910,7 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) return err; } - if (config->install_signal_handlers) { + if (interp->config.install_signal_handlers) { err = initsigs(); /* Signal handling stuff, including initintr() */ if (_Py_INIT_FAILED(err)) { return err; @@ -942,7 +972,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib) } /* TODO: Print any exceptions raised by these operations */ - err = _Py_ReadMainInterpreterConfig(&config); + err = _PyMainInterpreterConfig_Read(&config); if (_Py_INIT_FAILED(err)) { return err; } @@ -1235,6 +1265,9 @@ Py_FinalizeEx(void) #endif call_ll_exitfuncs(); + + _PyPathConfig_Fini(); + _PyRuntime_Finalize(); return status; } @@ -1264,6 +1297,7 @@ new_interpreter(PyThreadState **tstate_p) PyInterpreterState *interp; PyThreadState *tstate, *save_tstate; PyObject *bimod, *sysmod; + _PyInitError err; if (!_PyRuntime.initialized) { return _Py_INIT_ERR("Py_Initialize must be called first"); @@ -1299,8 +1333,13 @@ new_interpreter(PyThreadState **tstate_p) interp->config = main_interp->config; } - /* XXX The following is lax in error checking */ + err = _PyPathConfig_Init(&interp->config); + if (_Py_INIT_FAILED(err)) { + return err; + } + wchar_t *sys_path = Py_GetPath(); + /* XXX The following is lax in error checking */ PyObject *modules = PyDict_New(); if (modules == NULL) { return _Py_INIT_ERR("can't make modules dictionary"); @@ -1314,7 +1353,7 @@ new_interpreter(PyThreadState **tstate_p) goto handle_error; Py_INCREF(interp->sysdict); PyDict_SetItemString(interp->sysdict, "modules", modules); - PySys_SetPath(Py_GetPath()); + PySys_SetPath(sys_path); _PySys_EndInit(interp->sysdict); } @@ -1331,7 +1370,6 @@ new_interpreter(PyThreadState **tstate_p) if (bimod != NULL && sysmod != NULL) { PyObject *pstderr; - _PyInitError err; /* Set up a preliminary stderr printer until we have enough infrastructure for the io module in place. */ @@ -1466,7 +1504,6 @@ Py_GetProgramName(void) } static wchar_t *default_home = NULL; -static wchar_t env_home[MAXPATHLEN+1]; void Py_SetPythonHome(wchar_t *home) @@ -1474,21 +1511,32 @@ Py_SetPythonHome(wchar_t *home) default_home = home; } -wchar_t * + +wchar_t* Py_GetPythonHome(void) { - wchar_t *home = default_home; - if (home == NULL && !Py_IgnoreEnvironmentFlag) { - char* chome = Py_GETENV("PYTHONHOME"); - if (chome) { - size_t size = Py_ARRAY_LENGTH(env_home); - size_t r = mbstowcs(env_home, chome, size); - if (r != (size_t)-1 && r < size) - home = env_home; - } + /* Use a static buffer to avoid heap memory allocation failure. + Py_GetPythonHome() doesn't allow to report error, and the caller + doesn't release memory. */ + static wchar_t buffer[MAXPATHLEN+1]; + if (default_home) { + return default_home; } - return home; + + char *home = Py_GETENV("PYTHONHOME"); + if (!home) { + return NULL; + } + + size_t size = Py_ARRAY_LENGTH(buffer); + size_t r = mbstowcs(buffer, home, size); + if (r == (size_t)-1 || r >= size) { + /* conversion failed or the static buffer is too small */ + return NULL; + } + + return buffer; } /* Add the __main__ module */ @@ -2000,7 +2048,7 @@ fatal_output_debug(const char *msg) } #endif -static void +static void _Py_NO_RETURN fatal_error(const char *prefix, const char *msg, int status) { const int fd = fileno(stderr); diff --git a/Python/pystate.c b/Python/pystate.c index 807ac4eb9d18bcf..ecf921d0c25c53f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -40,8 +40,6 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) { memset(runtime, 0, sizeof(*runtime)); - _PyObject_Initialize(&runtime->obj); - _PyMem_Initialize(&runtime->mem); _PyGC_Initialize(&runtime->gc); _PyEval_Initialize(&runtime->ceval); @@ -106,55 +104,60 @@ PyInterpreterState_New(void) PyInterpreterState *interp = (PyInterpreterState *) PyMem_RawMalloc(sizeof(PyInterpreterState)); - if (interp != NULL) { - interp->modules = NULL; - interp->modules_by_index = NULL; - interp->sysdict = NULL; - interp->builtins = NULL; - interp->builtins_copy = NULL; - interp->tstate_head = NULL; - interp->check_interval = 100; - interp->num_threads = 0; - interp->pythread_stacksize = 0; - interp->codec_search_path = NULL; - interp->codec_search_cache = NULL; - interp->codec_error_registry = NULL; - interp->codecs_initialized = 0; - interp->fscodec_initialized = 0; - interp->importlib = NULL; - interp->import_func = NULL; - interp->eval_frame = _PyEval_EvalFrameDefault; - interp->co_extra_user_count = 0; + if (interp == NULL) { + return NULL; + } + + + interp->modules = NULL; + interp->modules_by_index = NULL; + interp->sysdict = NULL; + interp->builtins = NULL; + interp->builtins_copy = NULL; + interp->tstate_head = NULL; + interp->check_interval = 100; + interp->num_threads = 0; + interp->pythread_stacksize = 0; + interp->codec_search_path = NULL; + interp->codec_search_cache = NULL; + interp->codec_error_registry = NULL; + interp->codecs_initialized = 0; + interp->fscodec_initialized = 0; + interp->core_config = _PyCoreConfig_INIT; + interp->config = _PyMainInterpreterConfig_INIT; + interp->importlib = NULL; + interp->import_func = NULL; + interp->eval_frame = _PyEval_EvalFrameDefault; + interp->co_extra_user_count = 0; #ifdef HAVE_DLOPEN #if HAVE_DECL_RTLD_NOW - interp->dlopenflags = RTLD_NOW; + interp->dlopenflags = RTLD_NOW; #else - interp->dlopenflags = RTLD_LAZY; + interp->dlopenflags = RTLD_LAZY; #endif #endif #ifdef HAVE_FORK - interp->before_forkers = NULL; - interp->after_forkers_parent = NULL; - interp->after_forkers_child = NULL; + interp->before_forkers = NULL; + interp->after_forkers_parent = NULL; + interp->after_forkers_child = NULL; #endif - HEAD_LOCK(); - interp->next = _PyRuntime.interpreters.head; - if (_PyRuntime.interpreters.main == NULL) { - _PyRuntime.interpreters.main = interp; - } - _PyRuntime.interpreters.head = interp; - if (_PyRuntime.interpreters.next_id < 0) { - /* overflow or Py_Initialize() not called! */ - PyErr_SetString(PyExc_RuntimeError, - "failed to get an interpreter ID"); - interp = NULL; - } else { - interp->id = _PyRuntime.interpreters.next_id; - _PyRuntime.interpreters.next_id += 1; - } - HEAD_UNLOCK(); + HEAD_LOCK(); + interp->next = _PyRuntime.interpreters.head; + if (_PyRuntime.interpreters.main == NULL) { + _PyRuntime.interpreters.main = interp; + } + _PyRuntime.interpreters.head = interp; + if (_PyRuntime.interpreters.next_id < 0) { + /* overflow or Py_Initialize() not called! */ + PyErr_SetString(PyExc_RuntimeError, + "failed to get an interpreter ID"); + interp = NULL; + } else { + interp->id = _PyRuntime.interpreters.next_id; + _PyRuntime.interpreters.next_id += 1; } + HEAD_UNLOCK(); return interp; } diff --git a/Tools/c-globals/ignored-globals.txt b/Tools/c-globals/ignored-globals.txt index 7b5add865c166c7..ce6d1d805147b67 100644 --- a/Tools/c-globals/ignored-globals.txt +++ b/Tools/c-globals/ignored-globals.txt @@ -393,7 +393,6 @@ Py_NoUserSiteDirectory Py_OptimizeFlag Py_QuietFlag Py_UnbufferedStdioFlag -Py_UseClassExceptionsFlag Py_VerboseFlag diff --git a/configure b/configure index 5e0522476e71268..d02675742d27d91 100755 --- a/configure +++ b/configure @@ -778,7 +778,6 @@ infodir docdir oldincludedir includedir -runstatedir localstatedir sharedstatedir sysconfdir @@ -890,7 +889,6 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' -runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1143,15 +1141,6 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; - -runstatedir | --runstatedir | --runstatedi | --runstated \ - | --runstate | --runstat | --runsta | --runst | --runs \ - | --run | --ru | --r) - ac_prev=runstatedir ;; - -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ - | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ - | --run=* | --ru=* | --r=*) - runstatedir=$ac_optarg ;; - -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1289,7 +1278,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir runstatedir + libdir localedir mandir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1442,7 +1431,6 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] - --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -5623,7 +5611,6 @@ $as_echo "$ac_cv_safe_to_define___extensions__" >&6; } $as_echo_n "checking for the Android API level... " >&6; } cat >> conftest.c < android_api = __ANDROID_API__ arm_arch = __ARM_ARCH #else @@ -5636,6 +5623,10 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then _arm_arch=`sed -n -e '/__ARM_ARCH/d' -e 's/^arm_arch = //p' conftest.out` { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ANDROID_API_LEVEL" >&5 $as_echo "$ANDROID_API_LEVEL" >&6; } + if test -z "$ANDROID_API_LEVEL"; then + echo 'Fatal: you must define __ANDROID_API__' + exit 1 + fi cat >>confdefs.h <<_ACEOF #define ANDROID_API_LEVEL $ANDROID_API_LEVEL @@ -11155,7 +11146,8 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ futimens futimes gai_strerror getentropy \ getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ - initgroups kill killpg lchmod lchown linkat lstat lutimes mmap \ + if_nameindex \ + initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise pread \ @@ -12585,80 +12577,6 @@ else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - -# On Android API level 24 with android-ndk-r13, if_nameindex() is available, -# but the if_nameindex structure is not defined. -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for if_nameindex" >&5 -$as_echo_n "checking for if_nameindex... " >&6; } -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -#include -#ifdef STDC_HEADERS -# include -# include -#else -# ifdef HAVE_STDLIB_H -# include -# endif -#endif -#ifdef HAVE_SYS_SOCKET_H -# include -#endif -#ifdef HAVE_NET_IF_H -# include -#endif - -int -main () -{ -struct if_nameindex *ni = if_nameindex(); int x = ni[0].if_index; - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - -$as_echo "#define HAVE_IF_NAMEINDEX 1" >>confdefs.h - - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - -# Issue #28762: lockf() is available on Android API level 24, but the F_LOCK -# macro is not defined in android-ndk-r13. -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for lockf" >&5 -$as_echo_n "checking for lockf... " >&6; } -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main () -{ -lockf(0, F_LOCK, 0); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - -$as_echo "#define HAVE_LOCKF 1" >>confdefs.h - - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext diff --git a/configure.ac b/configure.ac index 3464212eddc019b..68a95c3be6af4a3 100644 --- a/configure.ac +++ b/configure.ac @@ -885,7 +885,6 @@ AC_USE_SYSTEM_EXTENSIONS AC_MSG_CHECKING([for the Android API level]) cat >> conftest.c < android_api = __ANDROID_API__ arm_arch = __ARM_ARCH #else @@ -897,6 +896,10 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then ANDROID_API_LEVEL=`sed -n -e '/__ANDROID_API__/d' -e 's/^android_api = //p' conftest.out` _arm_arch=`sed -n -e '/__ARM_ARCH/d' -e 's/^arm_arch = //p' conftest.out` AC_MSG_RESULT([$ANDROID_API_LEVEL]) + if test -z "$ANDROID_API_LEVEL"; then + echo 'Fatal: you must define __ANDROID_API__' + exit 1 + fi AC_DEFINE_UNQUOTED(ANDROID_API_LEVEL, $ANDROID_API_LEVEL, [The Android API level.]) AC_MSG_CHECKING([for the Android arm ABI]) @@ -3411,7 +3414,8 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ futimens futimes gai_strerror getentropy \ getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ - initgroups kill killpg lchmod lchown linkat lstat lutimes mmap \ + if_nameindex \ + initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise pread \ @@ -3760,40 +3764,6 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ AC_MSG_RESULT(no) ]) -# On Android API level 24 with android-ndk-r13, if_nameindex() is available, -# but the if_nameindex structure is not defined. -AC_MSG_CHECKING(for if_nameindex) -AC_LINK_IFELSE([AC_LANG_PROGRAM([[ -#include -#ifdef STDC_HEADERS -# include -# include -#else -# ifdef HAVE_STDLIB_H -# include -# endif -#endif -#ifdef HAVE_SYS_SOCKET_H -# include -#endif -#ifdef HAVE_NET_IF_H -# include -#endif -]], [[struct if_nameindex *ni = if_nameindex(); int x = ni[0].if_index;]])], - [AC_DEFINE(HAVE_IF_NAMEINDEX, 1, Define to 1 if you have the 'if_nameindex' function.) - AC_MSG_RESULT(yes)], - [AC_MSG_RESULT(no) -]) - -# Issue #28762: lockf() is available on Android API level 24, but the F_LOCK -# macro is not defined in android-ndk-r13. -AC_MSG_CHECKING(for lockf) -AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]],[[lockf(0, F_LOCK, 0);]])], - [AC_DEFINE(HAVE_LOCKF, 1, Define to 1 if you have the 'lockf' function and the F_LOCK macro.) - AC_MSG_RESULT(yes)], - [AC_MSG_RESULT(no) -]) - # On OSF/1 V5.1, getaddrinfo is available, but a define # for [no]getaddrinfo in netdb.h. AC_MSG_CHECKING(for getaddrinfo) diff --git a/pyconfig.h.in b/pyconfig.h.in index 6e0f3e8eeb37b53..66b9e8882745009 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -601,7 +601,7 @@ /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_VM_SOCKETS_H -/* Define to 1 if you have the 'lockf' function and the F_LOCK macro. */ +/* Define to 1 if you have the `lockf' function. */ #undef HAVE_LOCKF /* Define to 1 if you have the `log1p' function. */ diff --git a/setup.py b/setup.py index cbe6139ddec69f1..c22de17f9533964 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,31 @@ def add_dir_to_list(dirlist, dir): return dirlist.insert(0, dir) +def sysroot_paths(make_vars, subdirs): + """Get the paths of sysroot sub-directories. + + * make_vars: a sequence of names of variables of the Makefile where + sysroot may be set. + * subdirs: a sequence of names of subdirectories used as the location for + headers or libraries. + """ + + dirs = [] + for var_name in make_vars: + var = sysconfig.get_config_var(var_name) + if var is not None: + m = re.search(r'--sysroot=([^"]\S*|"[^"]+")', var) + if m is not None: + sysroot = m.group(1).strip('"') + for subdir in subdirs: + if os.path.isabs(subdir): + subdir = subdir[1:] + path = os.path.join(sysroot, subdir) + if os.path.isdir(path): + dirs.append(path) + break + return dirs + def macosx_sdk_root(): """ Return the directory of the current OSX SDK, @@ -559,18 +584,23 @@ def detect_modules(self): add_dir_to_list(self.compiler.include_dirs, sysconfig.get_config_var("INCLUDEDIR")) + system_lib_dirs = ['/lib64', '/usr/lib64', '/lib', '/usr/lib'] + system_include_dirs = ['/usr/include'] # lib_dirs and inc_dirs are used to search for files; # if a file is found in one of those directories, it can # be assumed that no additional -I,-L directives are needed. if not cross_compiling: - lib_dirs = self.compiler.library_dirs + [ - '/lib64', '/usr/lib64', - '/lib', '/usr/lib', - ] - inc_dirs = self.compiler.include_dirs + ['/usr/include'] + lib_dirs = self.compiler.library_dirs + system_lib_dirs + inc_dirs = self.compiler.include_dirs + system_include_dirs else: - lib_dirs = self.compiler.library_dirs[:] - inc_dirs = self.compiler.include_dirs[:] + # Add the sysroot paths. 'sysroot' is a compiler option used to + # set the logical path of the standard system headers and + # libraries. + lib_dirs = (self.compiler.library_dirs + + sysroot_paths(('LDFLAGS', 'CC'), system_lib_dirs)) + inc_dirs = (self.compiler.include_dirs + + sysroot_paths(('CPPFLAGS', 'CFLAGS', 'CC'), + system_include_dirs)) exts = [] missing = [] @@ -1650,12 +1680,11 @@ class db_found(Exception): pass # Build the _uuid module if possible uuid_incs = find_file("uuid.h", inc_dirs, ["/usr/include/uuid"]) - if uuid_incs: + if uuid_incs is not None: if self.compiler.find_library_file(lib_dirs, 'uuid'): uuid_libs = ['uuid'] else: uuid_libs = [] - if uuid_incs: self.extensions.append(Extension('_uuid', ['_uuidmodule.c'], libraries=uuid_libs, include_dirs=uuid_incs))