diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index e37afd6d0b6d5a..db5fae2006678a 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -637,25 +637,22 @@ are set. .. versionadded:: 3.14 -To highlight inline code in your description or epilog text, you can use -backticks:: +To highlight inline code in your description, epilog, or argument ``help`` +text, you can use single or double backticks:: >>> parser = argparse.ArgumentParser( ... formatter_class=argparse.RawDescriptionHelpFormatter, + ... description='Run ``python -m myapp`` to start.', ... epilog='''Examples: ... `python -m myapp --verbose` - ... `python -m myapp --config settings.json` + ... ``python -m myapp --config settings.json`` ... ''') + >>> parser.add_argument('--foo', help='set the `foo` value') When colors are enabled, the text inside backticks will be displayed in a distinct color to help examples stand out. When colors are disabled, backticks are preserved as-is, which is readable in plain text. -.. note:: - - Backtick markup only applies to description and epilog text. It does not - apply to individual argument ``help`` strings. - .. versionadded:: 3.15 diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 492c65ae209d92..1fa7fc13baa539 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -18,7 +18,7 @@ This module provides a single class, :class:`RobotFileParser`, which answers questions about whether or not a particular user agent can fetch a URL on the website that published the :file:`robots.txt` file. For more details on the -structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. +structure of :file:`robots.txt` files, see :rfc:`9309`. .. class:: RobotFileParser(url='') diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 3baae534041446..7f12cc04a460d4 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -492,6 +492,47 @@ Improved error messages ^^^^^^^^^^^^^^ AttributeError: 'Container' object has no attribute 'area'. Did you mean '.inner.area' instead of '.area'? +* When an :exc:`AttributeError` on a builtin type has no close match via + Levenshtein distance, the error message now checks a static table of common + method names from other languages (JavaScript, Java, Ruby, C#) and suggests + the Python equivalent: + + .. doctest:: + + >>> [1, 2, 3].push(4) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'list' object has no attribute 'push'. Did you mean '.append'? + + >>> 'hello'.toUpperCase() # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'str' object has no attribute 'toUpperCase'. Did you mean '.upper'? + + When the Python equivalent is a language construct rather than a method, + the hint describes the construct directly: + + .. doctest:: + + >>> {}.put("a", 1) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'dict' object has no attribute 'put'. Use d[k] = v. + + When a mutable method is called on an immutable type, the hint suggests + the mutable counterpart: + + .. doctest:: + + >>> (1, 2, 3).append(4) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'tuple' object has no attribute 'append'. Did you mean to use a 'list' object? + + These hints also work for subclasses of builtin types. + + (Contributed by Matt Van Horn in :gh:`146406`.) + * The interpreter now tries to provide a suggestion when :func:`delattr` fails due to a missing attribute. When an attribute name that closely resembles an existing attribute is used, @@ -715,10 +756,14 @@ argparse default to ``True``. This enables suggestions for mistyped arguments by default. (Contributed by Jakob Schluse in :gh:`140450`.) -* Added backtick markup support in description and epilog text to highlight - inline code when color output is enabled. +* Added backtick markup support in :class:`~argparse.ArgumentParser` description + and epilog text to highlight inline code when color output is enabled. (Contributed by Savannah Ostrowski in :gh:`142390`.) +* Extended backtick markup to argument ``help`` text and added support for + double backticks (RST inline-literal style). + (Contributed by Hugo van Kemenade in :gh:`149375`.) + array ----- @@ -1045,7 +1090,7 @@ mimetypes (Contributed by Benedikt Johannes, Charlie Lin, Foolbar, Gil Forcada and John Franey in :gh:`144217`, :gh:`145720`, :gh:`140937`, :gh:`139959`, :gh:`145698`, - :gh:`145718` and :gh:`144213`.) + :gh:`145718`, :gh:`145918`, and :gh:`144213`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. (Contributed by Charlie Lin in :gh:`140165`.) * Changed the MIME type for ``.ai`` files to ``application/pdf``. diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 1dd10f8d94cfd8..18490f98a918a7 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -158,8 +158,16 @@ typedef struct _Py_DebugOffsets { uint64_t tp_name; uint64_t tp_repr; uint64_t tp_flags; + uint64_t tp_basicsize; + uint64_t tp_dictoffset; } type_object; + // PyHeapTypeObject offset; + struct _heap_type_object { + uint64_t size; + uint64_t ht_cached_keys; + } heap_type_object; + // PyTuple object offset; struct _tuple_object { uint64_t size; @@ -330,6 +338,12 @@ typedef struct _Py_DebugOffsets { .tp_name = offsetof(PyTypeObject, tp_name), \ .tp_repr = offsetof(PyTypeObject, tp_repr), \ .tp_flags = offsetof(PyTypeObject, tp_flags), \ + .tp_basicsize = offsetof(PyTypeObject, tp_basicsize), \ + .tp_dictoffset = offsetof(PyTypeObject, tp_dictoffset), \ + }, \ + .heap_type_object = { \ + .size = sizeof(PyHeapTypeObject), \ + .ht_cached_keys = offsetof(PyHeapTypeObject, ht_cached_keys), \ }, \ .tuple_object = { \ .size = sizeof(PyTupleObject), \ diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index e105677cd2e674..bfe52f42f1141c 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -223,12 +223,14 @@ static inline void _PyObject_GC_TRACK( "object is in generation which is garbage collected", filename, lineno, __func__); - PyGC_Head *generation0 = _PyInterpreterState_GET()->gc.generation0; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + PyGC_Head *generation0 = gcstate->generation0; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); _PyGCHead_SET_NEXT(gc, generation0); generation0->_gc_prev = (uintptr_t)gc; + gcstate->heap_size++; #endif } @@ -263,6 +265,8 @@ static inline void _PyObject_GC_UNTRACK( _PyGCHead_SET_PREV(next, prev); gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + gcstate->heap_size--; #endif } diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 86f018e328656e..2d04c173e85abe 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -191,6 +191,8 @@ struct gc_generation_stats { Py_ssize_t candidates; // Total duration of the collection in seconds: double duration; + /* heap_size on the start of the collection */ + Py_ssize_t heap_size; }; #ifdef Py_GIL_DISABLED @@ -226,7 +228,6 @@ struct _gc_runtime_state { /* linked lists of container objects */ #ifndef Py_GIL_DISABLED struct gc_generation generations[NUM_GENERATIONS]; - PyGC_Head *generation0; #else struct gc_generation young; struct gc_generation old[2]; @@ -244,6 +245,9 @@ struct _gc_runtime_state { /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; + /* The number of live objects. */ + Py_ssize_t heap_size; + /* This is the number of objects that survived the last full collection. It approximates the number of long lived objects tracked by the GC. @@ -269,6 +273,8 @@ struct _gc_runtime_state { /* Mutex held for gc_should_collect_mem_usage(). */ PyMutex mutex; +#else + PyGC_Head *generation0; #endif }; @@ -278,7 +284,8 @@ struct _gc_runtime_state { { .threshold = 2000, }, \ { .threshold = 10, }, \ { .threshold = 10, }, \ - }, + }, \ + .heap_size = 0, #else #define GC_GENERATION_INIT \ .young = { .threshold = 2000, }, \ diff --git a/Lib/argparse.py b/Lib/argparse.py index d91707d9eec546..9bc3ea64431e52 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -529,7 +529,7 @@ def _apply_text_markup(self, text): """Apply color markup to text. Supported markup: - `...` - inline code (rendered with prog_extra color) + `...` or ``...`` - inline code (rendered with prog_extra color) When colors are disabled, backticks are preserved as-is. """ @@ -537,8 +537,8 @@ def _apply_text_markup(self, text): if not t.reset: return text text = _re.sub( - r'`([^`]+)`', - rf'{t.prog_extra}\1{t.reset}', + r'(`{1,2})([^`]+)\1', + rf'{t.prog_extra}\2{t.reset}', text, ) return text @@ -682,7 +682,7 @@ def _format_args(self, action, default_metavar): def _expand_help(self, action): help_string = self._get_help_string(action) if '%' not in help_string: - return help_string + return self._apply_text_markup(help_string) params = dict(vars(action), prog=self._prog) for name in list(params): value = params[name] @@ -726,7 +726,9 @@ def colorize(match): # bare %s etc. - format with full params dict, no colorization return spec % params - return _re.sub(fmt_spec, colorize, help_string, flags=_re.VERBOSE) + return self._apply_text_markup( + _re.sub(fmt_spec, colorize, help_string, flags=_re.VERBOSE) + ) def _iter_indented_subactions(self, action): try: @@ -2758,7 +2760,7 @@ def _check_value(self, action, value): if value not in choices: args = {'value': str(value), - 'choices': ', '.join(map(str, action.choices))} + 'choices': ', '.join(repr(str(choice)) for choice in action.choices)} msg = _('invalid choice: %(value)r (choose from %(choices)s)') if self.suggest_on_error and isinstance(value, str): diff --git a/Lib/ast.py b/Lib/ast.py index ba4ee0197b85d2..4f88a554344cc9 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -21,7 +21,6 @@ :license: Python License. """ from _ast import * -lazy from _colorize import can_colorize, get_theme def parse(source, filename='', mode='exec', *, @@ -142,6 +141,8 @@ def dump( If show_empty is False, then empty lists and fields that are None will be omitted from the output for better readability. """ + from _colorize import get_theme + t = get_theme(force_color=color, force_no_color=not color).ast def _format(node, level=0): @@ -665,7 +666,7 @@ def main(args=None): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('infile', nargs='?', default='-', - help='the file to parse; defaults to stdin') + help='the file to parse; defaults to `stdin`') parser.add_argument('-m', '--mode', default='exec', choices=('exec', 'single', 'eval', 'func_type'), help='specify what kind of code must be parsed') @@ -678,8 +679,8 @@ def main(args=None): help='indentation of nodes (number of spaces)') parser.add_argument('--feature-version', type=str, default=None, metavar='VERSION', - help='Python version in the format 3.x ' - '(for example, 3.10)') + help='Python version in the format `3.x` ' + '(for example, `3.10`)') parser.add_argument('-O', '--optimize', type=int, default=-1, metavar='LEVEL', help='optimization level for parser') @@ -708,6 +709,7 @@ def main(args=None): tree = parse(source, name, args.mode, type_comments=args.no_type_comments, feature_version=feature_version, optimize=args.optimize) + from _colorize import can_colorize print(dump(tree, include_attributes=args.include_attributes, color=can_colorize(file=sys.stdout), indent=args.indent, show_empty=args.show_empty)) diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 37eba9657ac5a8..7f0565d0b8ddc7 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -159,7 +159,6 @@ def interrupt(self) -> None: parser = argparse.ArgumentParser( prog="python3 -m asyncio", description="Interactive asyncio shell and CLI tools", - color=True, ) subparsers = parser.add_subparsers(help="sub-commands", dest="command") ps = subparsers.add_parser( diff --git a/Lib/calendar.py b/Lib/calendar.py index fa9775ab040b14..92fe6b7723fe26 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -922,7 +922,7 @@ def main(args=None): "-t", "--type", default="text", choices=("text", "html"), - help="output type (text or html)" + help="output type (`text` or `html`)" ) parser.add_argument( "-f", "--first-weekday", diff --git a/Lib/code.py b/Lib/code.py index f7e275d8801b7c..df1d7199e33934 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -385,7 +385,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=Fa if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument('-q', action='store_true', help="don't print version and copyright messages") args = parser.parse_args() diff --git a/Lib/compileall.py b/Lib/compileall.py index c452aed135838f..812a496611e043 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -326,7 +326,6 @@ def main(): parser = argparse.ArgumentParser( description='Utilities to support installing Python libraries.', - color=True, ) parser.add_argument('-l', action='store_const', const=0, default=None, dest='maxlevels', @@ -338,10 +337,10 @@ def main(): parser.add_argument('-f', action='store_true', dest='force', help='force rebuild even if timestamps are up to date') parser.add_argument('-q', action='count', dest='quiet', default=0, - help='output only error messages; -qq will suppress ' + help='output only error messages; `-qq` will suppress ' 'the error messages as well.') parser.add_argument('-b', action='store_true', dest='legacy', - help='use legacy (pre-PEP3147) compiled file locations') + help='use legacy (pre-PEP 3147) compiled file locations') parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, help=('directory to prepend to file paths for use in ' 'compile-time tracebacks and in runtime ' @@ -367,28 +366,28 @@ def main(): 'of each file considered for compilation')) parser.add_argument('-i', metavar='FILE', dest='flist', help=('add all the files and directories listed in ' - 'FILE to the list considered for compilation; ' - 'if "-", names are read from stdin')) + '`FILE` to the list considered for compilation; ' + 'if `"-"`, names are read from `stdin`')) parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', help=('zero or more file and directory names ' 'to compile; if no arguments given, defaults ' - 'to the equivalent of -l sys.path')) + 'to the equivalent of `-l` `sys.path`')) parser.add_argument('-j', '--workers', default=1, type=int, help='Run compileall concurrently') invalidation_modes = [mode.name.lower().replace('_', '-') for mode in py_compile.PycInvalidationMode] parser.add_argument('--invalidation-mode', choices=sorted(invalidation_modes), - help=('set .pyc invalidation mode; defaults to ' - '"checked-hash" if the SOURCE_DATE_EPOCH ' + help=('set `.pyc` invalidation mode; defaults to ' + '`"checked-hash"` if the `SOURCE_DATE_EPOCH` ' 'environment variable is set, and ' - '"timestamp" otherwise.')) + '`"timestamp"` otherwise.')) parser.add_argument('-o', action='append', type=int, dest='opt_levels', help=('Optimization levels to run compilation with. ' - 'Default is -1 which uses the optimization level ' - 'of the Python interpreter itself (see -O).')) + 'Default is `-1` which uses the optimization level ' + 'of the Python interpreter itself (see `-O`).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', - help='Ignore symlinks pointing outsite of the DIR') + help='Ignore symlinks pointing outsite of the `DIR`') parser.add_argument('--hardlink-dupes', action='store_true', dest='hardlink_dupes', help='Hardlink duplicated pyc files') diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index d67cc4dd1b19ab..dbfabded2e47aa 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -6,7 +6,6 @@ import abc from reprlib import recursive_repr lazy import copy -lazy import inspect lazy import re @@ -981,6 +980,7 @@ def __get__(self, _obj, cls): try: # In some cases fetching a signature is not possible. # But, we surely should not fail in this case. + import inspect text_sig = str(inspect.signature( cls, annotation_format=annotationlib.Format.FORWARDREF, @@ -1391,6 +1391,7 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): # If this is a wrapped function, unwrap it. if not isinstance(member, type) and hasattr(member, '__wrapped__'): + import inspect member = inspect.unwrap(member) if isinstance(member, types.FunctionType): diff --git a/Lib/dis.py b/Lib/dis.py index 64f3450da3071b..d60507ae473453 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -1146,7 +1146,7 @@ def dis(self): def main(args=None): import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument('-C', '--show-caches', action='store_true', help='show inline caches') parser.add_argument('-O', '--show-offsets', action='store_true', diff --git a/Lib/doctest.py b/Lib/doctest.py index 0fcfa1e3e97144..05acac1745ace9 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -2951,7 +2951,7 @@ def get(self): def _test(): import argparse - parser = argparse.ArgumentParser(description="doctest runner", color=True) + parser = argparse.ArgumentParser(description="doctest runner") parser.add_argument('-v', '--verbose', action='store_true', default=False, help='print very verbose output for all tests') parser.add_argument('-o', '--option', action='append', @@ -2961,8 +2961,8 @@ def _test(): ' than once to apply multiple options')) parser.add_argument('-f', '--fail-fast', action='store_true', help=('stop running tests after first failure (this' - ' is a shorthand for -o FAIL_FAST, and is' - ' in addition to any other -o options)')) + ' is a shorthand for `-o FAIL_FAST`, and is' + ' in addition to any other `-o` options)')) parser.add_argument('file', nargs='+', help='file containing the tests to run') args = parser.parse_args() diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 5a55525d6bd235..ab6b95e0ba60c0 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "26.1" +_PIP_VERSION = "26.1.1" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora @@ -217,7 +217,7 @@ def _uninstall_helper(*, verbosity=0): def _main(argv=None): import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", diff --git a/Lib/ensurepip/_bundled/pip-26.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-26.1.1-py3-none-any.whl similarity index 94% rename from Lib/ensurepip/_bundled/pip-26.1-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-26.1.1-py3-none-any.whl index b51afa14f7c0ad..ab0307c7716212 100644 Binary files a/Lib/ensurepip/_bundled/pip-26.1-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-26.1.1-py3-none-any.whl differ diff --git a/Lib/gzip.py b/Lib/gzip.py index 8a2faf846bf894..971063aa24f871 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -664,15 +664,14 @@ def decompress(data): def main(): from argparse import ArgumentParser parser = ArgumentParser(description= - "A simple command line interface for the gzip module: act like gzip, " + "A simple command line interface for the `gzip` module: act like `gzip`, " "but do not delete the input file.", - color=True, ) group = parser.add_mutually_exclusive_group() group.add_argument('--fast', action='store_true', help='compress faster') group.add_argument('--best', action='store_true', help='compress better') group.add_argument("-d", "--decompress", action="store_true", - help="act like gunzip instead of gzip") + help="act like `gunzip` instead of `gzip`") parser.add_argument("args", nargs="*", default=["-"], metavar='file') args = parser.parse_args() diff --git a/Lib/http/server.py b/Lib/http/server.py index 16ea7f3f93693f..ebc85052aecb90 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -1081,7 +1081,7 @@ def _main(args=None): import argparse import contextlib - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument('-b', '--bind', metavar='ADDRESS', help='bind to this address ' '(default: all interfaces)') diff --git a/Lib/inspect.py b/Lib/inspect.py index af304f186a69bc..9eb87b0d277918 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -3445,11 +3445,11 @@ def _main(): import argparse import importlib - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument( 'object', help="The object to be analysed. " - "It supports the 'module:qualname' syntax") + "It supports the `module:qualname` syntax") parser.add_argument( '-d', '--details', action='store_true', help='Display info about the module rather than its source code') diff --git a/Lib/json/tool.py b/Lib/json/tool.py index e56a601c581ae5..6385d971f73304 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -44,13 +44,13 @@ def _replace_match_callback(match): def main(): description = ('A simple command line interface for json module ' 'to validate and pretty-print JSON objects.') - parser = argparse.ArgumentParser(description=description, color=True) + parser = argparse.ArgumentParser(description=description) parser.add_argument('infile', nargs='?', help='a JSON file to be validated or pretty-printed; ' - 'defaults to stdin', + 'defaults to `stdin`', default='-') parser.add_argument('outfile', nargs='?', - help='write the output of infile to outfile', + help='write the output of `infile` to `outfile`', default=None) parser.add_argument('--sort-keys', action='store_true', default=False, help='sort the output of dictionaries alphabetically by key') @@ -58,7 +58,7 @@ def main(): help='disable escaping of non-ASCII characters') parser.add_argument('--json-lines', action='store_true', default=False, help='parse input using the JSON Lines format. ' - 'Use with --no-indent or --compact to produce valid JSON Lines output.') + 'Use with `--no-indent` or `--compact` to produce valid JSON Lines output.') group = parser.add_mutually_exclusive_group() group.add_argument('--indent', default=4, type=int, help='separate items with newlines and use this number ' diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index a834826114614d..6d9278bccf927e 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -599,10 +599,14 @@ def _default_mime_types(): '.ra' : 'audio/x-pn-realaudio', '.wav' : 'audio/vnd.wave', '.weba' : 'audio/webm', + '.ttc' : 'font/collection', '.otf' : 'font/otf', '.ttf' : 'font/ttf', '.woff' : 'font/woff', '.woff2' : 'font/woff2', + '.hjif' : 'haptics/hjif', + '.hmpg' : 'haptics/hmpg', + '.ivs' : 'haptics/ivs', '.avif' : 'image/avif', '.bmp' : 'image/bmp', '.emf' : 'image/emf', @@ -708,7 +712,7 @@ def _parse_args(args): from argparse import ArgumentParser parser = ArgumentParser( - description='map filename extensions to MIME types', color=True + description='map filename extensions to MIME types', ) parser.add_argument( '-e', '--extension', diff --git a/Lib/pdb.py b/Lib/pdb.py index 569599d349c49e..bdc9caf80ec26e 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -3757,13 +3757,12 @@ def parse_args(): description=_usage, formatter_class=argparse.RawDescriptionHelpFormatter, allow_abbrev=False, - color=True, ) # Get all the commands out first. For backwards compatibility, we allow # -c commands to be after the target. parser.add_argument('-c', '--command', action='append', default=[], metavar='command', dest='commands', - help='pdb commands to execute as if given in a .pdbrc file') + help='pdb commands to execute as if given in a `.pdbrc` file') opts, args = parser.parse_known_args() diff --git a/Lib/pickle.py b/Lib/pickle.py index 95836afdc2b43e..f92b1fde768fc7 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -1937,7 +1937,6 @@ def _main(args=None): import pprint parser = argparse.ArgumentParser( description='display contents of the pickle files', - color=True, ) parser.add_argument( 'pickle_file', diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 976e218db19298..a9711538dae342 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -2899,7 +2899,7 @@ def _main(args=None): help='preserve memo between disassemblies') parser.add_argument( '-l', '--indentlevel', default=4, type=int, - help='the number of blanks by which to indent a new MARK level') + help='the number of blanks by which to indent a new `MARK` level') parser.add_argument( '-a', '--annotate', action='store_true', help='annotate each line with a short opcode description') diff --git a/Lib/platform.py b/Lib/platform.py index 9d7aa5c66a91cb..36489d4fdd98ae 100644 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1400,7 +1400,7 @@ def invalidate_caches(): def _parse_args(args: list[str] | None): import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument("args", nargs="*", choices=["nonaliased", "terse"]) parser.add_argument( "--terse", diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index bc879c43e15965..9900415ae8a927 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -389,13 +389,13 @@ def _add_sampling_options(parser): sampling_group.add_argument( "--native", action="store_true", - help='Include artificial "" frames to denote calls to non-Python code', + help='Include artificial `` frames to denote calls to non-Python code', ) sampling_group.add_argument( "--no-gc", action="store_false", dest="gc", - help='Don\'t include artificial "" frames to denote active garbage collection', + help='Don\'t include artificial `` frames to denote active garbage collection', ) sampling_group.add_argument( "--opcodes", @@ -432,14 +432,14 @@ def _add_mode_options(parser): help="Sampling mode: wall (all samples), cpu (only samples when thread is on CPU), " "gil (only samples when thread holds the GIL), " "exception (only samples when thread has an active exception). " - "Incompatible with --async-aware", + "Incompatible with `--async-aware`", ) mode_group.add_argument( "--async-mode", choices=["running", "all"], default="running", help='Async profiling mode: "running" (only running task) ' - 'or "all" (all tasks including waiting). Requires --async-aware', + 'or "all" (all tasks including waiting). Requires `--async-aware`', ) @@ -486,7 +486,7 @@ def _add_format_options(parser, include_compression=True, include_binary=True): "--diff-flamegraph", metavar="BASELINE", action=DiffFlamegraphAction, - help="Generate differential flamegraph comparing current profile to BASELINE binary file", + help="Generate differential flamegraph comparing current profile to `BASELINE` binary file", ) if include_binary: format_group.add_argument( @@ -494,7 +494,7 @@ def _add_format_options(parser, include_compression=True, include_binary=True): action="store_const", const="binary", dest="format", - help="Generate high-performance binary format (use 'replay' command to convert)", + help="Generate high-performance binary format (use `replay` command to convert)", ) parser.set_defaults(format="pstats", diff_baseline=None) @@ -510,14 +510,14 @@ def _add_format_options(parser, include_compression=True, include_binary=True): "-o", "--output", dest="outfile", - help="Output path (default: stdout for pstats text; with -o, pstats is binary). " - "Auto-generated for other formats. For heatmap: directory name (default: heatmap_PID)", + help="Output path (default: `stdout` for `pstats` text; with `-o`, `pstats` is binary). " + "Auto-generated for other formats. For heatmap: directory name (default: `heatmap_PID`)", ) output_group.add_argument( "--browser", action="store_true", help="Automatically open HTML output (flamegraph, heatmap) in browser. " - "When using --subprocesses, only the main process opens the browser", + "When using `--subprocesses`, only the main process opens the browser", ) @@ -564,13 +564,13 @@ def _add_dump_options(parser): dump_group.add_argument( "--native", action="store_true", - help='Include artificial "" frames to denote calls to non-Python code', + help='Include artificial `` frames to denote calls to non-Python code', ) dump_group.add_argument( "--no-gc", action="store_false", dest="gc", - help='Don\'t include artificial "" frames to denote active garbage collection', + help='Don\'t include artificial `` frames to denote active garbage collection', ) dump_group.add_argument( "--opcodes", @@ -588,7 +588,7 @@ def _add_dump_options(parser): default=argparse.SUPPRESS, help='Async stack mode: "running" (only running task) ' 'or "all" (all tasks including waiting, default for dump). ' - "Requires --async-aware", + "Requires `--async-aware`", ) dump_group.add_argument( "--blocking", @@ -998,7 +998,7 @@ def _main(): "-m", "--module", action="store_true", - help="Run target as a module (like python -m)", + help="Run target as a module (like `python -m`)", ) run_parser.add_argument( "target", diff --git a/Lib/py_compile.py b/Lib/py_compile.py index 694ea9304da9f9..7ca479141e01e4 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -177,7 +177,7 @@ def main(): import argparse description = 'A simple command-line interface for py_compile module.' - parser = argparse.ArgumentParser(description=description, color=True) + parser = argparse.ArgumentParser(description=description) parser.add_argument( '-q', '--quiet', action='store_true', diff --git a/Lib/random.py b/Lib/random.py index 726a71e782893c..4541267bab866a 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -1016,26 +1016,26 @@ def _test(N=10_000): def _parse_args(arg_list: list[str] | None): import argparse parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, color=True) + formatter_class=argparse.RawTextHelpFormatter) group = parser.add_mutually_exclusive_group() group.add_argument( "-c", "--choice", nargs="+", help="print a random choice") group.add_argument( "-i", "--integer", type=int, metavar="N", - help="print a random integer between 1 and N inclusive") + help="print a random integer between 1 and `N` inclusive") group.add_argument( "-f", "--float", type=float, metavar="N", - help="print a random floating-point number between 0 and N inclusive") + help="print a random floating-point number between 0 and `N` inclusive") group.add_argument( "--test", type=int, const=10_000, nargs="?", help=argparse.SUPPRESS) parser.add_argument("input", nargs="*", help="""\ if no options given, output depends on the input - string or multiple: same as --choice - integer: same as --integer - float: same as --float""") + string or multiple: same as `--choice` + integer: same as `--integer` + float: same as `--float`""") args = parser.parse_args(arg_list) return args, parser.format_help() diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index 8805442b69e080..ec72c694390717 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -87,14 +87,11 @@ def runsource(self, source, filename="", symbol="single"): def main(*args): - parser = ArgumentParser( - description="Python sqlite3 CLI", - color=True, - ) + parser = ArgumentParser(description="Python sqlite3 CLI") parser.add_argument( "filename", type=str, default=":memory:", nargs="?", help=( - "SQLite database to open (defaults to ':memory:'). " + "SQLite database to open (defaults to `:memory:`). " "A new database is created if the file does not previously exist." ), ) @@ -102,7 +99,7 @@ def main(*args): "sql", type=str, nargs="?", help=( "An SQL query to execute. " - "Any returned rows are printed to stdout." + "Any returned rows are printed to `stdout`." ), ) parser.add_argument( diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 4f47aaab9028d0..d0e7dec5575047 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -3067,7 +3067,7 @@ def main(): import argparse description = 'A simple command-line interface for tarfile module.' - parser = argparse.ArgumentParser(description=description, color=True) + parser = argparse.ArgumentParser(description=description) parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose output') parser.add_argument('--filter', metavar='', diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 45e229eb19f0f9..64c035307e6654 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -245,7 +245,7 @@ def _create_parser(): 'buildbot workers') group.add_argument('--timeout', metavar='TIMEOUT', help='dump the traceback and exit if a test takes ' - 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'more than `TIMEOUT` seconds; disabled if `TIMEOUT` ' 'is negative or equals to zero') group.add_argument('--wait', action='store_true', help='wait for user input, e.g., allow a debugger ' @@ -261,11 +261,11 @@ def _create_parser(): group = parser.add_argument_group('Verbosity') group.add_argument('-v', '--verbose', action='count', - help='run tests in verbose mode with output to stdout') + help='run tests in verbose mode with output to `stdout`') group.add_argument('-w', '--rerun', action='store_true', help='re-run failed tests in verbose mode') group.add_argument('--verbose2', action='store_true', dest='rerun', - help='deprecated alias to --rerun') + help='deprecated alias to `--rerun`') group.add_argument('-W', '--verbose3', action='store_true', help='display test output on failure') group.add_argument('-q', '--quiet', action='store_true', @@ -295,22 +295,22 @@ def _create_parser(): more_details) group.add_argument('-m', '--match', metavar='PAT', dest='match_tests', action=FilterAction, const=True, - help='match test cases and methods with glob pattern PAT') + help='match test cases and methods with glob pattern `PAT`') group.add_argument('-i', '--ignore', metavar='PAT', dest='match_tests', action=FilterAction, const=False, - help='ignore test cases and methods with glob pattern PAT') + help='ignore test cases and methods with glob pattern `PAT`') group.add_argument('--matchfile', metavar='FILENAME', dest='match_tests', action=FromFileFilterAction, const=True, - help='similar to --match but get patterns from a ' + help='similar to `--match` but get patterns from a ' 'text file, one pattern per line') group.add_argument('--ignorefile', metavar='FILENAME', dest='match_tests', action=FromFileFilterAction, const=False, - help='similar to --matchfile but it receives patterns ' + help='similar to `--matchfile` but it receives patterns ' 'from text file to ignore') group.add_argument('-G', '--failfast', action='store_true', - help='fail as soon as a test fails (only with -v or -W)') + help='fail as soon as a test fails (only with `-v` or `-W`)') group.add_argument('-u', '--use', metavar='RES1,RES2,...', action='extend', type=resources_list, help='specify which special resource intensive tests ' @@ -325,7 +325,7 @@ def _create_parser(): group = parser.add_argument_group('Special runs') group.add_argument('-L', '--runleaks', action='store_true', - help='run the leaks(1) command just before exit.' + + help='run the `leaks(1)` command just before exit.' + more_details) group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', type=huntrleaks, @@ -333,20 +333,20 @@ def _create_parser(): 'very slow).' + more_details) group.add_argument('-j', '--multiprocess', metavar='PROCESSES', dest='use_mp', type=int, - help='run PROCESSES processes at once') + help='run `PROCESSES` processes at once') group.add_argument('--single-process', action='store_true', dest='single_process', help='always run all tests sequentially in ' - 'a single process, ignore -jN option, ' + 'a single process, ignore `-jN` option, ' 'and failed tests are also rerun sequentially ' 'in the same process') group.add_argument('--parallel-threads', metavar='PARALLEL_THREADS', type=int, - help='run copies of each test in PARALLEL_THREADS at ' + help='run copies of each test in `PARALLEL_THREADS` at ' 'once') group.add_argument('-T', '--coverage', action='store_true', dest='trace', - help='turn on code coverage tracing using the trace ' + help='turn on code coverage tracing using the `trace` ' 'module') group.add_argument('-D', '--coverdir', metavar='DIR', type=relative_filename, @@ -356,18 +356,18 @@ def _create_parser(): help='put coverage files alongside modules') group.add_argument('-t', '--threshold', metavar='THRESHOLD', type=int, - help='call gc.set_threshold(THRESHOLD)') + help='call `gc.set_threshold(THRESHOLD)`') group.add_argument('-n', '--nowindows', action='store_true', help='suppress error message boxes on Windows') group.add_argument('-F', '--forever', action='store_true', help='run the specified tests in a loop, until an ' - 'error happens; imply --failfast') + 'error happens; imply `--failfast`') group.add_argument('--list-tests', action='store_true', help="only write the name of tests that will be run, " "don't execute them") group.add_argument('--list-cases', action='store_true', - help='only write the name of test cases that will be run' - ' , don\'t execute them') + help='only write the name of test cases that will be run, ' + 'don\'t execute them') group.add_argument('-P', '--pgo', dest='pgo', action='store_true', help='enable Profile Guided Optimization (PGO) training') group.add_argument('--pgo-extended', action='store_true', @@ -390,11 +390,11 @@ def _create_parser(): group.add_argument('--tempdir', metavar='PATH', help='override the working directory for the test run') group.add_argument('--cleanup', action='store_true', - help='remove old test_python_* directories') + help='remove old `test_python_*` directories') group.add_argument('--bisect', action='store_true', - help='if some tests fail, run test.bisect_cmd on them') + help='if some tests fail, run `test.bisect_cmd` on them') group.add_argument('--pythoninfo', action='store_true', - help="run python -m test.pythoninfo before tests") + help="run `python -m test.pythoninfo` before tests") group.add_argument('--dont-add-python-opts', dest='_add_python_opts', action='store_false', help="internal option, don't use it") diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index e0c32976fd6f0d..88c1a21aa28551 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1123,7 +1123,7 @@ def test_invalid_enum_value_raises_error(self): parser.add_argument('--color', choices=self.Color) self.assertRaisesRegex( argparse.ArgumentError, - r"invalid choice: 'yellow' \(choose from red, green, blue\)", + r"invalid choice: 'yellow' \(choose from 'red', 'green', 'blue'\)", parser.parse_args, ['--color', 'yellow'], ) @@ -2392,7 +2392,7 @@ def test_wrong_argument_error_with_suggestions(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('bazz',)) self.assertIn( - "error: argument foo: invalid choice: 'bazz', maybe you meant 'baz'? (choose from bar, baz)", + "error: argument foo: invalid choice: 'bazz', maybe you meant 'baz'? (choose from 'bar', 'baz')", excinfo.exception.stderr ) @@ -2402,7 +2402,7 @@ def test_wrong_argument_error_no_suggestions(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('bazz',)) self.assertIn( - "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + "error: argument foo: invalid choice: 'bazz' (choose from 'bar', 'baz')", excinfo.exception.stderr, ) @@ -2415,7 +2415,7 @@ def test_wrong_argument_subparsers_with_suggestions(self): parser.parse_args(('baz',)) self.assertIn( "error: argument {foo,bar}: invalid choice: 'baz', maybe you meant" - " 'bar'? (choose from foo, bar)", + " 'bar'? (choose from 'foo', 'bar')", excinfo.exception.stderr, ) @@ -2427,7 +2427,7 @@ def test_wrong_argument_subparsers_no_suggestions(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('baz',)) self.assertIn( - "error: argument {foo,bar}: invalid choice: 'baz' (choose from foo, bar)", + "error: argument {foo,bar}: invalid choice: 'baz' (choose from 'foo', 'bar')", excinfo.exception.stderr, ) @@ -2438,7 +2438,7 @@ def test_wrong_argument_with_suggestion_explicit(self): parser.parse_args(('bazz',)) self.assertIn( "error: argument foo: invalid choice: 'bazz', maybe you meant" - " 'baz'? (choose from bar, baz)", + " 'baz'? (choose from 'bar', 'baz')", excinfo.exception.stderr, ) @@ -2458,7 +2458,7 @@ def test_suggestions_choices_int(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('3',)) self.assertIn( - "error: argument foo: invalid choice: '3' (choose from 1, 2)", + "error: argument foo: invalid choice: '3' (choose from '1', '2')", excinfo.exception.stderr, ) @@ -2468,7 +2468,7 @@ def test_suggestions_choices_mixed_types(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('3',)) self.assertIn( - "error: argument foo: invalid choice: '3' (choose from 1, 2)", + "error: argument foo: invalid choice: '3' (choose from '1', '2')", excinfo.exception.stderr, ) @@ -7620,21 +7620,25 @@ def test_backtick_markup_in_description(self): parser = argparse.ArgumentParser( prog='PROG', color=True, - description='Run `python -m myapp` to start.', + description='Run `python myapp` or ``python -m myapp`` to start.', ) prog_extra = self.theme.prog_extra reset = self.theme.reset help_text = parser.format_help() - self.assertIn(f'Run {prog_extra}python -m myapp{reset} to start.', - help_text) + self.assertIn( + f'Run {prog_extra}python myapp{reset} or ' + f'{prog_extra}python -m myapp{reset} to start.', + help_text, + ) + self.assertNotIn("`", help_text) def test_backtick_markup_multiple(self): parser = argparse.ArgumentParser( prog='PROG', color=True, - epilog='Try `app run` or `app test`.', + epilog='Try `app run` or ``app test``.', ) prog_extra = self.theme.prog_extra @@ -7643,17 +7647,19 @@ def test_backtick_markup_multiple(self): help_text = parser.format_help() self.assertIn(f'{prog_extra}app run{reset}', help_text) self.assertIn(f'{prog_extra}app test{reset}', help_text) + self.assertNotIn('`', help_text) def test_backtick_markup_not_applied_when_color_disabled(self): # When color is disabled, backticks are preserved as-is parser = argparse.ArgumentParser( prog='PROG', color=False, - epilog='Example: `python -m myapp`', + epilog='Examples: `python -m myapp` or ``python -m myapp --x``', ) help_text = parser.format_help() self.assertIn('`python -m myapp`', help_text) + self.assertIn('``python -m myapp --x``', help_text) self.assertNotIn('\x1b[', help_text) def test_backtick_markup_with_format_string(self): @@ -7696,6 +7702,39 @@ def test_backtick_markup_special_regex_chars(self): help_text = parser.format_help() self.assertIn(f'{prog_extra}grep "foo.*bar" | sort{reset}', help_text) + def test_backtick_markup_in_argument_help(self): + parser = argparse.ArgumentParser(prog="PROG", color=True) + parser.add_argument("--foo", help="set the `foo` value") + parser.add_argument("--bar", help="set the ``bar`` value") + + prog_extra = self.theme.prog_extra + reset = self.theme.reset + + help_text = parser.format_help() + self.assertIn(f"set the {prog_extra}foo{reset} value", help_text) + self.assertIn(f"set the {prog_extra}bar{reset} value", help_text) + self.assertNotIn("`", help_text) + + def test_backtick_markup_in_argument_help_with_format(self): + parser = argparse.ArgumentParser(prog="PROG", color=True) + parser.add_argument( + "--foo", default="bar", help="set `foo` (default: %(default)s)" + ) + + prog_extra = self.theme.prog_extra + reset = self.theme.reset + + help_text = parser.format_help() + self.assertIn(f"set {prog_extra}foo{reset}", help_text) + + def test_backtick_markup_in_argument_help_color_disabled(self): + parser = argparse.ArgumentParser(prog="PROG", color=False) + parser.add_argument("--foo", help="set the `foo` value") + + help_text = parser.format_help() + self.assertIn("set the `foo` value", help_text) + self.assertNotIn("\x1b[", help_text) + def test_help_with_format_specifiers(self): # GH-142950: format specifiers like %x should work with color=True parser = argparse.ArgumentParser(prog='PROG', color=True) diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 88d265cbc21709..3fc084ea6e9c6e 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1288,6 +1288,15 @@ def test_tuple_untrack_counts(self): # Use n // 2 just in case some other objects were collected. self.assertTrue(new_count - count > (n // 2)) + @requires_gil_enabled('need generational GC') + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_heap_size(self): + count = _testinternalcapi.get_tracked_heap_size() + l = [] + self.assertEqual(count + 1, _testinternalcapi.get_tracked_heap_size()) + del l + self.assertEqual(count, _testinternalcapi.get_tracked_heap_size()) + class GCCallbackTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_gc_stats.py b/Lib/test/test_gc_stats.py index 59365ad45b32c9..bd75924397e76e 100644 --- a/Lib/test/test_gc_stats.py +++ b/Lib/test/test_gc_stats.py @@ -22,7 +22,7 @@ GC_STATS_FIELDS = ( "gen", "iid", "ts_start", "ts_stop", "collections", "collected", - "uncollectable", "candidates", "duration") + "uncollectable", "candidates", "heap_size", "duration") def get_interpreter_identifiers(gc_stats) -> tuple[int,...]: diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index a9a8cd143e0d75..1d1d2e00bd733f 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -1034,6 +1034,50 @@ def test_cli_lazy_imports_none_forces_all_imports_eager(self): self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") self.assertIn("EAGER", result.stdout) + @support.requires_resource("cpu") + def test_cli_lazy_imports_modes_import_stdlib_modules(self): + """-X lazy_imports modes should import available stdlib modules.""" + # Do not smoke-test modules with intentional import-time effects. + import_side_effect_modules = {"antigravity", "this"} + importable = [] + + for module in sorted(sys.stdlib_module_names): + if module in import_side_effect_modules: + continue + + with self.subTest(module=module): + code = f"import {module}; print({module})" + baseline = subprocess.run( + [sys.executable, "-I", "-c", code], + capture_output=True, + text=True, + timeout=60, + ) + if baseline.returncode: + # sys.stdlib_module_names includes modules for other + # platforms and optional extension modules not built here. + continue + importable.append(module) + + for mode in ("normal", "none"): + with self.subTest(module=module, mode=mode): + result = subprocess.run( + [ + sys.executable, + "-I", + "-X", + f"lazy_imports={mode}", + "-c", + code, + ], + capture_output=True, + text=True, + timeout=60, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + self.assertGreater(len(importable), 100) + def test_cli_lazy_imports_normal_respects_lazy_keyword_only(self): """-X lazy_imports=normal should respect lazy keyword only.""" # Note: Use test modules instead of stdlib modules to avoid diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index e33723cc70c877..3ea0ec66fbfbe9 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -15,14 +15,18 @@ class BaseRobotTest: good = [] bad = [] site_maps = None + expected_output = None def __init_subclass__(cls): super().__init_subclass__() # Remove tests that do nothing. - if not cls.good: - cls.test_good_urls = None - if not cls.bad: - cls.test_bad_urls = None + if issubclass(cls, unittest.TestCase): + if not cls.good: + cls.test_good_urls = None + if not cls.bad: + cls.test_bad_urls = None + if cls.expected_output is None: + cls.test_string_formatting = None def setUp(self): lines = io.StringIO(self.robots_txt).readlines() @@ -50,6 +54,8 @@ def test_bad_urls(self): def test_site_maps(self): self.assertEqual(self.parser.site_maps(), self.site_maps) + def test_string_formatting(self): + self.assertEqual(str(self.parser), self.expected_output) class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ @@ -61,6 +67,56 @@ class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase): good = ['/', '/test.html'] bad = ['/cyberworld/map/index.html', '/tmp/xxx', '/foo.html'] +class SimpleExampleTest(BaseRobotTest, unittest.TestCase): + # Example from RFC 9309, section 5.1. + robots_txt = """\ +User-Agent: * +Disallow: *.gif$ +Disallow: /example/ +Allow: /publications/ + +User-Agent: foobot +Disallow:/ +Allow:/example/page.html +Allow:/example/allowed.gif + +User-Agent: barbot +User-Agent: bazbot +Disallow: /example/page.html + +User-Agent: quxbot + """ + good = [ + '/', '/publications/', + ('foobot', '/example/page.html'), ('foobot', '/example/allowed.gif'), + ('barbot', '/'), ('barbot', '/example/'), + ('barbot', '/example/allowed.gif'), + ('barbot', '/example/disallowed.gif'), + ('barbot', '/publications/'), + ('barbot', '/publications/allowed.gif'), + ('bazbot', '/'), ('bazbot', '/example/'), + ('bazbot', '/example/allowed.gif'), + ('bazbot', '/example/disallowed.gif'), + ('bazbot', '/publications/'), + ('bazbot', '/publications/allowed.gif'), + ('quxbot', '/'), ('quxbot', '/example/'), + ('quxbot', '/example/page.html'), ('quxbot', '/example/allowed.gif'), + ('quxbot', '/example/disallowed.gif'), + ('quxbot', '/publications/'), + ('quxbot', '/publications/allowed.gif'), + ] + bad = [ + '/example/', '/example/page.html', '/example/allowed.gif', + '/example/disallowed.gif', + '/publications/allowed.gif', + ('foobot', '/'), ('foobot', '/example/'), + ('foobot', '/example/disallowed.gif'), + ('foobot', '/publications/'), + ('foobot', '/publications/allowed.gif'), + ('barbot', '/example/page.html'), + ('bazbot', '/example/page.html'), + ] + class CrawlDelayAndCustomAgentTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ @@ -102,7 +158,7 @@ class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase): User-agent: * Disallow: / """ - good = [] + good = ['/robots.txt'] bad = ['/cyberworld/map/index.html', '/', '/tmp/'] @@ -137,6 +193,7 @@ def test_request_rate(self): class EmptyFileTest(BaseRequestRateTest, unittest.TestCase): robots_txt = '' good = ['/foo'] + expected_output = '' class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase): @@ -203,35 +260,209 @@ class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase): class UserAgentOrderingTest(BaseRobotTest, unittest.TestCase): - # the order of User-agent should be correct. note - # that this file is incorrect because "Googlebot" is a - # substring of "Googlebot-Mobile" + # the order of User-agent should not matter robots_txt = """\ User-agent: Googlebot Disallow: / +Allow: /folder1/ User-agent: Googlebot-Mobile Allow: / +Disallow: /folder1/ """ agent = 'Googlebot' bad = ['/something.jpg'] + good = ['/folder1/myfile.html'] class UserAgentGoogleMobileTest(UserAgentOrderingTest): - agent = 'Googlebot-Mobile' + agent = 'Googlebot-mobile' + bad = ['/folder1/myfile.html'] + good = ['/something.jpg'] -class GoogleURLOrderingTest(BaseRobotTest, unittest.TestCase): - # Google also got the order wrong. You need - # to specify the URLs from more specific to more general +class LongestMatchTest(BaseRobotTest, unittest.TestCase): + # Based on example from RFC 9309, section 5.2. robots_txt = """\ -User-agent: Googlebot -Allow: /folder1/myfile.html -Disallow: /folder1/ +User-agent: * +Allow: /example/page/ +Disallow: /example/page/disallowed.gif +Allow: /example/ """ - agent = 'googlebot' - good = ['/folder1/myfile.html'] - bad = ['/folder1/anotherfile.html'] + good = ['/example/', '/example/page/'] + bad = ['/example/page/disallowed.gif'] + + +class LongestMatchWildcardTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Allow: /example/page/ +Disallow: *.gif +Allow: /example/ + """ + good = ['/example/', '/example/page/'] + bad = ['/example/page/disallowed.gif', '/x.gif'] + + +class AllowWinsEqualMatchTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /spam +Allow: /spam +Disallow: /spam + """ + good = ['/spam', '/spam/'] + + +class AllowWinsEqualFullMatchTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /spam +Allow: /spam$ +Disallow: /spam +Disallow: /eggs$ +Allow: /eggs +Disallow: /eggs$ + """ + good = ['/spam', '/eggs', '/eggs/'] + bad = ['/spam/'] + + +class AllowWinsEqualMatchWildcardTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /spam +Allow: *am +Disallow: /spam +Disallow: *gs +Allow: /eggs +Disallow: *gs + """ + good = ['/spam', '/eggs', '/spam/', '/eggs/'] + + +class MergeGroupsTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot +Disallow: /some/path + +User-agent: spambot +Disallow: /another/path + """ + agent = 'spambot' + bad = ['/some/path', '/another/path'] + + +class UserAgentStartsGroupTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot +Disallow: /some/path +User-agent: eggsbot +Disallow: /another/path + """ + good = [('spambot', '/'), ('spambot', '/another/path'), + ('eggsbot', '/'), ('eggsbot', '/some/path')] + bad = [('spambot', '/some/path'), ('eggsbot', '/another/path')] + expected_output = """\ +User-agent: spambot +Disallow: /some/path + +User-agent: eggsbot +Disallow: /another/path\ +""" + +class IgnoreEmptyLinesTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot + +User-agent: eggsbot +Disallow: /some/path + +Disallow: /another/path + """ + good = [('spambot', '/'), ('eggsbot', '/')] + bad = [ + ('spambot', '/some/path'), ('spambot', '/another/path'), + ('eggsbot', '/some/path'), ('eggsbot', '/another/path'), + ] + expected_output = """\ +User-agent: spambot +User-agent: eggsbot +Disallow: /some/path +Disallow: /another/path\ +""" + + +class IgnoreRulesWithoutUserAgentTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +Disallow: /some/path + +User-agent: * +Disallow: /another/path + """ + good = ['/', '/some/path'] + bad = ['/another/path'] + expected_output = """\ +User-agent: * +Disallow: /another/path\ +""" + + +class EmptyGroupTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /some/path + +User-agent: spambot + """ + agent = 'spambot' + good = ['/', '/some/path'] + expected_output = """\ +User-agent: * +Disallow: /some/path + +User-agent: spambot +Allow:\ +""" + + +class WeirdPathTest(BaseRobotTest, unittest.TestCase): + robots_txt = f"""\ +User-agent: * +Disallow: /a$$$ +Disallow: /b$z +Disallow: /c*** +Disallow: /d***z +Disallow: /e*$**$$ +Disallow: /f*$**$$z +Disallow: /g$*$$** +Disallow: /h$*$$**z + """ + good = ['/ax', '/a$$', '/b', '/bz', '/b$z', '/d', '/f', '/fz', + '/f$$$z', '/fx$y$$z', '/gx', '/g$$$', '/g$x$$y', '/h', '/hz', + '/h$$$z', '/h$x$$yz'] + bad = ['/a', '/c', '/cxy', '/dz', '/dxyz', '/dxzy', '/e', '/exy', + '/e$$', '/ex$y$', '/g'] + expected_output = """\ +User-agent: * +Disallow: /a$ +Disallow: /c* +Disallow: /d*z +Disallow: /e*$ +Disallow: /g$\ +""" + + +class PathWithManyWildcardsTest(BaseRobotTest, unittest.TestCase): + # This test would take many years if use naive translation to regular + # expression (* -> .*). + N = 50 + robots_txt = f"""\ +User-agent: * +Disallow: /{'*a'*N}*b + """ + good = ['/' + 'a'*N + 'a'] + bad = ['/' + 'a'*N + 'b'] class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): @@ -245,25 +476,13 @@ class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): good = ['/some/path', '/some/path?', '/some/path%3Fname=value', '/some/path?name%3Dvalue', '/another/path', '/another/path%3F', - '/yet/one/path?name=value%26more'] + '/yet/one/path?name=value%26more', + '/some/pathxname=value'] bad = ['/some/path?name=value' '/another/path?', '/another/path?name=value', '/yet/one/path?name=value&more'] -class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase): - # obey first * entry (#4108) - robots_txt = """\ -User-agent: * -Disallow: /some/path - -User-agent: * -Disallow: /another/path - """ - good = ['/another/path'] - bad = ['/some/path'] - - class PercentEncodingTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ User-agent: * @@ -365,17 +584,60 @@ class StringFormattingTest(BaseRobotTest, unittest.TestCase): """ expected_output = """\ -User-agent: cybermapper -Disallow: /some/path - User-agent: * Crawl-delay: 1 Request-rate: 3/15 -Disallow: /cyberworld/map/\ +Disallow: /cyberworld/map/ + +User-agent: cybermapper +Disallow: /some/path\ """ - def test_string_formatting(self): - self.assertEqual(str(self.parser), self.expected_output) + +class ConstructedStringFormattingTest(unittest.TestCase): + def test_empty(self): + parser = urllib.robotparser.RobotFileParser() + self.assertEqual(str(parser), '') + + def test_group_without_rules(self): + parser = urllib.robotparser.RobotFileParser() + entry = urllib.robotparser.Entry() + entry.useragents = ['spambot'] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.useragents = ['hambot'] + entry.rulelines = [urllib.robotparser.RuleLine('/ham', False)] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.useragents = ['eggsbot'] + parser._add_entry(entry) + self.assertEqual(str(parser), """\ +User-agent: spambot +Allow: + +User-agent: hambot +Disallow: /ham + +User-agent: eggsbot +Allow:\ +""") + + def test_group_without_user_agent(self): + parser = urllib.robotparser.RobotFileParser() + entry = urllib.robotparser.Entry() + entry.rulelines = [urllib.robotparser.RuleLine('/ham', False)] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.useragents = ['spambot'] + entry.rulelines = [urllib.robotparser.RuleLine('/spam', False)] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.rulelines = [urllib.robotparser.RuleLine('/eggs', False)] + parser._add_entry(entry) + self.assertEqual(str(parser), """\ +User-agent: spambot +Disallow: /spam\ +""") @unittest.skipUnless( @@ -495,7 +757,7 @@ def test_basic(self): def test_can_fetch(self): self.assertTrue(self.parser.can_fetch('*', self.url('elsewhere'))) self.assertFalse(self.parser.can_fetch('Nutch', self.base_url)) - self.assertFalse(self.parser.can_fetch('Nutch', self.url('brian'))) + self.assertTrue(self.parser.can_fetch('Nutch', self.url('brian'))) self.assertFalse(self.parser.can_fetch('Nutch', self.url('webstats'))) self.assertFalse(self.parser.can_fetch('*', self.url('webstats'))) self.assertTrue(self.parser.can_fetch('*', self.base_url)) diff --git a/Lib/test/test_timeit.py b/Lib/test/test_timeit.py index 81f1a9c97393d1..a2a09f9de61490 100644 --- a/Lib/test/test_timeit.py +++ b/Lib/test/test_timeit.py @@ -359,7 +359,7 @@ def test_main_with_time_unit(self): seconds_per_increment=0.003, switches=["-u", "parsec"] ) self.assertIn( - "choose from nsec, usec, msec, sec", error_stringio.getvalue() + "choose from 'nsec', 'usec', 'msec', 'sec'", error_stringio.getvalue() ) def test_main_exception(self): diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 909808825f055e..6624191f164bc1 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -4565,6 +4565,95 @@ def __init__(self): actual = self.get_suggestion(Outer(), 'target') self.assertIn("'.normal.target'", actual) + @force_not_colorized + def test_cross_language(self): + cases = [ + # (type, attr, hint_attr) + (list, 'push', 'append'), + (list, 'concat', 'extend'), + (list, 'addAll', 'extend'), + (str, 'toUpperCase', 'upper'), + (str, 'toLowerCase', 'lower'), + (str, 'trimStart', 'lstrip'), + (str, 'trimEnd', 'rstrip'), + (dict, 'keySet', 'keys'), + (dict, 'entrySet', 'items'), + (dict, 'entries', 'items'), + (dict, 'putAll', 'update'), + ] + for test_type, attr, hint_attr in cases: + with self.subTest(type=test_type.__name__, attr=attr): + obj = test_type() + actual = self.get_suggestion(obj, attr) + self.assertEndsWith(actual, f"Did you mean '.{hint_attr}'?") + + cases = [ + # (type, attr, hint) + (list, 'contains', "Use 'x in list'."), + (list, 'add', "Did you mean to use a 'set' object?"), + (dict, 'put', "Use d[k] = v."), + ] + for test_type, attr, expected in cases: + with self.subTest(type=test_type, attr=attr): + obj = test_type() + actual = self.get_suggestion(obj, attr) + self.assertEndsWith(actual, expected) + + @force_not_colorized + def test_cross_language_levenshtein_fallback(self): + # When no cross-language entry exists, Levenshtein still works + # (e.g., trim->strip is not in the table but Levenshtein catches it) + actual = self.get_suggestion('', 'trim') + self.assertIn("strip", actual) + + @force_not_colorized + def test_cross_language_no_hint_for_unknown_attr(self): + actual = self.get_suggestion([], 'completely_unknown_method') + self.assertNotIn("Did you mean", actual) + + @force_not_colorized + def test_cross_language_works_for_subclasses(self): + # isinstance() check means subclasses also get hints + class MyList(list): + pass + actual = self.get_suggestion(MyList(), 'push') + self.assertEndsWith(actual, "Did you mean '.append'?") + + class MyDict(dict): + pass + actual = self.get_suggestion(MyDict(), 'keySet') + self.assertEndsWith(actual, "Did you mean '.keys'?") + + @force_not_colorized + def test_cross_language_mutable_on_immutable(self): + # Mutable method on immutable type suggests the mutable counterpart + cases = [ + (tuple, 'append', "Did you mean to use a 'list' object?"), + (tuple, 'extend', "Did you mean to use a 'list' object?"), + (tuple, 'insert', "Did you mean to use a 'list' object?"), + (tuple, 'remove', "Did you mean to use a 'list' object?"), + (frozenset, 'add', "Did you mean to use a 'set' object?"), + (frozenset, 'discard', "Did you mean to use a 'set' object?"), + (frozenset, 'remove', "Did you mean to use a 'set' object?"), + (frozenset, 'update', "Did you mean to use a 'set' object?"), + (frozendict, 'update', "Did you mean to use a 'dict' object?"), + ] + for test_type, attr, expected in cases: + with self.subTest(type=test_type.__name__, attr=attr): + obj = test_type() + actual = self.get_suggestion(obj, attr) + self.assertEndsWith(actual, expected) + + @force_not_colorized + def test_cross_language_float_bitwise(self): + # Bitwise operators on float suggest using int + cases = ['__or__', '__and__', '__xor__', '__lshift__', '__rshift__'] + for attr in cases: + with self.subTest(attr=attr): + actual = self.get_suggestion(1.0, attr) + self.assertIn("'int'", actual) + self.assertIn("Bitwise operators", actual) + def make_module(self, code): tmpdir = Path(tempfile.mkdtemp()) self.addCleanup(shutil.rmtree, tmpdir) diff --git a/Lib/timeit.py b/Lib/timeit.py index a897d9663c24e2..01bdfd901e30d5 100644 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -299,14 +299,14 @@ def main(args=None, *, _wrap_timer=None): "-p", "--process", action="store_true", - help="use time.process_time() (default is time.perf_counter())", + help="use `time.process_time()` (default is `time.perf_counter()`)", ) parser.add_argument( "-t", "--target-time", type=float, default=default_target_time, - help="if --number is 0 the code will run until it takes " + help="if `--number` is 0 the code will run until it takes " "at least this many seconds (default %(default)s)", ) parser.add_argument( diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 52cf3f0b7ccaa9..3545d92c4f5d7f 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -578,7 +578,7 @@ def error(message, filename=None, location=None): parser = argparse.ArgumentParser() parser.add_argument(dest='filename', nargs='?', metavar='filename.py', - help='the file to tokenize; defaults to stdin') + help='the file to tokenize; defaults to `stdin`') parser.add_argument('-e', '--exact', dest='exact', action='store_true', help='display token names using the exact type') args = parser.parse_args(args) diff --git a/Lib/trace.py b/Lib/trace.py index cd3a6d30661da3..43ec201c4696d1 100644 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -604,7 +604,7 @@ def results(self): def main(): import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument('--version', action='version', version='trace 2.0') grp = parser.add_argument_group('Main options', @@ -612,27 +612,27 @@ def main(): grp.add_argument('-c', '--count', action='store_true', help='Count the number of times each line is executed and write ' - 'the counts to .cover for each module executed, in ' - 'the module\'s directory. See also --coverdir, --file, ' - '--no-report below.') + 'the counts to `.cover` for each module executed, in ' + 'the module\'s directory. See also `--coverdir`, `--file`, ' + '`--no-report` below.') grp.add_argument('-t', '--trace', action='store_true', - help='Print each line to sys.stdout before it is executed') + help='Print each line to `sys.stdout` before it is executed') grp.add_argument('-l', '--listfuncs', action='store_true', help='Keep track of which functions are executed at least once ' - 'and write the results to sys.stdout after the program exits. ' - 'Cannot be specified alongside --trace or --count.') + 'and write the results to `sys.stdout` after the program exits. ' + 'Cannot be specified alongside `--trace` or `--count`.') grp.add_argument('-T', '--trackcalls', action='store_true', help='Keep track of caller/called pairs and write the results to ' - 'sys.stdout after the program exits.') + '`sys.stdout` after the program exits.') grp = parser.add_argument_group('Modifiers') _grp = grp.add_mutually_exclusive_group() _grp.add_argument('-r', '--report', action='store_true', help='Generate a report from a counts file; does not execute any ' - 'code. --file must specify the results file to read, which ' - 'must have been created in a previous run with --count ' - '--file=FILE') + 'code. `--file` must specify the results file to read, which ' + 'must have been created in a previous run with `--count` ' + '`--file=FILE`') _grp.add_argument('-R', '--no-report', action='store_true', help='Do not generate the coverage report files. ' 'Useful if you want to accumulate over several runs.') @@ -641,14 +641,14 @@ def main(): help='File to accumulate counts over several runs') grp.add_argument('-C', '--coverdir', help='Directory where the report files go. The coverage report ' - 'for . will be written to file ' - '//.cover') + 'for `.` will be written to file ' + '`//.cover`') grp.add_argument('-m', '--missing', action='store_true', help='Annotate executable lines that were not executed with ' '">>>>>> "') grp.add_argument('-s', '--summary', action='store_true', - help='Write a brief summary for each file to sys.stdout. ' - 'Can only be used with --count or --report') + help='Write a brief summary for each file to `sys.stdout`. ' + 'Can only be used with `--count` or `--report`') grp.add_argument('-g', '--timing', action='store_true', help='Prefix each line with the time since the program started. ' 'Only used while tracing') @@ -661,7 +661,7 @@ def main(): 'module names.') grp.add_argument('--ignore-dir', action='append', default=[], help='Ignore files in the given directory ' - '(multiple directories can be joined by os.pathsep).') + '(multiple directories can be joined by `os.pathsep`).') parser.add_argument('--module', action='store_true', default=False, help='Trace a module. ') diff --git a/Lib/traceback.py b/Lib/traceback.py index 343d0e5f108c35..66e88d0a588af3 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1187,12 +1187,20 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, elif exc_type and issubclass(exc_type, AttributeError) and \ getattr(exc_value, "name", None) is not None: wrong_name = getattr(exc_value, "name", None) - suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) - if suggestion: - if suggestion.isascii(): - self._str += f". Did you mean '.{suggestion}' instead of '.{wrong_name}'?" - else: - self._str += f". Did you mean '.{suggestion}' ({suggestion!a}) instead of '.{wrong_name}' ({wrong_name!a})?" + # Check cross-language/wrong-type hints first (more specific), + # then fall back to Levenshtein distance suggestions. + hint = None + if hasattr(exc_value, 'obj'): + hint = _get_cross_language_hint(exc_value.obj, wrong_name) + if hint: + self._str += f". {hint}" + else: + suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) + if suggestion: + if suggestion.isascii(): + self._str += f". Did you mean '.{suggestion}' instead of '.{wrong_name}'?" + else: + self._str += f". Did you mean '.{suggestion}' ({suggestion!a}) instead of '.{wrong_name}' ({wrong_name!a})?" elif exc_type and issubclass(exc_type, NameError) and \ getattr(exc_value, "name", None) is not None: wrong_name = getattr(exc_value, "name", None) @@ -1689,6 +1697,62 @@ def print(self, *, file=None, chain=True, **kwargs): _MOVE_COST = 2 _CASE_COST = 1 +# Cross-language method suggestions for builtin types. +# Consulted as a fallback when Levenshtein-based suggestions find no match. +# +# Inclusion criteria: +# +# 1. Must have evidence of real cross-language confusion (Stack Overflow +# traffic, bug reports in production repos, developer survey data). +# 2. Must not be catchable by Levenshtein distance (too different from +# the correct Python method name). +# +# Each entry maps a wrong method name to a list of (type, suggestion, is_raw) +# tuples. The lookup checks isinstance() so subclasses are also matched. +# If is_raw is False, the suggestion is wrapped in "Did you mean '.X'?". +# If is_raw is True, the suggestion is rendered as-is. +# +# See https://github.com/python/cpython/issues/146406. +_CROSS_LANGUAGE_HINTS = frozendict({ + # list -- JavaScript/Ruby equivalents + "push": ((list, "append", False),), + "concat": ((list, "extend", False),), + # list -- Java/C# equivalents + "addAll": ((list, "extend", False),), + "contains": ((list, "Use 'x in list'.", True),), + # list -- wrong-type suggestion (user expected a set) + "add": ((list, "Did you mean to use a 'set' object?", True), + (frozenset, "Did you mean to use a 'set' object?", True)), + # str -- JavaScript equivalents + "toUpperCase": ((str, "upper", False),), + "toLowerCase": ((str, "lower", False),), + "trimStart": ((str, "lstrip", False),), + "trimEnd": ((str, "rstrip", False),), + # dict -- Java/JavaScript equivalents + "keySet": ((dict, "keys", False),), + "entrySet": ((dict, "items", False),), + "entries": ((dict, "items", False),), + "putAll": ((dict, "update", False),), + "put": ((dict, "Use d[k] = v.", True),), + # tuple -- mutable method on immutable type (user expected a list) + "append": ((tuple, "Did you mean to use a 'list' object?", True),), + "extend": ((tuple, "Did you mean to use a 'list' object?", True),), + "insert": ((tuple, "Did you mean to use a 'list' object?", True),), + "remove": ((tuple, "Did you mean to use a 'list' object?", True), + (frozenset, "Did you mean to use a 'set' object?", True)), + # frozenset -- mutable method on immutable type (user expected a set) + "discard": ((frozenset, "Did you mean to use a 'set' object?", True),), + # frozendict -- mutable method on immutable type (user expected a dict) + "update": ((frozenset, "Did you mean to use a 'set' object?", True), + (frozendict, "Did you mean to use a 'dict' object?", True)), + # float -- bitwise operators belong to int + "__or__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), + "__and__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), + "__xor__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), + "__lshift__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), + "__rshift__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), +}) + def _substitution_cost(ch_a, ch_b): if ch_a == ch_b: @@ -1751,6 +1815,24 @@ def _check_for_nested_attribute(obj, wrong_name, attrs): return None +def _get_cross_language_hint(obj, wrong_name): + """Check if wrong_name is a common method name from another language, + a mutable method on an immutable type, or a method tried on None. + + Uses isinstance() so subclasses of builtin types also get hints. + Returns a formatted hint string, or None. + """ + entries = _CROSS_LANGUAGE_HINTS.get(wrong_name) + if entries is None: + return None + for check_type, hint, is_raw in entries: + if isinstance(obj, check_type): + if is_raw: + return hint + return f"Did you mean '.{hint}'?" + return None + + def _get_safe___dir__(obj): # Use obj.__dir__() to avoid a TypeError when calling dir(obj). # See gh-131001 and gh-139933. diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index be99d93c78cca6..6eeebf9657a3c7 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -172,7 +172,7 @@ def _getParentArgParser(self): help='Show local variables in tracebacks') parser.add_argument('--durations', dest='durations', type=int, default=None, metavar="N", - help='Show the N slowest test cases (N=0 for all)') + help='Show the `N` slowest test cases (`N=0` for all)') if self.failfast is None: parser.add_argument('-f', '--failfast', dest='failfast', action='store_true', @@ -181,12 +181,12 @@ def _getParentArgParser(self): if self.catchbreak is None: parser.add_argument('-c', '--catch', dest='catchbreak', action='store_true', - help='Catch Ctrl-C and display results so far') + help='Catch `Ctrl-C` and display results so far') self.catchbreak = False if self.buffer is None: parser.add_argument('-b', '--buffer', dest='buffer', action='store_true', - help='Buffer stdout and stderr during tests') + help='Buffer `stdout` and `stderr` during tests') self.buffer = False if self.testNamePatterns is None: parser.add_argument('-k', dest='testNamePatterns', @@ -197,7 +197,7 @@ def _getParentArgParser(self): return parser def _getMainArgParser(self, parent): - parser = argparse.ArgumentParser(parents=[parent], color=True) + parser = argparse.ArgumentParser(parents=[parent]) parser.prog = self.progName parser.print_help = self._print_help @@ -208,16 +208,16 @@ def _getMainArgParser(self, parent): return parser def _getDiscoveryArgParser(self, parent): - parser = argparse.ArgumentParser(parents=[parent], color=True) + parser = argparse.ArgumentParser(parents=[parent]) parser.prog = '%s discover' % self.progName parser.epilog = ('For test discovery all test modules must be ' 'importable from the top level directory of the ' 'project.') parser.add_argument('-s', '--start-directory', dest='start', - help="Directory to start discovery ('.' default)") + help="Directory to start discovery (`.` default)") parser.add_argument('-p', '--pattern', dest='pattern', - help="Pattern to match tests ('test*.py' default)") + help="Pattern to match tests (`test*.py` default)") parser.add_argument('-t', '--top-level-directory', dest='top', help='Top level directory of project (defaults to ' 'start directory)') diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 4009fd6b58f594..e70eae80036784 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -7,7 +7,7 @@ 2) PSF license for Python 2.2 The robots.txt Exclusion Protocol is implemented as specified in - http://www.robotstxt.org/norobots-rfc.txt + RFC 9309 """ import collections @@ -21,19 +21,6 @@ RequestRate = collections.namedtuple("RequestRate", "requests seconds") -def normalize(path): - unquoted = urllib.parse.unquote(path, errors='surrogateescape') - return urllib.parse.quote(unquoted, errors='surrogateescape') - -def normalize_path(path): - path, sep, query = path.partition('?') - path = normalize(path) - if sep: - query = re.sub(r'[^=&]+', lambda m: normalize(m[0]), query) - path += '?' + query - return path - - class RobotFileParser: """ This class provides a set of methods to read, parse and answer questions about a single robots.txt file. @@ -42,6 +29,7 @@ class RobotFileParser: def __init__(self, url=''): self.entries = [] + self.groups = {} self.sitemaps = [] self.default_entry = None self.disallow_all = False @@ -86,13 +74,13 @@ def read(self): self.parse(raw.decode("utf-8", "surrogateescape").splitlines()) def _add_entry(self, entry): - if "*" in entry.useragents: - # the default entry is considered last - if self.default_entry is None: - # the first default entry wins - self.default_entry = entry - else: - self.entries.append(entry) + self.entries.append(entry) + for agent in entry.useragents: + agent = agent.lower() + if agent not in self.groups: + self.groups[agent] = entry + else: + self.groups[agent] = merge_entries(self.groups[agent], entry) def parse(self, lines): """Parse the input lines from a robots.txt file. @@ -100,6 +88,7 @@ def parse(self, lines): We allow that a user-agent: line is not preceded by one or more blank lines. """ + entries = [] # states: # 0: start state # 1: saw user-agent line @@ -109,14 +98,6 @@ def parse(self, lines): self.modified() for line in lines: - if not line: - if state == 1: - entry = Entry() - state = 0 - elif state == 2: - self._add_entry(entry) - entry = Entry() - state = 0 # remove optional comment and strip line i = line.find('#') if i >= 0: @@ -132,16 +113,23 @@ def parse(self, lines): if state == 2: self._add_entry(entry) entry = Entry() - entry.useragents.append(line[1]) + product_token = line[1] + entry.useragents.append(product_token) state = 1 elif line[0] == "disallow": if state != 0: - entry.rulelines.append(RuleLine(line[1], False)) state = 2 + try: + entry.rulelines.append(RuleLine(line[1], False)) + except ValueError: + pass elif line[0] == "allow": if state != 0: - entry.rulelines.append(RuleLine(line[1], True)) state = 2 + try: + entry.rulelines.append(RuleLine(line[1], True)) + except ValueError: + pass elif line[0] == "crawl-delay": if state != 0: # before trying to convert to int we need to make @@ -164,9 +152,18 @@ def parse(self, lines): # so it doesn't matter where you place it in your file." # Therefore we do not change the state of the parser. self.sitemaps.append(line[1]) - if state == 2: + if state != 0: self._add_entry(entry) + def _find_entry(self, useragent): + entry = self.groups.get(useragent.lower()) + if entry is not None: + return entry + for entry in self.groups.values(): + if entry.applies_to(useragent): + return entry + return self.groups.get('*') + def can_fetch(self, useragent, url): """using the parsed robots.txt decide if useragent can fetch url""" if self.disallow_all: @@ -179,43 +176,36 @@ def can_fetch(self, useragent, url): # calls can_fetch() before calling read(). if not self.last_checked: return False - # search for given user agent matches - # the first match counts # TODO: The private API is used in order to preserve an empty query. # This is temporary until the public API starts supporting this feature. parsed_url = urllib.parse._urlsplit(url, '') url = urllib.parse._urlunsplit(None, None, *parsed_url[2:]) - url = normalize_path(url) + url = normalize_uri(url) if not url: url = "/" - for entry in self.entries: - if entry.applies_to(useragent): - return entry.allowance(url) - # try the default entry last - if self.default_entry: - return self.default_entry.allowance(url) - # agent not found ==> access granted - return True + if url == '/robots.txt': + # The /robots.txt URI is implicitly allowed. + return True + entry = self._find_entry(useragent) + if entry is None: + return True + return entry.allowance(url) def crawl_delay(self, useragent): if not self.mtime(): return None - for entry in self.entries: - if entry.applies_to(useragent): - return entry.delay - if self.default_entry: - return self.default_entry.delay - return None + entry = self._find_entry(useragent) + if entry is None: + return None + return entry.delay def request_rate(self, useragent): if not self.mtime(): return None - for entry in self.entries: - if entry.applies_to(useragent): - return entry.req_rate - if self.default_entry: - return self.default_entry.req_rate - return None + entry = self._find_entry(useragent) + if entry is None: + return None + return entry.req_rate def site_maps(self): if not self.sitemaps: @@ -226,7 +216,7 @@ def __str__(self): entries = self.entries if self.default_entry is not None: entries = entries + [self.default_entry] - return '\n\n'.join(map(str, entries)) + return '\n\n'.join(filter(None, map(str, entries))) class RuleLine: """A rule line is a single "Allow:" (allowance==True) or "Disallow:" @@ -235,14 +225,42 @@ def __init__(self, path, allowance): if path == '' and not allowance: # an empty value means allow all allowance = True - self.path = normalize_path(path) + path = re.sub(r'[*]{2,}', '*', path) + path = re.sub(r'[$][$*]+', '$', path) + path = normalize_pattern(path) + self.fullmatch = path.endswith('$') + path = path.rstrip('$') + if '$' in path: + raise ValueError('$ not at the end of path') + self.matcher = None + if '*' in path: + pattern = re.compile(translate_pattern(path), re.DOTALL) + if self.fullmatch: + self.matcher = pattern.fullmatch + else: + self.matcher = pattern.match + self.path = path self.allowance = allowance def applies_to(self, filename): - return self.path == "*" or filename.startswith(self.path) + # If the filename matches the rule, return the matching length plus 1. + # If it does not match, return 0. + if self.matcher is not None: + m = self.matcher(filename) + if m: + return m.end() + 1 + else: + if self.fullmatch: + if filename == self.path: + return len(self.path) + 1 + else: + if filename.startswith(self.path): + return len(self.path) + 1 + return 0 def __str__(self): - return ("Allow" if self.allowance else "Disallow") + ": " + self.path + return (("Allow" if self.allowance else "Disallow") + ": " + self.path + + ('$' if self.fullmatch else '')) class Entry: @@ -254,6 +272,8 @@ def __init__(self): self.req_rate = None def __str__(self): + if not self.useragents: + return '' ret = [] for agent in self.useragents: ret.append(f"User-agent: {agent}") @@ -262,27 +282,74 @@ def __str__(self): if self.req_rate is not None: rate = self.req_rate ret.append(f"Request-rate: {rate.requests}/{rate.seconds}") - ret.extend(map(str, self.rulelines)) + if self.rulelines: + ret.extend(map(str, self.rulelines)) + else: + ret.append("Allow:") return '\n'.join(ret) def applies_to(self, useragent): """check if this entry applies to the specified agent""" + if useragent is None: + return '*' in self.useragents # split the name token and make it lower case useragent = useragent.split("/")[0].lower() for agent in self.useragents: - if agent == '*': - # we have the catch-all agent - return True - agent = agent.lower() - if agent in useragent: - return True + if agent != '*': + agent = agent.lower() + if agent in useragent: + return True return False def allowance(self, filename): """Preconditions: - our agent applies to this entry - - filename is URL encoded""" + - filename is URL encoded + """ + best_match = -1 + allowance = True for line in self.rulelines: - if line.applies_to(filename): - return line.allowance - return True + m = line.applies_to(filename) + if m: + if m > best_match: + best_match = m + allowance = line.allowance + elif m == best_match and not allowance: + allowance = line.allowance + return allowance + + +def normalize(path): + unquoted = urllib.parse.unquote(path, errors='surrogateescape') + return urllib.parse.quote(unquoted, errors='surrogateescape') + +def normalize_uri(path): + path, sep, query = path.partition('?') + path = normalize(path) + if sep: + query = re.sub(r'[^=&]+', lambda m: normalize(m[0]), query) + path += '?' + query + return path + +def normalize_pattern(path): + path, sep, query = path.partition('?') + path = re.sub(r'[^*$]+', lambda m: normalize(m[0]), path) + if sep: + query = re.sub(r'[^=&*$]+', lambda m: normalize(m[0]), query) + path += '?' + query + return path + +def translate_pattern(path): + parts = list(map(re.escape, path.split('*'))) + for i in range(1, len(parts)-1): + parts[i] = f'(?>.*?{parts[i]})' + parts[-1] = f'.*{parts[-1]}' + return ''.join(parts) + +def merge_entries(e1, e2): + entry = Entry() + entry.useragents = list(filter(set(e2.useragents).__contains__, e1.useragents)) + entry.rulelines = e1.rulelines + e2.rulelines + entry.delay = e1.delay if e2.delay is None else e2.delay + entry.req_rate = e1.req_rate if e2.req_rate is None else e2.req_rate + return entry diff --git a/Lib/uuid.py b/Lib/uuid.py index 8c59581464b0d0..4bdcb67775a2ea 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -955,7 +955,6 @@ def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description="Generate a UUID using the selected UUID function.", - color=True, ) parser.add_argument("-u", "--uuid", choices=uuid_funcs.keys(), @@ -963,14 +962,14 @@ def main(): help="function to generate the UUID") parser.add_argument("-n", "--namespace", metavar=f"{{any UUID,{','.join(namespaces)}}}", - help="uuid3/uuid5 only: " + help="`uuid3`/`uuid5` only: " "a UUID, or a well-known predefined UUID addressed " "by namespace name") parser.add_argument("-N", "--name", - help="uuid3/uuid5 only: " + help="`uuid3`/`uuid5` only: " "name used as part of generating the UUID") parser.add_argument("-C", "--count", metavar="NUM", type=int, default=1, - help="generate NUM fresh UUIDs") + help="generate `NUM` fresh UUIDs") args = parser.parse_args() uuid_func = uuid_funcs[args.uuid] diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 002f4ebc988a3b..0653a43a8b1776 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -621,7 +621,6 @@ def main(args=None): 'activate it, e.g. by ' 'sourcing an activate script ' 'in its bin directory.', - color=True, ) parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', help='A directory to create the environment in.') diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index 97aad6eea509eb..c2ee0df0ef8885 100644 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -734,7 +734,7 @@ def open(self, url, new=0, autoraise=True): def parse_args(arg_list: list[str] | None): import argparse parser = argparse.ArgumentParser( - description="Open URL in a web browser.", color=True, + description="Open URL in a web browser.", ) parser.add_argument("url", help="URL to open") diff --git a/Lib/zipapp.py b/Lib/zipapp.py index 7a4ef96ea0f077..a1cef18ada9d05 100644 --- a/Lib/zipapp.py +++ b/Lib/zipapp.py @@ -187,16 +187,16 @@ def main(args=None): """ import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument('--output', '-o', default=None, help="The name of the output archive. " - "Required if SOURCE is an archive.") + "Required if `SOURCE` is an archive.") parser.add_argument('--python', '-p', default=None, help="The name of the Python interpreter to use " "(default: no shebang line).") parser.add_argument('--main', '-m', default=None, help="The main function of the application " - "(default: use an existing __main__.py).") + "(default: use an existing `__main__.py`).") parser.add_argument('--compress', '-c', action='store_true', help="Compress files with the deflate method. " "Files are stored uncompressed by default.") diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 1e0cc5f6234f28..86c3bc36b695c7 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -2328,7 +2328,7 @@ def main(args=None): import argparse description = 'A simple command-line interface for zipfile module.' - parser = argparse.ArgumentParser(description=description, color=True) + parser = argparse.ArgumentParser(description=description) group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-l', '--list', metavar='', help='Show listing of a zipfile') @@ -2341,7 +2341,7 @@ def main(args=None): group.add_argument('-t', '--test', metavar='', help='Test if a zipfile is valid') parser.add_argument('--metadata-encoding', metavar='', - help='Specify encoding of member names for -l, -e and -t') + help='Specify encoding of member names for `-l`, `-e` and `-t`') args = parser.parse_args(args) encoding = args.metadata_encoding diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-15-12-00-00.gh-issue-146462.1YfK6v.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-15-12-00-00.gh-issue-146462.1YfK6v.rst new file mode 100644 index 00000000000000..44019b7562a344 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-15-12-00-00.gh-issue-146462.1YfK6v.rst @@ -0,0 +1,3 @@ +Added ``PyTypeObject.tp_basicsize``, ``PyTypeObject.tp_dictoffset``, and +``PyHeapTypeObject.ht_cached_keys`` offsets to :c:type:`!_Py_DebugOffsets` to +support version-independent read-only dict introspection tools. diff --git a/Misc/NEWS.d/next/Library/2026-02-19-04-40-57.gh-issue-130750.0hW52O.rst b/Misc/NEWS.d/next/Library/2026-02-19-04-40-57.gh-issue-130750.0hW52O.rst new file mode 100644 index 00000000000000..8bca48ab159476 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-19-04-40-57.gh-issue-130750.0hW52O.rst @@ -0,0 +1,2 @@ +Restore quoting of choices in :mod:`argparse` error messages for improved clarity and consistency with documentation. + diff --git a/Misc/NEWS.d/next/Library/2026-03-13-14-23-33.gh-issue-145917.TooGKx.rst b/Misc/NEWS.d/next/Library/2026-03-13-14-23-33.gh-issue-145917.TooGKx.rst new file mode 100644 index 00000000000000..23933a633f2391 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-13-14-23-33.gh-issue-145917.TooGKx.rst @@ -0,0 +1,2 @@ +Add MIME types for TTC and Haptics formats to :mod:`mimetypes`. +(Contributed by Charlie Lin in :gh:`145918`.) diff --git a/Misc/NEWS.d/next/Library/2026-03-25-07-17-41.gh-issue-146406.ydsmqe.rst b/Misc/NEWS.d/next/Library/2026-03-25-07-17-41.gh-issue-146406.ydsmqe.rst new file mode 100644 index 00000000000000..0f8107d2383ba9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-25-07-17-41.gh-issue-146406.ydsmqe.rst @@ -0,0 +1,6 @@ +Cross-language method suggestions are now shown for :exc:`AttributeError` on +builtin types and their subclasses. +For example, ``[].push()`` suggests ``append``, +``(1,2).append(3)`` suggests using a ``list``, +``None.keys()`` suggests expecting a ``dict``, +and ``1.0.__or__`` suggests using an ``int``. diff --git a/Misc/NEWS.d/next/Library/2026-04-25-14-11-24.gh-issue-138907.u21Wnh.rst b/Misc/NEWS.d/next/Library/2026-04-25-14-11-24.gh-issue-138907.u21Wnh.rst new file mode 100644 index 00000000000000..cc996a85f1c167 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-25-14-11-24.gh-issue-138907.u21Wnh.rst @@ -0,0 +1 @@ +Support :rfc:`9309` in :mod:`urllib.robotparser`. diff --git a/Misc/NEWS.d/next/Library/2026-05-03-12-00-00.gh-issue-149321.fUaxrz.rst b/Misc/NEWS.d/next/Library/2026-05-03-12-00-00.gh-issue-149321.fUaxrz.rst new file mode 100644 index 00000000000000..8fd4bf60cf32a7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-03-12-00-00.gh-issue-149321.fUaxrz.rst @@ -0,0 +1,2 @@ +Fix import cycles exposed by running standard library modules with +``-X lazy_imports=none``. diff --git a/Misc/NEWS.d/next/Library/2026-05-04-18-01-35.gh-issue-142389.4Faqpq.rst b/Misc/NEWS.d/next/Library/2026-05-04-18-01-35.gh-issue-142389.4Faqpq.rst new file mode 100644 index 00000000000000..725f2debe2c615 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-04-18-01-35.gh-issue-142389.4Faqpq.rst @@ -0,0 +1,2 @@ +Add backtick markup support in :mod:`argparse` option help text to highlight +inline code when color output is enabled. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-05-04-19-28-48.gh-issue-149377.WNlc8Y.rst b/Misc/NEWS.d/next/Library/2026-05-04-19-28-48.gh-issue-149377.WNlc8Y.rst new file mode 100644 index 00000000000000..7bab1c049e67ff --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-04-19-28-48.gh-issue-149377.WNlc8Y.rst @@ -0,0 +1 @@ +Update bundled pip to 26.1.1 diff --git a/Misc/NEWS.d/next/Library/2026-05-05-00-30-04.gh-issue-142389.4daLzc.rst b/Misc/NEWS.d/next/Library/2026-05-05-00-30-04.gh-issue-142389.4daLzc.rst new file mode 100644 index 00000000000000..4079854ef29349 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-05-00-30-04.gh-issue-142389.4daLzc.rst @@ -0,0 +1,2 @@ +Add backticks to stdlib argparse help to display in colour. Patch by Hugo +van Kemenade. diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h index 179a7b97dd4e2f..1133db808efaec 100644 --- a/Modules/_remote_debugging/clinic/module.c.h +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -601,6 +601,7 @@ PyDoc_STRVAR(_remote_debugging_GCMonitor_get_gc_stats__doc__, " - collected: Total number of collected objects.\n" " - uncollectable: Total number of uncollectable objects.\n" " - candidates: Total objects considered and traversed.\n" +" - heap_size: number of live objects.\n" " - duration: Total collection time, in seconds.\n" "\n" "Raises:\n" @@ -1563,4 +1564,4 @@ _remote_debugging_get_gc_stats(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=1151e58683dab9f4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=36674f4cb8a653f3 input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/debug_offsets_validation.h b/Modules/_remote_debugging/debug_offsets_validation.h index 1507026306192e..f070f03ac459dc 100644 --- a/Modules/_remote_debugging/debug_offsets_validation.h +++ b/Modules/_remote_debugging/debug_offsets_validation.h @@ -31,7 +31,7 @@ #define FIELD_SIZE(type, member) sizeof(((type *)0)->member) enum { - PY_REMOTE_DEBUG_OFFSETS_TOTAL_SIZE = 848, + PY_REMOTE_DEBUG_OFFSETS_TOTAL_SIZE = 880, PY_REMOTE_ASYNC_DEBUG_OFFSETS_TOTAL_SIZE = 104, }; diff --git a/Modules/_remote_debugging/gc_stats.c b/Modules/_remote_debugging/gc_stats.c index 852dc866153192..d5d05edb8ecf5e 100644 --- a/Modules/_remote_debugging/gc_stats.c +++ b/Modules/_remote_debugging/gc_stats.c @@ -53,6 +53,7 @@ read_gc_stats(struct gc_stats *stats, int64_t iid, PyObject *result, SET_FIELD(PyLong_FromSsize_t, items->collected); SET_FIELD(PyLong_FromSsize_t, items->uncollectable); SET_FIELD(PyLong_FromSsize_t, items->candidates); + SET_FIELD(PyLong_FromSsize_t, items->heap_size); SET_FIELD(PyFloat_FromDouble, items->duration); diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index c840c59971c478..c694e587e7cccb 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -143,6 +143,7 @@ static PyStructSequence_Field GCStatsInfo_fields[] = { {"collected", "Total number of collected objects"}, {"uncollectable", "Total number of uncollectable objects"}, {"candidates", "Total objects considered and traversed"}, + {"heap_size", "Number of live objects"}, {"duration", "Total collection time, in seconds"}, {NULL} }; @@ -151,7 +152,7 @@ PyStructSequence_Desc GCStatsInfo_desc = { "_remote_debugging.GCStatsInfo", "Information about a garbage collector stats sample", GCStatsInfo_fields, - 9 + 10 }; /* ============================================================================ @@ -1225,6 +1226,7 @@ Returns a list of GCStatsInfo objects with GC statistics data. - collected: Total number of collected objects. - uncollectable: Total number of uncollectable objects. - candidates: Total objects considered and traversed. + - heap_size: number of live objects. - duration: Total collection time, in seconds. Raises: @@ -1235,7 +1237,7 @@ Returns a list of GCStatsInfo objects with GC statistics data. static PyObject * _remote_debugging_GCMonitor_get_gc_stats_impl(GCMonitorObject *self, int all_interpreters) -/*[clinic end generated code: output=f73f365725224f7a input=09e647719c65f9e4]*/ +/*[clinic end generated code: output=f73f365725224f7a input=12f7c1a288cf2741]*/ { RemoteDebuggingState *st = RemoteDebugging_GetStateFromType(Py_TYPE(self)); return get_gc_stats(&self->offsets, all_interpreters, st->GCStatsInfo_Type); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index a07675bb66d8cc..d85b9eb5f7da89 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2731,8 +2731,7 @@ has_deferred_refcount(PyObject *self, PyObject *op) static PyObject * get_tracked_heap_size(PyObject *self, PyObject *Py_UNUSED(ignored)) { - // Generational GC doesn't track heap_size, return -1. - return PyLong_FromInt64(-1); + return PyLong_FromInt64(PyInterpreterState_Get()->gc.heap_size); } static PyObject * diff --git a/Python/gc.c b/Python/gc.c index 134da107e1b61d..54ac1b089e503d 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1405,13 +1405,13 @@ add_stats(GCState *gcstate, int gen, struct gc_generation_stats *stats) memcpy(cur_stats, prev_stats, sizeof(struct gc_generation_stats)); cur_stats->ts_start = stats->ts_start; - cur_stats->collections += 1; cur_stats->collected += stats->collected; cur_stats->uncollectable += stats->uncollectable; cur_stats->candidates += stats->candidates; cur_stats->duration += stats->duration; + cur_stats->heap_size = stats->heap_size; /* Publish ts_stop last so remote readers do not select a partially updated stats record as the latest collection. */ cur_stats->ts_stop = stats->ts_stop; @@ -1471,6 +1471,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) invoke_gc_callback(tstate, "start", generation, &stats); } + stats.heap_size = gcstate->heap_size; // ignore error: don't interrupt the GC if reading the clock fails (void)PyTime_PerfCounterRaw(&stats.ts_start); if (gcstate->debug & _PyGC_DEBUG_STATS) { @@ -2097,6 +2098,8 @@ PyObject_GC_Del(void *op) PyGC_Head *g = AS_GC(op); if (_PyObject_GC_IS_TRACKED(op)) { gc_list_remove(g); + GCState *gcstate = get_gc_state(); + gcstate->heap_size--; #ifdef Py_DEBUG PyObject *exc = PyErr_GetRaisedException(); if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0,