bpo-36540: PEP 570 -- Implementation (GH-12701)

This commit contains the implementation of PEP570: Python positional-only parameters. * Update Grammar/Grammar with new typedarglist and varargslist * Regenerate grammar files * Update and regenerate AST related files * Update code object * Update marshal.c * Update compiler and symtable * Regenerate importlib files * Update callable objects * Implement positional-only args logic in ceval.c * Regenerate frozen data * Update standard library to account for positional-only args * Add test file for positional-only args * Update other test files to account for positional-only args * Add News entry * Update inspect module and related tests
python · Apr 29, 2019 · 8c77b8cb9188165a123f2512026e3629bf03dc9b · 8c77b8c
1 parent 99fcc61
commit 8c77b8cb9188165a123f2512026e3629bf03dc9b
diff --git a/Grammar/Grammar b/Grammar/Grammar
@@ -22,13 +22,55 @@ async_funcdef: ASYNC funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] func_body_suite
 parameters: '(' [typedargslist] ')'
-  
-typedargslist: (tfpdef ['=' test] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] [
+  
+
+  
+# The following definition for typedarglist is equivalent to this set of rules:
+  
+#
+  
+#     arguments = argument (',' [TYPE_COMMENT] argument)*
+  
+#     argument = tfpdef ['=' test]
+  
+#     kwargs = '**' tfpdef [','] [TYPE_COMMENT]
+  
+#     args = '*' [tfpdef]
+  
+#     kwonly_kwargs = (',' [TYPE_COMMENT] argument)* (TYPE_COMMENT | [',' [TYPE_COMMENT] [kwargs]])
+  
+#     args_kwonly_kwargs = args kwonly_kwargs | kwargs
+  
+#     poskeyword_args_kwonly_kwargs = arguments ( TYPE_COMMENT | [',' [TYPE_COMMENT] [args_kwonly_kwargs]])
+  
+#     typedargslist_no_posonly  = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs
+  
+#     typedarglist = (arguments ',' [TYPE_COMMENT] '/' [',' [[TYPE_COMMENT] typedargslist_no_posonly]])|(typedargslist_no_posonly)"
+  
+#
+  
+# It needs to be fully expanded to allow our LL(1) parser to work on it.
+  
+
+  
+typedargslist: (
+  
+  (tfpdef ['=' test] (',' [TYPE_COMMENT] tfpdef ['=' test])* ',' [TYPE_COMMENT] '/' [',' [ [TYPE_COMMENT] tfpdef ['=' test] (
+  
+        ',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] [
        '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]])
      | '**' tfpdef [','] [TYPE_COMMENT]]])
  | '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]])
+  
+  | '**' tfpdef [','] [TYPE_COMMENT]]] )
+  
+|  (tfpdef ['=' test] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] [
+  
+   '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]])
+  
+  | '**' tfpdef [','] [TYPE_COMMENT]]])
+  
+  | '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]])
  | '**' tfpdef [','] [TYPE_COMMENT])
+  
+)
 tfpdef: NAME [':' test]
-  
-varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+  
+
+  
+# The following definition for varargslist is equivalent to this set of rules:
+  
+#
+  
+#     arguments = argument (',' argument )*
+  
+#     argument = vfpdef ['=' test]
+  
+#     kwargs = '**' vfpdef [',']
+  
+#     args = '*' [vfpdef]
+  
+#     kwonly_kwargs = (',' argument )* [',' [kwargs]]
+  
+#     args_kwonly_kwargs = args kwonly_kwargs | kwargs
+  
+#     poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]]
+  
+#     vararglist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs
+  
+#     varargslist = arguments ',' '/' [','[(vararglist_no_posonly)]] | (vararglist_no_posonly)
+  
+#
+  
+# It needs to be fully expanded to allow our LL(1) parser to work on it.
+  
+
+  
+varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+  
+        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  
+      | '**' vfpdef [',']]]
+  
+  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  
+  | '**' vfpdef [',']) ]] | (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
      | '**' vfpdef [',']]]
  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]

diff --git a/Include/Python-ast.h b/Include/Python-ast.h
diff --git a/Include/code.h b/Include/code.h
@@ -21,6 +21,7 @@ typedef uint16_t _Py_CODEUNIT;
 typedef struct {
    PyObject_HEAD
    int co_argcount;            /* #arguments, except *args */
+  
+    int co_posonlyargcount;     /* #positional only arguments */
    int co_kwonlyargcount;      /* #keyword only arguments */
    int co_nlocals;             /* #local variables */
    int co_stacksize;           /* #entries needed for evaluation stack */
@@ -102,7 +103,7 @@ PyAPI_DATA(PyTypeObject) PyCode_Type;
 /* Public interface */
 PyAPI_FUNC(PyCodeObject *) PyCode_New(
-  
-        int, int, int, int, int, PyObject *, PyObject *,
+  
+        int, int, int, int, int, int, PyObject *, PyObject *,
        PyObject *, PyObject *, PyObject *, PyObject *,
        PyObject *, PyObject *, int, PyObject *);
        /* same as struct above */

diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py
@@ -80,9 +80,9 @@ class struct_frozen(Structure):
                continue
            items.append((entry.name.decode("ascii"), entry.size))
-  
-        expected = [("__hello__", 139),
-  
-                    ("__phello__", -139),
-  
-                    ("__phello__.spam", 139),
+  
+        expected = [("__hello__", 141),
+  
+                    ("__phello__", -141),
+  
+                    ("__phello__.spam", 141),
                    ]
        self.assertEqual(items, expected, "PyImport_FrozenModules example "
            "in Doc/library/ctypes.rst may be out of date")

diff --git a/Lib/dis.py b/Lib/dis.py
@@ -157,6 +157,7 @@ def _format_code_info(co):
    lines.append("Name:              %s" % co.co_name)
    lines.append("Filename:          %s" % co.co_filename)
    lines.append("Argument count:    %s" % co.co_argcount)
+  
+    lines.append("Positional-only arguments: %s" % co.co_posonlyargcount)
    lines.append("Kw-only arguments: %s" % co.co_kwonlyargcount)
    lines.append("Number of locals:  %s" % co.co_nlocals)
    lines.append("Stack size:        %s" % co.co_stacksize)

diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
@@ -265,6 +265,7 @@ def _write_atomic(path, data, mode=0o666):
 #                         this might affected the first line number #32911)
 #     Python 3.8a1  3400 (move frame block handling to compiler #17611)
 #     Python 3.8a1  3401 (add END_ASYNC_FOR #33041)
+  
+#     Python 3.8a1  3410 (PEP570 Python Positional-Only Parameters #36540)
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
 # longer be understood by older implementations of the eval loop (usually
@@ -273,7 +274,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
-  
-MAGIC_NUMBER = (3401).to_bytes(2, 'little') + b'\r\n'
+  
+MAGIC_NUMBER = (3410).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 _PYCACHE = '__pycache__'

diff --git a/Lib/inspect.py b/Lib/inspect.py
@@ -272,6 +272,7 @@ def iscode(object):
                            | 16=nested | 32=generator | 64=nofree | 128=coroutine
                            | 256=iterable_coroutine | 512=async_generator
        co_freevars         tuple of names of free variables
+  
+        co_posonlyargcount  number of positional only arguments
        co_kwonlyargcount   number of keyword only arguments (not including ** arg)
        co_lnotab           encoded mapping of line numbers to bytecode indices
        co_name             name with which this code object was defined
@@ -1031,26 +1032,20 @@ def getargs(co):
    'args' is the list of argument names. Keyword-only arguments are
    appended. 'varargs' and 'varkw' are the names of the * and **
    arguments or None."""
-  
-    args, varargs, kwonlyargs, varkw = _getfullargs(co)
-  
-    return Arguments(args + kwonlyargs, varargs, varkw)
-  
-
-  
-def _getfullargs(co):
-  
-    """Get information about the arguments accepted by a code object.
-  
-
-  
-    Four things are returned: (args, varargs, kwonlyargs, varkw), where
-  
-    'args' and 'kwonlyargs' are lists of argument names, and 'varargs'
-  
-    and 'varkw' are the names of the * and ** arguments or None."""
-  
-
    if not iscode(co):
        raise TypeError('{!r} is not a code object'.format(co))
-  
-    nargs = co.co_argcount
    names = co.co_varnames
+  
+    nargs = co.co_argcount
+  
+    nposonlyargs = co.co_posonlyargcount
    nkwargs = co.co_kwonlyargcount
-  
-    args = list(names[:nargs])
-  
-    kwonlyargs = list(names[nargs:nargs+nkwargs])
+  
+    nposargs = nargs + nposonlyargs
+  
+    posonlyargs = list(names[:nposonlyargs])
+  
+    args = list(names[nposonlyargs:nposonlyargs+nargs])
+  
+    kwonlyargs = list(names[nposargs:nposargs+nkwargs])
    step = 0
+  
+    nargs += nposonlyargs
    nargs += nkwargs
    varargs = None
    if co.co_flags & CO_VARARGS:
@@ -1059,8 +1054,7 @@ def _getfullargs(co):
    varkw = None
    if co.co_flags & CO_VARKEYWORDS:
        varkw = co.co_varnames[nargs]
-  
-    return args, varargs, kwonlyargs, varkw
-  
-
+  
+    return Arguments(posonlyargs + args + kwonlyargs, varargs, varkw)
 ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults')
@@ -1087,15 +1081,16 @@ def getargspec(func):
    warnings.warn("inspect.getargspec() is deprecated since Python 3.0, "
                  "use inspect.signature() or inspect.getfullargspec()",
                  DeprecationWarning, stacklevel=2)
-  
-    args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \
-  
-        getfullargspec(func)
-  
-    if kwonlyargs or ann:
-  
-        raise ValueError("Function has keyword-only parameters or annotations"
-  
-                         ", use getfullargspec() API which can support them")
+  
+    args, varargs, varkw, defaults, posonlyargs, kwonlyargs, \
+  
+    kwonlydefaults, ann = getfullargspec(func)
+  
+    if posonlyargs or kwonlyargs or ann:
+  
+        raise ValueError("Function has positional-only, keyword-only parameters"
+  
+                         " or annotations, use getfullargspec() API which can"
+  
+                         " support them")
    return ArgSpec(args, varargs, varkw, defaults)
 FullArgSpec = namedtuple('FullArgSpec',
-  
-    'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations')
+  
+    'args, varargs, varkw, defaults, posonlyargs, kwonlyargs, kwonlydefaults, annotations')
 def getfullargspec(func):
    """Get the names and default values of a callable object's parameters.
@@ -1145,6 +1140,7 @@ def getfullargspec(func):
    args = []
    varargs = None
    varkw = None
+  
+    posonlyargs = []
    kwonlyargs = []
    defaults = ()
    annotations = {}
@@ -1159,7 +1155,9 @@ def getfullargspec(func):
        name = param.name
        if kind is _POSITIONAL_ONLY:
-  
-            args.append(name)
+  
+            posonlyargs.append(name)
+  
+            if param.default is not param.empty:
+  
+                defaults += (param.default,)
        elif kind is _POSITIONAL_OR_KEYWORD:
            args.append(name)
            if param.default is not param.empty:
@@ -1185,7 +1183,7 @@ def getfullargspec(func):
        defaults = None
    return FullArgSpec(args, varargs, varkw, defaults,
-  
-                       kwonlyargs, kwdefaults, annotations)
+  
+                       posonlyargs, kwonlyargs, kwdefaults, annotations)
 ArgInfo = namedtuple('ArgInfo', 'args varargs keywords locals')
@@ -1216,7 +1214,8 @@ def _formatannotation(annotation):
    return _formatannotation
 def formatargspec(args, varargs=None, varkw=None, defaults=None,
-  
-                  kwonlyargs=(), kwonlydefaults={}, annotations={},
+  
+                  posonlyargs=(), kwonlyargs=(), kwonlydefaults={},
+  
+                  annotations={},
                  formatarg=str,
                  formatvarargs=lambda name: '*' + name,
                  formatvarkw=lambda name: '**' + name,
@@ -1249,12 +1248,17 @@ def formatargandannotation(arg):
        return result
    specs = []
    if defaults:
-  
-        firstdefault = len(args) - len(defaults)
-  
-    for i, arg in enumerate(args):
+  
+        firstdefault = len(posonlyargs) + len(args) - len(defaults)
+  
+    posonly_left = len(posonlyargs)
+  
+    for i, arg in enumerate([*posonlyargs, *args]):
        spec = formatargandannotation(arg)
        if defaults and i >= firstdefault:
            spec = spec + formatvalue(defaults[i - firstdefault])
        specs.append(spec)
+  
+        posonly_left -= 1
+  
+        if posonlyargs and posonly_left == 0:
+  
+            specs.append('/')
+  
+
    if varargs is not None:
        specs.append(formatvarargs(formatargandannotation(varargs)))
    else:
@@ -1342,7 +1346,8 @@ def getcallargs(*func_and_positional, **named):
    func = func_and_positional[0]
    positional = func_and_positional[1:]
    spec = getfullargspec(func)
-  
-    args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = spec
+  
+    (args, varargs, varkw, defaults, posonlyargs,
+  
+     kwonlyargs, kwonlydefaults, ann) = spec
    f_name = func.__name__
    arg2value = {}
@@ -1351,12 +1356,16 @@ def getcallargs(*func_and_positional, **named):
        # implicit 'self' (or 'cls' for classmethods) argument
        positional = (func.__self__,) + positional
    num_pos = len(positional)
+  
+    num_posonlyargs = len(posonlyargs)
    num_args = len(args)
    num_defaults = len(defaults) if defaults else 0
+  
+    n = min(num_pos, num_posonlyargs)
+  
+    for i in range(num_posonlyargs):
+  
+        arg2value[posonlyargs[i]] = positional[i]
    n = min(num_pos, num_args)
    for i in range(n):
-  
-        arg2value[args[i]] = positional[i]
+  
+        arg2value[args[i]] = positional[num_posonlyargs+i]
    if varargs:
        arg2value[varargs] = tuple(positional[n:])
    possible_kwargs = set(args + kwonlyargs)
@@ -2137,9 +2146,12 @@ def _signature_from_function(cls, func):
    func_code = func.__code__
    pos_count = func_code.co_argcount
    arg_names = func_code.co_varnames
-  
-    positional = tuple(arg_names[:pos_count])
+  
+    posonly_count = func_code.co_posonlyargcount
+  
+    positional_count = posonly_count + pos_count
+  
+    positional_only = tuple(arg_names[:posonly_count])
+  
+    positional = tuple(arg_names[posonly_count:positional_count])
    keyword_only_count = func_code.co_kwonlyargcount
-  
-    keyword_only = arg_names[pos_count:(pos_count + keyword_only_count)]
+  
+    keyword_only = arg_names[positional_count:(positional_count + keyword_only_count)]
    annotations = func.__annotations__
    defaults = func.__defaults__
    kwdefaults = func.__kwdefaults__
@@ -2151,23 +2163,33 @@ def _signature_from_function(cls, func):
    parameters = []
+  
+    non_default_count = positional_count - pos_default_count
+  
+    all_positional = positional_only + positional
+  
+
+  
+    posonly_left = posonly_count
+  
+
    # Non-keyword-only parameters w/o defaults.
-  
-    non_default_count = pos_count - pos_default_count
-  
-    for name in positional[:non_default_count]:
+  
+    for name in all_positional[:non_default_count]:
+  
+        kind = _POSITIONAL_ONLY if posonly_left else _POSITIONAL_OR_KEYWORD
        annotation = annotations.get(name, _empty)
        parameters.append(Parameter(name, annotation=annotation,
-  
-                                    kind=_POSITIONAL_OR_KEYWORD))
+  
+                                    kind=kind))
+  
+        if posonly_left:
+  
+            posonly_left -= 1
    # ... w/ defaults.
-  
-    for offset, name in enumerate(positional[non_default_count:]):
+  
+    for offset, name in enumerate(all_positional[non_default_count:]):
+  
+        kind = _POSITIONAL_ONLY if posonly_left else _POSITIONAL_OR_KEYWORD
        annotation = annotations.get(name, _empty)
        parameters.append(Parameter(name, annotation=annotation,
-  
-                                    kind=_POSITIONAL_OR_KEYWORD,
+  
+                                    kind=kind,
                                    default=defaults[offset]))
+  
+        if posonly_left:
+  
+            posonly_left -= 1
    # *args
    if func_code.co_flags & CO_VARARGS:
-  
-        name = arg_names[pos_count + keyword_only_count]
+  
+        name = arg_names[positional_count + keyword_only_count]
        annotation = annotations.get(name, _empty)
        parameters.append(Parameter(name, annotation=annotation,
                                    kind=_VAR_POSITIONAL))
@@ -2184,7 +2206,7 @@ def _signature_from_function(cls, func):
                                    default=default))
    # **kwargs
    if func_code.co_flags & CO_VARKEYWORDS:
-  
-        index = pos_count + keyword_only_count
+  
+        index = positional_count + keyword_only_count
        if func_code.co_flags & CO_VARARGS:
            index += 1

diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py
@@ -619,8 +619,9 @@ def replace_paths_in_code(self, co):
            if isinstance(consts[i], type(co)):
                consts[i] = self.replace_paths_in_code(consts[i])
-  
-        return types.CodeType(co.co_argcount, co.co_kwonlyargcount,
-  
-                              co.co_nlocals, co.co_stacksize, co.co_flags,
+  
+        return types.CodeType(co.co_argcount, co.co_posonlyargcount,
+  
+                              co.co_kwonlyargcount, co.co_nlocals,
+  
+                              co.co_stacksize, co.co_flags,
                              co.co_code, tuple(consts), co.co_names,
                              co.co_varnames, new_filename, co.co_name,
                              co.co_firstlineno, co.co_lnotab, co.co_freevars,

diff --git a/Lib/test/inspect_fodder.py b/Lib/test/inspect_fodder.py
@@ -5,7 +5,7 @@
 # line 5
 # line 7
-  
-def spam(a, b, c, d=3, e=4, f=5, *g, **h):
+  
+def spam(a, /, b, c, d=3, e=4, f=5, *g, **h):
    eggs(b + d, c + f)
 # line 11