[mypyc] Use faster METH_FASTCALL wrapper functions on Python 3.7+

LICENSE

@@ -28,8 +28,9 @@ DEALINGS IN THE SOFTWARE.
 
 Portions of mypy and mypyc are licensed under different licenses.  The
 files under stdlib-samples as well as the files
-mypyc/lib-rt/pythonsupport.h and mypyc/lib-rt/getargs.c are licensed
-under the PSF 2 License, reproduced below.
+mypyc/lib-rt/pythonsupport.h, mypyc/lib-rt/getargs.c and
+mypyc/lib-rt/getargsfast.c are licensed under the PSF 2 License, reproduced
+below.
 
 = = = = =
 

mypyc/codegen/emitclass.py

@@ -4,7 +4,7 @@
 from typing import Optional, List, Tuple, Dict, Callable, Mapping, Set
 from mypy.ordered_dict import OrderedDict
 
-from mypyc.common import PREFIX, NATIVE_PREFIX, REG_PREFIX
+from mypyc.common import PREFIX, NATIVE_PREFIX, REG_PREFIX, USE_FASTCALL
 from mypyc.codegen.emit import Emitter, HeaderDeclaration
 from mypyc.codegen.emitfunc import native_function_header
 from mypyc.codegen.emitwrapper import (
@@ -644,7 +644,11 @@ def generate_methods_table(cl: ClassIR,
             continue
         emitter.emit_line('{{"{}",'.format(fn.name))
         emitter.emit_line(' (PyCFunction){}{},'.format(PREFIX, fn.cname(emitter.names)))
-        flags = ['METH_VARARGS', 'METH_KEYWORDS']
+        if USE_FASTCALL:
+            flags = ['METH_FASTCALL']
+        else:
+            flags = ['METH_VARARGS']
+        flags.append('METH_KEYWORDS')
         if fn.decl.kind == FUNC_STATICMETHOD:
             flags.append('METH_STATIC')
         elif fn.decl.kind == FUNC_CLASSMETHOD:

mypyc/codegen/emitmodule.py

@@ -23,14 +23,16 @@
 from mypyc.irbuild.prepare import load_type_map
 from mypyc.irbuild.mapper import Mapper
 from mypyc.common import (
-    PREFIX, TOP_LEVEL_NAME, INT_PREFIX, MODULE_PREFIX, RUNTIME_C_FILES, shared_lib_name,
+    PREFIX, TOP_LEVEL_NAME, INT_PREFIX, MODULE_PREFIX, RUNTIME_C_FILES, USE_FASTCALL,
+    shared_lib_name,
 )
 from mypyc.codegen.cstring import encode_as_c_string, encode_bytes_as_c_string
 from mypyc.codegen.emit import EmitterContext, Emitter, HeaderDeclaration
 from mypyc.codegen.emitfunc import generate_native_function, native_function_header
 from mypyc.codegen.emitclass import generate_class_type_decl, generate_class
 from mypyc.codegen.emitwrapper import (
     generate_wrapper_function, wrapper_function_header,
+    generate_legacy_wrapper_function, legacy_wrapper_function_header,
 )
 from mypyc.ir.ops import LiteralsMap, DeserMaps
 from mypyc.ir.rtypes import RType, RTuple
@@ -419,8 +421,12 @@ def generate_function_declaration(fn: FuncIR, emitter: Emitter) -> None:
         '{};'.format(native_function_header(fn.decl, emitter)),
         needs_export=True)
     if fn.name != TOP_LEVEL_NAME:
-        emitter.context.declarations[PREFIX + fn.cname(emitter.names)] = HeaderDeclaration(
-            '{};'.format(wrapper_function_header(fn, emitter.names)))
+        if is_fastcall_supported(fn):
+            emitter.context.declarations[PREFIX + fn.cname(emitter.names)] = HeaderDeclaration(
+                '{};'.format(wrapper_function_header(fn, emitter.names)))
+        else:
+            emitter.context.declarations[PREFIX + fn.cname(emitter.names)] = HeaderDeclaration(
+                '{};'.format(legacy_wrapper_function_header(fn, emitter.names)))
 
 
 def pointerize(decl: str, name: str) -> str:
@@ -529,9 +535,12 @@ def generate_c_for_modules(self) -> List[Tuple[str, str]]:
                 generate_native_function(fn, emitter, self.source_paths[module_name], module_name)
                 if fn.name != TOP_LEVEL_NAME:
                     emitter.emit_line()
-                    generate_wrapper_function(
-                        fn, emitter, self.source_paths[module_name], module_name)
-
+                    if is_fastcall_supported(fn):
+                        generate_wrapper_function(
+                            fn, emitter, self.source_paths[module_name], module_name)
+                    else:
+                        generate_legacy_wrapper_function(
+                            fn, emitter, self.source_paths[module_name], module_name)
             if multi_file:
                 name = ('__native_{}.c'.format(emitter.names.private_name(module_name)))
                 file_contents.append((name, ''.join(emitter.fragments)))
@@ -837,12 +846,17 @@ def generate_module_def(self, emitter: Emitter, module_name: str, module: Module
         for fn in module.functions:
             if fn.class_name is not None or fn.name == TOP_LEVEL_NAME:
                 continue
+            if is_fastcall_supported(fn):
+                flag = 'METH_FASTCALL'
+            else:
+                flag = 'METH_VARARGS'
             emitter.emit_line(
-                ('{{"{name}", (PyCFunction){prefix}{cname}, METH_VARARGS | METH_KEYWORDS, '
+                ('{{"{name}", (PyCFunction){prefix}{cname}, {flag} | METH_KEYWORDS, '
                  'NULL /* docstring */}},').format(
-                    name=fn.name,
-                    cname=fn.cname(emitter.names),
-                    prefix=PREFIX))
+                     name=fn.name,
+                     cname=fn.cname(emitter.names),
+                     prefix=PREFIX,
+                     flag=flag))
         emitter.emit_line('{NULL, NULL, 0, NULL}')
         emitter.emit_line('};')
         emitter.emit_line()
@@ -1054,3 +1068,8 @@ def visit(item: T) -> None:
         visit(item)
 
     return result
+
+
+def is_fastcall_supported(fn: FuncIR) -> bool:
+    # TODO: Support METH_FASTCALL for all methods.
+    return USE_FASTCALL and (fn.class_name is None or fn.name not in ('__init__', '__call__'))

mypyc/codegen/emitwrapper.py

@@ -1,4 +1,14 @@
-"""Generate CPython API wrapper function for a native function."""
+"""Generate CPython API wrapper functions for native functions.
+
+The wrapper functions are used by the CPython runtime when calling
+native functions from interpreted code, and when the called function
+can't be determined statically in compiled code. They validate, match,
+unbox and type check function arguments, and box return values as
+needed. All wrappers accept and return 'PyObject *' (boxed) values.
+
+The wrappers aren't used for most calls between two native functions
+or methods in a single compilation unit.
+"""
 
 from typing import List, Optional
 
@@ -14,15 +24,85 @@
 from mypyc.namegen import NameGenerator
 
 
+# Generic vectorcall wrapper functions (Python 3.7+)
+#
+# A wrapper function has a signature like this:
+#
+# PyObject *fn(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+#
+# The function takes a self object, pointer to an array of arguments,
+# the number of positional arguments, and a tuple of keyword argument
+# names (that are stored starting in args[nargs]).
+#
+# It returns the returned object, or NULL on an exception.
+#
+# These are more efficient than legacy wrapper functions, since
+# usually no tuple or dict objects need to be created for the
+# arguments. Vectorcalls also use pre-constructed str objects for
+# keyword argument names and other pre-computed information, instead
+# of processing the argument format string on each call.
+
+
 def wrapper_function_header(fn: FuncIR, names: NameGenerator) -> str:
-    return 'PyObject *{prefix}{name}(PyObject *self, PyObject *args, PyObject *kw)'.format(
-        prefix=PREFIX,
-        name=fn.cname(names))
+    """Return header of a vectorcall wrapper function.
+
+    See comment above for a summary of the arguments.
+    """
+    return (
+        'PyObject *{prefix}{name}('
+        'PyObject *self, PyObject *const *args, size_t nargs, PyObject *kwnames)').format(
+            prefix=PREFIX,
+            name=fn.cname(names))
+
+
+def generate_traceback_code(fn: FuncIR,
+                            emitter: Emitter,
+                            source_path: str,
+                            module_name: str) -> str:
+    # If we hit an error while processing arguments, then we emit a
+    # traceback frame to make it possible to debug where it happened.
+    # Unlike traceback frames added for exceptions seen in IR, we do this
+    # even if there is no `traceback_name`. This is because the error will
+    # have originated here and so we need it in the traceback.
+    globals_static = emitter.static_name('globals', module_name)
+    traceback_code = 'CPy_AddTraceback("%s", "%s", %d, %s);' % (
+        source_path.replace("\\", "\\\\"),
+        fn.traceback_name or fn.name,
+        fn.line,
+        globals_static)
+    return traceback_code
+
+
+def make_arg_groups(args: List[RuntimeArg]) -> List[List[RuntimeArg]]:
+    """Group arguments by kind."""
+    return [[arg for arg in args if arg.kind == k] for k in range(ARG_NAMED_OPT + 1)]
+
+
+def reorder_arg_groups(groups: List[List[RuntimeArg]]) -> List[RuntimeArg]:
+    """Reorder argument groups to match their order in a format string."""
+    return groups[ARG_POS] + groups[ARG_OPT] + groups[ARG_NAMED_OPT] + groups[ARG_NAMED]
+
+
+def make_static_kwlist(args: List[RuntimeArg]) -> str:
+    arg_names = ''.join('"{}", '.format(arg.name) for arg in args)
+    return 'static const char * const kwlist[] = {{{}0}};'.format(arg_names)
 
 
 def make_format_string(func_name: str, groups: List[List[RuntimeArg]]) -> str:
-    # Construct the format string. Each group requires the previous
-    # groups delimiters to be present first.
+    """Return a format string that specifies the accepted arguments.
+
+    The format string is an extended subset of what is supported by
+    PyArg_ParseTupleAndKeywords(). Only the type 'O' is used, and we
+    also support some extensions:
+
+    - Required keyword-only arguments are introduced after '@'
+    - If the function receives *args or **kwargs, we add a '%' prefix
+
+    Each group requires the previous groups' delimiters to be present
+    first.
+
+    These are used by both vectorcall and legacy wrapper functions.
+    """
     main_format = ''
     if groups[ARG_STAR] or groups[ARG_STAR2]:
         main_format += '%'
@@ -40,24 +120,80 @@ def generate_wrapper_function(fn: FuncIR,
                               emitter: Emitter,
                               source_path: str,
                               module_name: str) -> None:
-    """Generates a CPython-compatible wrapper function for a native function.
+    """Generate a CPython-compatible vectorcall wrapper for a native function.
 
     In particular, this handles unboxing the arguments, calling the native function, and
     then boxing the return value.
     """
     emitter.emit_line('{} {{'.format(wrapper_function_header(fn, emitter.names)))
 
-    # If we hit an error while processing arguments, then we emit a
-    # traceback frame to make it possible to debug where it happened.
-    # Unlike traceback frames added for exceptions seen in IR, we do this
-    # even if there is no `traceback_name`. This is because the error will
-    # have originated here and so we need it in the traceback.
-    globals_static = emitter.static_name('globals', module_name)
-    traceback_code = 'CPy_AddTraceback("%s", "%s", %d, %s);' % (
-        source_path.replace("\\", "\\\\"),
-        fn.traceback_name or fn.name,
-        fn.line,
-        globals_static)
+    # If fn is a method, then the first argument is a self param
+    real_args = list(fn.args)
+    if fn.class_name and not fn.decl.kind == FUNC_STATICMETHOD:
+        arg = real_args.pop(0)
+        emitter.emit_line('PyObject *obj_{} = self;'.format(arg.name))
+
+    # Need to order args as: required, optional, kwonly optional, kwonly required
+    # This is because CPyArg_ParseStackAndKeywords format string requires
+    # them grouped in that way.
+    groups = make_arg_groups(real_args)
+    reordered_args = reorder_arg_groups(groups)
+
+    emitter.emit_line(make_static_kwlist(reordered_args))
+    fmt = make_format_string(fn.name, groups)
+    # Define the arguments the function accepts (but no types yet)
+    emitter.emit_line('static CPyArg_Parser parser = {{"{}", kwlist, 0}};'.format(fmt))
+
+    for arg in real_args:
+        emitter.emit_line('PyObject *obj_{}{};'.format(
+                          arg.name, ' = NULL' if arg.optional else ''))
+
+    cleanups = ['CPy_DECREF(obj_{});'.format(arg.name)
+                for arg in groups[ARG_STAR] + groups[ARG_STAR2]]
+
+    arg_ptrs = []  # type: List[str]
+    if groups[ARG_STAR] or groups[ARG_STAR2]:
+        arg_ptrs += ['&obj_{}'.format(groups[ARG_STAR][0].name) if groups[ARG_STAR] else 'NULL']
+        arg_ptrs += ['&obj_{}'.format(groups[ARG_STAR2][0].name) if groups[ARG_STAR2] else 'NULL']
+    arg_ptrs += ['&obj_{}'.format(arg.name) for arg in reordered_args]
+
+    emitter.emit_lines(
+        'if (!CPyArg_ParseStackAndKeywords(args, nargs, kwnames, &parser{})) {{'.format(
+            ''.join(', ' + n for n in arg_ptrs)),
+        'return NULL;',
+        '}')
+    traceback_code = generate_traceback_code(fn, emitter, source_path, module_name)
+    generate_wrapper_core(fn, emitter, groups[ARG_OPT] + groups[ARG_NAMED_OPT],
+                          cleanups=cleanups,
+                          traceback_code=traceback_code)
+
+    emitter.emit_line('}')
+
+
+# Legacy generic wrapper functions
+#
+# These take a self object, a Python tuple of positional arguments,
+# and a dict of keyword arguments. These are a lot slower than
+# vectorcall wrappers, especially in calls involving keyword
+# arguments.
+
+
+def legacy_wrapper_function_header(fn: FuncIR, names: NameGenerator) -> str:
+    return 'PyObject *{prefix}{name}(PyObject *self, PyObject *args, PyObject *kw)'.format(
+        prefix=PREFIX,
+        name=fn.cname(names))
+
+
+def generate_legacy_wrapper_function(fn: FuncIR,
+                                     emitter: Emitter,
+                                     source_path: str,
+                                     module_name: str) -> None:
+    """Generates a CPython-compatible legacy wrapper for a native function.
+
+    In particular, this handles unboxing the arguments, calling the native function, and
+    then boxing the return value.
+    """
+    emitter.emit_line('{} {{'.format(legacy_wrapper_function_header(fn, emitter.names)))
 
     # If fn is a method, then the first argument is a self param
     real_args = list(fn.args)
@@ -68,11 +204,10 @@ def generate_wrapper_function(fn: FuncIR,
     # Need to order args as: required, optional, kwonly optional, kwonly required
     # This is because CPyArg_ParseTupleAndKeywords format string requires
     # them grouped in that way.
-    groups = [[arg for arg in real_args if arg.kind == k] for k in range(ARG_NAMED_OPT + 1)]
-    reordered_args = groups[ARG_POS] + groups[ARG_OPT] + groups[ARG_NAMED_OPT] + groups[ARG_NAMED]
+    groups = make_arg_groups(real_args)
+    reordered_args = reorder_arg_groups(groups)
 
-    arg_names = ''.join('"{}", '.format(arg.name) for arg in reordered_args)
-    emitter.emit_line('static char *kwlist[] = {{{}0}};'.format(arg_names))
+    emitter.emit_line(make_static_kwlist(reordered_args))
     for arg in real_args:
         emitter.emit_line('PyObject *obj_{}{};'.format(
                           arg.name, ' = NULL' if arg.optional else ''))
@@ -91,13 +226,17 @@ def generate_wrapper_function(fn: FuncIR,
             make_format_string(fn.name, groups), ''.join(', ' + n for n in arg_ptrs)),
         'return NULL;',
         '}')
+    traceback_code = generate_traceback_code(fn, emitter, source_path, module_name)
     generate_wrapper_core(fn, emitter, groups[ARG_OPT] + groups[ARG_NAMED_OPT],
                           cleanups=cleanups,
                           traceback_code=traceback_code)
 
     emitter.emit_line('}')
 
 
+# Specialized wrapper functions
+
+
 def generate_dunder_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str:
     """Generates a wrapper for native __dunder__ methods to be able to fit into the mapping
     protocol slot. This specifically means that the arguments are taken as *PyObjects and returned
@@ -214,12 +353,16 @@ def generate_bool_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str:
     return name
 
 
+# Helpers
+
+
 def generate_wrapper_core(fn: FuncIR, emitter: Emitter,
                           optional_args: Optional[List[RuntimeArg]] = None,
                           arg_names: Optional[List[str]] = None,
                           cleanups: Optional[List[str]] = None,
                           traceback_code: Optional[str] = None) -> None:
     """Generates the core part of a wrapper function for a native function.
+
     This expects each argument as a PyObject * named obj_{arg} as a precondition.
     It converts the PyObject *s to the necessary types, checking and unboxing if necessary,
     makes the call, then boxes the result if necessary and returns it.

mypyc/common.py

@@ -50,10 +50,14 @@
 MAX_LITERAL_SHORT_INT = (sys.maxsize >> 1 if not IS_MIXED_32_64_BIT_BUILD
                          else 2**30 - 1)  # type: Final
 
+# We can use faster wrapper functions on Python 3.7+ (fastcall/vectorcall).
+USE_FASTCALL = sys.version_info >= (3, 7)
+
 # Runtime C library files
 RUNTIME_C_FILES = [
     'init.c',
     'getargs.c',
+    'getargsfast.c',
     'int_ops.c',
     'list_ops.c',
     'dict_ops.c',