Support Python 3 stable ABI to allow mixed version interoperatbility

.github/workflows/ci.yml

@@ -50,9 +50,15 @@ jobs:
             shadow: ./src/shadow
           - features: huge
             coverage: true
+          - features: huge
+            compiler: clang
+            extra: none
+            interface: dynamic
+            python3: stable-abi
           - features: huge
             compiler: gcc
             coverage: true
+            interface: dynamic
             extra: testgui
             uchar: true
             luaver: lua5.4
@@ -141,7 +147,16 @@ jobs:
             ;;
           huge)
             echo "TEST=scripttests test_libvterm"
-            echo "CONFOPT=--enable-perlinterp --enable-pythoninterp --enable-python3interp --enable-rubyinterp --enable-luainterp --enable-tclinterp"
+            if ${{ matrix.interface == 'dynamic' }}; then
+              if ${{ matrix.python3 == 'stable-abi' }}; then
+                PYTHON3_FLAGS="--with-python3-stable-abi=3.8"
+              else
+                PYTHON3_FLAGS=""
+              fi
+              echo "CONFOPT=--enable-perlinterp=dynamic --enable-pythoninterp=dynamic --enable-python3interp=dynamic --enable-rubyinterp=dynamic --enable-luainterp=dynamic --enable-tclinterp=dynamic ${PYTHON3_FLAGS}"
+            else
+              echo "CONFOPT=--enable-perlinterp --enable-pythoninterp --enable-python3interp --enable-rubyinterp --enable-luainterp --enable-tclinterp"
+            fi
             ;;
           esac
 
@@ -370,8 +385,8 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { features: HUGE,   toolchain: msvc,  VIMDLL: no,  GUI: no,  arch: x64 }
-          - { features: HUGE,   toolchain: mingw, VIMDLL: yes, GUI: yes, arch: x86, coverage: yes }
+          - { features: HUGE,   toolchain: msvc,  VIMDLL: no,  GUI: no,  arch: x64, python3: stable }
+          - { features: HUGE,   toolchain: mingw, VIMDLL: yes, GUI: yes, arch: x86, python3: stable, coverage: yes }
           - { features: HUGE,   toolchain: msvc,  VIMDLL: no,  GUI: yes, arch: x86 }
           - { features: HUGE,   toolchain: mingw, VIMDLL: yes, GUI: no,  arch: x64, coverage: yes }
           - { features: NORMAL, toolchain: msvc,  VIMDLL: yes, GUI: no,  arch: x86 }
@@ -502,13 +517,19 @@ jobs:
           ) else (
             set GUI=${{ matrix.GUI }}
           )
+          if "${{ matrix.python3 }}"=="stable" (
+            set PYTHON3_STABLE=yes
+          ) else (
+            set PYTHON3_STABLE=no
+          )
           if "${{ matrix.features }}"=="HUGE" (
             nmake -nologo -f Make_mvc.mak ^
               FEATURES=${{ matrix.features }} ^
               GUI=%GUI% IME=yes ICONV=yes VIMDLL=${{ matrix.VIMDLL }} ^
               DYNAMIC_LUA=yes LUA=%LUA_DIR% ^
               DYNAMIC_PYTHON=yes PYTHON=%PYTHON_DIR% ^
               DYNAMIC_PYTHON3=yes PYTHON3=%PYTHON3_DIR% ^
+              DYNAMIC_PYTHON3_STABLE_ABI=%PYTHON3_STABLE% ^
               DYNAMIC_SODIUM=yes SODIUM=%SODIUM_DIR%
           ) else (
             nmake -nologo -f Make_mvc.mak ^
@@ -526,13 +547,19 @@ jobs:
           else
             GUI=${{ matrix.GUI }}
           fi
+          if [ "${{ matrix.python3 }}" = "stable" ]; then
+            PYTHON3_STABLE=yes
+          else
+            PYTHON3_STABLE=no
+          fi
           if [ "${{ matrix.features }}" = "HUGE" ]; then
             mingw32-make -f Make_ming.mak -j2 \
               FEATURES=${{ matrix.features }} \
               GUI=$GUI IME=yes ICONV=yes VIMDLL=${{ matrix.VIMDLL }} \
               DYNAMIC_LUA=yes LUA=${LUA_DIR_SLASH} \
               DYNAMIC_PYTHON=yes PYTHON=${PYTHON_DIR} \
               DYNAMIC_PYTHON3=yes PYTHON3=${PYTHON3_DIR} \
+              DYNAMIC_PYTHON3_STABLE_ABI=${PYTHON3_STABLE} \
               DYNAMIC_SODIUM=yes SODIUM=${SODIUM_DIR} \
               STATIC_STDCPLUS=yes COVERAGE=${{ matrix.coverage }}
           else

runtime/doc/builtin.txt

@@ -10977,6 +10977,7 @@ python_dynamic		Python 2.x interface is dynamically loaded. |has-python|
 python3			Python 3.x interface available. |has-python|
 python3_compiled	Compiled with Python 3.x interface. |has-python|
 python3_dynamic		Python 3.x interface is dynamically loaded. |has-python|
+python3_stable		Python 3.x interface is using Python Stable ABI. |has-python|
 pythonx			Python 2.x and/or 3.x interface available. |python_x|
 qnx			QNX version of Vim.
 quickfix		Compiled with |quickfix| support.

runtime/doc/eval.txt

@@ -2424,6 +2424,25 @@ v:progpath	Contains the command with which Vim was invoked, in a form
 		".exe" is not added to v:progpath.
 		Read-only.
 
+				    *v:python3_version* *python3-version-variable*
+v:python3_version
+		Version of Python 3 that Vim was built against.  When
+		Python is loaded dynamically (|python-dynamic|), this version
+		should exactly match the Python library up to the minor
+		version (e.g.  3.10.2 and 3.10.3 are compatible as the minor
+		version is "10", whereas 3.9.4 and 3.10.3 are not compatible).
+		When |python-stable-abi| is used, this will be the minimum Python
+		version that you can use instead. (e.g. if v:python3_version
+		indicates 3.9, you can use 3.9, 3.10, or anything above).
+
+		This number is encoded as a hex number following Python ABI
+		versioning conventions.  Do the following to have a
+		human-readable full version in hex: >
+			echo printf("%08X", v:python3_version)
+<		You can obtain only the minor version by doing: >
+			echo and(v:python3_version>>16,0xff)
+<		Read-only.
+
 					*v:register* *register-variable*
 v:register	The name of the register in effect for the current normal mode
 		command (regardless of whether that command actually used a

runtime/doc/if_pyth.txt

@@ -769,7 +769,19 @@ Unix ~
 The 'pythondll' or 'pythonthreedll' option can be used to specify the Python
 shared library file instead of DYNAMIC_PYTHON_DLL or DYNAMIC_PYTHON3_DLL file
 what were specified at compile time.  The version of the shared library must
-match the Python 2.x or Python 3 version Vim was compiled with.
+match the Python 2.x or Python 3 version (|v:python3_version|) Vim was
+compiled with unless using |python3-stable-abi|.
+
+
+Stable ABI and mixing Python versions ~
+			*python-stable* *python-stable-abi* *python3-stable-abi*
+If Vim was not compiled with Stable ABI (only available for Python 3), the
+version of the Python shared library must match the version that Vim was
+compiled with.  Otherwise, mixing versions could result in unexpected crashes
+and failures.  With Stable ABI, this restriction is relaxed, and any Python 3
+library with version of at least |v:python3_version| will work.  See
+|has-python| for how to check if Stable ABI is supported, or see if version
+output includes |+python3/dyn-stable|.
 
 ==============================================================================
 10. Python 3						*python3*
@@ -881,6 +893,18 @@ python support: >
 	  endif
 	endif
 
+When loading the library dynamically, Vim can be compiled to support Python 3
+Stable ABI (|python3-stable-abi|) which allows you to load a different version
+of Python 3 library than the one Vim was compiled with.  To check it: >
+	if has('python3_dynamic')
+	  if has('python3_stable')
+	    echo 'support Python 3 Stable ABI.'
+	  else
+	    echo 'does not support Python 3 Stable ABI.'
+	    echo 'only use Python 3 version ' .. v:python3_version
+	  endif
+	endif
+
 This also tells you whether Python is dynamically loaded, which will fail if
 the runtime library cannot be found.
 

runtime/doc/tags

@@ -1434,6 +1434,7 @@ $quote	eval.txt	/*$quote*
 +python/dyn	various.txt	/*+python\/dyn*
 +python3	various.txt	/*+python3*
 +python3/dyn	various.txt	/*+python3\/dyn*
++python3/dyn-stable	various.txt	/*+python3\/dyn-stable*
 +quickfix	various.txt	/*+quickfix*
 +reltime	various.txt	/*+reltime*
 +rightleft	various.txt	/*+rightleft*
@@ -9294,6 +9295,8 @@ python-path_hook	if_pyth.txt	/*python-path_hook*
 python-pyeval	if_pyth.txt	/*python-pyeval*
 python-range	if_pyth.txt	/*python-range*
 python-special-path	if_pyth.txt	/*python-special-path*
+python-stable	if_pyth.txt	/*python-stable*
+python-stable-abi	if_pyth.txt	/*python-stable-abi*
 python-strwidth	if_pyth.txt	/*python-strwidth*
 python-tabpage	if_pyth.txt	/*python-tabpage*
 python-tabpages	if_pyth.txt	/*python-tabpages*
@@ -9306,6 +9309,8 @@ python.vim	syntax.txt	/*python.vim*
 python2-directory	if_pyth.txt	/*python2-directory*
 python3	if_pyth.txt	/*python3*
 python3-directory	if_pyth.txt	/*python3-directory*
+python3-stable-abi	if_pyth.txt	/*python3-stable-abi*
+python3-version-variable	eval.txt	/*python3-version-variable*
 python_x	if_pyth.txt	/*python_x*
 python_x-special-comments	if_pyth.txt	/*python_x-special-comments*
 pythonx	if_pyth.txt	/*pythonx*
@@ -10632,6 +10637,7 @@ v:prevcount	eval.txt	/*v:prevcount*
 v:profiling	eval.txt	/*v:profiling*
 v:progname	eval.txt	/*v:progname*
 v:progpath	eval.txt	/*v:progpath*
+v:python3_version	eval.txt	/*v:python3_version*
 v:register	eval.txt	/*v:register*
 v:scrollstart	eval.txt	/*v:scrollstart*
 v:searchforward	eval.txt	/*v:searchforward*

runtime/doc/various.txt

@@ -450,6 +450,8 @@ m  *+python*		Python 2 interface |python|
 m  *+python/dyn*	Python 2 interface |python-dynamic| |/dyn|
 m  *+python3*		Python 3 interface |python|
 m  *+python3/dyn*	Python 3 interface |python-dynamic| |/dyn|
+m  *+python3/dyn-stable*
+			Python 3 interface |python-dynamic| |python-stable| |/dyn|
 N  *+quickfix*		|:make| and |quickfix| commands
 N  *+reltime*		|reltime()| function, 'hlsearch'/'incsearch' timeout,
 			'redrawtime' option

src/Make_cyg_ming.mak

@@ -412,6 +412,9 @@ PYTHON3INC=-I $(PYTHON3)/include
   else
 PYTHON3INC=-I $(PYTHON3)/win32inc
   endif
+  ifeq ($(DYNAMIC_PYTHON3_STABLE_ABI),yes)
+PYTHON3INC += -DPy_LIMITED_API=0x3080000
+  endif
  endif
 endif
 
@@ -594,6 +597,9 @@ ifdef PYTHON3
 CFLAGS += -DFEAT_PYTHON3
  ifeq (yes, $(DYNAMIC_PYTHON3))
 CFLAGS += -DDYNAMIC_PYTHON3 -DDYNAMIC_PYTHON3_DLL=\"$(DYNAMIC_PYTHON3_DLL)\"
+  ifeq (yes, $(DYNAMIC_PYTHON3_STABLE_ABI))
+CFLAGS += -DDYNAMIC_PYTHON3_STABLE_ABI
+  endif
  else
 CFLAGS += -DPYTHON3_DLL=\"$(DYNAMIC_PYTHON3_DLL)\"
  endif

src/Make_mvc.mak

@@ -950,7 +950,13 @@ PYTHON3_INC = /I "$(PYTHON3)\Include" /I "$(PYTHON3)\PC"
 ! if "$(DYNAMIC_PYTHON3)" == "yes"
 CFLAGS = $(CFLAGS) -DDYNAMIC_PYTHON3 \
 		-DDYNAMIC_PYTHON3_DLL=\"$(DYNAMIC_PYTHON3_DLL)\"
+!  if "$(DYNAMIC_PYTHON3_STABLE_ABI)" == "yes"
+CFLAGS = $(CFLAGS) -DDYNAMIC_PYTHON3_STABLE_ABI
+PYTHON3_INC = $(PYTHON3_INC) -DPy_LIMITED_API=0x3080000
+PYTHON3_LIB = /nodefaultlib:python3.lib
+!  else
 PYTHON3_LIB = /nodefaultlib:python$(PYTHON3_VER).lib
+!  endif
 ! else
 CFLAGS = $(CFLAGS) -DPYTHON3_DLL=\"$(DYNAMIC_PYTHON3_DLL)\"
 PYTHON3_LIB = "$(PYTHON3)\libs\python$(PYTHON3_VER).lib"

src/auto/configure

@@ -680,6 +680,7 @@ PYTHON3_SRC
 PYTHON3_CFLAGS_EXTRA
 PYTHON3_CFLAGS
 PYTHON3_LIBS
+vi_cv_var_python3_stable_abi
 vi_cv_path_python3
 PYTHON_OBJ
 PYTHON_SRC
@@ -811,6 +812,7 @@ with_python_command
 with_python_config_dir
 enable_python3interp
 with_python3_command
+with_python3_stable_abi
 with_python3_config_dir
 enable_tclinterp
 with_tclsh
@@ -1531,6 +1533,7 @@ Optional Packages:
   --with-python-command=NAME  name of the Python 2 command (default: python2 or python)
   --with-python-config-dir=PATH  Python's config directory (deprecated)
   --with-python3-command=NAME  name of the Python 3 command (default: python3 or python)
+  --with-python3-stable-abi=VERSION  stable ABI version to target (e.g. 3.8)
   --with-python3-config-dir=PATH  Python's config directory (deprecated)
   --with-tclsh=PATH       which tclsh to use (default: tclsh8.0)
   --with-ruby-command=RUBY  name of the Ruby command (default: ruby)
@@ -6753,6 +6756,34 @@ $as_echo_n "checking Python is 3.0 or better... " >&6; }
       { $as_echo "$as_me:${as_lineno-$LINENO}: result: yep" >&5
 $as_echo "yep" >&6; }
 
+            { $as_echo "$as_me:${as_lineno-$LINENO}: checking --with-python3-stable-abi argument" >&5
+$as_echo_n "checking --with-python3-stable-abi argument... " >&6; }
+
+
+# Check whether --with-python3-stable-abi was given.
+if test "${with_python3_stable_abi+set}" = set; then :
+  withval=$with_python3_stable_abi; vi_cv_var_python3_stable_abi="$withval"; { $as_echo "$as_me:${as_lineno-$LINENO}: result: $vi_cv_var_python3_stable_abi" >&5
+$as_echo "$vi_cv_var_python3_stable_abi" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+      if test "X$vi_cv_var_python3_stable_abi" != "X"; then
+        if ${vi_cv_var_python3_stable_abi_hex+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+
+         vi_cv_var_python3_stable_abi_hex=`
+         ${vi_cv_path_python3} -c \
+         "major_minor='${vi_cv_var_python3_stable_abi}'.split('.'); print('0x{0:X}'.format( (int(major_minor.__getitem__(0))<<24) + (int(major_minor.__getitem__(1))<<16) ))"`
+fi
+
+        if test "X$vi_cv_var_python3_stable_abi_hex" == "X"; then
+          as_fn_error $? "can't parse Python 3 stable ABI version. It should be \"<major>.<minor>\"" "$LINENO" 5
+        fi
+      fi
+
             { $as_echo "$as_me:${as_lineno-$LINENO}: checking Python's abiflags" >&5
 $as_echo_n "checking Python's abiflags... " >&6; }
 if ${vi_cv_var_python3_abiflags+:} false; then :
@@ -6897,9 +6928,12 @@ $as_echo "$vi_cv_dll_name_python3" >&6; }
         else
           PYTHON3_CFLAGS="-I${vi_cv_path_python3_pfx}/include/python${vi_cv_var_python3_version}${vi_cv_var_python3_abiflags} -I${vi_cv_path_python3_epfx}/include/python${vi_cv_var_python3_version}${vi_cv_var_python3_abiflags}"
         fi
-	if test "X$have_python3_config_dir" = "X1" -a "$enable_python3interp" = "dynamic"; then
-	  	  PYTHON3_CFLAGS="${PYTHON3_CFLAGS}  -DPYTHON3_HOME='L\"${vi_cv_path_python3_pfx}\"'"
-	fi
+        if test "X$have_python3_config_dir" = "X1" -a "$enable_python3interp" = "dynamic"; then
+                    PYTHON3_CFLAGS="${PYTHON3_CFLAGS}  -DPYTHON3_HOME='L\"${vi_cv_path_python3_pfx}\"'"
+        fi
+        if test "X$vi_cv_var_python3_stable_abi_hex" != "X"; then
+          PYTHON3_CFLAGS="${PYTHON3_CFLAGS} -DPy_LIMITED_API=${vi_cv_var_python3_stable_abi_hex}"
+        fi
         PYTHON3_SRC="if_python3.c"
         PYTHON3_OBJ="objects/if_python3.o"
 
@@ -7009,6 +7043,10 @@ if test "$python_ok" = yes && test "$python3_ok" = yes; then
 
   $as_echo "#define DYNAMIC_PYTHON3 1" >>confdefs.h
 
+  if test "X$vi_cv_var_python3_stable_abi_hex" != "X"; then
+    $as_echo "#define DYNAMIC_PYTHON3_STABLE_ABI 1" >>confdefs.h
+
+  fi
   { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we can do without RTLD_GLOBAL for Python" >&5
 $as_echo_n "checking whether we can do without RTLD_GLOBAL for Python... " >&6; }
   cflags_save=$CFLAGS
@@ -7190,6 +7228,10 @@ rm -f core conftest.err conftest.$ac_objext \
 elif test "$python3_ok" = yes && test "$enable_python3interp" = "dynamic"; then
   $as_echo "#define DYNAMIC_PYTHON3 1" >>confdefs.h
 
+  if test "X$vi_cv_var_python3_stable_abi_hex" != "X"; then
+    $as_echo "#define DYNAMIC_PYTHON3_STABLE_ABI 1" >>confdefs.h
+
+  fi
   PYTHON3_SRC="if_python3.c"
   PYTHON3_OBJ="objects/if_python3.o"
   PYTHON3_CFLAGS="$PYTHON3_CFLAGS -DDYNAMIC_PYTHON3_DLL=\\\"${vi_cv_dll_name_python3}\\\""

src/config.h.in

@@ -354,6 +354,9 @@
 /* Define for linking via dlopen() or LoadLibrary() */
 #undef DYNAMIC_PYTHON3
 
+/* Define if compiled against Python 3 stable ABI / limited API */
+#undef DYNAMIC_PYTHON3_STABLE_ABI
+
 /* Define if dynamic python does not require RTLD_GLOBAL */
 #undef PY_NO_RTLD_GLOBAL