Use Git tags, not the development versions

doc/pythoncapi.py

@@ -1,15 +1,12 @@
 import glob
 import os.path
 import re
+import subprocess
 
 
-# Checkout of Python Git repository, one directory per branch:
-#
-# - 2.7/ = Python 2.7 branch
-# - 3.6/ = Python 3.6 branch
-# - main/ = Python main branch
-# - etc.
-PYTHON_ROOT = '/home/vstinner/python'
+# Checkout of Python Git repository
+CPYTHON_URL = 'https://github.com/python/cpython'
+GIT_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', 'cpython_git'))
 
 
 PATH_LIMITED_API = 'Include'
@@ -34,6 +31,56 @@
 }
 
 
+def run_command(cmd, cwd):
+    subprocess.run(cmd,
+                   stdout=subprocess.DEVNULL,
+                   stderr=subprocess.DEVNULL,
+                   check=True,
+                   cwd=cwd)
+
+
+def git_clone():
+    if os.path.exists(GIT_DIR):
+        return
+
+    dst_name = os.path.basename(GIT_DIR)
+    cmd = ['git', 'clone', CPYTHON_URL, dst_name]
+    run_command(cwd=os.path.dirname(GIT_DIR))
+
+
+_CLEANED = False
+_FETCHED = False
+
+def git_switch_branch(branch):
+    git_clone()
+
+    global _CLEANED
+    if not _CLEANED:
+        cmd = ['git', 'clean', '-fdx']
+        run_command(cmd, cwd=GIT_DIR)
+
+        cmd = ['git', 'checkout', '.']
+        run_command(cmd, cwd=GIT_DIR)
+
+        _CLEANED = True
+
+    if branch == 'main':
+        cmd = ['git', 'switch', branch]
+        run_command(cmd, cwd=GIT_DIR)
+
+        global _FETCHED
+        if not _FETCHED:
+            cmd = ['git', 'fetch']
+            run_command(cmd, cwd=GIT_DIR)
+            _FETCHED = True
+
+        cmd = ['git', 'merge', '--ff']
+        run_command(cmd, cwd=GIT_DIR)
+    else:
+        cmd = ['git', 'checkout', branch]
+        run_command(cmd, cwd=GIT_DIR)
+
+
 def list_files(path):
     if not os.path.exists(path):
         return []
@@ -67,13 +114,20 @@ def _get_types(filename, names):
         names.add(name)
 
 
-def get_types(directory):
+def get_types_path(directory):
     names = set()
     for filename in list_files(directory):
         _get_types(filename, names)
     return sorted(names)
 
 
+def get_types():
+    limited = len(get_types_path(PATH_LIMITED_API))
+    cpython = len(get_types_path(PATH_CPYTHON_API))
+    internal = len(get_types_path(PATH_INTERNAL_API))
+    return (limited, cpython, internal)
+
+
 def grep(regex, filenames, group=0):
     for filename in filenames:
         with open(filename, encoding='utf-8') as fp:
@@ -154,8 +208,9 @@ def get_variables():
     RE_VARIABLE = (
         # 'name'
         # 'name, name2'
+        # 'name, *name2'
         fr'(?:const *)?'
-        fr'({RE_IDENTIFIER}(?:, *{RE_IDENTIFIER})*)'
+        fr'({RE_IDENTIFIER}(?:, *\*? *{RE_IDENTIFIER})*)'
         # '[]', '[256]', '[PY_EXECUTABLE_KINDS+1]'
         fr'(?:\[[^]]*\])?'
     )
@@ -225,3 +280,10 @@ def get(path):
     cpython = get(PATH_CPYTHON_API)
     internal = get(PATH_INTERNAL_API)
     return (limited, cpython, internal)
+
+
+def get_file_numbers():
+    limited = len(list_files(PATH_LIMITED_API))
+    cpython = len(list_files(PATH_CPYTHON_API))
+    internal = len(list_files(PATH_INTERNAL_API))
+    return (limited, cpython, internal)

doc/stats.py

@@ -3,25 +3,26 @@
 import os
 
 from pythoncapi import (
-    PYTHON_ROOT, PATH_LIMITED_API, PATH_CPYTHON_API, PATH_INTERNAL_API,
+    GIT_DIR, git_switch_branch,
+    PATH_LIMITED_API, PATH_CPYTHON_API, PATH_INTERNAL_API,
     list_files,
     get_types, get_macros_static_inline_funcs,
-    get_functions, get_variables, get_line_numbers)
+    get_functions, get_variables, get_line_numbers, get_file_numbers)
 
 
-RST_FILENAME = 'stats.rst'
-MAIN_BRANCH = '3.13'
 BRANCHES = [
-    '2.7',
-    '3.6',
-    '3.7',
-    '3.8',
-    '3.9',
-    '3.10',
-    '3.11',
-    '3.12',
-    'main',
+    ('v2.7', '2.7.0'),
+    ('v3.6.0', '3.6.0'),
+    ('v3.7.0', '3.7.0'),
+    ('v3.8.0', '3.8.0'),
+    ('v3.9.0', '3.9.0'),
+    ('v3.10.0', '3.10.0'),
+    ('v3.11.0', '3.11.0'),
+    #('v3.12.0', '3.12'),
+    ('3.12', '3.12 (dev)'),
+    ('main', 'main (3.13)'),
 ]
+RST_FILENAME = os.path.normpath(os.path.join(os.path.dirname(__file__), 'stats.rst'))
 COLUMNS = ['Python', 'Limited API', 'CPython API', 'Internal API', 'Total']
 TABLE_SPACE = '  '
 
@@ -31,22 +32,11 @@ def log(msg=''):
     output.append(msg)
 
 
-@contextlib.contextmanager
-def change_directory(path):
-    old_dir = os.getcwd()
-    try:
-        os.chdir(path)
-        yield
-    finally:
-        os.chdir(old_dir)
-
-
 def iter_branches():
-    for name in BRANCHES:
-        with change_directory(name):
-            if name == 'main':
-                name = MAIN_BRANCH
-            yield name
+    for branch, version in BRANCHES:
+        git_switch_branch(branch)
+        os.chdir(GIT_DIR)
+        yield version
 
 
 def display_title(title):
@@ -155,9 +145,7 @@ def file_numbers():
     paragraph('Number of header file numbers per Python version:')
     lines = [COLUMNS]
     for name in iter_branches():
-        limited = len(list_files(PATH_LIMITED_API))
-        cpython = len(list_files(PATH_CPYTHON_API))
-        internal = len(list_files(PATH_INTERNAL_API))
+        limited, cpython, internal = get_file_numbers()
         line = [name, limited, cpython, internal, limited + cpython + internal]
         lines.append(line)
     table_compute_diff(lines)
@@ -221,10 +209,9 @@ def structures():
 
     lines = [COLUMNS]
     for name in iter_branches():
-        limited = len(get_types(PATH_LIMITED_API))
-        cpython = len(get_types(PATH_CPYTHON_API))
-        internal = len(get_types(PATH_INTERNAL_API))
-        line = [name, limited, cpython, internal, limited + cpython + internal]
+        limited, cpython, internal = get_types()
+        total = limited + cpython + internal
+        line = [name, limited, cpython, internal, total]
         lines.append(line)
     table_compute_diff(lines)
     render_table(lines)
@@ -243,14 +230,14 @@ def render_page():
 
 
 def main():
-    with change_directory(PYTHON_ROOT):
-        render_page()
+    render_page()
 
-    with open(RST_FILENAME, 'w') as fp:
+    filename = RST_FILENAME
+    with open(filename, 'w') as fp:
         for line in output:
             print(line, file=fp)
 
-    print(f"Write into {RST_FILENAME}")
+    print(f"Write into {filename}")