bpo-35482: Fixes HTML escaping in CHM index and build location of NEW…

…S file (GH-11224) (GH-11251)

Doc/make.bat

@@ -16,7 +16,7 @@ if not defined SPHINXBUILD (
         %PYTHON% -m pip install sphinx
         if errorlevel 1 exit /B
     )
-    set SPHINXBUILD=%PYTHON% -c "import sphinx, sys; sys.argv[0] = 'sphinx-build'; sys.exit(sphinx.main())"
+    set SPHINXBUILD=%PYTHON% -c "import sphinx.cmd.build, sys; sys.exit(sphinx.cmd.build.main())"
 )
 
 %PYTHON% -c "import python_docs_theme" > nul 2> nul
@@ -115,13 +115,15 @@ goto end
 :build
 if not exist "%BUILDDIR%" mkdir "%BUILDDIR%"
 
+rem PY_MISC_NEWS_DIR is also used by our Sphinx extension in tools/extensions/pyspecific.py
+if not defined PY_MISC_NEWS_DIR set PY_MISC_NEWS_DIR=%BUILDDIR%\%1
 if exist ..\Misc\NEWS (
-    echo.Copying Misc\NEWS to build\NEWS
-    copy ..\Misc\NEWS build\NEWS > nul
+    echo.Copying Misc\NEWS to %PY_MISC_NEWS_DIR%\NEWS
+    copy ..\Misc\NEWS "%PY_MISC_NEWS_DIR%\NEWS" > nul
 ) else if exist ..\Misc\NEWS.D (
     if defined BLURB (
         echo.Merging Misc/NEWS with %BLURB%
-        %BLURB% merge -f build\NEWS
+        %BLURB% merge -f "%PY_MISC_NEWS_DIR%\NEWS"
     ) else (
         echo.No Misc/NEWS file and Blurb is not available.
         exit /B 1

Doc/tools/extensions/escape4chm.py

@@ -8,6 +8,8 @@
 import re
 from html.entities import codepoint2name
 
+from sphinx.util.logging import getLogger
+
 # escape the characters which codepoint > 0x7F
 def _process(string):
     def escape(matchobj):
@@ -23,17 +25,33 @@ def escape(matchobj):
 
 def escape_for_chm(app, pagename, templatename, context, doctree):
     # only works for .chm output
-    if not hasattr(app.builder, 'name') or app.builder.name != 'htmlhelp':
+    if getattr(app.builder, 'name', '') != 'htmlhelp':
         return
 
     # escape the `body` part to 7-bit ASCII
     body = context.get('body')
     if body is not None:
         context['body'] = _process(body)
 
+def fixup_keywords(app, exception):
+    # only works for .chm output
+    if getattr(app.builder, 'name', '') != 'htmlhelp' or exception:
+        return
+
+    getLogger(__name__).info('fixing HTML escapes in keywords file...')
+    outdir = app.builder.outdir
+    outname = app.builder.config.htmlhelp_basename
+    with app.builder.open_file(outdir, outname + '.hhk', 'r') as f:
+        index = f.read()
+    with app.builder.open_file(outdir, outname + '.hhk', 'w') as f:
+        f.write(index.replace(''', '''))
+
 def setup(app):
     # `html-page-context` event emitted when the HTML builder has
     # created a context dictionary to render a template with.
     app.connect('html-page-context', escape_for_chm)
+    # `build-finished` event emitted when all the files have been
+    # output.
+    app.connect('build-finished', fixup_keywords)
 
     return {'version': '1.0', 'parallel_read_safe': True}

Doc/tools/extensions/pyspecific.py

@@ -11,7 +11,7 @@
 
 import re
 import codecs
-from os import path
+from os import getenv, path
 from time import asctime
 from pprint import pformat
 from docutils.io import StringOutput
@@ -253,7 +253,9 @@ def run(self):
         fname = self.arguments[0]
         source = self.state_machine.input_lines.source(
             self.lineno - self.state_machine.input_offset - 1)
-        source_dir = path.dirname(path.abspath(source))
+        source_dir = getenv('PY_MISC_NEWS_DIR')
+        if not source_dir:
+            source_dir = path.dirname(path.abspath(source))
         fpath = path.join(source_dir, fname)
         self.state.document.settings.record_dependencies.add(fpath)
         try: