From a3a696bcd1dafe291f2bd6adedbe7e400bb1a0de Mon Sep 17 00:00:00 2001 From: Andrew Shteren <38960745+2trvl@users.noreply.github.com> Date: Fri, 31 Jan 2025 17:37:19 +0300 Subject: [PATCH 01/11] perf: replace _Line with _LineParser --- Lib/configparser.py | 75 +++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/Lib/configparser.py b/Lib/configparser.py index 9dc4fa515cfcbea..71d0be099aeb24e 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -561,36 +561,32 @@ def __init__(self): self.errors = list() -class _Line(str): +class _LineParser: - def __new__(cls, val, *args, **kwargs): - return super().__new__(cls, val) - - def __init__(self, val, prefixes): - self.prefixes = prefixes - - @functools.cached_property - def clean(self): - return self._strip_full() and self._strip_inline() + def __init__(self, comments): + self.comments = comments @property - def has_comments(self): - return self.strip() != self.clean + def value(self): + return self._value - def _strip_inline(self): - """ - Search for the earliest prefix at the beginning of the line or following a space. - """ - matcher = re.compile( - '|'.join(fr'(^|\s)({re.escape(prefix)})' for prefix in self.prefixes.inline) - # match nothing if no prefixes - or '(?!)' - ) - match = matcher.search(self) - return self[:match.start() if match else None].strip() + @value.setter + def value(self, string): + self._value = string + string = string.strip() + self.clean = self._strip_full(string) and self._strip_inline(string) + self.has_comments = string != self.clean + + def _strip_full(self, string): + return '' if any(map(string.startswith, self.comments.full)) else True - def _strip_full(self): - return '' if any(map(self.strip().startswith, self.prefixes.full)) else True + def _strip_inline(self, string): + match = None + if self.comments.inline: + match = self.comments.inline.search(string) + if match: + return string[:match.start()].rstrip() + return string class RawConfigParser(MutableMapping): @@ -659,9 +655,17 @@ def __init__(self, defaults=None, dict_type=_default_dict, else: self._optcre = re.compile(self._OPT_TMPL.format(delim=d), re.VERBOSE) - self._prefixes = types.SimpleNamespace( - full=tuple(comment_prefixes or ()), - inline=tuple(inline_comment_prefixes or ()), + comment_prefixes = tuple(comment_prefixes or ()) + if inline_comment_prefixes: + # prefix at the beginning of the line or following a space + inline_comment_cre = re.compile( + '|'.join(fr'(^|\s)({re.escape(prefix)})' + for prefix in inline_comment_prefixes)) + else: + inline_comment_cre = None + self._comments = types.SimpleNamespace( + full=comment_prefixes, + inline=inline_comment_cre, ) self._strict = strict self._allow_no_value = allow_no_value @@ -1057,7 +1061,6 @@ def _read(self, fp, fpname): in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ - try: ParsingError._raise_all(self._read_inner(fp, fpname)) finally: @@ -1065,9 +1068,9 @@ def _read(self, fp, fpname): def _read_inner(self, fp, fpname): st = _ReadState() + line = _LineParser(self._comments) - Line = functools.partial(_Line, prefixes=self._prefixes) - for st.lineno, line in enumerate(map(Line, fp), start=1): + for st.lineno, line.value in enumerate(fp, start=1): if not line.clean: if self._empty_lines_in_values: # add empty line to the value, but only if there was no @@ -1082,7 +1085,7 @@ def _read_inner(self, fp, fpname): st.indent_level = sys.maxsize continue - first_nonspace = self.NONSPACECRE.search(line) + first_nonspace = self.NONSPACECRE.search(line.value) st.cur_indent_level = first_nonspace.start() if first_nonspace else 0 if self._handle_continuation_line(st, line, fpname): @@ -1098,7 +1101,7 @@ def _handle_continuation_line(self, st, line, fpname): st.cur_indent_level > st.indent_level) if is_continue: if st.cursect[st.optname] is None: - raise MultilineContinuationError(fpname, st.lineno, line) + raise MultilineContinuationError(fpname, st.lineno, line.value) st.cursect[st.optname].append(line.clean) return is_continue @@ -1112,7 +1115,7 @@ def _handle_rest(self, st, line, fpname): mo = self.SECTCRE.match(line.clean) if not mo and st.cursect is None: - raise MissingSectionHeaderError(fpname, st.lineno, line) + raise MissingSectionHeaderError(fpname, st.lineno, line.value) self._handle_header(st, mo.group('header'), fpname) if mo else self._handle_option(st, line, fpname) @@ -1144,12 +1147,12 @@ def _handle_option(self, st, line, fpname): # exception but keep going. the exception will be # raised at the end of the file and will contain a # list of all bogus lines - st.errors.append(ParsingError(fpname, st.lineno, line)) + st.errors.append(ParsingError(fpname, st.lineno, line.value)) return st.optname, vi, optval = mo.group('option', 'vi', 'value') if not st.optname: - st.errors.append(ParsingError(fpname, st.lineno, line)) + st.errors.append(ParsingError(fpname, st.lineno, line.value)) st.optname = self.optionxform(st.optname.rstrip()) if (self._strict and (st.sectname, st.optname) in st.elements_added): From 710085761657f9431f937b938ac63880f1f461c2 Mon Sep 17 00:00:00 2001 From: Andrew Shteren <38960745+2trvl@users.noreply.github.com> Date: Mon, 3 Feb 2025 00:41:43 +0300 Subject: [PATCH 02/11] docs: add news entry --- .../next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst diff --git a/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst b/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst new file mode 100644 index 000000000000000..c0bf4939c9acd7f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst @@ -0,0 +1,2 @@ +Reduce :mod:`configparser`. :class:`RawConfigParser`. :meth:`read` performance +regression that occurred after refactoring from 130% to 25%. From cff9f4499c02dd578b73537fef41e98cdd5cdaf8 Mon Sep 17 00:00:00 2001 From: Andrew Shteren <38960745+2trvl@users.noreply.github.com> Date: Mon, 3 Feb 2025 02:20:29 +0300 Subject: [PATCH 03/11] refactor: apply requested changes --- Lib/configparser.py | 5 ++--- .../Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/configparser.py b/Lib/configparser.py index 71d0be099aeb24e..fb87298690ceed6 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -581,11 +581,10 @@ def _strip_full(self, string): return '' if any(map(string.startswith, self.comments.full)) else True def _strip_inline(self, string): - match = None if self.comments.inline: match = self.comments.inline.search(string) - if match: - return string[:match.start()].rstrip() + if match: + return string[:match.start()].rstrip() return string diff --git a/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst b/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst index c0bf4939c9acd7f..a3ddaacb70ec5c2 100644 --- a/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst +++ b/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst @@ -1,2 +1 @@ -Reduce :mod:`configparser`. :class:`RawConfigParser`. :meth:`read` performance -regression that occurred after refactoring from 130% to 25%. +Improve performance of :meth:`configparser.RawConfigParser.read` by up to 45%. From 45c7a0eee3716ce5df967cc801b6248c2d8cd746 Mon Sep 17 00:00:00 2001 From: Andrew Shteren <38960745+2trvl@users.noreply.github.com> Date: Mon, 3 Feb 2025 02:35:11 +0300 Subject: [PATCH 04/11] docs: use ConfigParser instead of RawConfigParser --- .../next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst b/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst index a3ddaacb70ec5c2..9c173461d199c1a 100644 --- a/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst +++ b/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst @@ -1 +1 @@ -Improve performance of :meth:`configparser.RawConfigParser.read` by up to 45%. +Improve performance of :meth:`configparser.ConfigParser.read` by up to 45%. From dc751b8bfba7c83de19c03d95c3e7b7c428bf26b Mon Sep 17 00:00:00 2001 From: Andrew Shteren <38960745+2trvl@users.noreply.github.com> Date: Thu, 13 Feb 2025 21:38:08 +0300 Subject: [PATCH 05/11] refactor: apply requested changes --- Lib/configparser.py | 20 +++++++++---------- ...-02-02-23-47-35.gh-issue-128641.GFs673.rst | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Lib/configparser.py b/Lib/configparser.py index fb87298690ceed6..df88609d9234158 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -581,10 +581,11 @@ def _strip_full(self, string): return '' if any(map(string.startswith, self.comments.full)) else True def _strip_inline(self, string): + match = None if self.comments.inline: match = self.comments.inline.search(string) - if match: - return string[:match.start()].rstrip() + if match: + return string[:match.start()].rstrip() return string @@ -654,17 +655,14 @@ def __init__(self, defaults=None, dict_type=_default_dict, else: self._optcre = re.compile(self._OPT_TMPL.format(delim=d), re.VERBOSE) - comment_prefixes = tuple(comment_prefixes or ()) - if inline_comment_prefixes: - # prefix at the beginning of the line or following a space - inline_comment_cre = re.compile( - '|'.join(fr'(^|\s)({re.escape(prefix)})' - for prefix in inline_comment_prefixes)) - else: - inline_comment_cre = None + # prefix at the beginning of the line or following a space + inline_tmpl = lambda prefix: fr'(^|\s)({re.escape(prefix)})' + inline_comm = '|'.join(map(inline_tmpl, inline_comment_prefixes or ())) + # optional cre used with _LineParser for best performance (gh-128641) + inline_comment_cre = re.compile(inline_comm) if inline_comm else None self._comments = types.SimpleNamespace( - full=comment_prefixes, inline=inline_comment_cre, + full=tuple(comment_prefixes or ()) ) self._strict = strict self._allow_no_value = allow_no_value diff --git a/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst b/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst index 9c173461d199c1a..bfc0f8fe6d04fb4 100644 --- a/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst +++ b/Misc/NEWS.d/next/Library/2025-02-02-23-47-35.gh-issue-128641.GFs673.rst @@ -1 +1 @@ -Improve performance of :meth:`configparser.ConfigParser.read` by up to 45%. +Restore :meth:`configparser.ConfigParser.read` performance. From a01e0a929ab589867bde7d1e6d68587f85d7f61a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 15 Feb 2025 12:05:17 -0500 Subject: [PATCH 06/11] Move comment handling into a _CommentSpec class. Rely on re.sub to perform the substitutions in a unified way across full and inline prefixes. --- Lib/configparser.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/Lib/configparser.py b/Lib/configparser.py index df88609d9234158..a7b6a6b049267be 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -154,7 +154,6 @@ import os import re import sys -import types __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", @@ -574,19 +573,30 @@ def value(self): def value(self, string): self._value = string string = string.strip() - self.clean = self._strip_full(string) and self._strip_inline(string) + self.clean = self.comments.strip(string) self.has_comments = string != self.clean - def _strip_full(self, string): - return '' if any(map(string.startswith, self.comments.full)) else True - def _strip_inline(self, string): - match = None - if self.comments.inline: - match = self.comments.inline.search(string) - if match: - return string[:match.start()].rstrip() - return string +class _CommentSpec: + def __init__(self, full_prefixes, inline_prefixes): + if not full_prefixes and not inline_prefixes: + # performance optimization when no prefixes (gh-128641) + self.strip = lambda text: text + return + full_patterns = ( + # prefix at the beginning of a line + fr'^({re.escape(prefix)}).*' + for prefix in full_prefixes + ) + inline_patterns = ( + # prefix at the beginning of the line or following a space + fr'(^|\s)({re.escape(prefix)}.*)' + for prefix in inline_prefixes + ) + self.pattern = re.compile('|'.join(itertools.chain(full_patterns, inline_patterns))) + + def strip(self, text): + return self.pattern.sub('', text).rstrip() class RawConfigParser(MutableMapping): @@ -655,15 +665,7 @@ def __init__(self, defaults=None, dict_type=_default_dict, else: self._optcre = re.compile(self._OPT_TMPL.format(delim=d), re.VERBOSE) - # prefix at the beginning of the line or following a space - inline_tmpl = lambda prefix: fr'(^|\s)({re.escape(prefix)})' - inline_comm = '|'.join(map(inline_tmpl, inline_comment_prefixes or ())) - # optional cre used with _LineParser for best performance (gh-128641) - inline_comment_cre = re.compile(inline_comm) if inline_comm else None - self._comments = types.SimpleNamespace( - inline=inline_comment_cre, - full=tuple(comment_prefixes or ()) - ) + self._comments = _CommentSpec(comment_prefixes or (), inline_comment_prefixes or ()) self._strict = strict self._allow_no_value = allow_no_value self._empty_lines_in_values = empty_lines_in_values From f74543c2cbd3012cad6152f01a39e4775448a00c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 15 Feb 2025 12:06:44 -0500 Subject: [PATCH 07/11] Remove the optimization, as it has no effect. --- Lib/configparser.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/configparser.py b/Lib/configparser.py index a7b6a6b049267be..e9181a20d8ee33c 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -579,10 +579,6 @@ def value(self, string): class _CommentSpec: def __init__(self, full_prefixes, inline_prefixes): - if not full_prefixes and not inline_prefixes: - # performance optimization when no prefixes (gh-128641) - self.strip = lambda text: text - return full_patterns = ( # prefix at the beginning of a line fr'^({re.escape(prefix)}).*' From a098e6974678f0dbd6490195d08bd310035ba9a9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 15 Feb 2025 12:07:57 -0500 Subject: [PATCH 08/11] Rename variable from string to text. Use 'trimmed' to avoid masking the variable name. --- Lib/configparser.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/configparser.py b/Lib/configparser.py index e9181a20d8ee33c..63c27a11ff57b1a 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -570,11 +570,11 @@ def value(self): return self._value @value.setter - def value(self, string): - self._value = string - string = string.strip() - self.clean = self.comments.strip(string) - self.has_comments = string != self.clean + def value(self, text): + self._value = text + trimmed = text.strip() + self.clean = self.comments.strip(trimmed) + self.has_comments = trimmed != self.clean class _CommentSpec: From 218db85a20877dd74aa9b8141fc8c313cc3a6d17 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 15 Feb 2025 12:41:14 -0500 Subject: [PATCH 09/11] Restored Line as str subclass. Use 'slots' to avoid '__dict__'. --- Lib/configparser.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/Lib/configparser.py b/Lib/configparser.py index 63c27a11ff57b1a..4815d5399baffa0 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -560,20 +560,15 @@ def __init__(self): self.errors = list() -class _LineParser: +class _Line(str): + __slots__ = 'clean', 'has_comments' - def __init__(self, comments): - self.comments = comments + def __new__(cls, val, *args, **kwargs): + return super().__new__(cls, val) - @property - def value(self): - return self._value - - @value.setter - def value(self, text): - self._value = text - trimmed = text.strip() - self.clean = self.comments.strip(trimmed) + def __init__(self, val, comments): + trimmed = val.strip() + self.clean = comments.strip(trimmed) self.has_comments = trimmed != self.clean @@ -594,6 +589,9 @@ def __init__(self, full_prefixes, inline_prefixes): def strip(self, text): return self.pattern.sub('', text).rstrip() + def load(self, text): + return _Line(text, self) + class RawConfigParser(MutableMapping): """ConfigParser that does not do interpolation.""" @@ -1063,9 +1061,8 @@ def _read(self, fp, fpname): def _read_inner(self, fp, fpname): st = _ReadState() - line = _LineParser(self._comments) - for st.lineno, line.value in enumerate(fp, start=1): + for st.lineno, line in enumerate(map(self._comments.load, fp), start=1): if not line.clean: if self._empty_lines_in_values: # add empty line to the value, but only if there was no @@ -1080,7 +1077,7 @@ def _read_inner(self, fp, fpname): st.indent_level = sys.maxsize continue - first_nonspace = self.NONSPACECRE.search(line.value) + first_nonspace = self.NONSPACECRE.search(line) st.cur_indent_level = first_nonspace.start() if first_nonspace else 0 if self._handle_continuation_line(st, line, fpname): @@ -1096,7 +1093,7 @@ def _handle_continuation_line(self, st, line, fpname): st.cur_indent_level > st.indent_level) if is_continue: if st.cursect[st.optname] is None: - raise MultilineContinuationError(fpname, st.lineno, line.value) + raise MultilineContinuationError(fpname, st.lineno, line) st.cursect[st.optname].append(line.clean) return is_continue @@ -1110,7 +1107,7 @@ def _handle_rest(self, st, line, fpname): mo = self.SECTCRE.match(line.clean) if not mo and st.cursect is None: - raise MissingSectionHeaderError(fpname, st.lineno, line.value) + raise MissingSectionHeaderError(fpname, st.lineno, line) self._handle_header(st, mo.group('header'), fpname) if mo else self._handle_option(st, line, fpname) @@ -1142,12 +1139,12 @@ def _handle_option(self, st, line, fpname): # exception but keep going. the exception will be # raised at the end of the file and will contain a # list of all bogus lines - st.errors.append(ParsingError(fpname, st.lineno, line.value)) + st.errors.append(ParsingError(fpname, st.lineno, line)) return st.optname, vi, optval = mo.group('option', 'vi', 'value') if not st.optname: - st.errors.append(ParsingError(fpname, st.lineno, line.value)) + st.errors.append(ParsingError(fpname, st.lineno, line)) st.optname = self.optionxform(st.optname.rstrip()) if (self._strict and (st.sectname, st.optname) in st.elements_added): From 8610c7158c10d6ac7bcf03cb605e10d0f25a4e9f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 15 Feb 2025 13:17:14 -0500 Subject: [PATCH 10/11] Normalize whitespace in diff. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/configparser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/configparser.py b/Lib/configparser.py index 4815d5399baffa0..28e336bf2ea8b8e 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -583,7 +583,7 @@ def __init__(self, full_prefixes, inline_prefixes): # prefix at the beginning of the line or following a space fr'(^|\s)({re.escape(prefix)}.*)' for prefix in inline_prefixes - ) + ) self.pattern = re.compile('|'.join(itertools.chain(full_patterns, inline_patterns))) def strip(self, text): From 1f70178cecd19fcaf468028f81cc5422f3ef44a8 Mon Sep 17 00:00:00 2001 From: Andrew Shteren <38960745+2trvl@users.noreply.github.com> Date: Mon, 24 Feb 2025 02:55:46 +0300 Subject: [PATCH 11/11] refactor: comments wrap --- Lib/configparser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/configparser.py b/Lib/configparser.py index 28e336bf2ea8b8e..e510db5e0260b47 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -589,7 +589,7 @@ def __init__(self, full_prefixes, inline_prefixes): def strip(self, text): return self.pattern.sub('', text).rstrip() - def load(self, text): + def wrap(self, text): return _Line(text, self) @@ -1062,7 +1062,7 @@ def _read(self, fp, fpname): def _read_inner(self, fp, fpname): st = _ReadState() - for st.lineno, line in enumerate(map(self._comments.load, fp), start=1): + for st.lineno, line in enumerate(map(self._comments.wrap, fp), start=1): if not line.clean: if self._empty_lines_in_values: # add empty line to the value, but only if there was no