Skip to content

Commit f029e8d

Browse files
gh-136065: Fix quadratic complexity in os.path.expandvars() (GH-134952)
Co-authored-by: Łukasz Langa <lukasz@langa.pl>
1 parent 81cec22 commit f029e8d

File tree

5 files changed

+96
-116
lines changed

5 files changed

+96
-116
lines changed

Lib/ntpath.py

Lines changed: 41 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -400,17 +400,23 @@ def expanduser(path):
400400
# XXX With COMMAND.COM you can use any characters in a variable name,
401401
# XXX except '^|<>='.
402402

403+
_varpattern = r"'[^']*'?|%(%|[^%]*%?)|\$(\$|[-\w]+|\{[^}]*\}?)"
404+
_varsub = None
405+
_varsubb = None
406+
403407
def expandvars(path):
404408
"""Expand shell variables of the forms $var, ${var} and %var%.
405409
406410
Unknown variables are left unchanged."""
407411
path = os.fspath(path)
412+
global _varsub, _varsubb
408413
if isinstance(path, bytes):
409414
if b'$' not in path and b'%' not in path:
410415
return path
411-
import string
412-
varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii')
413-
quote = b'\''
416+
if not _varsubb:
417+
import re
418+
_varsubb = re.compile(_varpattern.encode(), re.ASCII).sub
419+
sub = _varsubb
414420
percent = b'%'
415421
brace = b'{'
416422
rbrace = b'}'
@@ -419,94 +425,44 @@ def expandvars(path):
419425
else:
420426
if '$' not in path and '%' not in path:
421427
return path
422-
import string
423-
varchars = string.ascii_letters + string.digits + '_-'
424-
quote = '\''
428+
if not _varsub:
429+
import re
430+
_varsub = re.compile(_varpattern, re.ASCII).sub
431+
sub = _varsub
425432
percent = '%'
426433
brace = '{'
427434
rbrace = '}'
428435
dollar = '$'
429436
environ = os.environ
430-
res = path[:0]
431-
index = 0
432-
pathlen = len(path)
433-
while index < pathlen:
434-
c = path[index:index+1]
435-
if c == quote: # no expansion within single quotes
436-
path = path[index + 1:]
437-
pathlen = len(path)
438-
try:
439-
index = path.index(c)
440-
res += c + path[:index + 1]
441-
except ValueError:
442-
res += c + path
443-
index = pathlen - 1
444-
elif c == percent: # variable or '%'
445-
if path[index + 1:index + 2] == percent:
446-
res += c
447-
index += 1
448-
else:
449-
path = path[index+1:]
450-
pathlen = len(path)
451-
try:
452-
index = path.index(percent)
453-
except ValueError:
454-
res += percent + path
455-
index = pathlen - 1
456-
else:
457-
var = path[:index]
458-
try:
459-
if environ is None:
460-
value = os.fsencode(os.environ[os.fsdecode(var)])
461-
else:
462-
value = environ[var]
463-
except KeyError:
464-
value = percent + var + percent
465-
res += value
466-
elif c == dollar: # variable or '$$'
467-
if path[index + 1:index + 2] == dollar:
468-
res += c
469-
index += 1
470-
elif path[index + 1:index + 2] == brace:
471-
path = path[index+2:]
472-
pathlen = len(path)
473-
try:
474-
index = path.index(rbrace)
475-
except ValueError:
476-
res += dollar + brace + path
477-
index = pathlen - 1
478-
else:
479-
var = path[:index]
480-
try:
481-
if environ is None:
482-
value = os.fsencode(os.environ[os.fsdecode(var)])
483-
else:
484-
value = environ[var]
485-
except KeyError:
486-
value = dollar + brace + var + rbrace
487-
res += value
488-
else:
489-
var = path[:0]
490-
index += 1
491-
c = path[index:index + 1]
492-
while c and c in varchars:
493-
var += c
494-
index += 1
495-
c = path[index:index + 1]
496-
try:
497-
if environ is None:
498-
value = os.fsencode(os.environ[os.fsdecode(var)])
499-
else:
500-
value = environ[var]
501-
except KeyError:
502-
value = dollar + var
503-
res += value
504-
if c:
505-
index -= 1
437+
438+
def repl(m):
439+
lastindex = m.lastindex
440+
if lastindex is None:
441+
return m[0]
442+
name = m[lastindex]
443+
if lastindex == 1:
444+
if name == percent:
445+
return name
446+
if not name.endswith(percent):
447+
return m[0]
448+
name = name[:-1]
506449
else:
507-
res += c
508-
index += 1
509-
return res
450+
if name == dollar:
451+
return name
452+
if name.startswith(brace):
453+
if not name.endswith(rbrace):
454+
return m[0]
455+
name = name[1:-1]
456+
457+
try:
458+
if environ is None:
459+
return os.fsencode(os.environ[os.fsdecode(name)])
460+
else:
461+
return environ[name]
462+
except KeyError:
463+
return m[0]
464+
465+
return sub(repl, path)
510466

511467

512468
# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B.

Lib/posixpath.py

Lines changed: 20 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -285,56 +285,53 @@ def expanduser(path):
285285
# This expands the forms $variable and ${variable} only.
286286
# Non-existent variables are left unchanged.
287287

288-
_varprog = None
289-
_varprogb = None
288+
_varpattern = r'\$(\w+|\{[^}]*\}?)'
289+
_varsub = None
290+
_varsubb = None
290291

291292
def expandvars(path):
292293
"""Expand shell variables of form $var and ${var}. Unknown variables
293294
are left unchanged."""
294295
path = os.fspath(path)
295-
global _varprog, _varprogb
296+
global _varsub, _varsubb
296297
if isinstance(path, bytes):
297298
if b'$' not in path:
298299
return path
299-
if not _varprogb:
300+
if not _varsubb:
300301
import re
301-
_varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII)
302-
search = _varprogb.search
302+
_varsubb = re.compile(_varpattern.encode(), re.ASCII).sub
303+
sub = _varsubb
303304
start = b'{'
304305
end = b'}'
305306
environ = getattr(os, 'environb', None)
306307
else:
307308
if '$' not in path:
308309
return path
309-
if not _varprog:
310+
if not _varsub:
310311
import re
311-
_varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII)
312-
search = _varprog.search
312+
_varsub = re.compile(_varpattern, re.ASCII).sub
313+
sub = _varsub
313314
start = '{'
314315
end = '}'
315316
environ = os.environ
316-
i = 0
317-
while True:
318-
m = search(path, i)
319-
if not m:
320-
break
321-
i, j = m.span(0)
322-
name = m.group(1)
323-
if name.startswith(start) and name.endswith(end):
317+
318+
def repl(m):
319+
name = m[1]
320+
if name.startswith(start):
321+
if not name.endswith(end):
322+
return m[0]
324323
name = name[1:-1]
325324
try:
326325
if environ is None:
327326
value = os.fsencode(os.environ[os.fsdecode(name)])
328327
else:
329328
value = environ[name]
330329
except KeyError:
331-
i = j
330+
return m[0]
332331
else:
333-
tail = path[j:]
334-
path = path[:i] + value
335-
i = len(path)
336-
path += tail
337-
return path
332+
return value
333+
334+
return sub(repl, path)
338335

339336

340337
# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B.

Lib/test/test_genericpath.py

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@
99
import sys
1010
import unittest
1111
import warnings
12-
from test.support import (
13-
is_apple, os_helper, warnings_helper
14-
)
12+
from test import support
13+
from test.support import os_helper
14+
from test.support import warnings_helper
1515
from test.support.script_helper import assert_python_ok
1616
from test.support.os_helper import FakePath
1717

@@ -462,6 +462,19 @@ def check(value, expected):
462462
os.fsencode('$bar%s bar' % nonascii))
463463
check(b'$spam}bar', os.fsencode('%s}bar' % nonascii))
464464

465+
@support.requires_resource('cpu')
466+
def test_expandvars_large(self):
467+
expandvars = self.pathmodule.expandvars
468+
with os_helper.EnvironmentVarGuard() as env:
469+
env.clear()
470+
env["A"] = "B"
471+
n = 100_000
472+
self.assertEqual(expandvars('$A'*n), 'B'*n)
473+
self.assertEqual(expandvars('${A}'*n), 'B'*n)
474+
self.assertEqual(expandvars('$A!'*n), 'B!'*n)
475+
self.assertEqual(expandvars('${A}A'*n), 'BA'*n)
476+
self.assertEqual(expandvars('${'*10*n), '${'*10*n)
477+
465478
def test_abspath(self):
466479
self.assertIn("foo", self.pathmodule.abspath("foo"))
467480
with warnings.catch_warnings():
@@ -519,7 +532,7 @@ def test_nonascii_abspath(self):
519532
# directory (when the bytes name is used).
520533
and sys.platform not in {
521534
"win32", "emscripten", "wasi"
522-
} and not is_apple
535+
} and not support.is_apple
523536
):
524537
name = os_helper.TESTFN_UNDECODABLE
525538
elif os_helper.TESTFN_NONASCII:

Lib/test/test_ntpath.py

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import warnings
1010
from ntpath import ALL_BUT_LAST, ALLOW_MISSING
1111
from test import support
12-
from test.support import TestFailed, cpython_only, os_helper
12+
from test.support import os_helper
1313
from test.support.os_helper import FakePath
1414
from test import test_genericpath
1515
from tempfile import TemporaryFile
@@ -59,7 +59,7 @@ def tester(fn, wantResult):
5959
fn = fn.replace("\\", "\\\\")
6060
gotResult = eval(fn)
6161
if wantResult != gotResult and _norm(wantResult) != _norm(gotResult):
62-
raise TestFailed("%s should return: %s but returned: %s" \
62+
raise support.TestFailed("%s should return: %s but returned: %s" \
6363
%(str(fn), str(wantResult), str(gotResult)))
6464

6565
# then with bytes
@@ -75,7 +75,7 @@ def tester(fn, wantResult):
7575
warnings.simplefilter("ignore", DeprecationWarning)
7676
gotResult = eval(fn)
7777
if _norm(wantResult) != _norm(gotResult):
78-
raise TestFailed("%s should return: %s but returned: %s" \
78+
raise support.TestFailed("%s should return: %s but returned: %s" \
7979
%(str(fn), str(wantResult), repr(gotResult)))
8080

8181

@@ -1133,6 +1133,19 @@ def check(value, expected):
11331133
check('%spam%bar', '%sbar' % nonascii)
11341134
check('%{}%bar'.format(nonascii), 'ham%sbar' % nonascii)
11351135

1136+
@support.requires_resource('cpu')
1137+
def test_expandvars_large(self):
1138+
expandvars = ntpath.expandvars
1139+
with os_helper.EnvironmentVarGuard() as env:
1140+
env.clear()
1141+
env["A"] = "B"
1142+
n = 100_000
1143+
self.assertEqual(expandvars('%A%'*n), 'B'*n)
1144+
self.assertEqual(expandvars('%A%A'*n), 'BA'*n)
1145+
self.assertEqual(expandvars("''"*n + '%%'), "''"*n + '%')
1146+
self.assertEqual(expandvars("%%"*n), "%"*n)
1147+
self.assertEqual(expandvars("$$"*n), "$"*n)
1148+
11361149
def test_expanduser(self):
11371150
tester('ntpath.expanduser("test")', 'test')
11381151

@@ -1550,7 +1563,7 @@ def test_con_device(self):
15501563
self.assertTrue(os.path.exists(r"\\.\CON"))
15511564

15521565
@unittest.skipIf(sys.platform != 'win32', "Fast paths are only for win32")
1553-
@cpython_only
1566+
@support.cpython_only
15541567
def test_fast_paths_in_use(self):
15551568
# There are fast paths of these functions implemented in posixmodule.c.
15561569
# Confirm that they are being used, and not the Python fallbacks in
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix quadratic complexity in :func:`os.path.expandvars`.

0 commit comments

Comments
 (0)