Skip to content

Commit

Permalink
pythonGH-110109: Move tests for pathlib.Path.walk() into main test …
Browse files Browse the repository at this point in the history 
…classes.

This adds some test coverage to `pathlib._PathBase.walk()`
  • Loading branch information
@barneygale
barneygale committed Oct 10, 2023
1 parent da0a68a commit 17511b3
Showing 1 changed file with 162 additions and 185 deletions.
347 changes: 162 additions & 185 deletions Lib/test/test_pathlib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1801,6 +1801,51 @@ def setUp(self):
p.joinpath('dirB', 'linkD').symlink_to(pathmod.join('..', 'dirB'))
p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop')

def setUpWalk(self):
# Build:
# TESTFN/
# TEST1/ a file kid and two directory kids
# tmp1
# SUB1/ a file kid and a directory kid
# tmp2
# SUB11/ no kids
# SUB2/ a file kid and a dirsymlink kid
# tmp3
# link/ a symlink to TEST2
# broken_link
# broken_link2
# broken_link3
# TEST2/
# tmp4 a lone file
self.walk_path = self.cls(BASE, "TEST1")
self.sub1_path = self.walk_path / "SUB1"
self.sub11_path = self.sub1_path / "SUB11"
self.sub2_path = self.walk_path / "SUB2"
tmp1_path = self.walk_path / "tmp1"
tmp2_path = self.sub1_path / "tmp2"
tmp3_path = self.sub2_path / "tmp3"
self.link_path = self.sub2_path / "link"
t2_path = self.cls(BASE, "TEST2")
tmp4_path = self.cls(BASE, "TEST2", "tmp4")
broken_link_path = self.sub2_path / "broken_link"
broken_link2_path = self.sub2_path / "broken_link2"

self.sub11_path.mkdir(parents=True)
self.sub2_path.mkdir(parents=True)
t2_path.mkdir(parents=True)

for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path:
with path.open("w", encoding='utf-8') as f:
f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n")

if self.can_symlink:
self.link_path.symlink_to(t2_path)
broken_link_path.symlink_to('broken')
broken_link2_path.symlink_to(self.cls('tmp3', 'broken'))
self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"])
else:
self.sub2_tree = (self.sub2_path, [], ["tmp3"])

def tearDown(self):
cls = self.cls
cls._files.clear()
Expand Down Expand Up @@ -2614,6 +2659,119 @@ def test_complex_symlinks_relative(self):
def test_complex_symlinks_relative_dot_dot(self):
self._check_complex_symlinks(os.path.join('dirA', '..'))

def test_walk_topdown(self):
self.setUpWalk()
walker = self.walk_path.walk()
entry = next(walker)
entry[1].sort() # Ensure we visit SUB1 before SUB2
self.assertEqual(entry, (self.walk_path, ["SUB1", "SUB2"], ["tmp1"]))
entry = next(walker)
self.assertEqual(entry, (self.sub1_path, ["SUB11"], ["tmp2"]))
entry = next(walker)
self.assertEqual(entry, (self.sub11_path, [], []))
entry = next(walker)
entry[1].sort()
entry[2].sort()
self.assertEqual(entry, self.sub2_tree)
with self.assertRaises(StopIteration):
next(walker)

def test_walk_prune(self):
self.setUpWalk()
# Prune the search.
all = []
for root, dirs, files in self.walk_path.walk():
all.append((root, dirs, files))
if 'SUB1' in dirs:
# Note that this also mutates the dirs we appended to all!
dirs.remove('SUB1')

self.assertEqual(len(all), 2)
self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"]))

all[1][-1].sort()
all[1][1].sort()
self.assertEqual(all[1], self.sub2_tree)

def test_walk_bottom_up(self):
self.setUpWalk()
seen_testfn = seen_sub1 = seen_sub11 = seen_sub2 = False
for path, dirnames, filenames in self.walk_path.walk(top_down=False):
if path == self.walk_path:
self.assertFalse(seen_testfn)
self.assertTrue(seen_sub1)
self.assertTrue(seen_sub2)
self.assertEqual(sorted(dirnames), ["SUB1", "SUB2"])
self.assertEqual(filenames, ["tmp1"])
seen_testfn = True
elif path == self.sub1_path:
self.assertFalse(seen_testfn)
self.assertFalse(seen_sub1)
self.assertTrue(seen_sub11)
self.assertEqual(dirnames, ["SUB11"])
self.assertEqual(filenames, ["tmp2"])
seen_sub1 = True
elif path == self.sub11_path:
self.assertFalse(seen_sub1)
self.assertFalse(seen_sub11)
self.assertEqual(dirnames, [])
self.assertEqual(filenames, [])
seen_sub11 = True
elif path == self.sub2_path:
self.assertFalse(seen_testfn)
self.assertFalse(seen_sub2)
self.assertEqual(sorted(dirnames), sorted(self.sub2_tree[1]))
self.assertEqual(sorted(filenames), sorted(self.sub2_tree[2]))
seen_sub2 = True
else:
raise AssertionError(f"Unexpected path: {path}")
self.assertTrue(seen_testfn)

def test_walk_follow_symlinks(self):
if not self.can_symlink:
self.skipTest("symlinks required")
self.setUpWalk()
walk_it = self.walk_path.walk(follow_symlinks=True)
for root, dirs, files in walk_it:
if root == self.link_path:
self.assertEqual(dirs, [])
self.assertEqual(files, ["tmp4"])
break
else:
self.fail("Didn't follow symlink with follow_symlinks=True")

def test_walk_symlink_location(self):
if not self.can_symlink:
self.skipTest("symlinks required")
self.setUpWalk()
# Tests whether symlinks end up in filenames or dirnames depending
# on the `follow_symlinks` argument.
walk_it = self.walk_path.walk(follow_symlinks=False)
for root, dirs, files in walk_it:
if root == self.sub2_path:
self.assertIn("link", files)
break
else:
self.fail("symlink not found")

walk_it = self.walk_path.walk(follow_symlinks=True)
for root, dirs, files in walk_it:
if root == self.sub2_path:
self.assertIn("link", dirs)
break

def test_walk_above_recursion_limit(self):
self.setUpWalk()
recursion_limit = 40
# directory_depth > recursion_limit
directory_depth = recursion_limit + 10
base = self.cls(BASE, 'deep')
path = self.cls(base, *(['d'] * directory_depth))
path.mkdir(parents=True)

with set_recursion_limit(recursion_limit):
list(base.walk())
list(base.walk(top_down=False))

class DummyPathWithSymlinks(DummyPath):
def readlink(self):
Expand Down Expand Up @@ -3187,178 +3345,8 @@ def test_passing_kwargs_deprecated(self):
with self.assertWarns(DeprecationWarning):
self.cls(foo="bar")


class WalkTests(unittest.TestCase):

def setUp(self):
self.addCleanup(os_helper.rmtree, os_helper.TESTFN)

# Build:
# TESTFN/
# TEST1/ a file kid and two directory kids
# tmp1
# SUB1/ a file kid and a directory kid
# tmp2
# SUB11/ no kids
# SUB2/ a file kid and a dirsymlink kid
# tmp3
# SUB21/ not readable
# tmp5
# link/ a symlink to TEST2
# broken_link
# broken_link2
# broken_link3
# TEST2/
# tmp4 a lone file
self.walk_path = pathlib.Path(os_helper.TESTFN, "TEST1")
self.sub1_path = self.walk_path / "SUB1"
self.sub11_path = self.sub1_path / "SUB11"
self.sub2_path = self.walk_path / "SUB2"
sub21_path= self.sub2_path / "SUB21"
tmp1_path = self.walk_path / "tmp1"
tmp2_path = self.sub1_path / "tmp2"
tmp3_path = self.sub2_path / "tmp3"
tmp5_path = sub21_path / "tmp3"
self.link_path = self.sub2_path / "link"
t2_path = pathlib.Path(os_helper.TESTFN, "TEST2")
tmp4_path = pathlib.Path(os_helper.TESTFN, "TEST2", "tmp4")
broken_link_path = self.sub2_path / "broken_link"
broken_link2_path = self.sub2_path / "broken_link2"
broken_link3_path = self.sub2_path / "broken_link3"

os.makedirs(self.sub11_path)
os.makedirs(self.sub2_path)
os.makedirs(sub21_path)
os.makedirs(t2_path)

for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path:
with open(path, "x", encoding='utf-8') as f:
f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n")

if os_helper.can_symlink():
os.symlink(os.path.abspath(t2_path), self.link_path)
os.symlink('broken', broken_link_path, True)
os.symlink(pathlib.Path('tmp3', 'broken'), broken_link2_path, True)
os.symlink(pathlib.Path('SUB21', 'tmp5'), broken_link3_path, True)
self.sub2_tree = (self.sub2_path, ["SUB21"],
["broken_link", "broken_link2", "broken_link3",
"link", "tmp3"])
else:
self.sub2_tree = (self.sub2_path, ["SUB21"], ["tmp3"])

if not is_emscripten:
# Emscripten fails with inaccessible directories.
os.chmod(sub21_path, 0)
try:
os.listdir(sub21_path)
except PermissionError:
self.addCleanup(os.chmod, sub21_path, stat.S_IRWXU)
else:
os.chmod(sub21_path, stat.S_IRWXU)
os.unlink(tmp5_path)
os.rmdir(sub21_path)
del self.sub2_tree[1][:1]

def test_walk_topdown(self):
walker = self.walk_path.walk()
entry = next(walker)
entry[1].sort() # Ensure we visit SUB1 before SUB2
self.assertEqual(entry, (self.walk_path, ["SUB1", "SUB2"], ["tmp1"]))
entry = next(walker)
self.assertEqual(entry, (self.sub1_path, ["SUB11"], ["tmp2"]))
entry = next(walker)
self.assertEqual(entry, (self.sub11_path, [], []))
entry = next(walker)
entry[1].sort()
entry[2].sort()
self.assertEqual(entry, self.sub2_tree)
with self.assertRaises(StopIteration):
next(walker)

def test_walk_prune(self, walk_path=None):
if walk_path is None:
walk_path = self.walk_path
# Prune the search.
all = []
for root, dirs, files in walk_path.walk():
all.append((root, dirs, files))
if 'SUB1' in dirs:
# Note that this also mutates the dirs we appended to all!
dirs.remove('SUB1')

self.assertEqual(len(all), 2)
self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"]))

all[1][-1].sort()
all[1][1].sort()
self.assertEqual(all[1], self.sub2_tree)

def test_file_like_path(self):
self.test_walk_prune(FakePath(self.walk_path).__fspath__())

def test_walk_bottom_up(self):
seen_testfn = seen_sub1 = seen_sub11 = seen_sub2 = False
for path, dirnames, filenames in self.walk_path.walk(top_down=False):
if path == self.walk_path:
self.assertFalse(seen_testfn)
self.assertTrue(seen_sub1)
self.assertTrue(seen_sub2)
self.assertEqual(sorted(dirnames), ["SUB1", "SUB2"])
self.assertEqual(filenames, ["tmp1"])
seen_testfn = True
elif path == self.sub1_path:
self.assertFalse(seen_testfn)
self.assertFalse(seen_sub1)
self.assertTrue(seen_sub11)
self.assertEqual(dirnames, ["SUB11"])
self.assertEqual(filenames, ["tmp2"])
seen_sub1 = True
elif path == self.sub11_path:
self.assertFalse(seen_sub1)
self.assertFalse(seen_sub11)
self.assertEqual(dirnames, [])
self.assertEqual(filenames, [])
seen_sub11 = True
elif path == self.sub2_path:
self.assertFalse(seen_testfn)
self.assertFalse(seen_sub2)
self.assertEqual(sorted(dirnames), sorted(self.sub2_tree[1]))
self.assertEqual(sorted(filenames), sorted(self.sub2_tree[2]))
seen_sub2 = True
else:
raise AssertionError(f"Unexpected path: {path}")
self.assertTrue(seen_testfn)

@os_helper.skip_unless_symlink
def test_walk_follow_symlinks(self):
walk_it = self.walk_path.walk(follow_symlinks=True)
for root, dirs, files in walk_it:
if root == self.link_path:
self.assertEqual(dirs, [])
self.assertEqual(files, ["tmp4"])
break
else:
self.fail("Didn't follow symlink with follow_symlinks=True")

@os_helper.skip_unless_symlink
def test_walk_symlink_location(self):
# Tests whether symlinks end up in filenames or dirnames depending
# on the `follow_symlinks` argument.
walk_it = self.walk_path.walk(follow_symlinks=False)
for root, dirs, files in walk_it:
if root == self.sub2_path:
self.assertIn("link", files)
break
else:
self.fail("symlink not found")

walk_it = self.walk_path.walk(follow_symlinks=True)
for root, dirs, files in walk_it:
if root == self.sub2_path:
self.assertIn("link", dirs)
break

def test_walk_bad_dir(self):
self.setUpWalk()
errors = []
walk_it = self.walk_path.walk(on_error=errors.append)
root, dirs, files = next(walk_it)
Expand All @@ -3379,9 +3367,10 @@ def test_walk_bad_dir(self):
path1new.rename(path1)

def test_walk_many_open_files(self):
self.setUpWalk()
depth = 30
base = pathlib.Path(os_helper.TESTFN, 'deep')
path = pathlib.Path(base, *(['d']*depth))
base = self.cls(BASE, 'deep')
path = self.cls(base, *(['d']*depth))
path.mkdir(parents=True)

iters = [base.walk(top_down=False) for _ in range(100)]
Expand All @@ -3399,18 +3388,6 @@ def test_walk_many_open_files(self):
self.assertEqual(next(it), expected)
path = path / 'd'

def test_walk_above_recursion_limit(self):
recursion_limit = 40
# directory_depth > recursion_limit
directory_depth = recursion_limit + 10
base = pathlib.Path(os_helper.TESTFN, 'deep')
path = pathlib.Path(base, *(['d'] * directory_depth))
path.mkdir(parents=True)

with set_recursion_limit(recursion_limit):
list(base.walk())
list(base.walk(top_down=False))


@only_posix
class PosixPathTest(PathTest):
Expand Down

0 comments on commit 17511b3

Please sign in to comment.