Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix conan cache restore when restoring the same cache multiple times #15950

Merged
merged 3 commits into from
Mar 27, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions conan/api/subapi/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,14 +180,22 @@ def restore(self, path):
pkg_layout = cache.get_or_create_pkg_layout(pref)
pkg_folder = pref_bundle["package_folder"]
out.info(f"Restore: {pref} in {pkg_folder}")
# We need to put the package in the final location in the cache
shutil.move(os.path.join(cache.cache_folder, pkg_folder), pkg_layout.package())
if pkg_layout.package() != os.path.join(cache.cache_folder, pkg_folder):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not very evident what this comparison is trying to protect against or do. I would say that these could match, and still the package should be restored, but it seems in some cases it would be skipped.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now I have understood it is an optimization to not have to process folders already in place

# We need to put the package in the final location in the cache
if os.path.exists(pkg_layout.package()):
shutil.rmtree(pkg_layout.package())
shutil.move(os.path.join(cache.cache_folder, pkg_folder), pkg_layout.package())
pref_bundle["package_folder"] = os.path.relpath(pkg_layout.package(), cache.cache_folder)
metadata_folder = pref_bundle.get("metadata_folder")
if metadata_folder:
out.info(f"Restore: {pref} metadata in {metadata_folder}")
# We need to put the package in the final location in the cache
shutil.move(os.path.join(cache.cache_folder, metadata_folder),
pkg_layout.metadata())
if pkg_layout.metadata() != os.path.join(cache.cache_folder, metadata_folder):
# We need to put the package in the final location in the cache
if os.path.exists(pkg_layout.metadata()):
shutil.rmtree(pkg_layout.metadata())
shutil.move(os.path.join(cache.cache_folder, metadata_folder),
pkg_layout.metadata())
pref_bundle["metadata_folder"] = os.path.relpath(pkg_layout.metadata(), cache.cache_folder)

return package_list

Expand Down
45 changes: 45 additions & 0 deletions conans/test/integration/command_v2/test_cache_save_restore.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,36 @@ def test_cache_save_restore():
assert "\\" not in package_list


def test_cache_save_restore_with_package_file():
"""If we have some sources in the root (like the CMakeLists.txt)
we don't declare folders.source"""
conan_file = GenConanfile() \
.with_settings("os") \
.with_package_file("bin/file.txt", "content!!")

client = TestClient()
client.save({"conanfile.py": conan_file})
client.run("create . --name=pkg --version=1.0 -s os=Linux")
client.run("cache save pkg/*:* ")
cache_path = os.path.join(client.current_folder, "conan_cache_save.tgz")
assert os.path.exists(cache_path)

c2 = TestClient()
shutil.copy2(cache_path, c2.current_folder)
c2.run("cache restore conan_cache_save.tgz")
c2.run("list *:*#*")
assert "pkg/1.0" in c2.out
tree = _get_directory_tree(c2.base_folder)

# Restore again, expect the tree to be unchanged
c2.run("cache restore conan_cache_save.tgz")
c2.run("list *:*#*")
assert "pkg/1.0" in c2.out
tree2 = _get_directory_tree(c2.base_folder)

assert tree2 == tree


def test_cache_save_downloaded_restore():
""" what happens if we save packages downloaded from server, not
created
Expand All @@ -49,6 +79,18 @@ def test_cache_save_downloaded_restore():
_validate_restore(cache_path)


def _get_directory_tree(base_folder):
tree = []
for d, _, fs in os.walk(base_folder):
rel_d = os.path.relpath(d, base_folder) if d != base_folder else ""
if rel_d:
tree.append(rel_d)
for f in fs:
tree.append(os.path.join(rel_d, f))
tree.sort()
return tree


def _validate_restore(cache_path):
c2 = TestClient()
# Create a package in the cache to check put doesn't interact badly
Expand All @@ -61,6 +103,7 @@ def _validate_restore(cache_path):
assert "pkg/1.0" in c2.out
assert "pkg/1.1" in c2.out
assert "other/2.0" not in c2.out
tree = _get_directory_tree(c2.base_folder)

# Restore again, just in case
c2.run("cache restore conan_cache_save.tgz")
Expand All @@ -69,6 +112,8 @@ def _validate_restore(cache_path):
assert "pkg/1.0" in c2.out
assert "pkg/1.1" in c2.out
assert "other/2.0" not in c2.out
tree2 = _get_directory_tree(c2.base_folder)
assert tree2 == tree


def test_cache_save_restore_metadata():
Expand Down