Skip to content

Commit

Permalink
fix(examples): use filelock to prevent pooch from clobbering files wh…
Browse files Browse the repository at this point in the history
…en fetching concurrently
  • Loading branch information
cpcloud authored and gforsyth committed Jun 27, 2023
1 parent 21ffdcf commit e14662e
Show file tree
Hide file tree
Showing 3 changed files with 9 additions and 3 deletions.
8 changes: 7 additions & 1 deletion ibis/examples/__init__.py
Expand Up @@ -4,6 +4,8 @@
import os
from typing import TYPE_CHECKING, Optional

import filelock

import ibis
from ibis.common.grounds import Concrete

Expand Down Expand Up @@ -36,7 +38,11 @@ def fetch(
table_name: str | None = None,
backend: BaseBackend | None = None,
) -> ir.Table:
path = _EXAMPLES.fetch(self.key, progressbar=True)
key = self.key
# lock to ensure we don't clobber the file if fetched in another
# process
with filelock.FileLock(f"{key}.lock"):
path = _EXAMPLES.fetch(key, progressbar=True)

if backend is None:
backend = ibis.get_backend()
Expand Down
2 changes: 1 addition & 1 deletion poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pyproject.toml
Expand Up @@ -32,6 +32,7 @@ classifiers = [
python = "^3.8"
atpublic = ">=2.3,<5"
bidict = ">=0.22.1,<1"
filelock = ">=3.7.0,<4"
importlib-resources = { version = ">=5,<6", python = "<3.9" }
multipledispatch = ">=0.6,<1"
numpy = ">=1,<2"
Expand Down Expand Up @@ -110,7 +111,6 @@ ruff = ">=0.0.271,<0.273"

[tool.poetry.group.test.dependencies]
black = ">=22.1.0,<24"
filelock = ">=3.7.0,<4"
hypothesis = ">=6.58.0,<7"
packaging = ">=21.3,<24"
pytest = ">=7.0.0,<8"
Expand Down

0 comments on commit e14662e

Please sign in to comment.