Skip to content

Commit

Permalink
feat(deps): load examples lazily
Browse files Browse the repository at this point in the history
  • Loading branch information
cpcloud authored and jcrist committed May 9, 2023
1 parent 729b2d5 commit 4ea0ddb
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 16 deletions.
35 changes: 19 additions & 16 deletions ibis/examples/__init__.py
Expand Up @@ -4,8 +4,6 @@
import os
from typing import TYPE_CHECKING, Any, Optional

import pooch

import ibis
from ibis.common.grounds import Concrete

Expand All @@ -17,18 +15,9 @@
if TYPE_CHECKING:
import ibis.expr.types as ir


_EXAMPLES = pooch.create(
path=pooch.os_cache("ibis-framework"),
# the trailing slash matters here
base_url="https://storage.googleapis.com/ibis-examples/data/",
version=ibis.__version__,
)
with resources.files(__name__).joinpath("registry.txt").open(mode="r") as _f:
_EXAMPLES.load_registry(_f)
_EXAMPLES = None

_METADATA = json.loads(resources.files(__name__).joinpath("metadata.json").read_text())

_READER_FUNCS = {"csv": "read_csv", "csv.gz": "read_csv", "parquet": "read_parquet"}


Expand All @@ -42,10 +31,6 @@ def fetch(self, **kwargs: Any) -> ir.Table:
return reader(_EXAMPLES.fetch(self.key, progressbar=True), **kwargs)


def __dir__() -> list[str]:
return sorted(_METADATA.keys())


def _make_fetch_docstring(*, name: str, reader: str):
return f"""Fetch the {name} example.
Parameters
Expand All @@ -65,7 +50,25 @@ def _make_fetch_docstring(*, name: str, reader: str):
"""


def __dir__() -> list[str]:
return sorted(_METADATA.keys())


def __getattr__(name: str) -> Example:
global _EXAMPLES # noqa: PLW0603

if _EXAMPLES is None:
import pooch

_EXAMPLES = pooch.create(
path=pooch.os_cache("ibis-framework"),
# the trailing slash matters here
base_url="https://storage.googleapis.com/ibis-examples/data/",
version=ibis.__version__,
)
with resources.files(__name__).joinpath("registry.txt").open(mode="r") as _f:
_EXAMPLES.load_registry(_f)

spec = _METADATA.get(name, {})

if (key := spec.get("key")) is None:
Expand Down
1 change: 1 addition & 0 deletions ibis/examples/tests/test_examples.py
Expand Up @@ -9,6 +9,7 @@
pytestmark = pytest.mark.examples

duckdb = pytest.importorskip("duckdb")
pytest.importorskip("pooch")

# large files
ignored = frozenset(
Expand Down

0 comments on commit 4ea0ddb

Please sign in to comment.