Merge 3d893d7 into 668419b

mtg_ssm/scryfall/fetcher.py

@@ -7,6 +7,7 @@
 import pprint
 import uuid
 from concurrent.futures import ProcessPoolExecutor
+from pathlib import Path
 from typing import Any, Dict, List, Mapping, Union, cast
 
 import appdirs
@@ -30,7 +31,7 @@
 
 APP_AUTHOR = "gwax"
 APP_NAME = "mtg_ssm"
-CACHE_DIR = appdirs.user_cache_dir(APP_NAME, APP_AUTHOR)
+CACHE_DIR = Path(appdirs.user_cache_dir(APP_NAME, APP_AUTHOR))
 
 BULK_DATA_ENDPOINT = "https://api.scryfall.com/bulk-data"
 SETS_ENDPOINT = "https://api.scryfall.com/sets"
@@ -61,17 +62,17 @@ def _value_from_validation_error(data: JSON, verr: ValidationError) -> Dict[str,
     return values
 
 
-def _cache_path(endpoint: str, extension: str) -> str:
+def _cache_path(endpoint: str, extension: str) -> Path:
     if not extension.startswith("."):
         extension = "." + extension
     cache_id = uuid.uuid5(uuid.NAMESPACE_URL, endpoint)
-    return os.path.join(CACHE_DIR, f"{cache_id}{extension}")
+    return CACHE_DIR / f"{cache_id}{extension}"
 
 
 def _fetch_endpoint(endpoint: str, *, dirty: bool, write_cache: bool = True) -> JSON:
-    os.makedirs(CACHE_DIR, exist_ok=True)
+    CACHE_DIR.mkdir(parents=True, exist_ok=True)
     cache_path = _cache_path(endpoint, ".json.gz")
-    if not os.path.exists(cache_path):
+    if not cache_path.exists():
         dirty = True
     if dirty:
         print(f"Fetching {endpoint}")
@@ -120,15 +121,15 @@ def _deserialize_cards(card_jsons: List[JSON]) -> List[ScryCard]:
 def scryfetch() -> ScryfallDataSet:  # pylint: disable=too-many-locals
     """Retrieve and deserialize Scryfall object data."""
     cached_bulk_json = None
-    if os.path.exists(_cache_path(BULK_DATA_ENDPOINT, ".json.gz")):
+    if _cache_path(BULK_DATA_ENDPOINT, ".json.gz").exists():
         cached_bulk_json = _fetch_endpoint(BULK_DATA_ENDPOINT, dirty=False)
     bulk_json = _fetch_endpoint(BULK_DATA_ENDPOINT, dirty=True, write_cache=False)
     cache_dirty = bulk_json != cached_bulk_json
 
     object_cache_path = _cache_path(OBJECT_CACHE_URL, ".pickle.gz")
-    if os.path.exists(object_cache_path):
+    if object_cache_path.exists():
         if cache_dirty or DEBUG:
-            os.remove(object_cache_path)
+            object_cache_path.unlink()
         else:
             print("Loading cached scryfall data objects")
             try:
@@ -144,20 +145,20 @@ def scryfetch() -> ScryfallDataSet:  # pylint: disable=too-many-locals
     sets_list = ScryObjectList[ScrySet].parse_obj(
         _fetch_endpoint(SETS_ENDPOINT, dirty=cache_dirty)
     )
-    sets_data = list(sets_list.data)
-    while sets_list.has_more:
+    sets_data = sets_list.data
+    while sets_list.has_more and sets_list.next_page is not None:
         sets_list = ScryObjectList[ScrySet].parse_obj(
-            _fetch_endpoint(str(sets_list.next_page), dirty=cache_dirty)
+            _fetch_endpoint(sets_list.next_page, dirty=cache_dirty)
         )
         sets_data += sets_list.data
 
     migrations_list = ScryObjectList[ScryMigration].parse_obj(
         _fetch_endpoint(MIGRATIONS_ENDPOINT, dirty=cache_dirty)
     )
-    migrations_data = list(migrations_list.data)
-    while migrations_list.has_more:
+    migrations_data = migrations_list.data
+    while migrations_list.has_more and migrations_list.next_page is not None:
         migrations_list = ScryObjectList[ScryMigration].parse_obj(
-            _fetch_endpoint(str(migrations_list.next_page), dirty=cache_dirty)
+            _fetch_endpoint(migrations_list.next_page, dirty=cache_dirty)
         )
         migrations_data += migrations_list.data
 

mtg_ssm/scryfall/models.py

@@ -6,7 +6,7 @@
 from typing import Dict, Generic, List, Literal, Optional, TypeVar, Union
 from uuid import UUID
 
-from pydantic import AnyUrl, BaseModel
+from pydantic import BaseModel, HttpUrl
 from pydantic.generics import GenericModel
 
 
@@ -235,7 +235,7 @@ class ScryObjectList(GenericModel, Generic[T]):
     object: Literal["list"] = "list"
     data: List[T]
     has_more: bool
-    next_page: Optional[AnyUrl]
+    next_page: Optional[HttpUrl]
     total_cards: Optional[int]
     warnings: Optional[List[str]]
 
@@ -260,10 +260,10 @@ class ScrySet(BaseModel):
     digital: bool
     foil_only: bool
     nonfoil_only: Optional[bool]
-    icon_svg_uri: AnyUrl
-    search_uri: AnyUrl
-    scryfall_uri: AnyUrl
-    uri: AnyUrl
+    icon_svg_uri: HttpUrl
+    search_uri: HttpUrl
+    scryfall_uri: HttpUrl
+    uri: HttpUrl
 
 
 class ScryRelatedCard(BaseModel):
@@ -274,7 +274,7 @@ class ScryRelatedCard(BaseModel):
     component: str
     name: str
     type_line: str
-    uri: AnyUrl
+    uri: HttpUrl
 
 
 class ScryCardFace(BaseModel):
@@ -289,7 +289,7 @@ class ScryCardFace(BaseModel):
     flavor_name: Optional[str]
     flavor_text: Optional[str]
     illustration_id: Optional[UUID]
-    image_uris: Optional[Dict[str, AnyUrl]]
+    image_uris: Optional[Dict[str, HttpUrl]]
     layout: Optional[ScryCardLayout]
     loyalty: Optional[str]
     mana_cost: str
@@ -309,7 +309,7 @@ class CardPreviewBlock(BaseModel):
     """Model for card preview block."""
 
     source: str
-    source_uri: Union[AnyUrl, Literal[""], str]
+    source_uri: Union[HttpUrl, Literal[""], str]
     previewed_at: dt.date
 
 
@@ -328,10 +328,10 @@ class ScryCard(BaseModel):
     tcgplayer_etched_id: Optional[int]
     cardmarket_id: Optional[int]
     oracle_id: Optional[UUID]
-    prints_search_uri: AnyUrl
-    rulings_uri: AnyUrl
-    scryfall_uri: AnyUrl
-    uri: AnyUrl
+    prints_search_uri: HttpUrl
+    rulings_uri: HttpUrl
+    scryfall_uri: HttpUrl
+    uri: HttpUrl
     # Gameplay Fields
     all_parts: Optional[List[ScryRelatedCard]]
     card_faces: Optional[List[ScryCardFace]]
@@ -378,23 +378,23 @@ class ScryCard(BaseModel):
     highres_image: bool
     illustration_id: Optional[UUID]
     image_status: ScryImageStatus
-    image_uris: Optional[Dict[str, AnyUrl]]
+    image_uris: Optional[Dict[str, HttpUrl]]
     prices: Optional[Dict[str, Optional[Decimal]]]  # TODO: enum keys
     printed_name: Optional[str]
     printed_text: Optional[str]
     printed_type_line: Optional[str]
     promo: bool
     promo_types: Optional[List[str]]
-    purchase_uris: Optional[Dict[str, AnyUrl]]
+    purchase_uris: Optional[Dict[str, HttpUrl]]
     rarity: ScryRarity
-    related_uris: Optional[Dict[str, AnyUrl]]
+    related_uris: Optional[Dict[str, HttpUrl]]
     released_at: dt.date
     reprint: bool
-    scryfall_set_uri: AnyUrl
+    scryfall_set_uri: HttpUrl
     set_name: str
-    set_search_uri: AnyUrl
+    set_search_uri: HttpUrl
     set_type: str
-    set_uri: AnyUrl
+    set_uri: HttpUrl
     set: str
     set_id: UUID
     story_spotlight: bool
@@ -411,11 +411,11 @@ class ScryBulkData(BaseModel):
 
     object: Literal["bulk_data"] = "bulk_data"
     id: UUID
-    uri: AnyUrl
+    uri: HttpUrl
     type: str
     name: str
     description: str
-    download_uri: AnyUrl
+    download_uri: HttpUrl
     updated_at: dt.datetime
     compressed_size: Optional[int]
     content_type: str
@@ -427,7 +427,7 @@ class ScryMigration(BaseModel):
 
     object: Literal["migration"] = "migration"
     id: UUID
-    uri: AnyUrl
+    uri: HttpUrl
     performed_at: dt.date
     migration_strategy: ScryMigrationStrategy
     old_scryfall_id: UUID

tests/conftest.py

@@ -2,14 +2,13 @@
 # pylint: disable=redefined-outer-name
 
 import json
-import os
+from pathlib import Path
 from typing import Dict, Generator, List
 from uuid import UUID
 
 import pytest
 import responses
 from _pytest.monkeypatch import MonkeyPatch
-from py._path.local import LocalPath
 
 from mtg_ssm.containers.bundles import ScryfallDataSet
 from mtg_ssm.scryfall.models import (
@@ -20,17 +19,17 @@
     ScrySet,
 )
 
-TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
-SETS_DATA_FILE = os.path.join(TEST_DATA_DIR, "sets.json")
-CARDS_DATA_FILE = os.path.join(TEST_DATA_DIR, "cards.json")
-MIGRATIONS_DATA_FILE = os.path.join(TEST_DATA_DIR, "migrations.json")
+TEST_DATA_DIR = Path(__file__).parent / "data"
+SETS_DATA_FILE = TEST_DATA_DIR / "sets.json"
+CARDS_DATA_FILE = TEST_DATA_DIR / "cards.json"
+MIGRATIONS_DATA_FILE = TEST_DATA_DIR / "migrations.json"
 
 
 @pytest.fixture(autouse=True)
-def fetcher_cache_dir(tmpdir: LocalPath, monkeypatch: MonkeyPatch) -> LocalPath:
+def fetcher_cache_dir(tmp_path: Path, monkeypatch: MonkeyPatch) -> Path:
     """Patch fetcher cache dirs for testing."""
-    cache_path = tmpdir.mkdir("cache")
-    monkeypatch.setattr("mtg_ssm.scryfall.fetcher.CACHE_DIR", str(cache_path))
+    cache_path = tmp_path / "cache"
+    monkeypatch.setattr("mtg_ssm.scryfall.fetcher.CACHE_DIR", cache_path)
     return cache_path
 
 
@@ -44,23 +43,23 @@ def requests_mock() -> Generator[responses.RequestsMock, None, None]:
 @pytest.fixture(scope="session")
 def cards_data() -> List[ScryCard]:
     """Fixture containing all test card data."""
-    with open(CARDS_DATA_FILE, "rt", encoding="utf-8") as card_data_file:
+    with CARDS_DATA_FILE.open("rt", encoding="utf-8") as card_data_file:
         card_json = json.load(card_data_file)
     return ScryRootList[ScryCard].parse_obj(card_json).__root__
 
 
 @pytest.fixture(scope="session")
 def sets_data() -> List[ScrySet]:
     """Fixture containing all test set data."""
-    with open(SETS_DATA_FILE, "rt", encoding="utf-8") as sets_data_file:
+    with SETS_DATA_FILE.open("rt", encoding="utf-8") as sets_data_file:
         sets_json = json.load(sets_data_file)
     return ScryObjectList[ScrySet].parse_obj(sets_json).data
 
 
 @pytest.fixture(scope="session")
 def migrations_data() -> List[ScryMigration]:
     """Fixture containing all test migrations data."""
-    with open(MIGRATIONS_DATA_FILE, "rt", encoding="utf-8") as migrations_data_file:
+    with MIGRATIONS_DATA_FILE.open("rt", encoding="utf-8") as migrations_data_file:
         migrations_json = json.load(migrations_data_file)
     return ScryObjectList[ScryMigration].parse_obj(migrations_json).data
 

tests/data/bulk_data.json

@@ -4,12 +4,12 @@
       "content_encoding": "gzip",
       "content_type": "application/json",
       "description": "A JSON file containing every card object on Scryfall in English or the printed language if the card is only available in one language.",
-      "download_uri": "https://data.scryfall.io/default-cards/default-cards-20230104220557.json",
+      "download_uri": "https://data.scryfall.io/default-cards/default-cards-20230108220735.json",
       "id": "e2ef41e3-5778-4bc2-af3f-78eca4dd9c23",
       "name": "Default Cards",
       "object": "bulk_data",
       "type": "default_cards",
-      "updated_at": "2023-01-04T22:05:57.440000+00:00",
+      "updated_at": "2023-01-08T22:07:35.806000+00:00",
       "uri": "https://api.scryfall.com/bulk-data/e2ef41e3-5778-4bc2-af3f-78eca4dd9c23"
     }
   ],