Restructured configuration for app

.env.example

@@ -1,6 +1,6 @@
 DEBUG=True
 ALLOW_ORIGINS="http://localhost,http://localhost:8080,http://localhost:8081,http://127.0.0.1,http://127.0.0.1:8080,http://127.0.0.1:8081"
-DATASOURCE__DETA_KEY=<your Deta key should inserted here>
-DATASOURCE__DETA_BASE=<your Deta key should inserted here>
-STORAGE__DETA_KEY=<your Deta key should inserted here>
-STORAGE__DRIVE_NAME=<drive name>
+DATASOURCE__DETA__KEY=<your Deta key should inserted here>
+DATASOURCE__DETA__BASE=<your Deta key should inserted here>
+STORAGE__DETA__KEY=<your Deta key should inserted here>
+STORAGE__DETA__DRIVE=<drive name>

.env.test.example

@@ -1,3 +1,3 @@
-DATASOURCE__DETA_KEY=<your Deta key should inserted here>
-STORAGE__DETA_KEY=<your Deta key should inserted here>
-STORAGE__DETA_DRIVE_NAME=ytdl_test_downloads
+DATASOURCE__DETA__KEY=<your Deta key should inserted here>
+STORAGE__DETA__KEY=<your Deta key should inserted here>
+STORAGE__DETA__DRIVE=ytdl_test_downloads

.github/workflows/testing.yml

@@ -67,9 +67,9 @@ jobs:
       #----------------------------------------------
       - name: Run Pytest 
         env:
-            DATASOURCE__DETA_KEY: ${{secrets.DATASOURCE__DETA_KEY}}
-            STORAGE__DETA_KEY: ${{secrets.TEST_STORAGE__DETA_KEY}}
-            STORAGE__DETA_DRIVE_NAME: ${{secrets.TEST_STORAGE__DETA_DRIVE_NAME}}
+            DATASOURCE__DETA__KEY: ${{secrets.DATASOURCE__DETA_KEY}}
+            STORAGE__DETA__KEY: ${{secrets.TEST_STORAGE__DETA_KEY}}
+            STORAGE__DETA__DRIVE: ${{secrets.TEST_STORAGE__DETA_DRIVE_NAME}}
         run: poetry run pytest --cov=${{ github.workspace }} --cov-report=xml
       #----------------------------------------------
       #             upload coverage stats

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.14] - 2024-03-24
+### Changed
+- Restructured configuration for app.
+
 ## [1.4.13] - 2024-03-24
 ### Added
 - Sorting downloads by date in descending order in Deta datasource.

README.md

@@ -17,8 +17,8 @@ Before starting the application server you should create `.env` file which will
 ```shell
 DEBUG=True
 ALLOW_ORIGINS="http://localhost,http://localhost:8080,http://localhost:8081,http://127.0.0.1,http://127.0.0.1:8080,http://127.0.0.1:8081"
-DATASOURCE__DETA_KEY=<your Deta key should inserted here>
-DATASOURCE__DETA_BASE=<your deta base name should be inserted here>
+DATASOURCE__DETA__KEY=<your Deta key should inserted here>
+DATASOURCE__DETA__BASE=<your deta base name should be inserted here>
 MEDIA_PATH="/app/media"
 ```
 After you're done with configuration simply execute script below:
@@ -35,9 +35,9 @@ $ docker-compose up -d ytdl-api
 ## Running tests
 Before running tests for the first time create file `.env.test` inside project directory with following content. Replace placeholders with real values:
 ```
-DATASOURCE__DETA_KEY=<your Deta key should inserted here>
-STORAGE__DETA_KEY=<your Deta key should inserted here>
-STORAGE__DETA_DRIVE_NAME=ytdl_test_downloads
+DATASOURCE__DETA__KEY=<your Deta key should inserted here>
+STORAGE__DETA__KEY=<your Deta key should inserted here>
+STORAGE__DETA__DRIVE_NAME=ytdl_test_downloads
 ```
 Run `pytest` with command below. `--cov-report` flag will generate coverage report in HTML format.
 ```shell
@@ -55,8 +55,8 @@ $ fly secrets import
 # your secrets are passed below
 DEBUG=True
 ALLOW_ORIGINS="http://localhost,http://localhost:8080,http://localhost:8081,http://127.0.0.1,http://127.0.0.1:8080,http://127.0.0.1:8081"
-DATASOURCE__DETA_KEY=<your Deta key should inserted here>
-DATASOURCE__DETA_BASE=<your deta base name should be inserted here>
+DATASOURCE__DETA__KEY=<your Deta key should inserted here>
+DATASOURCE__DETA__BASE=<your deta base name should be inserted here>
 EOL
 ```
 
@@ -87,7 +87,5 @@ This section describes potential issues that may occur when using API.
 **Solution:** Open video in another player (VLC Player for example). Maybe QuickTime Player doesn't support audio codec used in video.
 
 ### 2. Issue: Test `test_deta_storage` fails with `OSError: [Errno 30] Read-only file system: '/app'`
-**Solution:** This error most likely occurs because in project you have both `.env` and `.env.test` files. For some weird reason ConfZ tries to read both files although in the test we change the source of env variables. This is most likely a bug in ConfZ library. Workaround for this issue is simply comment out content of`.env` file and run pytest separately for `tests/storage/test_deta_storage.py` file again:
-```shell
-$ pytest tests/storage/test_deta_storage.py
+**Solution:** This error most likely occurs because in project you have both `.env` and `.env.test` files. For some weird reason ConfZ tries to read both files although in the test we change the source of env variables. This is most likely a bug in ConfZ library. Workaround for this issue is simply comment out content of`.env` file and run pytest separately only containing configuration in `.env.test` file.
 ```

pyproject.toml

@@ -70,14 +70,14 @@ profile = "black"
 
 [tool.poetry]
 name = "ytdl-api"
-version = "1.4.13"
+version = "1.4.14"
 description = "API for web-based youtube-dl client"
 authors = ["Nazar Oleksiuk <nazarii.oleksiuk@gmail.com>"]
 license = "MIT"
 
 [tool.poetry.dependencies]
 python = "^3.11"
-fastapi = "^0.109.1"
+fastapi = "0.110.0"
 uvicorn = "^0.18.2"
 sse-starlette = "^0.6.1"
 aiofiles = "^0.6.0"

tests/conftest.py

@@ -67,8 +67,8 @@ def deta_testbase() -> str:
 def settings(fake_media_path: Path, monkeypatch: pytest.MonkeyPatch, deta_testbase: str) -> Iterable[Settings]:
     monkeypatch.setenv("DEBUG", True)
     monkeypatch.setenv("DOWNLOADER", "pytube")
-    monkeypatch.setenv("DATASOURCE__DETA_BASE", deta_testbase)
-    monkeypatch.setenv("STORAGE__PATH", fake_media_path.as_posix())
+    monkeypatch.setenv("DATASOURCE__DETA__BASE", deta_testbase)
+    monkeypatch.setenv("STORAGE__LOCAL__PATH", fake_media_path.as_posix())
     data_source = EnvSource(
         allow_all=True,
         deny=["title", "description", "version"],
@@ -82,8 +82,8 @@ def settings(fake_media_path: Path, monkeypatch: pytest.MonkeyPatch, deta_testba
 @pytest.fixture()
 def datasource(settings: Settings):
     return DetaDB(
-        deta_project_key=settings.datasource.deta_key,
-        base_name=settings.datasource.deta_base,
+        deta_project_key=settings.datasource.deta.key,
+        base_name=settings.datasource.deta.base,
     )
 
 

tests/downloaders/test_pytube_downloader.py

@@ -6,10 +6,9 @@
 import ffmpeg
 import pytest
 
-from ytdl_api.config import Settings
 from ytdl_api.constants import DownloadStatus
 from ytdl_api.datasource import DetaDB
-from ytdl_api.dependencies import get_downloader
+from ytdl_api.dependencies import get_pytube_downloader
 from ytdl_api.queue import NotificationQueue
 from ytdl_api.schemas.models import Download, DownloadStatusInfo
 from ytdl_api.storage import LocalFileStorage
@@ -26,7 +25,6 @@ def notification_queue() -> NotificationQueue:
 
 
 def test_video_download(
-    settings: Settings,
     mock_persisted_download: Download,
     local_storage: LocalFileStorage,
     datasource: DetaDB,
@@ -41,7 +39,7 @@ def test_video_download(
     assert mock_persisted_download.file_path is None
     assert mock_persisted_download.when_started_download is None
     assert mock_persisted_download.when_download_finished is None
-    pytube_downloader = get_downloader(settings, datasource, notification_queue, local_storage)
+    pytube_downloader = get_pytube_downloader(datasource, notification_queue, local_storage)
     pytube_downloader.download(mock_persisted_download)
     finished_download = datasource.get_download(mock_persisted_download.client_id, mock_persisted_download.media_id)
     assert finished_download.status == DownloadStatus.FINISHED
@@ -58,7 +56,6 @@ def _raise_ffmpeg_error(*args, **kwargs):
 
 
 def test_video_download_ffmpeg_failed(
-    settings: Settings,
     mock_persisted_download: Download,
     local_storage: LocalFileStorage,
     datasource: DetaDB,
@@ -67,7 +64,7 @@ def test_video_download_ffmpeg_failed(
     """
     Test code behaviour when ffmpeg failed.
     """
-    pytube_downloader = get_downloader(settings, datasource, notification_queue, local_storage)
+    pytube_downloader = get_pytube_downloader(datasource, notification_queue, local_storage)
     # mocking _merge_streams method for PytubeDownloader soi it will raise error
     pytube_downloader._merge_streams = _raise_ffmpeg_error
     pytube_downloader.download(mock_persisted_download)

tests/downloaders/test_ytdlp_downloader.py

@@ -26,7 +26,7 @@ def settings(
     monkeypatch: pytest.MonkeyPatch,
 ) -> Generator[Settings, None, None]:
     monkeypatch.setenv("DOWNLOADER", "yt-dlp")
-    monkeypatch.setenv("DATASOURCE__DETA_BASE", deta_testbase)
+    monkeypatch.setenv("DATASOURCE__DETA__BASE", deta_testbase)
     monkeypatch.setenv("STORAGE__PATH", fake_media_path.as_posix())
     data_source = EnvSource(
         allow_all=True,

tests/endpoints/test_get_downloads.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 
+import pytest
 from fastapi.testclient import TestClient
 
 from ytdl_api.schemas.models import Download
@@ -18,3 +19,12 @@ def test_get_downloads(uid: str, app_client: TestClient, mock_persisted_download
     assert (
         datetime.fromisoformat(json_response["downloads"][0]["whenSubmitted"]) == mock_persisted_download.when_submitted
     )
+
+
+@pytest.mark.skip(
+    "Test fails in Github Actions testing workflow: https://github.com/tiangolo/fastapi/discussions/11341"
+)
+def test_get_downloads_no_cookie(app_client: TestClient):
+    response = app_client.get("/api/downloads")
+    assert response.status_code == 403
+    assert response.json()["detail"] == "No cookie provided :("

tests/storage/test_deta_storage.py

@@ -18,8 +18,9 @@ def mocked_settings(
     monkeypatch: pytest.MonkeyPatch,
 ) -> Generator[Settings, None, None]:
     # ignoring datasource config as in this test it's not needed
-    monkeypatch.setenv("DATASOURCE__DETA_KEY", "*****")
-    monkeypatch.setenv("DATASOURCE__DETA_BASE", "*****")
+    monkeypatch.setenv("DATASOURCE__DETA__KEY", "*****")
+    monkeypatch.setenv("DATASOURCE__DETA__BASE", "*****")
+    monkeypatch.setenv("STORAGE__PREFFERED", "deta")
     data_source = EnvSource(
         allow_all=True,
         deny=["title", "description", "version"],

tests/test_config.py

@@ -13,10 +13,10 @@
 def settings(fake_media_path: Path) -> Iterable[Settings]:
     data_source = DataSource(
         data={
-            "allow_origins": ["*"],
+            "allow_origins": "*",
             "downloader": "mock",
-            "datasource": {"deta_key": "*****", "deta_base": "*****"},
-            "storage": {"path": fake_media_path},
+            "datasource": {"deta": {"key": "*****", "base": "*****"}},
+            "storage": {"local": {"path": fake_media_path}},
             "disable_docs": True,
         }
     )

ytdl_api/callbacks.py

@@ -16,7 +16,7 @@
 async def noop_callback(*args, **kwargs):  # pragma: no cover
     """
     Empty on downaload progess callback. Use as default/placeholder callback for
-    - on start downaload event
+    - on start download event
     - on download progress event
     - on start converting event
     - on download finish event

ytdl_api/config/__init__.py

@@ -0,0 +1,2 @@
+# Package containing the configurations for datasources, storage, and other settings.
+from .settings import REPO_PATH, Settings  # noqa: F401

ytdl_api/config/datasources.py

@@ -0,0 +1,41 @@
+from confz import BaseConfig
+from pydantic import root_validator
+
+from ..datasource import DetaDB, IDataSource
+
+
+class DetaBaseDataSourceConfig(BaseConfig):
+    """
+    Deta Base DB datasource config.
+    """
+
+    key: str
+    base: str
+
+    def get_datasource(self) -> IDataSource:
+        return DetaDB(self.key, self.base)
+
+    def __hash__(self):  # make hashable BaseModel subclass  # pragma: no cover
+        attrs = tuple(attr if not isinstance(attr, list) else ",".join(attr) for attr in self.__dict__.values())
+        return hash((type(self),) + attrs)
+
+
+class DataSourceConfig(BaseConfig):
+    """
+    Datasource config.
+    """
+
+    deta: DetaBaseDataSourceConfig | None = None
+
+    def get_datasource(self) -> IDataSource:
+        return self.deta.get_datasource()
+
+    @root_validator
+    def validate_datasources(cls, values):
+        if not any(values.values()):
+            raise ValueError("No datasource config provided.")
+        return values
+
+    def __hash__(self):  # make hashable BaseModel subclass  # pragma: no cover
+        attrs = tuple(hash(attr) for attr in self.__dict__.values())
+        return hash((type(self),) + attrs)