Sorting downloads by date in descending order in Deta datasource. (#26)

.github/workflows/testing.yml

@@ -10,13 +10,6 @@ on:
       - tests/**
       - poetry.lock
       - .github/workflows/**
-  push:
-    branches:
-      - master
-    paths:
-      - ytdl_api/**
-      - tests/**
-      - poetry.lock
   workflow_dispatch:
 jobs:
   ruff:

CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.13] - 2024-03-24
+### Added
+- Sorting downloads by date in descending order in Deta datasource.
+
 ## [1.4.12] - 2024-03-24
 ### Changed
 - Updated ruff to version 0.3.4 and config, removed isort from dev group.

pyproject.toml

@@ -70,7 +70,7 @@ profile = "black"
 
 [tool.poetry]
 name = "ytdl-api"
-version = "1.4.12"
+version = "1.4.13"
 description = "API for web-based youtube-dl client"
 authors = ["Nazar Oleksiuk <nazarii.oleksiuk@gmail.com>"]
 license = "MIT"

tests/storage/test_deta_storage.py

@@ -1,5 +1,4 @@
 import inspect
-from datetime import UTC, datetime
 from pathlib import Path
 from typing import Generator
 
@@ -11,6 +10,7 @@
 from ytdl_api.constants import DownloadStatus, MediaFormat
 from ytdl_api.schemas.models import Download
 from ytdl_api.storage import DetaDriveStorage
+from ytdl_api.utils import get_datetime_now
 
 
 @pytest.fixture()
@@ -89,7 +89,7 @@ def example_download() -> Download:
         "filesize": 1024,
         "status": DownloadStatus.FINISHED,
         "progress": 0,
-        "when_started_download": datetime.now(UTC),
+        "when_started_download": get_datetime_now(),
     }
     download = parse_obj_as(Download, download_data)
     return download

ytdl_api/callbacks.py

@@ -1,4 +1,3 @@
-from datetime import UTC, datetime
 from logging import Logger
 from pathlib import Path
 from typing import Any, Callable, Coroutine
@@ -11,7 +10,7 @@
 from .schemas.models import Download, DownloadStatusInfo
 from .storage import IStorage
 from .types import DownloadDataInfo
-from .utils import extract_percentage_progress, get_file_size
+from .utils import extract_percentage_progress, get_datetime_now, get_file_size
 
 
 async def noop_callback(*args, **kwargs):  # pragma: no cover
@@ -31,11 +30,12 @@ async def on_download_start_callback(
     queue: NotificationQueue,
 ):
     download.status = DownloadStatus.DOWNLOADING
-    download.when_started_download = datetime.now(UTC)
+    download.when_started_download = get_datetime_now()
     datasource.update_download(download)
     await queue.put(
         download.client_id,
         DownloadStatusInfo(
+            key=download.key,
             title=download.title,
             client_id=download.client_id,
             media_id=download.media_id,
@@ -56,6 +56,7 @@ async def on_pytube_progress_callback(
     Callback which will be used in Pytube's progress update callback
     """
     download_proress = DownloadStatusInfo(
+        key=download.key,
         title=download.title,
         client_id=download.client_id,
         media_id=download.media_id,
@@ -75,6 +76,7 @@ async def on_ytdlp_progress_callback(progress: DownloadDataInfo, **kwargs):
     queue: NotificationQueue = kwargs["queue"]
     progress = extract_percentage_progress(progress.get("_percent_str"))
     download_proress = DownloadStatusInfo(
+        key=download.key,
         title=download.title,
         client_id=download.client_id,
         media_id=download.media_id,
@@ -100,6 +102,7 @@ async def on_start_converting(
     await queue.put(
         download.client_id,
         DownloadStatusInfo(
+            key=download.key,
             title=download.title,
             client_id=download.client_id,
             media_id=download.media_id,
@@ -142,12 +145,13 @@ async def on_finish_callback(
     download.progress = 100
     download.filesize = file_size_bytes
     download.filesize_hr = file_size_hr
-    download.when_download_finished = datetime.now(UTC)
+    download.when_download_finished = get_datetime_now()
     datasource.update_download(download)
     logger.debug(f'Download status for ({download.media_id}): {download.filename} updated to "{status}"')
     await queue.put(
         download.client_id,
         DownloadStatusInfo(
+            key=download.key,
             title=download.title,
             filesize_hr=file_size_hr,
             client_id=download.client_id,
@@ -175,6 +179,7 @@ async def on_error_callback(
     await queue.put(
         download.client_id,
         download_progress=DownloadStatusInfo(
+            key=download.key,
             title=download.title,
             client_id=download.client_id,
             media_id=download.media_id,

ytdl_api/datasource.py

@@ -6,6 +6,7 @@
 
 from .constants import DownloadStatus
 from .schemas.models import Download, DownloadStatusInfo
+from .utils import get_datetime_now
 
 
 class IDataSource(ABC):
@@ -97,12 +98,11 @@ def __init__(self, deta_project_key: str, base_name: str = "ytdl"):
         self.base = deta.Base(base_name)
 
     def fetch_downloads(self, client_id: str) -> list[Download]:
-        downloads = self.base.fetch({"client_id": client_id, "status?ne": DownloadStatus.DELETED}).items
+        downloads = self.base.fetch({"client_id": client_id, "status?ne": DownloadStatus.DELETED}, desc=True).items
         return parse_obj_as(list[Download], downloads)
 
     def put_download(self, download: Download):
         data = download.dict()
-        key = data["media_id"]
         data["when_submitted"] = download.when_submitted.isoformat()
         if download.when_started_download:
             data["when_started_download"] = download.when_started_download.isoformat()
@@ -114,7 +114,7 @@ def put_download(self, download: Download):
             data["when_deleted"] = download.when_deleted.isoformat()
         if download.when_failed:
             data["when_failed"] = download.when_failed.isoformat()
-        self.base.put(data, key)
+        self.base.put(data, download.key)
 
     def get_download(self, client_id: str, media_id: str) -> Download | None:
         data = next(
@@ -124,7 +124,7 @@ def get_download(self, client_id: str, media_id: str) -> Download | None:
                         "client_id": client_id,
                         "media_id": media_id,
                         "status?ne": DownloadStatus.DELETED,
-                    }
+                    },
                 ).items
             ),
             None,
@@ -149,13 +149,13 @@ def update_download(self, download: Download):
             data["when_deleted"] = download.when_deleted.isoformat()
         if download.when_failed:
             data["when_failed"] = download.when_failed.isoformat()
-        self.base.update(data, download.media_id)
+        self.base.update(data, download.key)
 
     def update_download_progress(self, progress_obj: DownloadStatusInfo):
         media_id = progress_obj.media_id
         status = progress_obj.status
         progress = progress_obj.progress
-        self.base.update({"status": status, "progress": progress}, media_id)
+        self.base.update({"status": status, "progress": progress, "media_id": media_id}, progress_obj.key)
 
     def delete_download(
         self,
@@ -165,23 +165,23 @@ def delete_download(
         """
         Soft deleting download for now.
         """
-        when_deleted = when_deleted or datetime.datetime.now(datetime.UTC)
+        when_deleted = when_deleted or get_datetime_now()
         when_deleted_iso = when_deleted.isoformat()
         data = {"status": DownloadStatus.DELETED, "when_deleted": when_deleted_iso}
-        self.base.update(data, key=download.media_id)
+        self.base.update(data, key=download.key)
 
     def mark_as_downloaded(
         self,
         download: Download,
         when_file_downloaded: datetime.datetime | None = None,
     ):
-        when_file_downloaded = when_file_downloaded or datetime.datetime.now(datetime.UTC)
+        when_file_downloaded = when_file_downloaded or get_datetime_now()
         when_file_downloaded_iso = when_file_downloaded.isoformat()
         data = {
             "status": DownloadStatus.DOWNLOADED,
             "when_file_downloaded": when_file_downloaded_iso,
         }
-        self.base.update(data, download.media_id)
+        self.base.update(data, download.key)
 
     def clear_downloads(self):
         all_downloads = self.base.fetch().items
@@ -190,7 +190,7 @@ def clear_downloads(self):
         self.base.client.close()
 
     def mark_as_failed(self, download: Download, when_failed: datetime.datetime | None = None):
-        when_failed = when_failed or datetime.datetime.now(datetime.UTC)
+        when_failed = when_failed or get_datetime_now()
         when_failed_iso = when_failed.isoformat()
         data = {"status": DownloadStatus.FAILED, "when_failed": when_failed_iso}
-        self.base.update(data, download.media_id)
+        self.base.update(data, download.key)

ytdl_api/endpoints.py

@@ -159,7 +159,7 @@ async def _stream():
             except asyncio.QueueEmpty:
                 continue
             else:
-                yield data.json(by_alias=True)
+                yield data.json(exclude={"key"}, by_alias=True)
 
     return EventSourceResponse(_stream())
 

ytdl_api/schemas/models.py

@@ -1,11 +1,10 @@
 import datetime
-from functools import partial
 
 from pydantic import AnyHttpUrl, Field
 
 from ..constants import DownloadStatus, MediaFormat
 from ..types import YoutubeURL
-from ..utils import get_unique_id
+from ..utils import get_datetime_now, get_epoch_now, get_unique_id
 from .base import BaseModel_
 
 
@@ -23,6 +22,7 @@ class AudioStream(BaseStream):
 
 
 class Download(BaseModel_):
+    epoch: int = Field(description="Epoch timestamp", default_factory=get_epoch_now)
     client_id: str = Field(..., description="Client ID")
     media_id: str = Field(description="Download id", default_factory=get_unique_id)
     title: str = Field(..., description="Video title")
@@ -43,7 +43,7 @@ class Download(BaseModel_):
     file_path: str | None = Field(None, description="Path to file")
     progress: int = Field(0, description="Download progress in %")
     when_submitted: datetime.datetime = Field(
-        default_factory=partial(datetime.datetime.now, datetime.UTC),
+        default_factory=get_datetime_now,
         description="Date & time in UTC when download was submitted to API.",
     )
     when_started_download: datetime.datetime | None = Field(
@@ -62,6 +62,13 @@ class Download(BaseModel_):
         None, description="Date & time in UTC when error occured during download."
     )
 
+    @property
+    def key(self) -> str:
+        """
+        Key used for PK in database.
+        """
+        return f"{self.epoch}"
+
     @property
     def storage_filename(self) -> str:
         """
@@ -78,6 +85,7 @@ def filename(self) -> str:
 
 
 class DownloadStatusInfo(BaseModel_):
+    key: str = Field(..., description="Unique key used in database.")
     title: str = Field(..., description="Video/audio title")
     filesize_hr: str | None = Field(None, description="Video/audio file size in human-readable format")
     client_id: str = Field(..., description="Id of client")

ytdl_api/utils.py

@@ -1,6 +1,7 @@
 import logging
 import re
 import uuid
+from datetime import UTC, datetime
 from pathlib import Path
 from urllib.parse import quote
 
@@ -15,6 +16,14 @@ def get_unique_id() -> str:
     return uuid.uuid4().hex
 
 
+def get_datetime_now() -> datetime:
+    return datetime.now(UTC)
+
+
+def get_epoch_now() -> int:
+    return int(get_datetime_now().timestamp())
+
+
 def extract_percentage_progress(progress_string: str) -> int:
     """
     Extracts percentage progress from string.