_storage.py

import json
import logging
import sqlite3
from datetime import datetime
from datetime import timedelta
from itertools import chain
from typing import Any
from typing import Iterable
from typing import Mapping
from typing import Optional
from typing import Sequence
from typing import Tuple

from ._sqlite_utils import DBError
from ._sqlite_utils import open_sqlite_db
from ._sqlite_utils import rowcount_exactly_one
from ._sqlite_utils import wrap_exceptions
from ._types import EntryFilterOptions
from ._types import EntryForUpdate
from ._types import EntryUpdateIntent
from ._types import FeedForUpdate
from ._types import FeedUpdateIntent
from ._utils import returns_iter_list
from .exceptions import EntryNotFoundError
from .exceptions import FeedExistsError
from .exceptions import FeedNotFoundError
from .exceptions import MetadataNotFoundError
from .exceptions import StorageError
from .types import Content
from .types import Enclosure
from .types import Entry
from .types import EntrySortOrder
from .types import Feed
from .types import FeedSortOrder
from .types import JSONType


log = logging.getLogger('reader')


def create_db(db: sqlite3.Connection) -> None:
    create_feeds(db)
    create_entries(db)
    create_feed_metadata(db)


def create_feeds(db: sqlite3.Connection, name: str = 'feeds') -> None:
    db.execute(
        f"""
        CREATE TABLE {name} (

            -- feed data
            url TEXT PRIMARY KEY NOT NULL,
            title TEXT,
            link TEXT,
            updated TIMESTAMP,
            author TEXT,
            user_title TEXT,  -- except this one, which comes from reader
            http_etag TEXT,
            http_last_modified TEXT,

            -- reader data
            stale INTEGER NOT NULL DEFAULT 0,
            last_updated TIMESTAMP,  -- null if the feed was never updated
            added TIMESTAMP NOT NULL

        );
        """
    )


def create_entries(db: sqlite3.Connection, name: str = 'entries') -> None:
    # TODO: Add NOT NULL where applicable.
    db.execute(
        f"""
        CREATE TABLE {name} (

            -- entry data
            id TEXT NOT NULL,
            feed TEXT NOT NULL,
            title TEXT,
            link TEXT,
            updated TIMESTAMP NOT NULL,
            author TEXT,
            published TIMESTAMP,
            summary TEXT,
            content TEXT,
            enclosures TEXT,

            -- reader data
            read INTEGER NOT NULL DEFAULT 0,
            important INTEGER NOT NULL DEFAULT 0,
            last_updated TIMESTAMP NOT NULL,
            first_updated_epoch TIMESTAMP NOT NULL,
            feed_order INTEGER NOT NULL,

            PRIMARY KEY (id, feed),
            FOREIGN KEY (feed) REFERENCES feeds(url)
                ON UPDATE CASCADE
                ON DELETE CASCADE
        );
        """
    )


def create_feed_metadata(db: sqlite3.Connection) -> None:
    db.execute(
        """
        CREATE TABLE feed_metadata (
            feed TEXT NOT NULL,
            key TEXT NOT NULL,
            value TEXT NOT NULL,

            PRIMARY KEY (feed, key),
            FOREIGN KEY (feed) REFERENCES feeds(url)
                ON UPDATE CASCADE
                ON DELETE CASCADE
        );
        """
    )


def update_from_17_to_18(db: sqlite3.Connection) -> None:  # pragma: no cover
    # for https://github.com/lemon24/reader/issues/125
    db.execute("UPDATE feeds SET stale = 1;")


def open_db(path: str, timeout: Optional[float]) -> sqlite3.Connection:
    return open_sqlite_db(
        path,
        create=create_db,
        version=18,
        migrations={
            # 1-9 removed before 0.1 (last in e4769d8ba77c61ec1fe2fbe99839e1826c17ace7)
            # 10-16 removed before 1.0 (last in 618f158ebc0034eefb724a55a84937d21c93c1a7)
            17: update_from_17_to_18,
        },
        # Row value support was added in 3.15.
        minimum_sqlite_version=(3, 15),
        # We use the JSON1 extension for entries.content.
        required_sqlite_compile_options=["ENABLE_JSON1"],
        timeout=timeout,
    )


_GetEntriesLast = Optional[Tuple[Any, Any, Any, Any, Any, Any]]


class Storage:

    open_db = staticmethod(open_db)

    recent_threshold = timedelta(7)

    @wrap_exceptions(StorageError)
    def __init__(self, path: str, timeout: Optional[float] = None):
        try:
            self.db = self.open_db(path, timeout=timeout)
        except DBError as e:
            raise StorageError(str(e)) from e

        # TODO: If migrations happened, Storage-coupled Search needs to be notified.
        #
        # Even better, Search's migration should happen within the same
        # ddl_transaction open_db uses.
        #
        # Note that simply calling search.disable() and then search.enable() +
        # search.update() will probably not do the right thing, since
        # ddl_transaction is not reentrant.
        #
        # Also see "How does this interact with migrations?" in
        # https://github.com/lemon24/reader/issues/122#issuecomment-591302580

        self.path = path

    def close(self) -> None:
        self.db.close()

    @wrap_exceptions(StorageError)
    def add_feed(self, url: str, added: datetime) -> None:
        with self.db:
            try:
                self.db.execute(
                    "INSERT INTO feeds (url, added) VALUES (:url, :added);", locals(),
                )
            except sqlite3.IntegrityError:
                # FIXME: Match the error string.
                raise FeedExistsError(url)

    @wrap_exceptions(StorageError)
    def remove_feed(self, url: str) -> None:
        with self.db:
            cursor = self.db.execute("DELETE FROM feeds WHERE url = :url;", locals())
        rowcount_exactly_one(cursor, lambda: FeedNotFoundError(url))

    @wrap_exceptions(StorageError)
    @returns_iter_list
    def get_feeds(
        self, url: Optional[str] = None, sort: FeedSortOrder = 'title'
    ) -> Iterable[Feed]:
        where_url_snippet = '' if not url else "WHERE url = :url"

        if sort == 'title':
            order_by_snippet = "lower(coalesce(feeds.user_title, feeds.title)) ASC"
        elif sort == 'added':
            order_by_snippet = "feeds.added DESC"
        else:
            assert False, "shouldn't get here"  # noqa: B011; # pragma: no cover

        cursor = self.db.execute(
            f"""
            SELECT url, updated, title, link, author, user_title FROM feeds
            {where_url_snippet}
            ORDER BY
                {order_by_snippet},
                -- to make sure the order is deterministic
                feeds.url;
            """,
            locals(),
        )

        for row in cursor:
            yield Feed._make(row)

    @wrap_exceptions(StorageError)
    @returns_iter_list
    def get_feeds_for_update(
        self, url: Optional[str] = None, new_only: bool = False
    ) -> Iterable[FeedForUpdate]:
        if url or new_only:
            where_snippet = "WHERE 1"
        else:
            where_snippet = ''
        where_url_snippet = '' if not url else " AND url = :url"
        where_new_only_snippet = '' if not new_only else " AND last_updated is NULL"
        cursor = self.db.execute(
            f"""
            SELECT
                url,
                updated,
                http_etag,
                http_last_modified,
                stale,
                last_updated
            FROM feeds
            {where_snippet}
            {where_url_snippet}
            {where_new_only_snippet}
            ORDER BY feeds.url;
            """,
            locals(),
        )
        for row in cursor:
            yield FeedForUpdate._make(row)

    @returns_iter_list
    def _get_entries_for_update_n_queries(
        self, entries: Sequence[Tuple[str, str]]
    ) -> Iterable[Optional[EntryForUpdate]]:
        with self.db:
            for feed_url, id in entries:  # noqa: B007
                row = self.db.execute(
                    """
                    SELECT updated
                    FROM entries
                    WHERE feed = :feed_url
                        AND id = :id;
                    """,
                    locals(),
                ).fetchone()
                yield EntryForUpdate(row[0]) if row else None

    @returns_iter_list
    def _get_entries_for_update_one_query(
        self, entries: Sequence[Tuple[str, str]]
    ) -> Iterable[Optional[EntryForUpdate]]:
        if not entries:
            return []

        values_snippet = ', '.join(['(?, ?)'] * len(entries))
        parameters = list(chain.from_iterable(entries))

        rows = self.db.execute(
            f"""
            WITH
                input(feed, id) AS (
                    VALUES {values_snippet}
                )
                SELECT
                    entries.id IS NOT NULL,
                    entries.updated
                FROM input
                LEFT JOIN entries
                    ON (input.id, input.feed) == (entries.id, entries.feed);
            """,
            parameters,
        )

        return (EntryForUpdate(updated) if exists else None for exists, updated in rows)

    @wrap_exceptions(StorageError)
    def get_entries_for_update(
        self, entries: Iterable[Tuple[str, str]]
    ) -> Iterable[Optional[EntryForUpdate]]:
        # The reason there are two implementations for this method:
        # https://github.com/lemon24/reader/issues/109

        entries = list(entries)
        try:
            return self._get_entries_for_update_one_query(entries)
        except sqlite3.OperationalError as e:
            if "too many SQL variables" not in str(e):
                raise
        return self._get_entries_for_update_n_queries(entries)

    @wrap_exceptions(StorageError)
    def set_feed_user_title(self, url: str, title: Optional[str]) -> None:
        with self.db:
            cursor = self.db.execute(
                "UPDATE feeds SET user_title = :title WHERE url = :url;", locals(),
            )
        rowcount_exactly_one(cursor, lambda: FeedNotFoundError(url))

    @wrap_exceptions(StorageError)
    def mark_as_stale(self, url: str) -> None:
        with self.db:
            cursor = self.db.execute(
                "UPDATE feeds SET stale = 1 WHERE url = :url;", locals(),
            )
            rowcount_exactly_one(cursor, lambda: FeedNotFoundError(url))

    @wrap_exceptions(StorageError)
    def mark_as_read_unread(self, feed_url: str, entry_id: str, read: bool) -> None:
        with self.db:
            cursor = self.db.execute(
                """
                UPDATE entries
                SET read = :read
                WHERE feed = :feed_url AND id = :entry_id;
                """,
                locals(),
            )
        rowcount_exactly_one(cursor, lambda: EntryNotFoundError(feed_url, entry_id))

    @wrap_exceptions(StorageError)
    def mark_as_important_unimportant(
        self, feed_url: str, entry_id: str, important: bool
    ) -> None:
        with self.db:
            cursor = self.db.execute(
                """
                UPDATE entries
                SET important = :important
                WHERE feed = :feed_url AND id = :entry_id;
                """,
                locals(),
            )
        rowcount_exactly_one(cursor, lambda: EntryNotFoundError(feed_url, entry_id))

    @wrap_exceptions(StorageError)
    def update_feed(self, intent: FeedUpdateIntent) -> None:
        url, last_updated, feed, http_etag, http_last_modified = intent

        if feed:
            # TODO support updating feed URL
            # https://github.com/lemon24/reader/issues/149
            assert url == feed.url, "updating feed URL not supported"

            self._update_feed_full(intent)
            return

        assert http_etag is None, "http_etag must be none if feed is none"
        assert (
            http_last_modified is None
        ), "http_last_modified must be none if feed is none"
        self._update_feed_last_updated(url, last_updated)

    def _update_feed_full(self, intent: FeedUpdateIntent) -> None:
        url, last_updated, feed, http_etag, http_last_modified = intent

        assert feed is not None
        updated = feed.updated
        title = feed.title
        link = feed.link
        author = feed.author

        with self.db:
            cursor = self.db.execute(
                """
                UPDATE feeds
                SET
                    title = :title,
                    link = :link,
                    updated = :updated,
                    author = :author,
                    http_etag = :http_etag,
                    http_last_modified = :http_last_modified,
                    stale = 0,
                    last_updated = :last_updated
                WHERE url = :url;
                """,
                locals(),
            )

        rowcount_exactly_one(cursor, lambda: FeedNotFoundError(url))

    def _update_feed_last_updated(self, url: str, last_updated: datetime) -> None:
        with self.db:
            cursor = self.db.execute(
                """
                UPDATE feeds
                SET
                    last_updated = :last_updated
                WHERE url = :url;
                """,
                locals(),
            )
        rowcount_exactly_one(cursor, lambda: FeedNotFoundError(url))

    def _make_add_or_update_entries_args(
        self, intent: EntryUpdateIntent
    ) -> Mapping[str, Any]:
        entry, last_updated, first_updated_epoch, feed_order = intent

        updated = entry.updated
        published = entry.published
        feed_url = entry.feed_url
        id = entry.id
        title = entry.title
        link = entry.link
        author = entry.author
        summary = entry.summary
        content = (
            json.dumps([t._asdict() for t in entry.content]) if entry.content else None
        )
        enclosures = (
            json.dumps([t._asdict() for t in entry.enclosures])
            if entry.enclosures
            else None
        )
        return locals()

    @wrap_exceptions(StorageError)
    def add_or_update_entries(self, entry_tuples: Iterable[EntryUpdateIntent]) -> None:

        # We need to capture the last value for exception handling
        # (it's not guaranteed all the entries belong to the same tuple).
        last_param: Mapping[str, Any] = {}

        def make_params() -> Iterable[Mapping[str, Any]]:
            nonlocal last_param
            for last_param in map(self._make_add_or_update_entries_args, entry_tuples):
                yield last_param

        with self.db:

            try:
                self.db.executemany(
                    """
                    INSERT OR REPLACE INTO entries (
                        id,
                        feed,
                        title,
                        link,
                        updated,
                        author,
                        published,
                        summary,
                        content,
                        enclosures,
                        read,
                        important,
                        last_updated,
                        first_updated_epoch,
                        feed_order
                    ) VALUES (
                        :id,
                        :feed_url,
                        :title,
                        :link,
                        :updated,
                        :author,
                        :published,
                        :summary,
                        :content,
                        :enclosures,
                        (
                            SELECT read
                            FROM entries
                            WHERE id = :id AND feed = :feed_url
                        ),
                        (
                        SELECT important
                        FROM entries
                        WHERE id = :id AND feed = :feed_url
                        ),
                        :last_updated,
                        coalesce(:first_updated_epoch, (
                            SELECT first_updated_epoch
                            FROM entries
                            WHERE id = :id AND feed = :feed_url
                        )),
                        :feed_order
                    );
                    """,
                    make_params(),
                )
            except sqlite3.IntegrityError:
                # FIXME: Match the error string.

                feed_url = last_param['feed_url']
                entry_id = last_param['id']
                log.debug(
                    "add_entry %r of feed %r: got IntegrityError",
                    entry_id,
                    feed_url,
                    exc_info=True,
                )
                raise FeedNotFoundError(feed_url)

    def add_or_update_entry(self, intent: EntryUpdateIntent) -> None:
        # TODO: this method is for testing convenience only, maybe delete it?
        self.add_or_update_entries([intent])

    def get_entries(
        self,
        now: datetime,
        filter_options: EntryFilterOptions = EntryFilterOptions(),  # noqa: B008
        sort: EntrySortOrder = 'recent',
        chunk_size: Optional[int] = None,
        last: _GetEntriesLast = None,
    ) -> Iterable[Tuple[Entry, _GetEntriesLast]]:
        rv = self._get_entries(
            now=now,
            filter_options=filter_options,
            sort=sort,
            chunk_size=chunk_size,
            last=last,
        )

        # Equivalent to using @returns_iter_list, except when we don't have
        # a chunk_size (which disables pagination, but can block the database).
        # TODO: If we don't expose chunk_size, why have this special case?
        if chunk_size:
            rv = iter(list(rv))

        return rv

    def _get_entries(
        self,
        now: datetime,
        filter_options: EntryFilterOptions,
        sort: EntrySortOrder,
        chunk_size: Optional[int] = None,
        last: _GetEntriesLast = None,
    ) -> Iterable[Tuple[Entry, _GetEntriesLast]]:
        query = self._make_get_entries_query(filter_options, sort, chunk_size, last)

        feed_url, entry_id, read, important, has_enclosures = filter_options

        recent_threshold = now - self.recent_threshold
        if sort == 'recent':
            if last:
                (
                    last_entry_first_updated,
                    last_entry_updated,
                    last_feed_url,
                    last_entry_last_updated,
                    last_negative_feed_order,
                    last_entry_id,
                ) = last
        elif sort == 'random':
            assert not last, last  # pragma: no cover

        with wrap_exceptions(StorageError):
            cursor = self.db.execute(query, locals())
            for t in cursor:
                feed = t[0:6]
                feed = Feed._make(feed)
                entry = t[6:13] + (
                    tuple(Content(**d) for d in json.loads(t[13])) if t[13] else (),
                    tuple(Enclosure(**d) for d in json.loads(t[14])) if t[14] else (),
                    t[15] == 1,
                    t[16] == 1,
                    feed,
                )
                entry = Entry._make(entry)

                rv_last: _GetEntriesLast = None
                if sort == 'recent':
                    last_updated = t[17]
                    first_updated_epoch = t[18]
                    negative_feed_order = t[20]
                    rv_last = (
                        first_updated_epoch or entry.published or entry.updated,
                        entry.published or entry.updated,
                        entry.feed.url,
                        last_updated,
                        negative_feed_order,
                        entry.id,
                    )

                yield entry, rv_last

    def _make_get_entries_query(
        self,
        filter_options: EntryFilterOptions,
        sort: EntrySortOrder,
        chunk_size: Optional[int] = None,
        last: _GetEntriesLast = None,
    ) -> str:
        log.debug("_get_entries chunk_size=%s last=%s", chunk_size, last)

        feed_url, entry_id, read, important, has_enclosures = filter_options

        where_snippets = []

        if read is not None:
            where_snippets.append(f"{'' if read else 'NOT'} entries.read")

        # TODO: This needs some sort of query builder so badly.

        limit_snippet = ''
        if chunk_size:
            limit_snippet = "LIMIT :chunk_size"
            if sort == 'recent':
                if last:
                    where_snippets.append(
                        """
                        (
                            kinda_first_updated,
                            kinda_published,
                            feeds.url,
                            entries.last_updated,
                            negative_feed_order,
                            entries.id
                        ) < (
                            :last_entry_first_updated,
                            :last_entry_updated,
                            :last_feed_url,
                            :last_entry_last_updated,
                            :last_negative_feed_order,
                            :last_entry_id
                        )
                        """
                    )
            elif sort == 'random':
                assert not last, last  # pragma: no cover

        if feed_url:
            where_snippets.append("feeds.url = :feed_url")
            if entry_id:
                where_snippets.append("entries.id = :entry_id")

        if has_enclosures is not None:
            where_snippets.append(
                f"""
                {'NOT' if has_enclosures else ''}
                    (json_array_length(entries.enclosures) IS NULL
                        OR json_array_length(entries.enclosures) = 0)
                """
            )

        if important is not None:
            where_snippets.append(f"{'' if important else 'NOT'} entries.important")

        if any(where_snippets):
            where_keyword = 'WHERE'
            where_snippet = '\n                AND '.join(where_snippets)
        else:
            where_keyword = ''
            where_snippet = ''

        if sort == 'recent':
            order_by_snippet = """
                kinda_first_updated DESC,
                kinda_published DESC,
                feeds.url DESC,
                entries.last_updated DESC,
                negative_feed_order DESC,
                -- to make sure the order is deterministic;
                -- it's unlikely it'll be used, since the probability of a feed
                -- being updated twice during the same millisecond is very low
                entries.id DESC
            """

        elif sort == 'random':
            # TODO: "order by random()" always goes through the full result set, which is inefficient
            # details here https://github.com/lemon24/reader/issues/105#issue-409493128
            order_by_snippet = "random()"

        else:
            assert False, "shouldn't get here"  # noqa: B011; # pragma: no cover

        if sort == 'recent':
            select_snippet = """\
                ,
                entries.last_updated,
                coalesce(
                    CASE
                    WHEN
                        coalesce(entries.published, entries.updated)
                            >= :recent_threshold
                        THEN entries.first_updated_epoch
                    END,
                    entries.published, entries.updated
                ) as kinda_first_updated,
                coalesce(entries.published, entries.updated) as kinda_published,
                - entries.feed_order as negative_feed_order
            """
        else:
            select_snippet = ''

        query = f"""
            SELECT
                feeds.url,
                feeds.updated,
                feeds.title,
                feeds.link,
                feeds.author,
                feeds.user_title,
                entries.id,
                entries.updated,
                entries.title,
                entries.link,
                entries.author,
                entries.published,
                entries.summary,
                entries.content,
                entries.enclosures,
                entries.read,
                entries.important
                {select_snippet}
            FROM entries
            JOIN feeds ON feeds.url = entries.feed
            {where_keyword}
                {where_snippet}
            ORDER BY
                {order_by_snippet}
            {limit_snippet}
            ;
        """

        log.debug("_get_entries query\n%s\n", query)

        return query

    @wrap_exceptions(StorageError)
    @returns_iter_list
    def iter_feed_metadata(
        self, feed_url: str, key: Optional[str] = None
    ) -> Iterable[Tuple[str, JSONType]]:
        where_url_snippet = "WHERE feed = :feed_url"
        if key is not None:
            where_url_snippet += " AND key = :key"

        cursor = self.db.execute(
            f"""
            SELECT key, value FROM feed_metadata
            {where_url_snippet};
            """,
            locals(),
        )

        for mkey, value in cursor:
            yield mkey, json.loads(value)

    @wrap_exceptions(StorageError)
    def set_feed_metadata(self, feed_url: str, key: str, value: JSONType) -> None:
        value_json = json.dumps(value)

        with self.db:
            try:
                self.db.execute(
                    """
                    INSERT OR REPLACE INTO feed_metadata (
                        feed,
                        key,
                        value
                    ) VALUES (
                        :feed_url,
                        :key,
                        :value_json
                    );
                    """,
                    locals(),
                )
            except sqlite3.IntegrityError:
                # FIXME: Match the error string.
                raise FeedNotFoundError(feed_url)

    @wrap_exceptions(StorageError)
    def delete_feed_metadata(self, feed_url: str, key: str) -> None:
        with self.db:
            cursor = self.db.execute(
                """
                DELETE FROM feed_metadata
                WHERE feed = :feed_url AND key = :key;
                """,
                locals(),
            )
        rowcount_exactly_one(cursor, lambda: MetadataNotFoundError(feed_url, key))