Entry backlog (#93)

* add backlogged field

* add backlog route and query support

* exlcude backlogged from deletion

* add backlog section

* add entry command to send/pop from backlog

* autocomplete command to send to backlog

* add a daily task to pop an entry from the backlog

* try with a backlog icon shortcut instead of a menu command

* split backlog route

* fetch content when unbacklog

* add extra recycling logic

* update icon actions

* fix route

* fix add entry route

* first stab at separating unwrap and backlog again

* add backlog menu command

* embedded links and is unwrappable entry helpers

* remove from pinned actions

* fix command display conditions

* improve doc

* route doc

* add unit test

* fix commands

feedi/models.py

@@ -511,6 +511,7 @@ class Entry(db.Model):
     viewed = sa.Column(sa.TIMESTAMP, index=True)
     favorited = sa.Column(sa.TIMESTAMP, index=True)
     pinned = sa.Column(sa.TIMESTAMP, index=True)
+    backlogged = sa.Column(sa.TIMESTAMP, index=True)
 
     raw_data = sa.orm.deferred(sa.Column(sa.String,
                                          doc="The original entry data received from the feed, as JSON"))
@@ -571,10 +572,34 @@ def fetch_content(self):
             except Exception as e:
                 logger.debug("failed to fetch content %s", e)
 
+    def embedded_links(self):
+        "Return a list of (url, title) for links found in the content_short (excluding hashtags and mentions)."
+        if self.content_short:
+            return [url for (url, text) in scraping.extract_links(self.target_url, self.content_short)
+                    # skip hashtag and profile links
+                    if not text.startswith('#') and not text.startswith('@')]
+        return []
+
+    def is_unwrappable(self):
+        "Return whether there are embedded links that can be extracted from the content_shrot."
+        return bool(self.embedded_links())
+
+    def backlog(self):
+        "Put this entry in the backlog."
+        self.backlogged = datetime.datetime.utcnow()
+        self.pinned = None
+
+    def unbacklog(self):
+        "Pop this entry from the backlog."
+        self.backlogged = None
+        self.viewed = None
+        self.sort_date = datetime.datetime.utcnow()
+        self.fetch_content()
+
     @classmethod
-    def _filtered_query(cls, user_id, hide_seen=False, favorited=None,
-                        feed_name=None, username=None, folder=None,
-                        older_than=None, text=None):
+    def _filtered_query(cls, user_id, hide_seen=False, favorited=None, backlogged=None,
+                        feed_name=None, username=None, folder=None, older_than=None,
+                        text=None):
         """
         Return a base Entry query applying any combination of filters.
         """
@@ -594,6 +619,14 @@ def _filtered_query(cls, user_id, hide_seen=False, favorited=None,
         if favorited:
             query = query.filter(cls.favorited.is_not(None))
 
+        if backlogged:
+            query = query.filter(cls.backlogged.is_not(None))
+        elif hide_seen:
+            # exclude backlogged unless explicitly asking for backlog list
+            # or if the hide seen option is disabled.
+            # (abusing the fact that non mixed lists set this flag to false)
+            query = query.filter(cls.backlogged.is_(None))
+
         if feed_name:
             query = query.filter(cls.feed.has(name=feed_name))
 
@@ -631,6 +664,8 @@ def sorted_by(cls, user_id, ordering, start_at, **filters):
 
         if filters.get('favorited'):
             return query.order_by(cls.favorited.desc())
+        if filters.get('backlogged'):
+            return query.order_by(cls.backlogged)
 
         elif ordering == cls.ORDER_RECENCY:
             # reverse chronological order

feedi/routes.py

@@ -16,6 +16,7 @@
 
 @app.route("/users/<username>")
 @app.route("/favorites", defaults={'favorited': True}, endpoint='favorites')
+@app.route("/backlog", defaults={'backlogged': True}, endpoint='backlog')
 @app.route("/folder/<folder>")
 @app.route("/feeds/<feed_name>/entries")
 @app.route("/")
@@ -124,8 +125,8 @@ def autocomplete():
         # we can reasonably assume this is a url
 
         options += [
-            ('Add to feed', flask.url_for('entry_add', url=term), 'fas fa-download'),
-            ('View in reader', flask.url_for('entry_add', url=term, redirect=1), 'fas fa-book-reader'),
+            ('Add to feed', flask.url_for('entry_add', url=term), 'fas fa-download', 'POST'),
+            ('View in reader', flask.url_for('entry_add', url=term, redirect=1), 'fas fa-book-reader', 'POST'),
             ('Discover feed', flask.url_for('feed_add', url=term), 'fas fa-rss'),
         ]
         if current_user.has_kindle:
@@ -160,6 +161,7 @@ def autocomplete():
     static_options = [
         ('Home', flask.url_for('entry_list'), 'fas fa-home'),
         ('Favorites', flask.url_for('favorites', favorited=True), 'far fa-star'),
+        ('Backlog', flask.url_for('backlog', favorited=True), 'fa fa-archive'),
         ('Add Feed', flask.url_for('feed_add'), 'fas fa-plus'),
         ('Manage Feeds', flask.url_for('feed_list'), 'fas fa-edit'),
         ('Mastodon login', flask.url_for('mastodon_oauth'), 'fab fa-mastodon'),
@@ -188,6 +190,7 @@ def entry_pin(id):
     else:
         entry.fetch_content()
         entry.pinned = datetime.datetime.utcnow()
+        entry.backlogged = None
     db.session.commit()
 
     # get the new list of pinned based on filters
@@ -217,6 +220,34 @@ def entry_favorite(id):
     return '', 204
 
 
+@app.put("/backlog/<int:id>")
+@login_required
+def entry_backlog_push(id):
+    "Put the entry of the given id in the backlog."
+    entry = db.get_or_404(models.Entry, id)
+    if entry.user_id != current_user.id:
+        flask.abort(404)
+
+    entry.backlog()
+    db.session.commit()
+    return '', 204
+
+
+@app.delete("/backlog/<int:id>")
+@login_required
+def entry_backlog_pop(id):
+    "Remove the entry of the given id from the backlog, sending it back to the home feed."
+    entry = db.get_or_404(models.Entry, id)
+    if entry.user_id != current_user.id:
+        flask.abort(404)
+
+    if entry.backlogged:
+        entry.unbacklog()
+
+    db.session.commit()
+    return '', 204
+
+
 @app.put("/mastodon/favorites/<int:id>")
 @login_required
 def mastodon_favorite(id):
@@ -390,6 +421,7 @@ def feed_delete(feed_name):
     update = db.update(models.Entry)\
         .where((models.Entry.feed_id == feed.id) & (
             models.Entry.favorited.isnot(None) |
+            models.Entry.backlogged.isnot(None) |
             models.Entry.pinned.isnot(None)))\
         .values(feed_id=None)
     db.session.execute(update)
@@ -417,8 +449,7 @@ def feed_sync(feed_name):
     return response
 
 
-# this should be a .post but that complicates utilization from hyperscipt
-@app.get("/entries/")
+@app.post("/entries/")
 @login_required
 def entry_add():
     """
@@ -427,21 +458,49 @@ def entry_add():
     """
     # TODO sanitize?
     url = flask.request.args['url']
+    redirect = flask.request.args.get('redirect')
 
     try:
         entry = models.Entry.from_url(current_user.id, url)
     except Exception:
-        return redirect_response(url)
+        if redirect:
+            return redirect_response(url)
+        else:
+            return 'failed to parse entry', 500
 
     db.session.add(entry)
     db.session.commit()
 
-    if flask.request.args.get('redirect'):
+    if redirect:
         return redirect_response(flask.url_for('entry_view', id=entry.id))
     else:
         return '', 204
 
 
+@app.post("/entries/<int:id>")
+@login_required
+def entry_unwrap(id):
+    "If the entry has embedded links in its short content, extract the first and render it."
+    entry = db.get_or_404(models.Entry, id)
+    if entry.user_id != current_user.id:
+        flask.abort(404)
+
+    if entry.content_short:
+        # If there's an inline link, "unwrap it", e.g. remove the old entry and put the linked
+        # article entry in its place
+        for link in entry.embedded_links():
+            try:
+                subentry = models.Entry.from_url(current_user.id, link)
+                entry.viewed = datetime.datetime.now()
+                db.session.add(subentry)
+                db.session.commit()
+                return flask.render_template('entry_list_page.html',
+                                             entries=[subentry])
+            except Exception:
+                continue
+    return "Couldn't unwrap", 400
+
+
 @app.get("/entries/<int:id>")
 @login_required
 def entry_view(id):

feedi/tasks.py

@@ -121,6 +121,7 @@ def delete_old_entries():
         .join(models.Feed.entries)\
         .filter(models.Entry.sort_date < older_than_date,
                 models.Entry.favorited.is_(None),
+                models.Entry.backlogged.is_(None),
                 models.Entry.pinned.is_(None)
                 )\
         .group_by(models.Feed.id)\
@@ -143,6 +144,7 @@ def delete_old_entries():
         q = db.delete(models.Entry)\
               .where(
                   models.Entry.favorited.is_(None),
+                  models.Entry.backlogged.is_(None),
                   models.Entry.pinned.is_(None),
                   models.Entry.feed_id == feed_id,
                   models.Entry.sort_date < min_sort_date,
@@ -167,6 +169,24 @@ def delete_old_entries():
         app.logger.info("Deleted %s old standalone entries from", res.rowcount)
 
 
+@feed_cli.command('backlog')
+@huey_task(crontab(minute='0', hour='0'))
+def pop_backlog():
+    "Periodically pop an entry from the backlog into the home feed."
+    # TODO make this configurable
+    week_ago = datetime.datetime.utcnow() - datetime.timedelta(days=7)
+    backlogged_date = sa.func.min(models.Entry.backlogged).label('backlogged_date')
+    query = db.select(models.Entry)\
+        .group_by(models.Entry.user_id)\
+        .having(backlogged_date < week_ago)
+
+    for entry in db.session.scalars(query):
+        entry.unbacklog()
+        app.logger.info("Popped from user %s backlog: %s ", entry.user_id, entry.target_url)
+
+    db.session.commit()
+
+
 @feed_cli.command('debug')
 @click.argument('url')
 def debug_feed(url):

feedi/templates/base.html

@@ -24,6 +24,8 @@
                 {% endfor %}
                 <a class="dropdown-item {% if filters.favorited %}is-active{% endif %}" href="{{ url_for('favorites') }}">
                     <icon class="icon"><i class="far fa-star"></i></icon> favorites</a>
+                <a class="dropdown-item {% if filters.backlogged %}is-active{% endif %}" href="{{ url_for('backlog') }}">
+                    <icon class="icon"><i class="fas fa-archive"></i></icon> backlog</a>
                 <hr class="dropdown-divider"/>
                 {% endif %}
                 {{ self.sidebar_right() }}
@@ -84,7 +86,15 @@ <h1 class="title is-4">feedi</h1>
                                 </div>
                             </div>
                         </li>
-                    </ul>
+                        <li>
+                            <div class="level-left {% if filters.backlogged %}is-active{% endif %}">
+                                <div class="level-item">
+                                    <a href="{{ url_for('backlog') }}"><icon class="icon"><i class="fas fa-archive"></i></icon>
+                                        backlog</a>
+                                </div>
+                            </div>
+                        </li>
+                        </ul>
                 </aside>
                 {% endblock %}
             </div>

feedi/templates/entry_commands.html

@@ -9,24 +9,28 @@
 {% if entry.comments_url %}
 <a href="{{ entry.comments_url }}" target="_blank" class="dropdown-item"><span class="icon"><i class="far fa-comment-alt"></i></span> Go to comments</a>
 {% endif %}
-{% if entry.content_url %}
-{% if entry.id and request.path != url_for('entry_view', id=entry.id) %}
+{% if entry.content_url and entry.id and request.path != url_for('entry_view', id=entry.id) %}
 <a hx-boost="true" href="{{ url_for('entry_view', id=entry.id) }}" class="dropdown-item"><span class="icon"><i class="fas fa-book-reader"></i></span> View in reader</a>
 {% endif %}
+{% if not entry.backlogged %}
+<a hx-put="{{ url_for('entry_backlog_push', id=entry.id) }}" class="dropdown-item"
+   _="on htmx:afterRequest remove the closest .feed-entry"
+><span class="icon"><i class="fas fa-archive"></i></span> Send to backlog</a>
+{% endif %}
+<hr class="dropdown-divider">
 <a class="dropdown-item" _="on click writeText('{{ entry.content_url }}') into the navigator's clipboard"><span class="icon"><i class="fas fa-link"></i></span> Copy URL</a>
 
 {% if entry.is_external_link %}
 <a href="{{ url_for('feed_add', url=entry.content_url) }}" class="dropdown-item"><span class="icon"><i class="fas fa-rss"></i></span> Discover feed</a>
 {% endif %}
-{% if current_user.has_kindle %}
+{% if entry.content_url and current_user.has_kindle %}
 <a class="dropdown-item" hx-post="{{ url_for('send_to_kindle', url=entry.content_url ) }}"
    _="on htmx:beforeRequest or htmx:afterRequest toggle .fa-spin on <i/> in me"
 ><span class="icon"><i class="fas fa-tablet-alt "></i></span> Send to Kindle</a>
 {% endif %}
 
 <hr class="dropdown-divider">
 
-{% endif %}
 {% if entry.username %}
 <a class="dropdown-item" href="{{ url_for('entry_list', username=entry.username ) }}"><span class="icon"><i class="fas fa-user"></i></span> View {{ entry.username }}</a>
 {% endif %}
@@ -35,6 +39,6 @@
 <a href="{{ url_for('feed_edit', feed_name=entry.feed.name ) }}" class="dropdown-item"><span class="icon"><i class="far fa-edit"></i></span> Edit {{ entry.feed.name }}</a>
 <a hx-delete="{{ url_for('feed_delete', feed_name=entry.feed.name ) }}" _="on htmx:afterRequest go to url {{ url_for('entry_list') }}" class="dropdown-item"><span class="icon"><i class="far fa-trash-alt"></i></span> Delete {{ entry.feed.name }}</a>
 <hr class="dropdown-divider is-hidden-mobile">
-<a class="dropdown-item is-hidden-mobile" href="{{ url_for('raw_entry', id=entry.id ) }}" target="_blank"><span class="icon"><i class="fas fa-file-code"></i></span>  View raw entry data</a>
 <a class="dropdown-item is-hidden-mobile" href="{{ url_for('raw_feed', feed_name=entry.feed.name ) }}" target="_blank"><span class="icon"><i class="fas fa-file-code"></i></span>  View raw feed data</a>
 {% endif %}
+<a class="dropdown-item is-hidden-mobile" href="{{ url_for('raw_entry', id=entry.id ) }}" target="_blank"><span class="icon"><i class="fas fa-file-code"></i></span>  View raw entry data</a>

feedi/templates/entry_content.html

@@ -34,12 +34,10 @@
             >
                 <i class="fas fa-arrow-left fa-lg"></i>
             </a>
-        {% if entry.id %}
         <a class="level-item icon is-white is-rounded {% if entry.favorited %}toggled{% endif %}" title="Favorite"
            hx-put="{{ url_for('entry_favorite', id=entry.id )}}"
            _="on click toggle .toggled"
         ><i class="fas fa-star"></i></a>
-        {% endif %}
 
         {% if entry.comments_url %}
         <a class="is-white icon is-rounded level-item" title="Comment"
@@ -53,13 +51,11 @@
         </a>
         {% endif %}
 
-        {% if entry.id %}
         <a class="level-item icon hover-icon is-white is-rounded {% if entry.pinned %}toggled{% endif %} pin-button" title="Pin"
            hx-put="{{ url_for('entry_pin', id=entry.id, **filters) }}"
            hx-swap="none"
            _="on click toggle .toggled">
             <i class="fas fa-thumbtack"></i></a>
-        {% endif %}
 
         <a class="level-item icon hover-icon is-white is-rounded pin-button" title="More"
            _="on click toggle .is-active then toggle .is-active on the previous .navbar-menu"

feedi/templates/entry_header.html

@@ -83,6 +83,19 @@
            href="{{ entry.comments_url}}" target="_blank"
         ><i class="fas fa-comment-alt"></i></a>
         {% endif %}
+        {% if entry.backlogged %}
+        <a tabindex="-1" class="level-item icon hover-icon is-white is-rounded" title="Pop from backlog"
+           _="on click remove the closest .feed-entry"
+           hx-delete="{{ url_for('entry_backlog_pop', id=entry.id )}}"
+        ><i class="fas fa-undo"></i></a>
+        {% elif entry.is_unwrappable() %}
+        <a tabindex="-1" class="level-item icon hover-icon is-white is-rounded" title="Unwrap"
+           hx-post="{{ url_for('entry_unwrap', id=entry.id )}}"
+           hx-target="closest .feed-entry"
+           hx-swap="outerHTML"
+           _ = "on click set x to the closest .feed-entry then set x.style.opacity to 0.5"
+        ><i class="fas fa-envelope-open-text"></i></a>
+        {% endif %}
         <div class="dropdown is-right"
              _="on intersection(intersecting) having margin '0px 0px -50% 0px'
                     if intersecting remove .is-up else add .is-up -- show dropup up or dropdown depending position relative to middle of screen">

feedi/templates/entry_list_page.html

@@ -83,6 +83,20 @@
             <i class="fas fa-comment-alt"></i>
         </a>
         {% endif %}
+        {% if entry.backlogged %}
+        <a tabindex="-1" class="level-item icon  is-white is-rounded" title="Pop from backlog"
+           _="on click remove the closest .feed-entry"
+           hx-delete="{{ url_for('entry_backlog_pop', id=entry.id )}}"
+        ><i class="fas fa-undo"></i></a>
+        {% elif entry.is_unwrappable() %}
+        <a tabindex="-1" class="level-item icon is-white is-rounded" title="Unwrap"
+           hx-post="{{ url_for('entry_unwrap', id=entry.id )}}"
+           hx-target="closest .feed-entry"
+           hx-swap="outerHTML"
+           _ = "on click set x to the closest .feed-entry then set x.style.opacity to 0.5"
+        ><i class="fas fa-envelope-open-text"></i></a>
+        {% endif %}
+
         <a class="icon is-white is-rounded level-item"
            tabindex="-1"
            _="on click toggle .is-active on the next .dropdown then