Skip to content
Permalink
Browse files

Fix the way we dedupe websub too, and support websub on non-feed pages

  • Loading branch information...
fluffy-critter committed Aug 13, 2019
1 parent 2ae6438 commit 7c1c15eefbf614c335f6d19cb6be130e9bfa6697
Showing with 56 additions and 30 deletions.
  1. +0 −5 Makefile
  2. +19 −2 pushl/__init__.py
  3. +7 −1 pushl/entries.py
  4. +0 −22 pushl/feeds.py
  5. +30 −0 pushl/websub.py
@@ -1,10 +1,5 @@
all: format pylint flake8

.PHONY: format
format:
pipenv run isort -y
pipenv run autopep8 -r --in-place .

.PHONY: format
format:
pipenv run isort -y
@@ -3,7 +3,7 @@
import asyncio
import logging

from . import caching, entries, feeds, utils, webmentions
from . import caching, entries, feeds, utils, webmentions, websub

LOGGER = logging.getLogger("pushl")

@@ -26,6 +26,8 @@ def __init__(self, session, args):
self._processed_mentions = set()
self._feed_domains = set()

self._processed_websub = set()

self.session = session

async def process_feed(self, url, send_mentions=True):
@@ -71,7 +73,7 @@ def __init__(self, session, args):
if updated and link.get('rel') == 'hub' and not feed.is_archive:
LOGGER.debug("Found WebSub hub %s", link)
pending.append(
("update websub " + href, feed.update_websub(self, href)))
("update websub " + href, self.send_websub(feed.url, href)))
except (AttributeError, KeyError):
LOGGER.debug("Feed %s has no links", url)

@@ -138,6 +140,10 @@ def __init__(self, session, args):
else:
LOGGER.info("Ignoring non-local feed %s", feed)

for hub in entry.hubs:
pending.append(("send websub {} -> {}".format(url, hub),
self.send_websub(url, hub)))

LOGGER.debug("--- finish process_entry %s", url)

if pending:
@@ -154,6 +160,7 @@ def __init__(self, session, args):
if (entry.url, url) in self._processed_mentions:
LOGGER.debug(
"Skipping already processed mention %s -> %s", entry.url, url)
return
self._processed_mentions.add((entry.url, url))

LOGGER.debug("++WAIT: webmentions.get_target %s", url)
@@ -164,3 +171,13 @@ def __init__(self, session, args):
LOGGER.debug("++WAIT: Sending webmention %s -> %s", entry.url, url)
await target.send(self, entry)
LOGGER.debug("++DONE: Sending webmention %s -> %s", entry.url, url)

async def send_websub(self, url, hub):
""" send a websub notification """

if (url, hub) in self._processed_websub:
LOGGER.debug("Skipping already processed websub %s -> %s", url, hub)
return
self._processed_websub.add((url, hub))

websub.send(self, url, hub)
@@ -14,7 +14,7 @@

class Entry:
""" Encapsulates a scanned entry """
# pylint:disable=too-few-public-methods
# pylint:disable=too-few-public-methods,too-many-instance-attributes

def __init__(self, request):
""" Build an Entry from a completed request """
@@ -44,9 +44,15 @@ def __init__(self, request):
and 'type' in link.attrs
and link.attrs['type'] in ('application/rss.xml',
'application/atom+xml')]

self.hubs = [link.attrs['href'] for link in soup.find_all('link', rel='hub')]
if 'hub' in request.links:
self.hubs.append(request.links['hub']['url'])

else:
self._targets = []
self.feeds = []
self.hubs = []

self.schema = SCHEMA_VERSION

@@ -69,28 +69,6 @@ def is_archive(self):
('self' not in rels or
rels['self'] != rels['current']))

async def update_websub(self, config, hub):
""" Update WebSub hub to know about this feed """
try:
LOGGER.debug("WebSub: Notifying %s of %s", hub, self.url)
request = await utils.retry_post(
config,
hub,
data={
'hub.mode': 'publish',
'hub.url': self.url
})

if request.success:
LOGGER.info("%s: WebSub notification sent to %s",
self.url, hub)
else:
LOGGER.warning("%s: Hub %s returned status code %s: %s", self.url, hub,
request.status, request.text)
except Exception as err: # pylint:disable=broad-except
LOGGER.warning("WebSub %s: got %s: %s",
hub, err.__class__.__name__, err)


async def get_feed(config, url):
""" Get a feed
@@ -0,0 +1,30 @@
""" Functions for sending WebSub notifications """

import logging

from . import utils

LOGGER = logging.getLogger(__name__)


async def send(config, url, hub):
""" Update WebSub hub to know about this URL """
try:
LOGGER.debug("WebSub: Notifying %s of %s", hub, url)
request = await utils.retry_post(
config,
hub,
data={
'hub.mode': 'publish',
'hub.url': url
})

if request.success:
LOGGER.info("%s: WebSub notification sent to %s",
url, hub)
else:
LOGGER.warning("%s: Hub %s returned status code %s: %s", url, hub,
request.status, request.text)
except Exception as err: # pylint:disable=broad-except
LOGGER.warning("WebSub %s: got %s: %s",
hub, err.__class__.__name__, err)

0 comments on commit 7c1c15e

Please sign in to comment.
You can’t perform that action at this time.