add pixelfed!

for #927

README.md

@@ -42,6 +42,20 @@ dev_appserver.py --log_level debug --enable_host_checking false \
 
 Open [localhost:8080](http://localhost:8080/) and you should see the Bridgy home page!
 
+To test polling interactively, temporarily comment out the `manual_scaling` section in [background.yaml](https://github.com/snarfed/bridgy/blob/master/background.yaml), uncomment the `automatic_scaling` section, and use `curl` to run the task handler directly:
+
+```sh
+curl -d 'source_key=[KEY]&last_polled=1970-01-01-00-00-00' http://localhost:8081/_ah/queue/poll
+```
+
+...where `[KEY]` is the url-safe ndb key for the source entity you want to poll, e.g. `ndb.Key('Twitter', 'schnarfed').urlsafe()`.
+
+Similarly, to run a webmention (aka propagate) task, look in `dev_appserver`'s logs for lines beginning with _Would add task: ..._, grab the request body there, and plug it into a similar `curl` command, eg:
+
+```sh
+curl -d 'response_key=...' localhost:8081/_ah/queue/propagate
+```
+
 If you hit an error during setup, check out the [oauth-dropins Troubleshooting/FAQ section](https://github.com/snarfed/oauth-dropins#troubleshootingfaq). For searchability, here are a handful of error messages that [have solutions there](https://github.com/snarfed/oauth-dropins#troubleshootingfaq):
 
 ```

admin.py

@@ -16,7 +16,7 @@
 import webapp2
 
 # Import source class files so their metaclasses are initialized.
-import blogger, flickr, github, instagram, mastodon, medium, tumblr, twitter, wordpress_rest
+import blogger, flickr, github, instagram, mastodon, medium, pixelfed, tumblr, twitter, wordpress_rest
 
 
 class ResponsesHandler(handlers.TemplateHandler):
@@ -59,7 +59,7 @@ def template_file(self):
 
   def template_vars(self):
     CLASSES = (flickr.Flickr, github.GitHub, twitter.Twitter,
-               instagram.Instagram, mastodon.Mastodon)
+               instagram.Instagram, mastodon.Mastodon, pixelfed.Pixelfed)
     queries = [cls.query(Source.status == 'enabled',
                          Source.poll_status == 'error',
                          Source.rate_limited.IN((False, None)),

app.py

@@ -39,6 +39,7 @@
   'mastodon',
   'medium',
   'meetup',
+  'pixelfed',
   'publish',
   'tumblr',
   'twitter',
@@ -733,7 +734,7 @@ def get(self):
 routes += [
   ('/?', FrontPageHandler),
   ('/users/?', UsersHandler),
-  ('/(blogger|fake|fake_blog|flickr|github|instagram|mastodon|medium|meetup|tumblr|twitter|wordpress)/([^/]+)/?',
+  ('/(blogger|fake|fake_blog|flickr|github|instagram|mastodon|medium|meetup|pixelfed|tumblr|twitter|wordpress)/([^/]+)/?',
    UserHandler),
   ('/facebook/.*', FacebookIsDeadHandler),
   ('/googleplus/.*', GooglePlusIsDeadHandler),

docs/source/modules.rst

@@ -54,6 +54,10 @@ original_post_discovery
 -----------------------
 .. automodule:: original_post_discovery
 
+pixelfed
+--------
+.. automodule:: pixelfed
+
 publish
 -------
 .. automodule:: publish

handlers.py

@@ -35,7 +35,7 @@
 import util
 
 # Import source class files so their metaclasses are initialized.
-import blogger, flickr, github, instagram, mastodon, medium, tumblr, twitter, wordpress_rest
+import blogger, flickr, github, instagram, mastodon, medium, pixelfed, tumblr, twitter, wordpress_rest
 
 CACHE_TIME = 60 * 15  # 15m
 

mastodon.py

@@ -12,20 +12,6 @@
 import models
 import util
 
-# https://docs.joinmastodon.org/api/oauth-scopes/
-LISTEN_SCOPES = (
-  'read:accounts',
-  'read:blocks',
-  'read:notifications',
-  'read:search',
-  'read:statuses',
-)
-PUBLISH_SCOPES = LISTEN_SCOPES + (
-  'write:statuses',
-  'write:favourites',
-  'write:media',
-)
-
 
 class StartHandler(oauth_dropins.mastodon.StartHandler):
   """Abstract base OAuth starter class with our redirect URLs."""
@@ -56,12 +42,7 @@ class Mastodon(models.Source):
   SHORT_NAME = 'mastodon'
   CAN_PUBLISH = True
   HAS_BLOCKS = True
-  TYPE_LABELS = {
-    'post': 'toot',
-    'comment': 'reply',
-    'repost': 'boost',
-    'like': 'favorite',
-  }
+  TYPE_LABELS = GR_CLASS.TYPE_LABELS
   DISABLE_HTTP_CODES = ('401', '403')
 
   @property
@@ -71,8 +52,8 @@ def URL_CANONICALIZER(self):
       domain=self.gr_source.DOMAIN,
       headers=util.REQUEST_HEADERS)
 
-  @staticmethod
-  def new(handler, auth_entity=None, **kwargs):
+  @classmethod
+  def new(cls, handler, auth_entity=None, **kwargs):
     """Creates and returns a :class:`Mastodon` entity.
 
     Args:
@@ -81,12 +62,12 @@ def new(handler, auth_entity=None, **kwargs):
       kwargs: property values
     """
     user = json_loads(auth_entity.user_json)
-    return Mastodon(id=auth_entity.key_id(),
-                    auth_entity=auth_entity.key,
-                    url=user.get('url'),
-                    name=user.get('display_name') or user.get('username'),
-                    picture=user.get('avatar'),
-                    **kwargs)
+    return cls(id=auth_entity.key_id(),
+               auth_entity=auth_entity.key,
+               url=user.get('url'),
+               name=user.get('display_name') or user.get('username'),
+               picture=user.get('avatar'),
+              **kwargs)
 
   def username(self):
     """Returns the Mastodon username, e.g. alice."""
@@ -119,13 +100,18 @@ def button_html(cls, feature, **kwargs):
     source = kwargs.get('source')
     instance = source.instance() if source else ''
     return """\
-<form method="%s" action="/mastodon/start">
-  <input type="image" class="mastodon-button shadow" alt="Sign in with Mastodon"
-         src="/oauth_dropins/static/mastodon_large.png" />
-  <input name="feature" type="hidden" value="%s" />
-  <input name="instance" type="hidden" value="%s" />
+<form method="{method}" action="/{short_name}/start">
+  <input type="image" class="{short_name}-button shadow" alt="Sign in with {name}"
+         src="/oauth_dropins/static/{short_name}_large.png" />
+  <input name="feature" type="hidden" value="{feature}" />
+  <input name="instance" type="hidden" value="{instance}" />
 </form>
-""" % ('post' if instance else 'get', feature, instance)
+""".format(instance=instance,
+           method='post' if instance else 'get',
+           feature=feature,
+           name=cls.GR_CLASS.NAME,
+           short_name=cls.SHORT_NAME,
+           **kwargs)
 
   def is_private(self):
     """Returns True if this Mastodon account is protected.
@@ -166,20 +152,44 @@ def load_blocklist(self):
 
 class InstanceHandler(TemplateHandler, util.Handler):
   """Serves the "Enter your instance" form page."""
+  SITE = 'mastodon'
+  START_HANDLER = StartHandler
+
+  # https://docs.joinmastodon.org/api/oauth-scopes/
+  LISTEN_SCOPES = (
+    'read:accounts',
+    'read:blocks',
+    'read:notifications',
+    'read:search',
+    'read:statuses',
+  )
+  PUBLISH_SCOPES = LISTEN_SCOPES + (
+    'write:statuses',
+    'write:favourites',
+    'write:media',
+  )
+
   def template_file(self):
-    return 'mastodon_instance.html'
+    return 'choose_instance.html'
+
+  def template_vars(self):
+    return {
+      'site': self.SITE,
+      'logo_file': 'mastodon_logo_large.png',
+      'join_url': 'https://joinmastodon.org/#getting-started',
+    }
 
   def post(self):
     feature = self.request.get('feature')
-    start_cls = util.oauth_starter(StartHandler).to('/mastodon/callback',
-      scopes=PUBLISH_SCOPES if feature == 'publish' else LISTEN_SCOPES)
+    start_cls = util.oauth_starter(self.START_HANDLER).to('/%s/callback' % self.SITE,
+      scopes=self.PUBLISH_SCOPES if feature == 'publish' else self.LISTEN_SCOPES)
     start = start_cls(self.request, self.response)
 
     instance = util.get_required_param(self, 'instance')
     try:
       self.redirect(start.redirect_url(instance=instance))
     except ValueError as e:
-      logging.warning('Bad Mastodon instance', stack_info=True)
+      logging.warning('Bad %s instance' % self.SITE.capitalize(), stack_info=True)
       self.messages.add(util.linkify(str(e), pretty=True))
       return self.redirect(self.request.path)
 
@@ -194,6 +204,6 @@ def finish(self, auth_entity, state=None):
   ('/mastodon/start', InstanceHandler),
   ('/mastodon/callback', CallbackHandler),
   ('/mastodon/delete/finish', oauth_dropins.mastodon.CallbackHandler.to('/delete/finish')),
-  ('/mastodon/publish/start', StartHandler.to('/publish/mastodon/finish',
-                                              scopes=PUBLISH_SCOPES)),
+  ('/mastodon/publish/start', StartHandler.to(
+    '/publish/mastodon/finish', scopes=InstanceHandler.PUBLISH_SCOPES)),
 ]

models.py

@@ -213,7 +213,7 @@ def __getattr__(self, name):
         kwargs = {'user_id': self.key_id()}
       elif self.key.kind() == 'Instagram':
         kwargs = {'scrape': True, 'cookie': INSTAGRAM_SESSIONID_COOKIE}
-      elif self.key.kind() == 'Mastodon':
+      elif self.key.kind() in ('Mastodon', 'Pixelfed'):
         args = (auth_entity.instance(),) + args
         kwargs = {'user_id': json_loads(auth_entity.user_json).get('id')}
       elif self.key.kind() == 'Twitter':
@@ -499,10 +499,12 @@ def create_new(cls, handler, user_url=None, **kwargs):
     if existing:
       # merge some fields
       source.features = set(source.features + existing.features)
-      source.populate(**existing.to_dict(include=(
-            'created', 'last_hfeed_refetch', 'last_poll_attempt', 'last_polled',
-            'last_syndication_url', 'last_webmention_sent', 'superfeedr_secret',
-            'webmention_endpoint')))
+      props = ('created', 'last_hfeed_refetch', 'last_poll_attempt',
+               'last_polled', 'last_syndication_url', 'last_webmention_sent',
+               'superfeedr_secret', 'webmention_endpoint')
+      if existing.domain_urls and not source.domain_urls:
+        props += ('domains', 'domain_urls')
+      source.populate(**existing.to_dict(include=props))
       verb = 'Updated'
     else:
       verb = 'Added'

pixelfed.py

@@ -0,0 +1,73 @@
+"""Pixelfed source and datastore model classes."""
+from granary import pixelfed as gr_pixelfed
+import oauth_dropins.pixelfed
+import webapp2
+
+import mastodon
+import util
+
+
+class StartHandler(oauth_dropins.pixelfed.StartHandler):
+  """Abstract base OAuth starter class with our redirect URLs."""
+  REDIRECT_PATHS = (
+    '/pixelfed/callback',
+    # TODO: uncomment when https://github.com/pixelfed/pixelfed/issues/2106 is fixed
+    # '/publish/pixelfed/finish',
+    # '/pixelfed/delete/finish',
+    # '/delete/finish',
+  )
+
+  def app_name(self):
+    return 'Bridgy'
+
+  def app_url(self):
+    if self.request.host in util.OTHER_DOMAINS:
+      return util.HOST_URL
+
+    return super().app_url()
+
+
+class Pixelfed(mastodon.Mastodon):
+  """A Pixelfed account.
+
+  The key name is the fully qualified address, eg '@snarfed@piconic.co'.
+  """
+  GR_CLASS = gr_pixelfed.Pixelfed
+  OAUTH_START_HANDLER = StartHandler
+  SHORT_NAME = 'pixelfed'
+  CAN_PUBLISH = True
+  HAS_BLOCKS = False
+  TYPE_LABELS = GR_CLASS.TYPE_LABELS
+
+  def search_for_links(self):
+    return []
+
+class InstanceHandler(mastodon.InstanceHandler):
+  """Serves the "Enter your instance" form page."""
+  SITE = 'pixelfed'
+  START_HANDLER = StartHandler
+
+  # https://docs.pixelfed.org/technical-documentation/api-v1.html
+  LISTEN_SCOPES = ('read',)
+  PUBLISH_SCOPES = LISTEN_SCOPES + ('write',)
+
+  def template_vars(self):
+    return {
+      'site': self.SITE,
+      'logo_file': 'pixelfed_logo.png',
+      'join_url': 'https://pixelfed.org/join',
+    }
+
+
+class CallbackHandler(oauth_dropins.pixelfed.CallbackHandler, util.Handler):
+  def finish(self, auth_entity, state=None):
+    source = self.maybe_add_or_delete_source(Pixelfed, auth_entity, state)
+
+
+ROUTES = [
+  ('/pixelfed/start', InstanceHandler),
+  ('/pixelfed/callback', CallbackHandler),
+  ('/pixelfed/delete/finish', oauth_dropins.pixelfed.CallbackHandler.to('/delete/finish')),
+  ('/pixelfed/publish/start', StartHandler.to(
+    '/publish/pixelfed/finish', scopes=InstanceHandler.PUBLISH_SCOPES)),
+]

publish.py

@@ -19,6 +19,7 @@
   github as oauth_github,
   mastodon as oauth_mastodon,
   meetup as oauth_meetup,
+  pixelfed as oauth_pixelfed,
   twitter as oauth_twitter,
 )
 from oauth_dropins.webutil import appengine_info
@@ -32,13 +33,14 @@
 from mastodon import Mastodon
 from meetup import Meetup
 from models import Publish, PublishedPage
+from pixelfed import Pixelfed
 from twitter import Twitter
 import models
 import util
 import webmention
 
 
-SOURCES = (Flickr, GitHub, Mastodon, Meetup, Twitter)
+SOURCES = (Flickr, GitHub, Mastodon, Meetup, Pixelfed, Twitter)
 SOURCE_NAMES = {cls.SHORT_NAME: cls for cls in SOURCES}
 SOURCE_DOMAINS = {cls.GR_CLASS.DOMAIN: cls for cls in SOURCES}
 # image URLs matching this regexp should be ignored.
@@ -147,7 +149,7 @@ def _run(self):
     if (domain not in util.DOMAINS or
         len(path_parts) != 2 or path_parts[0] != '/publish' or not source_cls):
       return self.error(
-        'Target must be brid.gy/publish/{flickr,github,mastodon,meetup,twitter}')
+        'Target must be brid.gy/publish/{flickr,github,mastodon,meetup,pixelfed,twitter}')
     elif source_cls == Instagram:
       return self.error('Sorry, %s is not supported.' %
                         source_cls.GR_CLASS.NAME)
@@ -691,18 +693,17 @@ def error(self, error, html=None, status=400, data=None, report=False, **kwargs)
 class FlickrSendHandler(oauth_flickr.CallbackHandler, SendHandler):
   finish = SendHandler.finish
 
-
 class GitHubSendHandler(oauth_github.CallbackHandler, SendHandler):
   finish = SendHandler.finish
 
-
 class MastodonSendHandler(oauth_mastodon.CallbackHandler, SendHandler):
   finish = SendHandler.finish
 
-
 class MeetupSendHandler(oauth_meetup.CallbackHandler, SendHandler):
   finish = SendHandler.finish
 
+class PixelfedSendHandler(oauth_pixelfed.CallbackHandler, SendHandler):
+  finish = SendHandler.finish
 
 class TwitterSendHandler(oauth_twitter.CallbackHandler, SendHandler):
   finish = SendHandler.finish
@@ -741,11 +742,12 @@ def authorize(self):
 ROUTES = [
   ('/publish/preview', PreviewHandler),
   ('/publish/webmention', WebmentionHandler),
-  ('/publish/(flickr|github|mastodon|meetup|twitter)',
+  ('/publish/(flickr|github|mastodon|meetup|pixelfed|twitter)',
    webmention.WebmentionGetHandler),
   ('/publish/flickr/finish', FlickrSendHandler),
   ('/publish/github/finish', GitHubSendHandler),
   ('/publish/mastodon/finish', MastodonSendHandler),
   ('/meetup/publish/finish', MeetupSendHandler), # because Meetup's `redirect_uri` handling is a little more restrictive
+  ('/publish/pixelfed/finish', PixelfedSendHandler),
   ('/publish/twitter/finish', TwitterSendHandler),
 ]