expand retry even more: run OPD to find and use SyndicatedPosts. for #…

…524

i think the only part missing now is to actually fetch their site's h-feed during this OPD. i think i'm ok with that.

app.py

@@ -26,6 +26,7 @@
 from wordpress_rest import WordPress
 import models
 from models import BlogPost, BlogWebmention, Publish, Response, Source
+import original_post_discovery
 import util
 
 # import source model class definitions for template rendering
@@ -491,15 +492,30 @@ def post(self):
     if not entity:
       self.abort(400, 'key not found')
 
+    # start all target URLs over
     if entity.status == 'complete':
       entity.status = 'new'
-    entity.unsent += entity.sent + entity.skipped
-    entity.sent = entity.skipped = []
+
+    targets = set(entity.unsent + entity.sent + entity.skipped + entity.error +
+                  entity.failed)
+    entity.sent = entity.skipped = entity.error = entity.failed = []
+
+    # run OPD to pick up any new SyndicatedPosts. note that we don't refetch
+    # their h-feed, so if they've added a syndication URL since we last crawled,
+    # retry won't make us pick it up. meh. background in #524.
+    if entity.key.kind() == 'Response':
+      source = entity.source.get()
+      for activity in [json.loads(a) for a in entity.activities_json]:
+        originals, mentions = original_post_discovery.discover(
+          source, activity, fetch_hfeed=False, include_redirect_sources=False)
+        targets |= original_post_discovery.targets_for_response(
+          json.loads(entity.response_json), originals=originals, mentions=mentions)
+
+    entity.unsent = targets
     entity.put()
 
     # clear any cached webmention endpoints
-    memcache.delete_multi(util.webmention_endpoint_cache_key(url) for url in
-                          entity.unsent + entity.error + entity.failed)
+    memcache.delete_multi(util.webmention_endpoint_cache_key(url) for url in targets)
 
     if entity.key.kind() == 'Response':
       util.add_propagate_task(entity)

templates/social_user.html

@@ -202,13 +202,11 @@
      </a>
     {% endif %}
 
-    {% if response.error or response.failed or response.skipped or response.unsent or response.sent %}
     <form method="post" action="/retry">
       <input name="key" type="hidden" value="{{ response.key.urlsafe }}" />
       <button id="retry-button" type="submit" class="btn btn-default">
         Retry</button>
     </form>
-    {% endif %}
 
    </div><div class="col-sm-3">
     {% for label, links in response.links.items %}

test/test_app.py

@@ -1,6 +1,7 @@
 """Unit tests for app.py.
 """
 import datetime
+import json
 import urllib
 
 from google.appengine.api import memcache
@@ -10,6 +11,7 @@
 import webapp2
 
 import app
+import models
 import util
 import testutil
 from testutil import FakeAuthEntity
@@ -33,38 +35,55 @@ def test_poll_now(self):
     params = testutil.get_task_params(self.taskqueue_stub.GetTasks('poll-now')[0])
     self.assertEqual(key, params['source_key'])
 
-  def test_retry_response(self):
+  def test_retry(self):
     self.assertEqual([], self.taskqueue_stub.GetTasks('propagate'))
 
-    self.responses[0].status = 'complete'
-    self.responses[0].sent = ['http://sent']
-    self.responses[0].skipped = ['https://skipped']
-    self.responses[0].put()
+    source = self.sources[0]
+    # source.domains = ['orig']
+    source.domain_urls = ['http://orig']
+    source.put()
+
+    resp = self.responses[0]
+    resp.status = 'complete'
+    resp.unsent = ['http://unsent']
+    resp.sent = ['http://sent']
+    resp.error = ['http://error']
+    resp.failed = ['http://failed']
+    resp.skipped = ['https://skipped']
+
+    # SyndicatedPost with new target URLs
+    resp.activities_json = [json.dumps({'url': 'https://silo/1'}),
+                            json.dumps({'url': 'https://silo/2'}),
+                            ]
+    resp.put()
+    models.SyndicatedPost.insert(source, 'https://silo/1', 'https://orig/1')
+    models.SyndicatedPost.insert(source, 'https://silo/2', 'http://orig/2')
 
     # cached webmention endpoint
     memcache.set('W https skipped', 'asdf')
 
-    key = self.responses[0].key.urlsafe()
-    resp = app.application.get_response(
+    key = resp.key.urlsafe()
+    response = app.application.get_response(
       '/retry', method='POST', body='key=' + key)
-    self.assertEquals(302, resp.status_int)
-    self.assertEquals(self.sources[0].bridgy_url(self.handler),
-                      resp.headers['Location'].split('#')[0])
+    self.assertEquals(302, response.status_int)
+    self.assertEquals(source.bridgy_url(self.handler),
+                      response.headers['Location'].split('#')[0])
     params = testutil.get_task_params(self.taskqueue_stub.GetTasks('propagate')[0])
     self.assertEqual(key, params['response_key'])
 
     # status and URLs should be refreshed
-    got = self.responses[0].key.get()
+    got = resp.key.get()
     self.assertEqual('new', got.status)
     self.assertItemsEqual(
-      ['http://sent', 'https://skipped', 'http://target1/post/url'], got.unsent)
-    self.assertEqual([], got.sent)
-    self.assertEqual([], got.skipped)
+      ['http://unsent', 'http://sent', 'https://skipped', 'http://error',
+       'http://failed', 'https://orig/1', 'http://orig/2'],
+      got.unsent)
+    for field in got.sent, got.skipped, got.error, got.failed:
+      self.assertEqual([], field)
 
     # webmention endpoints for URL domains should be refreshed
     self.assertIsNone(memcache.get('W https skipped'))
 
-
   def test_poll_now_and_retry_response_missing_key(self):
     for endpoint in '/poll-now', '/retry':
       for body in '', 'key=' + self.responses[0].key.urlsafe():  # hasn't been stored