Merge pull request #2208 from devos50/recheck_rss_feeds_endpoint

Implemented endpoint to refresh rss feeds, written tests for Tribler.Core.Modules.channel package

Tribler/Core/Modules/channel/channel.py

@@ -39,6 +39,9 @@ def name(self):
     def get_rss_feed_url_list(self):
         return [url for url in self._rss_feed_dict.iterkeys()]
 
+    def refresh_all_feeds(self):
+        [feed.parse_feed() for feed in self._rss_feed_dict.itervalues()]
+
     @call_on_reactor_thread
     def initialize(self):
         # load existing rss_feeds

Tribler/Core/Modules/channel/channel_rss.py

@@ -1,6 +1,5 @@
 from binascii import hexlify
 import logging
-import tempfile
 import hashlib
 import json
 import time
@@ -9,6 +8,7 @@
 import feedparser
 
 from twisted.internet import reactor
+from twisted.internet.defer import DeferredList
 from twisted.web.client import getPage
 
 from Tribler.dispersy.taskmanager import TaskManager
@@ -33,7 +33,6 @@ def __init__(self, session, channel_community, rss_url, check_interval=DEFAULT_C
         self.rss_url = rss_url
         self.check_interval = check_interval
 
-        self._tmp_dir = None
         self._url_cache = None
 
         self._pending_metadata_requests = {}
@@ -55,9 +54,6 @@ def initialize(self):
         self._url_cache = SimpleCache(url_cache_path)
         self._url_cache.load()
 
-        # create temporary directory
-        self._tmp_dir = tempfile.mkdtemp()
-
         # schedule the scraping task
         self.register_task(u"rss_scrape",
                            reactor.callLater(2, self._task_scrape))
@@ -76,22 +72,29 @@ def shutdown(self):
         self._to_stop = True
         self.cancel_all_pending_tasks()
 
-        self._tmp_dir = None
         self._url_cache.save()
         self._url_cache = None
 
         self.channel_community = None
         self.session = None
 
-    def _task_scrape(self):
+    def parse_feed(self):
         rss_parser = RSSFeedParser()
 
+        def_list = []
+
         for rss_item in rss_parser.parse(self.rss_url, self._url_cache):
             if self._to_stop:
-                return
+                return None
 
             torrent_deferred = getPage(rss_item[u'torrent_url'].encode('utf-8'))
             torrent_deferred.addCallback(lambda t, r=rss_item: self.on_got_torrent(t, rss_item=r))
+            def_list.append(torrent_deferred)
+
+        return DeferredList(def_list)
+
+    def _task_scrape(self):
+        self.parse_feed()
 
         if not self._to_stop:
             # schedule the next scraping task

Tribler/Core/Modules/restapi/my_channel_endpoint.py

@@ -39,7 +39,8 @@ class MyChannelEndpoint(MyChannelBaseEndpoint):
     def __init__(self, session):
         MyChannelBaseEndpoint.__init__(self, session)
         child_handler_dict = {"overview": MyChannelOverviewEndpoint, "torrents": MyChannelTorrentsEndpoint,
-                              "rssfeeds": MyChannelRssFeedsEndpoint, "playlists": MyChannelPlaylistsEndpoint}
+                              "rssfeeds": MyChannelRssFeedsEndpoint, "playlists": MyChannelPlaylistsEndpoint,
+                              "recheckfeeds": MyChannelRecheckFeedsEndpoint}
         for path, child_cls in child_handler_dict.iteritems():
             self.putChild(path, child_cls(self.session))
 
@@ -130,6 +131,26 @@ def render_GET(self, request):
         return json.dumps({"rssfeeds": feeds_list})
 
 
+class MyChannelRecheckFeedsEndpoint(MyChannelBaseEndpoint):
+    """
+    This class is responsible for handling requests regarding refreshing rss feeds in your channel.
+    """
+
+    def render_POST(self, request):
+        """
+        Rechecks all rss feeds in your channel. Returns error 404 if you channel does not exist.
+        """
+        request.setHeader('Content-Type', 'text/json')
+
+        channel_obj = self.get_my_channel_object()
+        if channel_obj is None:
+            return MyChannelBaseEndpoint.return_404(request)
+
+        channel_obj.refresh_all_feeds()
+
+        return json.dumps({"rechecked": True})
+
+
 class MyChannelModifyRssFeedsEndpoint(MyChannelBaseEndpoint):
     """
     This class is responsible for methods that modify the list of RSS feed URLs (adding/removing feeds).

Tribler/Main/vwxGUI/channel.py

@@ -1536,6 +1536,9 @@ def OnDeleteRss(self, event):
         my_channel_object.remove_rss_feed(item.url)
 
     def OnRefreshRss(self, event):
+        my_channel_object = self.guiutility.utility.session.lm.channel_manager.get_my_channel(self.channel.id)
+        my_channel_object.refresh_all_feeds()
+
         button = event.GetEventObject()
         button.Enable(False)
         wx.CallLater(5000, button.Enable, True)

Tribler/Test/Core/Modules/Channel/__init__.py

@@ -0,0 +1,3 @@
+"""
+This package contains tests for the channel management objects.
+"""

Tribler/Test/Core/Modules/Channel/base_test_channel.py

@@ -0,0 +1,24 @@
+from Tribler.Test.Core.base_test import TriblerCoreTest, MockObject
+
+
+class BaseTestChannel(TriblerCoreTest):
+
+    def setUp(self, annotate=True):
+        """
+        Setup some classes and files that are used by the tests in this module.
+        """
+        super(BaseTestChannel, self).setUp(annotate=annotate)
+
+        self.fake_session = MockObject()
+        self.fake_session.get_state_dir = lambda: self.session_base_dir
+        self.fake_session.add_observer = lambda a, b, c: False
+
+        fake_notifier = MockObject()
+        fake_notifier.add_observer = lambda a, b, c, d: False
+        fake_notifier.notify = lambda a, b, c, d: False
+        self.fake_session.notifier = fake_notifier
+
+        self.fake_channel_community = MockObject()
+        self.fake_channel_community.get_channel_id = lambda: 42
+        self.fake_channel_community.cid = 'a' * 20
+        self.fake_channel_community.get_channel_name = lambda: "my fancy channel"

Tribler/Test/Core/Modules/Channel/test_channel.py

@@ -0,0 +1,27 @@
+from Tribler.Core.Modules.channel.channel import ChannelObject
+from Tribler.Core.Modules.channel.channel_rss import ChannelRssParser
+from Tribler.Test.Core.Modules.Channel.base_test_channel import BaseTestChannel
+
+
+class TestChannel(BaseTestChannel):
+    """
+    This class contains some tests for the ChannelObject class.
+    """
+
+    def setUp(self, annotate=True):
+        """
+        Setup the tests by creating the ChannelObject instance.
+        """
+        super(TestChannel, self).setUp(annotate=annotate)
+        self.channel_object = ChannelObject(self.fake_session, self.fake_channel_community)
+
+    def test_get_channel_id(self):
+        self.assertEqual(self.channel_object.channel_id, 42)
+
+    def test_get_channel_name(self):
+        self.assertEqual(self.channel_object.name, "my fancy channel")
+
+    def test_get_rss_feed_url_list(self):
+        rss_parser = ChannelRssParser(self.fake_session, self.fake_channel_community, 'a')
+        self.channel_object._rss_feed_dict['a'] = rss_parser
+        self.assertEqual(self.channel_object.get_rss_feed_url_list(), ['a'])

Tribler/Test/Core/Modules/Channel/test_channel_rss.py

@@ -0,0 +1,72 @@
+import os
+from twisted.internet.defer import DeferredList
+from Tribler.Core.Modules.channel.cache import SimpleCache
+from Tribler.Core.Modules.channel.channel_rss import ChannelRssParser, RSSFeedParser
+from Tribler.Test.Core.Modules.Channel.base_test_channel import BaseTestChannel
+from Tribler.Test.Core.base_test import TriblerCoreTest
+from Tribler.Test.test_as_server import TESTS_DATA_DIR
+
+
+class TestChannelRss(BaseTestChannel):
+
+    def setUp(self, annotate=True):
+        """
+        Setup the tests by creating the ChannelRssParser instance.
+        """
+        super(TestChannelRss, self).setUp(annotate=annotate)
+        self.channel_rss = ChannelRssParser(self.fake_session, self.fake_channel_community, 'a')
+
+    def test_task_scrape_no_stop(self):
+        self.channel_rss._task_scrape()
+        self.assertTrue(self.channel_rss.is_pending_task_active("rss_scrape"))
+
+    def test_task_scrape_stop(self):
+        self.channel_rss._to_stop = True
+        self.channel_rss._task_scrape()
+        self.assertFalse(self.channel_rss.is_pending_task_active("rss_scrape"))
+
+    def test_initialize(self):
+        self.channel_rss.initialize()
+        self.assertTrue(self.channel_rss.is_pending_task_active("rss_scrape"))
+
+    def test_shutdown(self):
+        self.channel_rss.initialize()
+        cache_path = self.channel_rss._url_cache._file_path
+        self.channel_rss._url_cache.add('a')
+        self.channel_rss.shutdown()
+        self.assertTrue(os.path.exists(cache_path))
+        self.assertFalse(self.channel_rss.is_pending_task_active("rss_scrape"))
+
+    def test_parse_rss_feed(self):
+        self.channel_rss.rss_url = os.path.join(TESTS_DATA_DIR, 'test_rss.xml')
+        self.channel_rss._url_cache = SimpleCache(os.path.join(self.session_base_dir, 'cache.txt'))
+        self.assertIsInstance(self.channel_rss.parse_feed(), DeferredList)
+
+    def test_parse_feed_stopped(self):
+        self.channel_rss.rss_url = os.path.join(TESTS_DATA_DIR, 'test_rss.xml')
+        self.channel_rss._url_cache = SimpleCache(os.path.join(self.session_base_dir, 'cache.txt'))
+        self.channel_rss._to_stop = True
+        self.assertIsNone(self.channel_rss.parse_feed())
+
+
+class TestRssParser(TriblerCoreTest):
+
+    def test_parse_html(self):
+        parser = RSSFeedParser()
+        self.assertEqual(parser._parse_html("<p>Hi</p>"), set())
+        self.assertEqual(parser._parse_html("<a href='abc'></a>"), {'abc'})
+        self.assertEqual(parser._parse_html("<a href='abc'><img src='def'/></a>"), {'abc', 'def'})
+
+    def test_html2plaintext(self):
+        parser = RSSFeedParser()
+        self.assertEqual(parser._html2plaintext("<p>test</p>"), "test\n")
+        self.assertEqual(parser._html2plaintext("test"), "test\n")
+        self.assertEqual(parser._html2plaintext("<p>test\ntest2</p><p>test3</p>"), "test\ntest2\ntest3\n")
+
+    def test_parse(self):
+        parser = RSSFeedParser()
+        for rss_item in parser.parse(os.path.join(TESTS_DATA_DIR, 'test_rss.xml'),
+                                     SimpleCache(os.path.join(self.session_base_dir, 'cache.txt'))):
+            self.assertEqual(len(rss_item['thumbnail_list']), 1)
+            self.assertEqual(rss_item['title'], "ubuntu-15.04-desktop-amd64.iso")
+            self.assertEqual(rss_item['description'], '')

Tribler/Test/Core/Modules/RestApi/test_my_channel_endpoints.py

@@ -197,6 +197,28 @@ def verify_rss_removed(_):
         return self.do_request('mychannel/rssfeeds/http%3A%2F%2Frssfeed.com%2Frss.xml', expected_code=200,
                                expected_json=expected_json, request_type='DELETE').addCallback(verify_rss_removed)
 
+    @deferred(timeout=10)
+    def test_recheck_rss_feeds_no_channel(self):
+        """
+        Testing whether the API returns a 404 if no channel has been created when rechecking rss feeds
+        """
+        self.session.lm.channel_manager = ChannelManager(self.session)
+        return self.do_request('mychannel/recheckfeeds', expected_code=404, request_type='POST')
+
+    @deferred(timeout=10)
+    def test_recheck_rss_feeds(self):
+        """
+        Testing whether the API returns a 200 if the rss feeds are rechecked in your channel
+        """
+        expected_json = {"rechecked": True}
+        my_channel_id = self.create_fake_channel("my channel", "this is a short description")
+        channel_obj = self.session.lm.channel_manager.get_my_channel(my_channel_id)
+        channel_obj._is_created = True
+        channel_obj.create_rss_feed("http://rssfeed.com/rss.xml")
+
+        return self.do_request('mychannel/recheckfeeds', expected_code=200,
+                               expected_json=expected_json, request_type='POST')
+
 
 class TestMyChannelPlaylistEndpoints(AbstractTestMyChannelEndpoints):
 

Tribler/Test/Core/base_test.py

@@ -1,5 +1,12 @@
 from Tribler.Test.test_as_server import AbstractServer
 
 
+class MockObject(object):
+    """
+    This class is used to create as base class for fake (mocked) objects.
+    """
+    pass
+
+
 class TriblerCoreTest(AbstractServer):
     pass

doc/Tribler REST API.md

@@ -33,6 +33,7 @@ curl -X PUT http://localhost:8085/mychannel/rssfeeds/http%3A%2F%2Frssfeed.com%2F
 | GET /mychannel/rssfeeds | Get a list of rss feeds used by your channel |
 | PUT /mychannel/rssfeeds/{feedurl} | Add a rss feed to your channel |
 | DELETE /mychannel/rssfeeds/{feedurl} | Remove a rss feed from your channel |
+| POST /mychannel/recheckfeeds | Recheck all rss feeds in your channel |
 | GET /mychannel/playlists | Get a list of playlists in your channel |
 
 ### Search
@@ -202,6 +203,10 @@ Add a RSS feed to your channel. Returns error 409 (Conflict) if the supplied RSS
 
 Delete a RSS feed from your channel. Returns error 404 if the RSS feed that is being removed does not exist. Note that the rss feed url should be URL-encoded.
 
+## `POST /mychannel/recheckfeeds`
+
+Rechecks all rss feeds in your channel. Returns error 404 if you channel does not exist.
+
 ## `GET /mychannel/playlists`
 
 Returns the playlists in your channel. Returns error 404 if you have not created a channel.