Merge pull request #87 from mozilla/86-check-logo

Performs existence check on Clearbit logo before returning (closes #86).

recommendation/search/classification/__init__.py

@@ -1,12 +1,12 @@
-from recommendation.search.classification.domain import DomainClassifier
 from recommendation.search.classification.embedly import (FaviconClassifier,
                                                           KeyImageClassifier)
+from recommendation.search.classification.logo import LogoClassifier
 from recommendation.search.classification.wikipedia import WikipediaClassifier
 
 
 CLASSIFIERS = [
-    DomainClassifier,
     FaviconClassifier,
     KeyImageClassifier,
+    LogoClassifier,
     WikipediaClassifier,
 ]

recommendation/search/classification/logo.py

@@ -0,0 +1,27 @@
+import requests
+
+from recommendation.search.classification.base import BaseClassifier
+
+
+class LogoClassifier(BaseClassifier):
+    """
+    Classifier that is applied if the returned result is the root address of a
+    domain name. Returns a URL to the logo or None.
+    """
+    type = 'logo'
+
+    def _get_logo(self):
+        return 'https://logo.clearbit.com/%s' % self.url.netloc
+
+    def _logo_exists(self, url):
+        response = requests.get(url)
+        return response.status_code < 400
+
+    def is_match(self, result):
+        return self.url.path in ['/', '']
+
+    def enhance(self):
+        logo = self._get_logo()
+        if not self._logo_exists(logo):
+            return None
+        return logo

recommendation/search/classification/tests/test_logo.py

@@ -0,0 +1,71 @@
+from unittest import TestCase
+from urllib.parse import ParseResult
+
+import responses
+from mock import patch
+from nose.tools import eq_, ok_
+
+from recommendation.search.classification.logo import LogoClassifier
+
+
+DOMAIN = 'www.mozilla.com'
+URL = 'http://%s/' % DOMAIN
+LOGO = 'https://logo.clearbit.com/%s' % DOMAIN
+
+
+class TestLogoClassifier(TestCase):
+    def _result(self, url):
+        return {
+            'url': url
+        }
+
+    def _classifier(self, url):
+        return LogoClassifier(self._result(url))
+
+    def _matches(self, url):
+        return self._classifier(url).is_match(self._result(url))
+
+    def _enhance(self, url):
+        return self._classifier(url).enhance()
+
+    def test_get_logo(self):
+        eq_(self._classifier(URL)._get_logo(), LOGO)
+
+    @responses.activate
+    def test_logo_exists_ok(self):
+        responses.add(responses.GET, LOGO, status=200)
+        eq_(self._classifier(URL)._logo_exists(LOGO), True)
+
+    @responses.activate
+    def test_logo_exists_invalid(self):
+        responses.add(responses.GET, LOGO, status=400)
+        eq_(self._classifier(URL)._logo_exists(LOGO), False)
+
+    @responses.activate
+    def test_logo_exists_missing(self):
+        responses.add(responses.GET, LOGO, status=404)
+        eq_(self._classifier(URL)._logo_exists(LOGO), False)
+
+    @responses.activate
+    def test_init(self):
+        responses.add(responses.GET, LOGO, status=200)
+        instance = self._classifier(URL)
+        ok_(instance.result)
+        eq_(instance.result['url'], URL)
+        ok_(isinstance(instance.url, ParseResult))
+        ok_(isinstance(instance.is_match(instance.result), bool))
+        ok_(isinstance(instance.enhance(), str))
+
+    def test_is_match(self):
+        eq_(self._matches('http://%s' % DOMAIN), True)
+        eq_(self._matches('http://%s/' % DOMAIN), True)
+        eq_(self._matches('http://%s/en_US' % DOMAIN), False)
+        eq_(self._matches('http://%s/en_US/' % DOMAIN), False)
+
+    @patch(('recommendation.search.classification.logo.LogoClassifier.'
+            '_logo_exists'))
+    def test_enhance(self, mock_logo_exists):
+        mock_logo_exists.return_value = False
+        eq_(self._enhance(URL), None)
+        mock_logo_exists.return_value = True
+        eq_(self._enhance(URL), LOGO)

recommendation/search/tests/test_recommendation.py

@@ -3,8 +3,10 @@
 from mock import patch
 from nose.tools import eq_, ok_
 
-from recommendation.search.classification.domain import DomainClassifier
-from recommendation.search.classification.tests.test_domain import DOMAIN
+from recommendation.search.classification.logo import LogoClassifier
+from recommendation.search.classification.embedly import (FaviconClassifier,
+                                                          KeyImageClassifier)
+from recommendation.search.classification.tests.test_logo import DOMAIN
 from recommendation.search.recommendation import SearchRecommendation
 from recommendation.search.suggest.base import BaseSuggestionEngine
 from recommendation.search.suggest.bing import BingSuggestionEngine
@@ -35,15 +37,19 @@ def test_get_query_engine(self):
         engine = self.instance.get_query_engine()
         ok_(issubclass(engine, BaseQueryEngine))
 
-    @patch('recommendation.search.classification.domain.DomainClassifier'
+    @patch('recommendation.search.classification.logo.LogoClassifier'
            '.is_match')
     def test_get_classifiers(self, mock_match):
         mock_match.return_value = True
         classifiers = self.instance.get_classifiers({
             'url': 'http://%s/' % DOMAIN
         })
         eq_(len(classifiers), 3)
-        ok_(isinstance(classifiers[0], DomainClassifier))
+
+        classifier_classes = [FaviconClassifier, KeyImageClassifier,
+                              LogoClassifier]
+        for index, classifier in enumerate(classifiers):
+            ok_(isinstance(classifier, classifier_classes[index]))
         return classifiers
 
     @patch(('recommendation.search.recommendation.SearchRecommendation'
@@ -83,7 +89,7 @@ def test_do_search_get_recommendation(self, mock_suggestions,
         suggestions = BING_RESULTS
         top_suggestion = BING_RESULTS[0]
         result = YAHOO_RESPONSE['bossresponse']['web']['results'][0]
-        classifiers = [DomainClassifier(result)]
+        classifiers = [LogoClassifier(result)]
 
         mock_suggestions.return_value = suggestions
         mock_top_suggestion.return_value = top_suggestion
@@ -94,7 +100,7 @@ def test_do_search_get_recommendation(self, mock_suggestions,
 
         ok_(all([k in search for k in ['enhancements', 'query', 'result']]))
         eq_(list(search['enhancements'].keys()), [c.type for c in classifiers])
-        eq_(search['enhancements']['domain'], classifiers[0].enhance())
+        eq_(search['enhancements']['logo'], classifiers[0].enhance())
         eq_(search['query']['completed'], top_suggestion)
         eq_(search['query']['original'], QUERY)
         eq_(search['result'], result)