Make similarity client a stateful class

apiv2/views.py

@@ -62,7 +62,7 @@
 from comments.models import Comment
 from geotags.models import GeoTag
 from ratings.models import SoundRating
-from similarity.client import Similarity
+from similarity.client import similarity_client
 from sounds.models import Sound, Pack, License
 from utils.downloads import download_sounds
 from utils.filesystem import generate_tree
@@ -1201,7 +1201,7 @@ def get_description(cls):
     def get(self, request,  *args, **kwargs):
         api_logger.info(self.log_message('available_audio_descriptors'))
         try:
-            descriptor_names = Similarity.get_descriptor_names()
+            descriptor_names = similarity_client.get_descriptor_names()
             del descriptor_names['all']
             for key, value in descriptor_names.items():
                 descriptor_names[key] = [item[1:] for item in value]  # remove initial dot from descriptor names

freesound/settings.py

@@ -394,9 +394,8 @@
 
 # -------------------------------------------------------------------------------
 # Similarity client settings
-SIMILARITY_ADDRESS = 'similarity'
-SIMILARITY_PORT = 8008
-SIMILARITY_INDEXING_SERVER_PORT = 8009
+SIMILARITY_ADDRESS = 'similarity:8008'
+INDEXING_SIMILARITY_ADDRESS = 'similarity:8009'
 
 # -------------------------------------------------------------------------------
 # Tag recommendation client settings

general/management/commands/report_index_statuses.py

@@ -23,11 +23,11 @@
 from sounds.models import Sound
 from utils.management_commands import LoggingBaseCommand
 from utils.search.search_general import get_all_sound_ids_from_solr, delete_sounds_from_solr
-from utils.similarity_utilities import Similarity
+from similarity.client import similarity_client
+import sys
 
 console_logger = logging.getLogger('console')
 
-
 class Command(LoggingBaseCommand):
     help = "This command checks the status of the solr and gaia index compared to the fs database. Reports about " \
            "sounds which are missing in gaia and solr and sounds that are in gaia or solr but not in fs dataset. " \
@@ -52,7 +52,7 @@ def handle(self,  *args, **options):
 
         # Get ell gaia ids
         console_logger.info("Getting gaia ids...")
-        gaia_ids = Similarity.get_all_sound_ids()
+        gaia_ids = similarity_client.get_all_sound_ids()
 
         console_logger.info("Getting freesound db data...")
         # Get all moderated and processed sound ids
@@ -114,7 +114,7 @@ def handle(self,  *args, **options):
                 N = len(in_gaia_not_in_fs)
                 for count, sid in enumerate(in_gaia_not_in_fs):
                     console_logger.info('\r\tDeleting sound %i of %i         ' % (count+1, N))
-                    Similarity.delete(sid)
+                    similarity_client.delete(sid)
 
         self.log_end({
             'n_sounds_in_db_moderated_processed': len(fs_mp),

general/management/commands/similarity_save_index.py

@@ -18,8 +18,10 @@
 #     See AUTHORS file.
 #
 
-from similarity.client import Similarity
 from utils.management_commands import LoggingBaseCommand
+from similarity.client import indexing_similarity_client, similarity_client
+import logging
+logger = logging.getLogger("web")
 
 
 class Command(LoggingBaseCommand):
@@ -37,7 +39,7 @@ def add_arguments(self, parser):
     def handle(self, *args, **options):
         self.log_start()
         if options['indexing_server']:
-            Similarity.save_indexing_server()
+            indexing_similarity_client.save()
         else:
-            Similarity.save()
+            similarity_client.save()
         self.log_end()

general/management/commands/similarity_update.py

@@ -22,7 +22,7 @@
 
 import yaml
 
-from similarity.client import Similarity
+from similarity.client import indexing_similarity_client, similarity_client
 from sounds.models import Sound
 from utils.management_commands import LoggingBaseCommand
 
@@ -106,9 +106,9 @@ def handle(self,  *args, **options):
 
             try:
                 if options['indexing_server']:
-                    result = Similarity.add_to_indeixing_server(sound.id, sound.locations('analysis.statistics.path'))
+                    result = indexing_similarity_client.add(sound.id, sound.locations('analysis.statistics.path'))
                 else:
-                    result = Similarity.add(sound.id, sound.locations('analysis.statistics.path'))
+                    result = similarity_client.add(sound.id, sound.locations('analysis.statistics.path'))
                     sound.set_similarity_state('OK')
                     sound.invalidate_template_caches()
                 console_logger.info("%s (%i of %i)" % (result, count+1, N))

similarity/client/__init__.py

@@ -18,23 +18,20 @@
 #     See AUTHORS file.
 #
 
+import requests
 from django.conf import settings
-import json
-import urllib2
-
-_BASE_URL                     = 'http://%s:%i/similarity/' % (settings.SIMILARITY_ADDRESS, settings.SIMILARITY_PORT)
-_BASE_INDEXING_SERVER_URL     = 'http://%s:%i/similarity/' % (settings.SIMILARITY_ADDRESS, settings.SIMILARITY_INDEXING_SERVER_PORT)
-_URL_ADD_POINT                = 'add_point/'
-_URL_DELETE_POINT             = 'delete_point/'
-_URL_GET_DESCRIPTOR_NAMES     = 'get_descriptor_names/'
-_URL_GET_ALL_SOUND_IDS        = 'get_all_point_names/'
-_URL_CONTAINS_POINT           = 'contains/'
-_URL_NNSEARCH                 = 'nnsearch/'
-_URL_API_SEARCH               = 'api_search/'
-_URL_SOUNDS_DESCRIPTORS       = 'get_sounds_descriptors/'
-_URL_SAVE                     = 'save/'
-_URL_RELOAD_GAIA_WRAPPER      = 'reload_gaia_wrapper/'
-_URL_CLEAR_MEMORY             = 'clear_memory/'
+
+_URL_ADD_POINT = 'add_point/'
+_URL_DELETE_POINT = 'delete_point/'
+_URL_GET_DESCRIPTOR_NAMES = 'get_descriptor_names/'
+_URL_GET_ALL_SOUND_IDS = 'get_all_point_names/'
+_URL_CONTAINS_POINT = 'contains/'
+_URL_NNSEARCH = 'nnsearch/'
+_URL_API_SEARCH = 'api_search/'
+_URL_SOUNDS_DESCRIPTORS = 'get_sounds_descriptors/'
+_URL_SAVE = 'save/'
+_URL_RELOAD_GAIA_WRAPPER = 'reload_gaia_wrapper/'
+_URL_CLEAR_MEMORY = 'clear_memory/'
 
 
 class SimilarityException(Exception):
@@ -46,14 +43,15 @@ def __init__(self, *args, **kwargs):
 
 
 def _get_url_as_json(url, data=None, timeout=None):
+    # TODO: (requests): If no timeout is specified explicitly, requests do not time out.
     kwargs = dict()
     if data is not None:
         kwargs['data'] = data
     if timeout is not None:
         kwargs['timeout'] = timeout
-    f = urllib2.urlopen(url.replace(" ", "%20"), **kwargs)
-    resp = f.read()
-    return json.loads(resp)
+    r = requests.get(url.replace(" ", "%20"), **kwargs)
+    r.raise_for_status()
+    return r.json()
 
 
 def _result_or_exception(result):
@@ -66,11 +64,13 @@ def _result_or_exception(result):
             raise SimilarityException(result['result'], status_code=500)
 
 
-class Similarity():
+class Similarity(object):
+
+    def __init__(self, host):
+        self.base_url = 'http://%s/similarity/' % host
 
-    @classmethod
-    def search(cls, sound_id, num_results = None, preset = None, offset = None):
-        url = _BASE_URL + _URL_NNSEARCH + '?' + 'sound_id=' + str(sound_id)
+    def search(self, sound_id, num_results = None, preset = None, offset = None):
+        url = self.base_url + _URL_NNSEARCH + '?' + 'sound_id=' + str(sound_id)
         if num_results:
             url += '&num_results=' + str(num_results)
         if preset:
@@ -79,9 +79,9 @@ def search(cls, sound_id, num_results = None, preset = None, offset = None):
             url += '&offset=' + str(offset)
         return _result_or_exception(_get_url_as_json(url))
 
-    @classmethod
-    def api_search(cls, target_type=None, target=None, filter=None, preset=None, metric_descriptor_names=None, num_results=None, offset=None, file=None, in_ids=None):
-        url = _BASE_URL + _URL_API_SEARCH + '?'
+    def api_search(self, target_type=None, target=None, filter=None, preset=None, metric_descriptor_names=None,
+                   num_results=None, offset=None, file=None, in_ids=None):
+        url = self.base_url + _URL_API_SEARCH + '?'
         if target_type:
             url += '&target_type=' + str(target_type)
         if target:
@@ -104,63 +104,31 @@ def api_search(cls, target_type=None, target=None, filter=None, preset=None, met
 
         return r
 
-    @classmethod
-    def add(cls, sound_id, yaml_path):
-        url = _BASE_URL + _URL_ADD_POINT + '?' + 'sound_id=' + str(sound_id) + '&location=' + str(yaml_path)
-        return _result_or_exception(_get_url_as_json(url))
-
-    @classmethod
-    def add_to_indeixing_server(cls, sound_id, yaml_path):
-        url = _BASE_INDEXING_SERVER_URL + _URL_ADD_POINT + '?' + 'sound_id=' + str(sound_id) + '&location=' + str(yaml_path)
+    def add(self, sound_id, yaml_path):
+        url = self.base_url + _URL_ADD_POINT + '?' + 'sound_id=' + str(sound_id) + '&location=' + str(yaml_path)
         return _result_or_exception(_get_url_as_json(url))
 
-    @classmethod
-    def get_all_sound_ids(cls):
-        url = _BASE_URL + _URL_GET_ALL_SOUND_IDS
+    def get_all_sound_ids(self):
+        url = self.base_url + _URL_GET_ALL_SOUND_IDS
         return _result_or_exception(_get_url_as_json(url))
 
-    @classmethod
-    def get_descriptor_names(cls):
-        url = _BASE_URL + _URL_GET_DESCRIPTOR_NAMES
+    def get_descriptor_names(self):
+        url = self.base_url + _URL_GET_DESCRIPTOR_NAMES
         return _result_or_exception(_get_url_as_json(url))
 
-    @classmethod
-    def delete(cls, sound_id):
-        url = _BASE_URL + _URL_DELETE_POINT + '?' + 'sound_id=' + str(sound_id)
+    def delete(self, sound_id):
+        url = self.base_url + _URL_DELETE_POINT + '?' + 'sound_id=' + str(sound_id)
         return _result_or_exception(_get_url_as_json(url))
 
-    @classmethod
-    def contains(cls, sound_id):
-        url = _BASE_URL + _URL_CONTAINS_POINT + '?' + 'sound_id=' + str(sound_id)
-        return _result_or_exception(_get_url_as_json(url))
-
-    @classmethod
-    def save(cls, filename = None):
-        url = _BASE_URL + _URL_SAVE
+    def save(self, filename=None):
+        url = self.base_url + _URL_SAVE
         if filename:
             url += '?' + 'filename=' + str(filename)
         return _result_or_exception(_get_url_as_json(url, timeout=60 * 5))
 
-    @classmethod
-    def save_indexing_server(cls, filename = None):
-        url = _BASE_INDEXING_SERVER_URL + _URL_SAVE
-        if filename:
-            url += '?' + 'filename=' + str(filename)
-        return _result_or_exception(_get_url_as_json(url))
-
-    @classmethod
-    def clear_indexing_server_memory(cls):
-        url = _BASE_INDEXING_SERVER_URL + _URL_CLEAR_MEMORY
-        return _result_or_exception(_get_url_as_json(url))
-
-    @classmethod
-    def reload_indexing_server_gaia_wrapper(cls):
-        url = _BASE_INDEXING_SERVER_URL + _URL_RELOAD_GAIA_WRAPPER
-        return _result_or_exception(_get_url_as_json(url))
-
-    @classmethod
-    def get_sounds_descriptors(cls, sound_ids, descriptor_names=None, normalization=True, only_leaf_descriptors=False):
-        url = _BASE_URL + _URL_SOUNDS_DESCRIPTORS + '?' + 'sound_ids=' + ','.join([str(sound_id) for sound_id in sound_ids])
+    def get_sounds_descriptors(self, sound_ids, descriptor_names=None, normalization=True, only_leaf_descriptors=False):
+        url = self.base_url + _URL_SOUNDS_DESCRIPTORS + '?' + 'sound_ids=' + ','.join(
+            [str(sound_id) for sound_id in sound_ids])
         if descriptor_names:
             url += '&descriptor_names=' + ','.join(descriptor_names)
         if normalization:
@@ -169,3 +137,7 @@ def get_sounds_descriptors(cls, sound_ids, descriptor_names=None, normalization=
             url += '&only_leaf_descriptors=1'
 
         return _result_or_exception(_get_url_as_json(url))
+
+
+similarity_client = Similarity(settings.SIMILARITY_ADDRESS)
+indexing_similarity_client = Similarity(settings.INDEXING_SIMILARITY_ADDRESS)

similarity/test/test_client.py

@@ -0,0 +1,83 @@
+import unittest
+
+import mock
+
+from similarity import client
+
+
+class SimilarityClientTest(unittest.TestCase):
+
+    def setUp(self):
+        self.client = client.Similarity("localhost")
+
+    @mock.patch('similarity.client._get_url_as_json')
+    def test_search(self, get_url):
+        get_url.return_value = {"error": False, "result": {"data": "here"}}
+
+        r = self.client.search(1)
+
+        self.assertEqual(r, {"data": "here"})
+        # requests.Request('GET', 'url', params=params).prepare().url
+        get_url.assert_called_with("http://localhost/similarity/nnsearch/?sound_id=1")
+        get_url.reset_mock()
+
+        self.client.search(1, num_results=10, preset='pr', offset=3)
+        get_url.assert_called_with("http://localhost/similarity/nnsearch/?sound_id=1&num_results=10&preset=pr&offset=3")
+
+    @mock.patch('similarity.client._get_url_as_json')
+    def test_api_search(self, get_url):
+        get_url.return_value = {"error": False, "result": {"data": "here"}}
+
+        self.client.api_search()
+        get_url.assert_called_with("http://localhost/similarity/api_search/?", data=None)
+        get_url.reset_mock()
+
+        self.client.api_search(target_type=1, target=2, filter=3, preset=4, metric_descriptor_names=5,
+                               num_results=6, offset=7, file='x', in_ids="7,8,9")
+        get_url.assert_called_with("http://localhost/similarity/api_search/?&target_type=1&target=2&filter=3&preset=4&metric_descriptor_names=5&num_results=6&offset=7&in_ids=7,8,9", data="x")
+
+    @mock.patch('similarity.client._get_url_as_json')
+    def test_add(self, get_url):
+        get_url.return_value = {"error": False, "result": {"data": "here"}}
+
+        self.client.add(10, 'data.yaml')
+        get_url.assert_called_with("http://localhost/similarity/add_point/?sound_id=10&location=data.yaml")
+
+    @mock.patch('similarity.client._get_url_as_json')
+    def test_get_all_sound_ids(self, get_url):
+        get_url.return_value = {"error": False, "result": {"data": "here"}}
+
+        self.client.get_all_sound_ids()
+        get_url.assert_called_with("http://localhost/similarity/get_all_point_names/")
+
+    @mock.patch('similarity.client._get_url_as_json')
+    def test_get_descriptor_names(self, get_url):
+        get_url.return_value = {"error": False, "result": {"data": "here"}}
+
+        self.client.get_descriptor_names()
+        get_url.assert_called_with("http://localhost/similarity/get_descriptor_names/")
+
+    @mock.patch('similarity.client._get_url_as_json')
+    def test_delete(self, get_url):
+        get_url.return_value = {"error": False, "result": {"data": "here"}}
+
+        self.client.delete(123)
+        get_url.assert_called_with("http://localhost/similarity/delete_point/?sound_id=123")
+
+    @mock.patch('similarity.client._get_url_as_json')
+    def test_save(self, get_url):
+        get_url.return_value = {"error": False, "result": {"data": "here"}}
+
+        self.client.save()
+        get_url.assert_called_with("http://localhost/similarity/save/", timeout=300)
+        get_url.reset_mock()
+
+        self.client.save('database.db')
+        get_url.assert_called_with("http://localhost/similarity/save/?filename=database.db", timeout=300)
+
+    @mock.patch('similarity.client._get_url_as_json')
+    def test_get_sounds_descriptors(self, get_url):
+        get_url.return_value = {"error": False, "result": {"data": "here"}}
+
+        self.client.get_sounds_descriptors([1, 2, 3])
+        get_url.assert_called_with("http://localhost/similarity/get_sounds_descriptors/?sound_ids=1,2,3&normalization=1")

sounds/tests/test_sound.py

@@ -769,7 +769,7 @@ def test_download(self, sendfile):
         self.assertInHTML('1 download', resp.content)
 
     # Similarity link (cached in display and view)
-    @mock.patch('general.management.commands.similarity_update.Similarity.add', return_value='Dummy response')
+    @mock.patch('general.management.commands.similarity_update.similarity_client.add', return_value='Dummy response')
     def _test_similarity_update(self, cache_keys, check_present, similarity_add):
         # Default analysis_state is 'PE', but for similarity update it should be 'OK', otherwise sound gets ignored
         self.sound.analysis_state = 'OK'