diff --git a/apiv2/views.py b/apiv2/views.py index 63c714fb4..6cbd21e3d 100755 --- a/apiv2/views.py +++ b/apiv2/views.py @@ -62,7 +62,7 @@ from comments.models import Comment from geotags.models import GeoTag from ratings.models import SoundRating -from similarity.client import Similarity +from similarity.client import similarity_client from sounds.models import Sound, Pack, License from utils.downloads import download_sounds from utils.filesystem import generate_tree @@ -1201,7 +1201,7 @@ def get_description(cls): def get(self, request, *args, **kwargs): api_logger.info(self.log_message('available_audio_descriptors')) try: - descriptor_names = Similarity.get_descriptor_names() + descriptor_names = similarity_client.get_descriptor_names() del descriptor_names['all'] for key, value in descriptor_names.items(): descriptor_names[key] = [item[1:] for item in value] # remove initial dot from descriptor names diff --git a/freesound/settings.py b/freesound/settings.py index a33569d50..f12883f81 100644 --- a/freesound/settings.py +++ b/freesound/settings.py @@ -394,9 +394,8 @@ # ------------------------------------------------------------------------------- # Similarity client settings -SIMILARITY_ADDRESS = 'similarity' -SIMILARITY_PORT = 8008 -SIMILARITY_INDEXING_SERVER_PORT = 8009 +SIMILARITY_ADDRESS = 'similarity:8008' +INDEXING_SIMILARITY_ADDRESS = 'similarity:8009' # ------------------------------------------------------------------------------- # Tag recommendation client settings diff --git a/general/management/commands/report_index_statuses.py b/general/management/commands/report_index_statuses.py index 69712b117..5722f25ee 100644 --- a/general/management/commands/report_index_statuses.py +++ b/general/management/commands/report_index_statuses.py @@ -23,11 +23,11 @@ from sounds.models import Sound from utils.management_commands import LoggingBaseCommand from utils.search.search_general import get_all_sound_ids_from_solr, delete_sounds_from_solr -from utils.similarity_utilities import Similarity +from similarity.client import similarity_client +import sys console_logger = logging.getLogger('console') - class Command(LoggingBaseCommand): help = "This command checks the status of the solr and gaia index compared to the fs database. Reports about " \ "sounds which are missing in gaia and solr and sounds that are in gaia or solr but not in fs dataset. " \ @@ -52,7 +52,7 @@ def handle(self, *args, **options): # Get ell gaia ids console_logger.info("Getting gaia ids...") - gaia_ids = Similarity.get_all_sound_ids() + gaia_ids = similarity_client.get_all_sound_ids() console_logger.info("Getting freesound db data...") # Get all moderated and processed sound ids @@ -114,7 +114,7 @@ def handle(self, *args, **options): N = len(in_gaia_not_in_fs) for count, sid in enumerate(in_gaia_not_in_fs): console_logger.info('\r\tDeleting sound %i of %i ' % (count+1, N)) - Similarity.delete(sid) + similarity_client.delete(sid) self.log_end({ 'n_sounds_in_db_moderated_processed': len(fs_mp), diff --git a/general/management/commands/similarity_save_index.py b/general/management/commands/similarity_save_index.py index f588abebb..a889c8392 100644 --- a/general/management/commands/similarity_save_index.py +++ b/general/management/commands/similarity_save_index.py @@ -18,8 +18,10 @@ # See AUTHORS file. # -from similarity.client import Similarity from utils.management_commands import LoggingBaseCommand +from similarity.client import indexing_similarity_client, similarity_client +import logging +logger = logging.getLogger("web") class Command(LoggingBaseCommand): @@ -37,7 +39,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): self.log_start() if options['indexing_server']: - Similarity.save_indexing_server() + indexing_similarity_client.save() else: - Similarity.save() + similarity_client.save() self.log_end() diff --git a/general/management/commands/similarity_update.py b/general/management/commands/similarity_update.py index 49bda7de3..760b3fef0 100644 --- a/general/management/commands/similarity_update.py +++ b/general/management/commands/similarity_update.py @@ -22,7 +22,7 @@ import yaml -from similarity.client import Similarity +from similarity.client import indexing_similarity_client, similarity_client from sounds.models import Sound from utils.management_commands import LoggingBaseCommand @@ -106,9 +106,9 @@ def handle(self, *args, **options): try: if options['indexing_server']: - result = Similarity.add_to_indexing_server(sound.id, sound.locations('analysis.statistics.path')) + result = indexing_similarity_client.add(sound.id, sound.locations('analysis.statistics.path')) else: - result = Similarity.add(sound.id, sound.locations('analysis.statistics.path')) + result = similarity_client.add(sound.id, sound.locations('analysis.statistics.path')) sound.set_similarity_state('OK') sound.invalidate_template_caches() console_logger.info("%s (%i of %i)" % (result, count+1, N)) diff --git a/similarity/client/__init__.py b/similarity/client/__init__.py index 49c2c24c2..d22e6fa70 100644 --- a/similarity/client/__init__.py +++ b/similarity/client/__init__.py @@ -19,18 +19,19 @@ # import requests +from django.conf import settings -_URL_ADD_POINT = 'add_point/' -_URL_DELETE_POINT = 'delete_point/' -_URL_GET_DESCRIPTOR_NAMES = 'get_descriptor_names/' -_URL_GET_ALL_SOUND_IDS = 'get_all_point_names/' -_URL_CONTAINS_POINT = 'contains/' -_URL_NNSEARCH = 'nnsearch/' -_URL_API_SEARCH = 'api_search/' -_URL_SOUNDS_DESCRIPTORS = 'get_sounds_descriptors/' -_URL_SAVE = 'save/' -_URL_RELOAD_GAIA_WRAPPER = 'reload_gaia_wrapper/' -_URL_CLEAR_MEMORY = 'clear_memory/' +_URL_ADD_POINT = 'add_point/' +_URL_DELETE_POINT = 'delete_point/' +_URL_GET_DESCRIPTOR_NAMES = 'get_descriptor_names/' +_URL_GET_ALL_SOUND_IDS = 'get_all_point_names/' +_URL_CONTAINS_POINT = 'contains/' +_URL_NNSEARCH = 'nnsearch/' +_URL_API_SEARCH = 'api_search/' +_URL_SOUNDS_DESCRIPTORS = 'get_sounds_descriptors/' +_URL_SAVE = 'save/' +_URL_RELOAD_GAIA_WRAPPER = 'reload_gaia_wrapper/' +_URL_CLEAR_MEMORY = 'clear_memory/' class SimilarityException(Exception): @@ -42,6 +43,7 @@ def __init__(self, *args, **kwargs): def _get_url_as_json(url, data=None, timeout=None): + # TODO: (requests): If no timeout is specified explicitly, requests do not time out. kwargs = dict() if data is not None: kwargs['data'] = data @@ -66,7 +68,6 @@ class Similarity(object): def __init__(self, host): self.base_url = 'http://%s/similarity/' % host - self.base_indexing_server_url = 'http://%s/similarity/' % host def search(self, sound_id, num_results = None, preset = None, offset = None): url = self.base_url + _URL_NNSEARCH + '?' + 'sound_id=' + str(sound_id) @@ -78,7 +79,8 @@ def search(self, sound_id, num_results = None, preset = None, offset = None): url += '&offset=' + str(offset) return _result_or_exception(_get_url_as_json(url)) - def api_search(self, target_type=None, target=None, filter=None, preset=None, metric_descriptor_names=None, num_results=None, offset=None, file=None, in_ids=None): + def api_search(self, target_type=None, target=None, filter=None, preset=None, metric_descriptor_names=None, + num_results=None, offset=None, file=None, in_ids=None): url = self.base_url + _URL_API_SEARCH + '?' if target_type: url += '&target_type=' + str(target_type) @@ -106,10 +108,6 @@ def add(self, sound_id, yaml_path): url = self.base_url + _URL_ADD_POINT + '?' + 'sound_id=' + str(sound_id) + '&location=' + str(yaml_path) return _result_or_exception(_get_url_as_json(url)) - def add_to_indexing_server(self, sound_id, yaml_path): - url = self.base_indexing_server_url + _URL_ADD_POINT + '?' + 'sound_id=' + str(sound_id) + '&location=' + str(yaml_path) - return _result_or_exception(_get_url_as_json(url)) - def get_all_sound_ids(self): url = self.base_url + _URL_GET_ALL_SOUND_IDS return _result_or_exception(_get_url_as_json(url)) @@ -122,32 +120,15 @@ def delete(self, sound_id): url = self.base_url + _URL_DELETE_POINT + '?' + 'sound_id=' + str(sound_id) return _result_or_exception(_get_url_as_json(url)) - def contains(self, sound_id): - url = self.base_url + _URL_CONTAINS_POINT + '?' + 'sound_id=' + str(sound_id) - return _result_or_exception(_get_url_as_json(url)) - - def save(self, filename = None): + def save(self, filename=None): url = self.base_url + _URL_SAVE if filename: url += '?' + 'filename=' + str(filename) return _result_or_exception(_get_url_as_json(url, timeout=60 * 5)) - def save_indexing_server(self, filename = None): - url = self.base_indexing_server_url + _URL_SAVE - if filename: - url += '?' + 'filename=' + str(filename) - return _result_or_exception(_get_url_as_json(url)) - - def clear_indexing_server_memory(self): - url = self.base_indexing_server_url + _URL_CLEAR_MEMORY - return _result_or_exception(_get_url_as_json(url)) - - def reload_indexing_server_gaia_wrapper(self): - url = self.base_indexing_server_url + _URL_RELOAD_GAIA_WRAPPER - return _result_or_exception(_get_url_as_json(url)) - def get_sounds_descriptors(self, sound_ids, descriptor_names=None, normalization=True, only_leaf_descriptors=False): - url = self.base_url + _URL_SOUNDS_DESCRIPTORS + '?' + 'sound_ids=' + ','.join([str(sound_id) for sound_id in sound_ids]) + url = self.base_url + _URL_SOUNDS_DESCRIPTORS + '?' + 'sound_ids=' + ','.join( + [str(sound_id) for sound_id in sound_ids]) if descriptor_names: url += '&descriptor_names=' + ','.join(descriptor_names) if normalization: @@ -156,3 +137,7 @@ def get_sounds_descriptors(self, sound_ids, descriptor_names=None, normalization url += '&only_leaf_descriptors=1' return _result_or_exception(_get_url_as_json(url)) + + +similarity_client = Similarity(settings.SIMILARITY_ADDRESS) +indexing_similarity_client = Similarity(settings.INDEXING_SIMILARITY_ADDRESS) diff --git a/similarity/test/__init__.py b/similarity/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/similarity/test/test_client.py b/similarity/test/test_client.py new file mode 100644 index 000000000..48c646384 --- /dev/null +++ b/similarity/test/test_client.py @@ -0,0 +1,83 @@ +import unittest + +import mock + +from similarity import client + + +class SimilarityClientTest(unittest.TestCase): + + def setUp(self): + self.client = client.Similarity("localhost") + + @mock.patch('similarity.client._get_url_as_json') + def test_search(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + r = self.client.search(1) + + self.assertEqual(r, {"data": "here"}) + # requests.Request('GET', 'url', params=params).prepare().url + get_url.assert_called_with("http://localhost/similarity/nnsearch/?sound_id=1") + get_url.reset_mock() + + self.client.search(1, num_results=10, preset='pr', offset=3) + get_url.assert_called_with("http://localhost/similarity/nnsearch/?sound_id=1&num_results=10&preset=pr&offset=3") + + @mock.patch('similarity.client._get_url_as_json') + def test_api_search(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.api_search() + get_url.assert_called_with("http://localhost/similarity/api_search/?", data=None) + get_url.reset_mock() + + self.client.api_search(target_type=1, target=2, filter=3, preset=4, metric_descriptor_names=5, + num_results=6, offset=7, file='x', in_ids="7,8,9") + get_url.assert_called_with("http://localhost/similarity/api_search/?&target_type=1&target=2&filter=3&preset=4&metric_descriptor_names=5&num_results=6&offset=7&in_ids=7,8,9", data="x") + + @mock.patch('similarity.client._get_url_as_json') + def test_add(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.add(10, 'data.yaml') + get_url.assert_called_with("http://localhost/similarity/add_point/?sound_id=10&location=data.yaml") + + @mock.patch('similarity.client._get_url_as_json') + def test_get_all_sound_ids(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.get_all_sound_ids() + get_url.assert_called_with("http://localhost/similarity/get_all_point_names/") + + @mock.patch('similarity.client._get_url_as_json') + def test_get_descriptor_names(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.get_descriptor_names() + get_url.assert_called_with("http://localhost/similarity/get_descriptor_names/") + + @mock.patch('similarity.client._get_url_as_json') + def test_delete(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.delete(123) + get_url.assert_called_with("http://localhost/similarity/delete_point/?sound_id=123") + + @mock.patch('similarity.client._get_url_as_json') + def test_save(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.save() + get_url.assert_called_with("http://localhost/similarity/save/", timeout=300) + get_url.reset_mock() + + self.client.save('database.db') + get_url.assert_called_with("http://localhost/similarity/save/?filename=database.db", timeout=300) + + @mock.patch('similarity.client._get_url_as_json') + def test_get_sounds_descriptors(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.get_sounds_descriptors([1, 2, 3]) + get_url.assert_called_with("http://localhost/similarity/get_sounds_descriptors/?sound_ids=1,2,3&normalization=1") diff --git a/sounds/tests/test_sound.py b/sounds/tests/test_sound.py index 67fc4bd91..60a6617e3 100644 --- a/sounds/tests/test_sound.py +++ b/sounds/tests/test_sound.py @@ -769,7 +769,7 @@ def test_download(self, sendfile): self.assertInHTML('1 download', resp.content) # Similarity link (cached in display and view) - @mock.patch('general.management.commands.similarity_update.Similarity.add', return_value='Dummy response') + @mock.patch('general.management.commands.similarity_update.similarity_client.add', return_value='Dummy response') def _test_similarity_update(self, cache_keys, check_present, similarity_add): # Default analysis_state is 'PE', but for similarity update it should be 'OK', otherwise sound gets ignored self.sound.analysis_state = 'OK' diff --git a/utils/similarity_utilities.py b/utils/similarity_utilities.py index 0cec71afd..b64e2b723 100644 --- a/utils/similarity_utilities.py +++ b/utils/similarity_utilities.py @@ -24,7 +24,6 @@ from django.conf import settings from django.core.cache import cache -from similarity.client import Similarity from similarity.similarity_settings import PRESETS, DEFAULT_PRESET, SIMILARITY_CACHE_TIME from utils.encryption import create_hash @@ -53,7 +52,7 @@ def get_similar_sounds(sound, preset=DEFAULT_PRESET, num_results=settings.SOUNDS if not similar_sounds: try: - result = Similarity.search(sound.id, preset=preset, num_results=num_results, offset=offset) + result = similarity_client.search(sound.id, preset=preset, num_results=num_results, offset=offset) similar_sounds = [[int(x[0]), float(x[1])] for x in result['results']] count = result['count'] except Exception as e: @@ -92,7 +91,7 @@ def api_search(target=None, filter=None, preset=None, metric_descriptor_names=No if not returned_sounds or target_file: if target_file: - # If there is a file attahced, set the file as the target + # If there is a file attached, set the file as the target target_type = 'file' target = None # If target is given as a file, we set target to None (just in case) else: @@ -102,7 +101,7 @@ def api_search(target=None, filter=None, preset=None, metric_descriptor_names=No else: target_type = 'descriptor_values' - result = Similarity.api_search( + result = similarity_client.api_search( target_type=target_type, target=target, filter=filter, @@ -138,7 +137,7 @@ def get_sounds_descriptors(sound_ids, descriptor_names, normalization=True, only # remove id form list so it is not included in similarity request not_cached_sound_ids.remove(id) try: - returned_data = Similarity.get_sounds_descriptors(not_cached_sound_ids, descriptor_names, normalization, only_leaf_descriptors) + returned_data = similarity_client.get_sounds_descriptors(not_cached_sound_ids, descriptor_names, normalization, only_leaf_descriptors) except Exception as e: web_logger.error('Something wrong occurred with the "get sound descriptors" request (%s)\n\t%s' %\ (e, traceback.format_exc())) @@ -157,7 +156,7 @@ def get_sounds_descriptors(sound_ids, descriptor_names, normalization=True, only def delete_sound_from_gaia(sound): web_logger.info("Deleting sound from gaia with id %d" % sound.id) try: - Similarity.delete(sound.id) + similarity_client.delete(sound.id) except Exception as e: web_logger.error("Could not delete sound from gaia with id %d (%s)" % (sound.id, str(e)))