Skip to content

Commit

Permalink
Use an instance of the similarity client when needed, add tests
Browse files Browse the repository at this point in the history
  • Loading branch information
alastair committed Jun 19, 2020
1 parent f498aad commit d76df58
Show file tree
Hide file tree
Showing 10 changed files with 127 additions and 59 deletions.
4 changes: 2 additions & 2 deletions apiv2/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
from comments.models import Comment
from geotags.models import GeoTag
from ratings.models import SoundRating
from similarity.client import Similarity
from similarity.client import similarity_client
from sounds.models import Sound, Pack, License
from utils.downloads import download_sounds
from utils.filesystem import generate_tree
Expand Down Expand Up @@ -1201,7 +1201,7 @@ def get_description(cls):
def get(self, request, *args, **kwargs):
api_logger.info(self.log_message('available_audio_descriptors'))
try:
descriptor_names = Similarity.get_descriptor_names()
descriptor_names = similarity_client.get_descriptor_names()
del descriptor_names['all']
for key, value in descriptor_names.items():
descriptor_names[key] = [item[1:] for item in value] # remove initial dot from descriptor names
Expand Down
5 changes: 2 additions & 3 deletions freesound/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -394,9 +394,8 @@

# -------------------------------------------------------------------------------
# Similarity client settings
SIMILARITY_ADDRESS = 'similarity'
SIMILARITY_PORT = 8008
SIMILARITY_INDEXING_SERVER_PORT = 8009
SIMILARITY_ADDRESS = 'similarity:8008'
INDEXING_SIMILARITY_ADDRESS = 'similarity:8009'

# -------------------------------------------------------------------------------
# Tag recommendation client settings
Expand Down
8 changes: 4 additions & 4 deletions general/management/commands/report_index_statuses.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@
from sounds.models import Sound
from utils.management_commands import LoggingBaseCommand
from utils.search.search_general import get_all_sound_ids_from_solr, delete_sounds_from_solr
from utils.similarity_utilities import Similarity
from similarity.client import similarity_client
import sys

console_logger = logging.getLogger('console')


class Command(LoggingBaseCommand):
help = "This command checks the status of the solr and gaia index compared to the fs database. Reports about " \
"sounds which are missing in gaia and solr and sounds that are in gaia or solr but not in fs dataset. " \
Expand All @@ -52,7 +52,7 @@ def handle(self, *args, **options):

# Get ell gaia ids
console_logger.info("Getting gaia ids...")
gaia_ids = Similarity.get_all_sound_ids()
gaia_ids = similarity_client.get_all_sound_ids()

console_logger.info("Getting freesound db data...")
# Get all moderated and processed sound ids
Expand Down Expand Up @@ -114,7 +114,7 @@ def handle(self, *args, **options):
N = len(in_gaia_not_in_fs)
for count, sid in enumerate(in_gaia_not_in_fs):
console_logger.info('\r\tDeleting sound %i of %i ' % (count+1, N))
Similarity.delete(sid)
similarity_client.delete(sid)

self.log_end({
'n_sounds_in_db_moderated_processed': len(fs_mp),
Expand Down
8 changes: 5 additions & 3 deletions general/management/commands/similarity_save_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@
# See AUTHORS file.
#

from similarity.client import Similarity
from utils.management_commands import LoggingBaseCommand
from similarity.client import indexing_similarity_client, similarity_client
import logging
logger = logging.getLogger("web")


class Command(LoggingBaseCommand):
Expand All @@ -37,7 +39,7 @@ def add_arguments(self, parser):
def handle(self, *args, **options):
self.log_start()
if options['indexing_server']:
Similarity.save_indexing_server()
indexing_similarity_client.save()
else:
Similarity.save()
similarity_client.save()
self.log_end()
6 changes: 3 additions & 3 deletions general/management/commands/similarity_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

import yaml

from similarity.client import Similarity
from similarity.client import indexing_similarity_client, similarity_client
from sounds.models import Sound
from utils.management_commands import LoggingBaseCommand

Expand Down Expand Up @@ -106,9 +106,9 @@ def handle(self, *args, **options):

try:
if options['indexing_server']:
result = Similarity.add_to_indexing_server(sound.id, sound.locations('analysis.statistics.path'))
result = indexing_similarity_client.add(sound.id, sound.locations('analysis.statistics.path'))
else:
result = Similarity.add(sound.id, sound.locations('analysis.statistics.path'))
result = similarity_client.add(sound.id, sound.locations('analysis.statistics.path'))
sound.set_similarity_state('OK')
sound.invalidate_template_caches()
console_logger.info("%s (%i of %i)" % (result, count+1, N))
Expand Down
59 changes: 22 additions & 37 deletions similarity/client/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,19 @@
#

import requests
from django.conf import settings

_URL_ADD_POINT = 'add_point/'
_URL_DELETE_POINT = 'delete_point/'
_URL_GET_DESCRIPTOR_NAMES = 'get_descriptor_names/'
_URL_GET_ALL_SOUND_IDS = 'get_all_point_names/'
_URL_CONTAINS_POINT = 'contains/'
_URL_NNSEARCH = 'nnsearch/'
_URL_API_SEARCH = 'api_search/'
_URL_SOUNDS_DESCRIPTORS = 'get_sounds_descriptors/'
_URL_SAVE = 'save/'
_URL_RELOAD_GAIA_WRAPPER = 'reload_gaia_wrapper/'
_URL_CLEAR_MEMORY = 'clear_memory/'
_URL_ADD_POINT = 'add_point/'
_URL_DELETE_POINT = 'delete_point/'
_URL_GET_DESCRIPTOR_NAMES = 'get_descriptor_names/'
_URL_GET_ALL_SOUND_IDS = 'get_all_point_names/'
_URL_CONTAINS_POINT = 'contains/'
_URL_NNSEARCH = 'nnsearch/'
_URL_API_SEARCH = 'api_search/'
_URL_SOUNDS_DESCRIPTORS = 'get_sounds_descriptors/'
_URL_SAVE = 'save/'
_URL_RELOAD_GAIA_WRAPPER = 'reload_gaia_wrapper/'
_URL_CLEAR_MEMORY = 'clear_memory/'


class SimilarityException(Exception):
Expand All @@ -42,6 +43,7 @@ def __init__(self, *args, **kwargs):


def _get_url_as_json(url, data=None, timeout=None):
# TODO: (requests): If no timeout is specified explicitly, requests do not time out.
kwargs = dict()
if data is not None:
kwargs['data'] = data
Expand All @@ -66,7 +68,6 @@ class Similarity(object):

def __init__(self, host):
self.base_url = 'http://%s/similarity/' % host
self.base_indexing_server_url = 'http://%s/similarity/' % host

def search(self, sound_id, num_results = None, preset = None, offset = None):
url = self.base_url + _URL_NNSEARCH + '?' + 'sound_id=' + str(sound_id)
Expand All @@ -78,7 +79,8 @@ def search(self, sound_id, num_results = None, preset = None, offset = None):
url += '&offset=' + str(offset)
return _result_or_exception(_get_url_as_json(url))

def api_search(self, target_type=None, target=None, filter=None, preset=None, metric_descriptor_names=None, num_results=None, offset=None, file=None, in_ids=None):
def api_search(self, target_type=None, target=None, filter=None, preset=None, metric_descriptor_names=None,
num_results=None, offset=None, file=None, in_ids=None):
url = self.base_url + _URL_API_SEARCH + '?'
if target_type:
url += '&target_type=' + str(target_type)
Expand Down Expand Up @@ -106,10 +108,6 @@ def add(self, sound_id, yaml_path):
url = self.base_url + _URL_ADD_POINT + '?' + 'sound_id=' + str(sound_id) + '&location=' + str(yaml_path)
return _result_or_exception(_get_url_as_json(url))

def add_to_indexing_server(self, sound_id, yaml_path):
url = self.base_indexing_server_url + _URL_ADD_POINT + '?' + 'sound_id=' + str(sound_id) + '&location=' + str(yaml_path)
return _result_or_exception(_get_url_as_json(url))

def get_all_sound_ids(self):
url = self.base_url + _URL_GET_ALL_SOUND_IDS
return _result_or_exception(_get_url_as_json(url))
Expand All @@ -122,32 +120,15 @@ def delete(self, sound_id):
url = self.base_url + _URL_DELETE_POINT + '?' + 'sound_id=' + str(sound_id)
return _result_or_exception(_get_url_as_json(url))

def contains(self, sound_id):
url = self.base_url + _URL_CONTAINS_POINT + '?' + 'sound_id=' + str(sound_id)
return _result_or_exception(_get_url_as_json(url))

def save(self, filename = None):
def save(self, filename=None):
url = self.base_url + _URL_SAVE
if filename:
url += '?' + 'filename=' + str(filename)
return _result_or_exception(_get_url_as_json(url, timeout=60 * 5))

def save_indexing_server(self, filename = None):
url = self.base_indexing_server_url + _URL_SAVE
if filename:
url += '?' + 'filename=' + str(filename)
return _result_or_exception(_get_url_as_json(url))

def clear_indexing_server_memory(self):
url = self.base_indexing_server_url + _URL_CLEAR_MEMORY
return _result_or_exception(_get_url_as_json(url))

def reload_indexing_server_gaia_wrapper(self):
url = self.base_indexing_server_url + _URL_RELOAD_GAIA_WRAPPER
return _result_or_exception(_get_url_as_json(url))

def get_sounds_descriptors(self, sound_ids, descriptor_names=None, normalization=True, only_leaf_descriptors=False):
url = self.base_url + _URL_SOUNDS_DESCRIPTORS + '?' + 'sound_ids=' + ','.join([str(sound_id) for sound_id in sound_ids])
url = self.base_url + _URL_SOUNDS_DESCRIPTORS + '?' + 'sound_ids=' + ','.join(
[str(sound_id) for sound_id in sound_ids])
if descriptor_names:
url += '&descriptor_names=' + ','.join(descriptor_names)
if normalization:
Expand All @@ -156,3 +137,7 @@ def get_sounds_descriptors(self, sound_ids, descriptor_names=None, normalization
url += '&only_leaf_descriptors=1'

return _result_or_exception(_get_url_as_json(url))


similarity_client = Similarity(settings.SIMILARITY_ADDRESS)
indexing_similarity_client = Similarity(settings.INDEXING_SIMILARITY_ADDRESS)
Empty file added similarity/test/__init__.py
Empty file.
83 changes: 83 additions & 0 deletions similarity/test/test_client.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
import unittest

import mock

from similarity import client


class SimilarityClientTest(unittest.TestCase):

def setUp(self):
self.client = client.Similarity("localhost")

@mock.patch('similarity.client._get_url_as_json')
def test_search(self, get_url):
get_url.return_value = {"error": False, "result": {"data": "here"}}

r = self.client.search(1)

self.assertEqual(r, {"data": "here"})
# requests.Request('GET', 'url', params=params).prepare().url
get_url.assert_called_with("http://localhost/similarity/nnsearch/?sound_id=1")
get_url.reset_mock()

self.client.search(1, num_results=10, preset='pr', offset=3)
get_url.assert_called_with("http://localhost/similarity/nnsearch/?sound_id=1&num_results=10&preset=pr&offset=3")

@mock.patch('similarity.client._get_url_as_json')
def test_api_search(self, get_url):
get_url.return_value = {"error": False, "result": {"data": "here"}}

self.client.api_search()
get_url.assert_called_with("http://localhost/similarity/api_search/?", data=None)
get_url.reset_mock()

self.client.api_search(target_type=1, target=2, filter=3, preset=4, metric_descriptor_names=5,
num_results=6, offset=7, file='x', in_ids="7,8,9")
get_url.assert_called_with("http://localhost/similarity/api_search/?&target_type=1&target=2&filter=3&preset=4&metric_descriptor_names=5&num_results=6&offset=7&in_ids=7,8,9", data="x")

@mock.patch('similarity.client._get_url_as_json')
def test_add(self, get_url):
get_url.return_value = {"error": False, "result": {"data": "here"}}

self.client.add(10, 'data.yaml')
get_url.assert_called_with("http://localhost/similarity/add_point/?sound_id=10&location=data.yaml")

@mock.patch('similarity.client._get_url_as_json')
def test_get_all_sound_ids(self, get_url):
get_url.return_value = {"error": False, "result": {"data": "here"}}

self.client.get_all_sound_ids()
get_url.assert_called_with("http://localhost/similarity/get_all_point_names/")

@mock.patch('similarity.client._get_url_as_json')
def test_get_descriptor_names(self, get_url):
get_url.return_value = {"error": False, "result": {"data": "here"}}

self.client.get_descriptor_names()
get_url.assert_called_with("http://localhost/similarity/get_descriptor_names/")

@mock.patch('similarity.client._get_url_as_json')
def test_delete(self, get_url):
get_url.return_value = {"error": False, "result": {"data": "here"}}

self.client.delete(123)
get_url.assert_called_with("http://localhost/similarity/delete_point/?sound_id=123")

@mock.patch('similarity.client._get_url_as_json')
def test_save(self, get_url):
get_url.return_value = {"error": False, "result": {"data": "here"}}

self.client.save()
get_url.assert_called_with("http://localhost/similarity/save/", timeout=300)
get_url.reset_mock()

self.client.save('database.db')
get_url.assert_called_with("http://localhost/similarity/save/?filename=database.db", timeout=300)

@mock.patch('similarity.client._get_url_as_json')
def test_get_sounds_descriptors(self, get_url):
get_url.return_value = {"error": False, "result": {"data": "here"}}

self.client.get_sounds_descriptors([1, 2, 3])
get_url.assert_called_with("http://localhost/similarity/get_sounds_descriptors/?sound_ids=1,2,3&normalization=1")
2 changes: 1 addition & 1 deletion sounds/tests/test_sound.py
Original file line number Diff line number Diff line change
Expand Up @@ -769,7 +769,7 @@ def test_download(self, sendfile):
self.assertInHTML('1 download', resp.content)

# Similarity link (cached in display and view)
@mock.patch('general.management.commands.similarity_update.Similarity.add', return_value='Dummy response')
@mock.patch('general.management.commands.similarity_update.similarity_client.add', return_value='Dummy response')
def _test_similarity_update(self, cache_keys, check_present, similarity_add):
# Default analysis_state is 'PE', but for similarity update it should be 'OK', otherwise sound gets ignored
self.sound.analysis_state = 'OK'
Expand Down
11 changes: 5 additions & 6 deletions utils/similarity_utilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
from django.conf import settings
from django.core.cache import cache

from similarity.client import Similarity
from similarity.similarity_settings import PRESETS, DEFAULT_PRESET, SIMILARITY_CACHE_TIME
from utils.encryption import create_hash

Expand Down Expand Up @@ -53,7 +52,7 @@ def get_similar_sounds(sound, preset=DEFAULT_PRESET, num_results=settings.SOUNDS

if not similar_sounds:
try:
result = Similarity.search(sound.id, preset=preset, num_results=num_results, offset=offset)
result = similarity_client.search(sound.id, preset=preset, num_results=num_results, offset=offset)
similar_sounds = [[int(x[0]), float(x[1])] for x in result['results']]
count = result['count']
except Exception as e:
Expand Down Expand Up @@ -92,7 +91,7 @@ def api_search(target=None, filter=None, preset=None, metric_descriptor_names=No

if not returned_sounds or target_file:
if target_file:
# If there is a file attahced, set the file as the target
# If there is a file attached, set the file as the target
target_type = 'file'
target = None # If target is given as a file, we set target to None (just in case)
else:
Expand All @@ -102,7 +101,7 @@ def api_search(target=None, filter=None, preset=None, metric_descriptor_names=No
else:
target_type = 'descriptor_values'

result = Similarity.api_search(
result = similarity_client.api_search(
target_type=target_type,
target=target,
filter=filter,
Expand Down Expand Up @@ -138,7 +137,7 @@ def get_sounds_descriptors(sound_ids, descriptor_names, normalization=True, only
# remove id form list so it is not included in similarity request
not_cached_sound_ids.remove(id)
try:
returned_data = Similarity.get_sounds_descriptors(not_cached_sound_ids, descriptor_names, normalization, only_leaf_descriptors)
returned_data = similarity_client.get_sounds_descriptors(not_cached_sound_ids, descriptor_names, normalization, only_leaf_descriptors)
except Exception as e:
web_logger.error('Something wrong occurred with the "get sound descriptors" request (%s)\n\t%s' %\
(e, traceback.format_exc()))
Expand All @@ -157,7 +156,7 @@ def get_sounds_descriptors(sound_ids, descriptor_names, normalization=True, only
def delete_sound_from_gaia(sound):
web_logger.info("Deleting sound from gaia with id %d" % sound.id)
try:
Similarity.delete(sound.id)
similarity_client.delete(sound.id)
except Exception as e:
web_logger.error("Could not delete sound from gaia with id %d (%s)" % (sound.id, str(e)))

Expand Down

0 comments on commit d76df58

Please sign in to comment.