diff --git a/jsonrepo/__init__.py b/jsonrepo/__init__.py index 8e07bb3..d457f12 100644 --- a/jsonrepo/__init__.py +++ b/jsonrepo/__init__.py @@ -5,5 +5,5 @@ Copyright (C) 2017 Romary Dupuis """ -__version__ = '0.1.3' +__version__ = '0.1.4' __all__ = ['repository', 'mixin', 'record'] diff --git a/jsonrepo/backend.py b/jsonrepo/backend.py index b363a08..ffd27db 100644 --- a/jsonrepo/backend.py +++ b/jsonrepo/backend.py @@ -8,8 +8,9 @@ class Backend(object): """ Basic backend class """ - def __init__(self, prefix): + def __init__(self, prefix, secondary_indexes): self._prefix = prefix + self._secondary_indexes = secondary_indexes def prefixed(self, key): """ build a prefixed key """ @@ -26,3 +27,6 @@ def delete(self, key, sort_key): def history(self, key, _from='-', _to='+', _desc=True): raise NotImplementedError + + def find(self, index, value): + raise NotImplementedError diff --git a/jsonrepo/backends/dynamodb.py b/jsonrepo/backends/dynamodb.py index 4310095..d2f3028 100644 --- a/jsonrepo/backends/dynamodb.py +++ b/jsonrepo/backends/dynamodb.py @@ -1,3 +1,4 @@ +import json import boto3 from boto3.dynamodb.conditions import Key from loggingmixin import LoggingMixin @@ -9,10 +10,13 @@ class DynamoDBBackend(Backend, LoggingMixin): """ Backend based on DynamoDB """ - def __init__(self, prefix, key, sort_key): + + def __init__(self, prefix, key, sort_key, + secondary_indexes): self._prefix = prefix self._key = key self._sort_key = sort_key + self._secondary_indexes = secondary_indexes @memoized def dynamodb_server(self): @@ -23,6 +27,7 @@ def get(self, key, sort_key): self.logger.debug('Storage - get {}'.format(self.prefixed(key))) res = self.dynamodb_server.get_item(Key={ self._key: self.prefixed(key), + self._sort_key: sort_key }) if 'Item' in res and 'value' in res['Item']: return res['Item']['value'] @@ -35,6 +40,12 @@ def set(self, key, sort_key, value): self._key: self.prefixed(key), 'value': value } + obj = json.loads(value) + for index in self._secondary_indexes: + if obj.get(index, None) not in ['', None]: + item.update({ + index: obj[index] + }) if sort_key is not None: item.update({ self._sort_key: sort_key @@ -44,7 +55,8 @@ def set(self, key, sort_key, value): def delete(self, key, sort_key): self.logger.debug('Storage - delete {}'.format(self.prefixed(key))) return self.dynamodb_server.delete_item(Key={ - self._key: self.prefixed(key) + self._key: self.prefixed(key), + self._sort_key: sort_key }) def history(self, key, _from='-', _to='+', _desc=True): @@ -83,3 +95,14 @@ def latest(self, key): return response['Items'][0]['value'] else: return None + + def find(self, index, value): + res = self.dynamodb_server.query( + KeyConditionExpression=Key(index).eq(value), + IndexName='{}-index'.format(index) + ) + self.logger.debug('{}'.format(res)) + return { + 'count': res['Count'], + 'items': [item['value'] for item in res['Items']] + } diff --git a/jsonrepo/backends/memory.py b/jsonrepo/backends/memory.py index eac1c11..1491a80 100644 --- a/jsonrepo/backends/memory.py +++ b/jsonrepo/backends/memory.py @@ -4,6 +4,7 @@ Author: Romary Dupuis Copyright (C) 2017 Romary Dupuis """ +import json from loggingmixin import LoggingMixin from awesomedecorators import memoized from jsonrepo.backend import Backend @@ -28,10 +29,19 @@ def get(self, key, sort_key): return None return self.cache[key] + def init_secondary_indexes(self): + if 'secondary_indexes' not in self.cache: + self.cache['secondary_indexes'] = {} + + def init_secondary_index(self, index): + if 'secondary_indexes' not in self.cache: + return + if index not in self.cache['secondary_indexes']: + self.cache['secondary_indexes'][index] = {} + def set(self, key, sort_key, value): primary_key = key key = self.prefixed('{}:{}'.format(key, sort_key)) - """ Set an element in dictionary """ self.logger.debug('Storage - set value {} for {}'.format(value, key)) if (self.prefixed(primary_key) not in self.cache.keys() and sort_key is not None): @@ -40,6 +50,23 @@ def set(self, key, sort_key, value): self.cache[self.prefixed(primary_key)].append(sort_key) self.cache[self.prefixed(primary_key)] = sorted( self.cache[self.prefixed(primary_key)]) + p_value = {} + if key in self.cache.keys(): + p_value = json.loads(self.cache[key]) + for index in self._secondary_indexes: + if index in p_value.keys(): + self.cache['secondary_indexes'][index][p_value[index]].remove( + key) + obj = json.loads(value) + if index in obj.keys(): + self.init_secondary_indexes() + self.init_secondary_index(index) + if obj[index] not in self.cache['secondary_indexes'][index]: + self.cache['secondary_indexes'][index][obj[index]] = [ + key] + else: + self.cache['secondary_indexes'][index][obj[index]].append( + key) self.cache[key] = value return self.cache[key] is value @@ -50,7 +77,13 @@ def delete(self, key, sort_key): self.logger.debug('Storage - delete {}'.format(key)) if sort_key is not None: self.cache[self.prefixed(primary_key)].remove(sort_key) + for index in self._secondary_indexes: + obj = json.loads(self.cache[key]) + if index in obj.keys(): + self.cache['secondary_indexes'][index][obj[index]].remove( + key) del(self.cache[key]) + return True def history(self, key, _from='-', _to='+', _desc=True): if _from == '-': @@ -76,3 +109,15 @@ def latest(self, key): if len(self.cache[self.prefixed(key)]) == 0: return None return self.get(key, self.cache[self.prefixed(key)][-1]) + + def find(self, index, value): + if ('secondary_indexes' in self.cache and + index in self.cache['secondary_indexes'] and + value in self.cache['secondary_indexes'][index]): + res = [self.cache[item] + for item in self.cache['secondary_indexes'][index][value]] + return { + 'count': len(res), + 'items': res + } + return {'count': 0, 'items': []} diff --git a/jsonrepo/backends/redis.py b/jsonrepo/backends/redis.py index 5bcbfcd..5f24cb1 100644 --- a/jsonrepo/backends/redis.py +++ b/jsonrepo/backends/redis.py @@ -5,6 +5,7 @@ Copyright (C) 2017 Romary Dupuis """ import os +import json import redis from loggingmixin import LoggingMixin from awesomedecorators import memoized @@ -46,6 +47,27 @@ def set(self, key, sort_key, value): ))) if sort_key is not None: self.redis_server.zadd(self.prefixed(key), 0.0, sort_key) + prev_value = self.get(key, sort_key) + prev_obj = None + if prev_value is not None: + prev_obj = json.loads(prev_value) + for sec_index in self._secondary_indexes: + if (prev_obj is not None and + sec_index in prev_obj.keys()): + self.redis_server.srem( + self.prefixed('secondary_indexes:{}:{}'.format( + sec_index, prev_obj[sec_index] + )), + self.prefixed('{}:{}'.format(key, sort_key)) + ) + obj = json.loads(value) + if sec_index in obj.keys(): + self.redis_server.sadd( + self.prefixed('secondary_indexes:{}:{}'.format( + sec_index, obj[sec_index] + )), + self.prefixed('{}:{}'.format(key, sort_key)) + ) return self.redis_server.set(self.prefixed( '{}:{}'.format(key, sort_key)), value) @@ -54,6 +76,19 @@ def delete(self, key, sort_key): '{}:{}'.format(key, sort_key)))) if sort_key is not None: self.redis_server.zrem(self.prefixed(key), sort_key) + prev_value = self.get(key, sort_key) + prev_obj = None + if prev_value is not None: + prev_obj = json.loads(prev_value) + for sec_index in self._secondary_indexes: + if (prev_obj is not None and + sec_index in prev_obj.keys()): + self.redis_server.srem( + self.prefixed('secondary_indexes:{}:{}'.format( + sec_index, prev_obj[sec_index] + )), + self.prefixed('{}:{}'.format(key, sort_key)) + ) return self.redis_server.delete(self.prefixed( '{}:{}'.format(key, sort_key))) @@ -87,3 +122,17 @@ def latest(self, key): def transaction(self, func, *watchs, **params): return self.redis_server.transaction(func, *watchs, **params) + + def find(self, index, value): + keys = self.redis_server.smembers( + self.prefixed('secondary_indexes:{}:{}'.format( + index, value + )) + ) + if keys is not None: + return { + 'count': len(keys), + 'items': [self.redis_server.get(key.decode('utf-8')) + for key in keys] + } + return {'count': 0, 'items': []} diff --git a/jsonrepo/mixin.py b/jsonrepo/mixin.py index 9457556..afaae2c 100644 --- a/jsonrepo/mixin.py +++ b/jsonrepo/mixin.py @@ -20,7 +20,8 @@ def storage(self): Instantiates and returns a storage instance """ if self.backend == 'redis': - return RedisBackend(self.prefix) + return RedisBackend(self.prefix, self.secondary_indexes) if self.backend == 'dynamodb': - return DynamoDBBackend(self.prefix, self.key, self.sort_key) - return DictBackend(self.prefix) + return DynamoDBBackend(self.prefix, self.key, self.sort_key, + self.secondary_indexes) + return DictBackend(self.prefix, self.secondary_indexes) diff --git a/jsonrepo/record.py b/jsonrepo/record.py index 528ec58..53ce9e1 100644 --- a/jsonrepo/record.py +++ b/jsonrepo/record.py @@ -32,6 +32,7 @@ class Record(object): """ Definition of a JSON serializable record for a repository """ + @classmethod def from_json(cls, json_dump): """ JSON deserialization diff --git a/jsonrepo/repository.py b/jsonrepo/repository.py index 88ddd0b..0d94c5c 100644 --- a/jsonrepo/repository.py +++ b/jsonrepo/repository.py @@ -21,6 +21,7 @@ class Repository(StorageMixin, LoggingMixin): klass = Record key = '' sort_key = '' + secondary_indexes = [] def storage_get(self, key, sort_key): return self.storage.get(key, sort_key) @@ -42,6 +43,12 @@ def save(self, key, sort_key, _object): """ return self.storage.set(key, sort_key, _object.to_json()) + def delete(self, key, sort_key): + """ + Saves a context object + """ + return self.storage.delete(key, sort_key) + def history(self, key, _from='-', _to='+', _desc=True): """ Retrives a list of records according to a datetime range @@ -54,3 +61,14 @@ def latest(self, key): Get the most recent record for a specific key """ return self.klass.from_json(self.storage.latest(key)) + + def find(self, index, value): + """ + Find record according to the value of a secondary index + """ + res = self.storage.find(index, value) + return { + 'count': res['count'], + 'items': [self.klass.from_json(_object) + for _object in res['items']] + } diff --git a/tests/tests.py b/tests/tests.py index 953da26..40bc6bd 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -2,10 +2,11 @@ Test Json repository Author: Romary Dupuis """ - +import os import unittest from collections import namedtuple import datetime +import time from jsonrepo.repository import Repository from jsonrepo.record import NamedtupleRecord @@ -15,7 +16,9 @@ except ImportError: import mock -fields = ['title', 'content'] +fields = ['title', 'content', 'date', 'ttl'] + +os.environ['AWS_DEFAULT_REGION'] = 'eu-west-1' class Message(namedtuple('Message', fields), @@ -26,12 +29,21 @@ class Message(namedtuple('Message', fields), def __new__(cls, **kwargs): default = {f: None for f in fields} default.update(kwargs) + if default['date'] is None: + default['date'] = datetime.datetime.utcnow().isoformat()[:-3] + if default['ttl'] is None: + default['ttl'] = int(time.mktime(time.strptime( + default['date'], "%Y-%m-%dT%H:%M:%S.%f"))) return super(Message, cls).__new__(cls, **default) class MyRepository(Repository): + backend = 'redis' prefix = 'example' klass = Message + secondary_indexes = ['title', 'ttl'] + key = 'key' + sort_key = 'date' class RepositoryDictTests(unittest.TestCase): @@ -57,9 +69,10 @@ def test_save_record(self): msg = Message(title='This is a title', content='and this is the content') now = datetime.datetime.utcnow().isoformat()[:-3] - res = my_repository.save('message-one', + res = my_repository.save('test_save_record', now, msg) self.assertTrue(res) + my_repository.delete('test_save_record', now) def test_get_record(self): """ @@ -69,16 +82,46 @@ def test_get_record(self): msg = Message(title='This is a title', content='and this is the content') now = datetime.datetime.utcnow().isoformat()[:-3] - res = my_repository.save('message-one', + res = my_repository.save('test_get_record', now, msg) self.assertTrue(res) - record = my_repository.get('message-one', now) + record = my_repository.get('test_get_record', now) self.assertEqual(record.title, msg.title) self.assertEqual(record.content, msg.content) + my_repository.delete('test_get_record', now) + + def test_delete_record(self): + """ + Assert delete a record + """ + my_repository = MyRepository() + msg = Message(title='This is a title', + content='and this is the content') + now = datetime.datetime.utcnow().isoformat()[:-3] + my_repository.save('test_delete_record', + now, msg) + my_repository.delete('test_delete_record', now) + result = my_repository.find('title', 'This is a title') + self.assertEqual(result['count'], 0) + + def test_find_record(self): + """ + Assert find a record + """ + my_repository = MyRepository() + msg = Message(title='This is a title', + content='and this is the content') + now = datetime.datetime.utcnow().isoformat()[:-3] + my_repository.save('test_find_record', + now, msg) + result = my_repository.find('title', 'This is a title') + self.assertEqual(result['count'], 1) + self.assertEqual(result['items'][0].content, msg.content) + my_repository.delete('test_find_record', now) def test_latest_record(self): """ - Assert that a record is deleted + Assert that latest record """ my_repository = MyRepository() msg1 = Message(title='Message1', @@ -86,17 +129,20 @@ def test_latest_record(self): msg2 = Message(title='Message2', content='and this is the content') now1 = datetime.datetime.utcnow().isoformat()[:-3] - my_repository.save('user-message', + my_repository.save('test_latest_record', now1, msg1) + time.sleep(1) now2 = datetime.datetime.utcnow().isoformat()[:-3] - my_repository.save('user-message', + my_repository.save('test_latest_record', now2, msg2) - record = my_repository.latest('user-message') + record = my_repository.latest('test_latest_record') self.assertEqual(record.title, msg2.title) + my_repository.delete('test_latest_record', now1) + my_repository.delete('test_latest_record', now2) def test_history(self): """ - Assert that a record is deleted + Assert history of records """ my_repository = MyRepository() msg1 = Message(title='Message1', @@ -104,10 +150,13 @@ def test_history(self): msg2 = Message(title='Message2', content='and this is the content') now1 = datetime.datetime.utcnow().isoformat()[:-3] - my_repository.save('user-message', + my_repository.save('test_history', now1, msg1) + time.sleep(1) now2 = datetime.datetime.utcnow().isoformat()[:-3] - my_repository.save('user-message', + my_repository.save('test_history', now2, msg2) - record = my_repository.latest('user-message') + record = my_repository.latest('test_history') self.assertEqual(record.title, msg2.title) + my_repository.delete('test_history', now1) + my_repository.delete('test_history', now2)