From 0634fca0cbe4bbf867d2226a815861b502634a21 Mon Sep 17 00:00:00 2001 From: Ariel Date: Thu, 6 Jun 2019 13:17:00 +0300 Subject: [PATCH 01/17] bloom client + tests --- .gitignore | 104 ++++++++ LICENSE | 29 +++ redisbloom/__init__.py | 0 redisbloom/client.py | 523 +++++++++++++++++++++++++++++++++++++++++ requirements.txt | 5 + rltest_commands.py | 187 +++++++++++++++ setup.cfg | 2 + 7 files changed, 850 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 redisbloom/__init__.py create mode 100644 redisbloom/client.py create mode 100644 requirements.txt create mode 100644 rltest_commands.py create mode 100644 setup.cfg diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..894a44c --- /dev/null +++ b/.gitignore @@ -0,0 +1,104 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ae6326f --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2019, RedisBloom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/redisbloom/__init__.py b/redisbloom/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/redisbloom/client.py b/redisbloom/client.py new file mode 100644 index 0000000..1952adb --- /dev/null +++ b/redisbloom/client.py @@ -0,0 +1,523 @@ +import six +import redis +from redis import Redis, RedisError +from redis.client import bool_ok +from redis._compat import (long, nativestr) +from redis.exceptions import DataError + +''' +class TSInfo(object): + chunkCount = None + labels = [] + lastTimeStamp = None + maxSamplesPerChunk = None + retentionSecs = None + rules = [] + + def __init__(self, args): + self.chunkCount = args['chunkCount'] + self.labels = list_to_dict(args['labels']) + self.lastTimeStamp = args['lastTimestamp'] + self.maxSamplesPerChunk = args['maxSamplesPerChunk'] + self.retentionSecs = args['retentionSecs'] + self.rules = args['rules'] +''' + + +class CMSInfo(object): + width = None + depth = None + count = None + + def __init__(self, args): + response = dict(zip(map(nativestr, args[::2]), args[1::2])) + self.width = response['width'] + self.depth = response['depth'] + self.count = response['count'] + +class TopKInfo(object): + k = None + width = None + depth = None + decay = None + + def __init__(self, args): + response = dict(zip(map(nativestr, args[::2]), args[1::2])) + self.k = response['k'] + self.width = response['width'] + self.depth = response['depth'] + self.decay = response['decay'] + +def spaceHolder(response): + return response + +def list_to_dict(aList): + return {nativestr(aList[i][0]):nativestr(aList[i][1]) + for i in range(len(aList))} + +def parse_range(response): + return [tuple((l[0], l[1].decode())) for l in response] + +def parse_m_range(response): + res = [] + for item in response: + res.append({ nativestr(item[0]) : [list_to_dict(item[1]), + parse_range(item[2])]}) + return res + +def parse_m_get(response): + res = [] + for item in response: + res.append({ nativestr(item[0]) : [list_to_dict(item[1]), + item[2], nativestr(item[3])]}) + return res + +def parse_info(response): + res = dict(zip(map(nativestr, response[::2]), response[1::2])) + info = TopKInfo(res) + return info + +def parseToList(response): + res = [] + for item in response: + res.append(nativestr(item)) + return res + +class Client(Redis): #changed from StrictRedis + """ + This class subclasses redis-py's `Redis` and implements + RedisBloom's commands. + The client allows to interact with RedisBloom and use all of + it's functionality. + Prefix is according to the DS used. + - BF for Bloom Filter + - CF for Cuckoo Filter + - CMS for Count-Min Sketch + - TopK for TopK Data Structure + """ + + MODULE_INFO = { + 'name': 'RedisBloom', + 'ver': '0.1.0' + } + + BF_RESERVE = 'BF.RESERVE' + BF_ADD = 'BF.ADD' + BF_MADD = 'BF.MADD' + BF_INSERT = 'BF.INSERT' + BF_EXISTS = 'BF.EXISTS' + BF_MEXISTS = 'BF.MEXISTS' + BF_SCANDUMP = 'BF.SCANDUMP' + BF_LOADCHUNK = 'BF.LOADCHUNK' + + CF_RESERVE = 'CF.RESERVE' + CF_ADD = 'CF.ADD' + CF_ADDNX = 'CF.ADDNX' + CF_INSERT = 'CF.INSERT' + CF_INSERTNX = 'CF.INSERTNX' + CF_EXISTS = 'CF.EXISTS' + CF_DEL = 'CF.DEL' + CF_COUNT = 'CF.COUNT' + CF_SCANDUMP = 'CF.SCANDUMP' + CF_LOADDUMP = 'CF.LOADDUMP' + + CMS_INITBYDIM = 'CMS.INITBYDIM' + CMS_INITBYPROB = 'CMS.INITBYPROB' + CMS_INCRBY = 'CMS.INCRBY' + CMS_QUERY = 'CMS.QUERY' + CMS_MERGE = 'CMS.MERGE' + CMS_INFO = 'CMS.INFO' + + TOPK_RESERVE = 'TOPK.RESERVE' + TOPK_ADD = 'TOPK.ADD' + TOPK_QUERY = 'TOPK.QUERY' + TOPK_COUNT = 'TOPK.COUNT' + TOPK_LIST = 'TOPK.LIST' + TOPK_INFO = 'TOPK.INFO' + + + CREATE_CMD = 'TS.CREATE' + ALTER_CMD = 'TS.ALTER' + ADD_CMD = 'TS.ADD' + INCRBY_CMD = 'TS.INCRBY' + DECRBY_CMD = 'TS.DECRBY' + CREATERULE_CMD = 'TS.CREATERULE' + DELETERULE_CMD = 'TS.DELETERULE' + RANGE_CMD = 'TS.RANGE' + MRANGE_CMD = 'TS.MRANGE' + GET_CMD = 'TS.GET' + MGET_CMD = 'TS.MGET' + INFO_CMD = 'TS.INFO' + QUERYINDEX_CMD = 'TS.QUERYINDEX' + + def __init__(self, *args, **kwargs): + """ + Creates a new RedisBloom client. + """ + Redis.__init__(self, *args, **kwargs) + + # Set the module commands' callbacks + MODULE_CALLBACKS = { + self.BF_RESERVE : bool_ok, + #self.BF_ADD : spaceHolder, + #self.BF_MADD : spaceHolder, + self.BF_INSERT : spaceHolder, + self.BF_EXISTS : spaceHolder, + self.BF_MEXISTS : spaceHolder, + self.BF_SCANDUMP : spaceHolder, + self.BF_LOADCHUNK : spaceHolder, + + self.CF_RESERVE : bool_ok, + self.CF_ADD : spaceHolder, + self.CF_ADDNX : spaceHolder, + self.CF_INSERT : spaceHolder, + self.CF_INSERTNX : spaceHolder, + self.CF_EXISTS : spaceHolder, + self.CF_DEL : spaceHolder, + self.CF_COUNT : spaceHolder, + self.CF_SCANDUMP : spaceHolder, + self.CF_LOADDUMP : spaceHolder, + + self.CMS_INITBYDIM : bool_ok, + self.CMS_INITBYPROB : bool_ok, + self.CMS_INCRBY : bool_ok, + self.CMS_QUERY : spaceHolder, + self.CMS_MERGE : bool_ok, + self.CMS_INFO : CMSInfo, + + self.TOPK_RESERVE : bool_ok, + self.TOPK_ADD : bool_ok, + self.TOPK_QUERY : spaceHolder, + self.TOPK_COUNT : spaceHolder, + self.TOPK_LIST : parseToList, + self.TOPK_INFO : TopKInfo, + } + for k, v in six.iteritems(MODULE_CALLBACKS): + self.set_response_callback(k, v) + + @staticmethod + def appendItems(params, items): + params.extend(['ITEMS', items]) + + @staticmethod + def appendError(params, error): + if error is not None: + params.extend(['ERROR', error]) + + @staticmethod + def appendCapacity(params, capacity): + if capacity is not None: + params.extend(['CAPACITY', capacity]) + + @staticmethod + def appendWeights(params, weights): + if len(weights) > 0: + params.append('WEIGHTS') + params += weights + + @staticmethod + def appendNoCreate(params, noCreate): + if noCreate is not None: + params.extend(['NOCREATE']) + + @staticmethod + def appendItemsAndIncrements(params, items, increments): + for i in range(len(items)): + params.append(items[i]) + params.append(increments[i]) + + +################## Bloom Filter Functions ###################### + + def bfCreate(self, key, errorRate, capacity): + """ + Creates a new Bloom Filter ``key`` with desired probability of false + positives ``errorRate`` expected entries to be inserted as ``capacity``. + """ + params = [key, errorRate, capacity] + + return self.execute_command(self.BF_RESERVE, *params) + + def bfAdd(self, key, item): + """ + Adds to a Bloom Filter ``key`` an ``item``. + """ + params = [key, item] + + return self.execute_command(self.BF_ADD, *params) + + def bfMAdd(self, key, *items): + """ + Adds to a Bloom Filter ``key`` multiple ``items``. + """ + params = [key] + params += items + + return self.execute_command(self.BF_MADD, *params) + + def bfInsert(self, key, items, capacity=None, error=None, noCreate=None, ): + """ + Adds to a Bloom Filter ``key`` multiple ``items``. If ``nocreate`` + remain ``None`` and ``key does not exist, a new Bloom Filter ``key`` + will be created with desired probability of false positives ``errorRate`` + and expected entries to be inserted as ``size``. + """ + params = [key] + self.appendCapacity(params, capacity) + self.appendError(params, error) + self.appendNoCreate(params, noCreate) + params.extend(['ITEMS']) + params += items + + return self.execute_command(self.BF_INSERT, *params) + + def bfExists(self, key, item): + """ + Checks whether an ``item`` exists in Bloom Filter ``key``. + """ + params = [key, item] + + return self.execute_command(self.BF_EXISTS, *params) + + def bfMExists(self, key, *items): + """ + Checks whether ``items`` exist in Bloom Filter ``key``. + """ + params = [key] + params += items + + return self.execute_command(self.BF_MEXISTS, *params) + + def bfScandump(self, key, iter): + """ + Begins an incremental save of the bloom filter ``key``. This is useful + for large bloom filters which cannot fit into the normal SAVE + and RESTORE model. + The first time this command is called, the value of ``iter`` should be 0. + This command will return successive (iter, data) pairs until + (0, NULL) to indicate completion. + """ + params = [key, iter] + + return self.execute_command(self.BF_SCANDUMP, *params) + + def bfLoadChunk(self, key, iter, data): + """ + Restores a filter previously saved using SCANDUMP. See the SCANDUMP + command for example usage. + This command will overwrite any bloom filter stored under key. + Ensure that the bloom filter will not be modified between invocations. + """ + params = [key, iter, data] + + return self.execute_command(self.BF_LOADCHUNK, *params) + + +################## Cuckoo Filter Functions ###################### + + def cfCreate(self, key, capacity): + """ + Creates a new Cuckoo Filter ``key`` with desired probability of false + positives ``errorRate`` expected entries to be inserted as ``size``. + """ + params = [key, capacity] + + return self.execute_command(self.CF_RESERVE, *params) + + def cfAdd(self, key, item): + """ + Adds an ``item`` to a Cuckoo Filter ``key``. + """ + params = [key, item] + + return self.execute_command(self.CF_ADD, *params) + + def cfAddNX(self, key, item): + """ + Adds an ``item`` to a Cuckoo Filter ``key`` if item does not yet exist. + Command might be slower that ``cfAdd``. + """ + params = [key, item] + + return self.execute_command(self.CF_ADDNX, *params) + + def cfInsert(self, key, items, capacity=None, nocreate=None): + """ + Adds multiple ``items`` to a Cuckoo Filter ``key``. + """ + params = [key] + self.appendCapacity(params, capacity) + self.appendNoCreate(params, nocreate) + params.extend(['ITEMS']) + params += items + + return self.execute_command(self.CF_INSERT, *params) + + def cfInsertNX(self, key, items, capacity=None, nocreate=None): + """ + + """ + params = [key] + self.appendCapacity(params, capacity) + self.appendNoCreate(params, nocreate) + params.extend(['ITEMS']) + params += items + + return self.execute_command(self.CF_INSERTNX, *params) + + def cfExists(self, key, item): + """ + Checks whether an ``item`` exists in Cuckoo Filter ``key``. + """ + params = [key, item] + + return self.execute_command(self.CF_EXISTS, *params) + + def cfDel(self, key, item): + """ + Deletes ``item`` from ``key``. + """ + params = [key, item] + + return self.execute_command(self.CF_DEL, *params) + + def cfCount(self, key, item): + """ + Checks whether ``items`` exist in Cuckoo Filter ``key``. + """ + params = [key, item] + + return self.execute_command(self.CF_COUNT, *params) + + def cfScandump(self, key, iter): + """ + Begins an incremental save of the Cuckoo filter ``key``. This is useful + for large Cuckoo filters which cannot fit into the normal SAVE + and RESTORE model. + The first time this command is called, the value of ``iter`` should be 0. + This command will return successive (iter, data) pairs until + (0, NULL) to indicate completion. + """ + params = [key, iter] + + return self.execute_command(self.CF_SCANDUMP, *params) + + def cfLoadChunk(self, key, iter, data): + """ + Restores a filter previously saved using SCANDUMP. See the SCANDUMP + command for example usage. + This command will overwrite any Cuckoo filter stored under key. + Ensure that the Cuckoo filter will not be modified between invocations. + """ + params = [key, iter, data] + + return self.execute_command(self.CF_LOADDUMP, *params) + + +################## Count-Min Sketch Functions ###################### + + def cmsInitByDim(self, key, width, depth): + """ + Creates a new Cuckoo Filter ``key`` with desired probability of false + positives ``errorRate`` expected entries to be inserted as ``size``. + """ + params = [key, width, depth] + + return self.execute_command(self.CMS_INITBYDIM, *params) + + def cmsInitByProb(self, key, error, probability): + """ + Creates a new Cuckoo Filter ``key`` with desired probability of false + positives ``errorRate`` expected entries to be inserted as ``size``. + """ + params = [key, error, probability] + + return self.execute_command(self.CMS_INITBYPROB, *params) + + def cmsIncrBy(self, key, items, increments): + """ + Adds an ``item`` to a Cuckoo Filter ``key``. + """ + params = [key] + self.appendItemsAndIncrements(params, items, increments) + + return self.execute_command(self.CMS_INCRBY, *params) + + def cmsQuery(self, key, *items): + """ + Adds an ``item`` to a Cuckoo Filter ``key`` if item does not yet exist. + Command might be slower that ``cmsAdd``. + """ + params = [key] + params += items + + return self.execute_command(self.CMS_QUERY, *params) + + def cmsMerge(self, destKey, numKeys, srcKeys, weights=[]): + """ + Adds multiple ``items`` to a Cuckoo Filter ``key``. + """ + params = [destKey, numKeys] + params += srcKeys + self.appendWeights(params, weights) + + return self.execute_command(self.CMS_MERGE, *params) + + def cmsInfo(self, key): + """ + + """ + + return self.execute_command(self.CMS_INFO, key) + + +################## Top-K Functions ###################### + + def topkReserve(self, key, k, width, depth, decay): + """ + Creates a new Cuckoo Filter ``key`` with desired probability of false + positives ``errorRate`` expected entries to be inserted as ``size``. + """ + params = [key, k, width, depth, decay] + + return self.execute_command(self.TOPK_RESERVE, *params) + + + def topkAdd(self, key, *items): + """ + Adds an ``item`` to a Cuckoo Filter ``key``. + """ + params = [key] + params += items + + return self.execute_command(self.TOPK_ADD, *params) + + def topkQuery(self, key, *items): + """ + Checks whether an ``item`` is one of Top-K items at ``key``. + """ + params = [key] + params += items + + return self.execute_command(self.TOPK_QUERY, *params) + + def topkCount(self, key, *items): + """ + Returns count for an ``item`` from ``key``. + """ + params = [key] + params += items + + return self.execute_command(self.TOPK_COUNT, *params) + + def topkList(self, key): + """ + Return full list of items in Top K list of ``key```. + """ + return self.execute_command(self.TOPK_LIST, key) + + def topkInfo(self, key): + """ + Returns k, width, depth and decay values of ``key``. + """ + return self.execute_command(self.TOPK_INFO, key) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b9fda89 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ + +hiredis>=0.2.0 +redis>=2.10 +rmtest>=0.2 +six>=1.10.0 diff --git a/rltest_commands.py b/rltest_commands.py new file mode 100644 index 0000000..cb2e641 --- /dev/null +++ b/rltest_commands.py @@ -0,0 +1,187 @@ +# run using: +# RLTest -t rltest_commands.py --module /rebloom.so -s + +from RLTest import Env +import time +from redisbloom.client import Client as RedisBloom +from redis import ResponseError + +''' +from time import sleep +from unittest import TestCase +''' +# def CreateConn(): +# port = 6379 +# rb = RedisBloom(port=port) +# rb.flushdb() +# return rb + +i = lambda l: [int(v) for v in l] + +class TestRedisBloom(): + def __init__(self): + self.env = Env() + self.rb = RedisBloom(port=6379) + + def testCreate(self): + '''Test CREATE/RESERVE calls''' + self.env.cmd("flushall") + rb = self.rb + self.env.assertTrue(rb.bfCreate('bloom', 0.01, 1000)) + self.env.assertTrue(rb.cfCreate('cuckoo', 1000)) + self.env.assertTrue(rb.cmsInitByDim('cmsDim', 100, 5)) + self.env.assertTrue(rb.cmsInitByProb('cmsProb', 0.01, 0.01)) + self.env.assertTrue(rb.topkReserve('topk', 5, 100, 5, 0.9)) + + ################### Test Bloom Filter ################### + def testBFAdd(self): + self.env.cmd("flushall") + rb = self.rb + self.env.assertTrue(rb.bfCreate('bloom', 0.01, 1000)) + self.env.assertEqual(1, rb.bfAdd('bloom', 'foo')) + self.env.assertEqual(0, rb.bfAdd('bloom', 'foo')) + self.env.assertEqual([0], i(rb.bfMAdd('bloom', 'foo'))) + self.env.assertEqual([0, 1], rb.bfMAdd('bloom', 'foo', 'bar')) + self.env.assertEqual([0, 0, 1], rb.bfMAdd('bloom', 'foo', 'bar', 'baz')) + self.env.assertEqual(1, rb.bfExists('bloom', 'foo')) + self.env.assertEqual(0, rb.bfExists('bloom', 'noexist')) + self.env.assertEqual([1, 0], i(rb.bfMExists('bloom', 'foo', 'noexist'))) + + def testBFInsert(self): + self.env.cmd("flushall") + rb = self.rb + self.env.assertTrue(rb.bfCreate('bloom', 0.01, 1000)) + self.env.assertEqual([1], i(rb.bfInsert('bloom', ['foo']))) + self.env.assertEqual([0, 1], i(rb.bfInsert('bloom', ['foo', 'bar']))) + self.env.assertEqual([1], i(rb.bfInsert('captest', ['foo'], capacity=1000))) + self.env.assertEqual([1], i(rb.bfInsert('errtest', ['foo'], error=0.01))) + self.env.assertEqual(1, rb.bfExists('bloom', 'foo')) + self.env.assertEqual(0, rb.bfExists('bloom', 'noexist')) + self.env.assertEqual([1, 0], i(rb.bfMExists('bloom', 'foo', 'noexist'))) + + def testBFDumpLoad(self): + self.env.cmd("flushall") + rb = self.rb + # Store a filter + rb.bfCreate('myBloom', '0.0001', '1000') + + # test is probabilistic and might fail. It is OK to change variables if + # certain to not break anything + def do_verify(): + res = 0 + for x in xrange(1000): + rb.bfAdd('myBloom', x) + rv = rb.bfExists('myBloom', x) + self.env.assertTrue(rv) + rv = rb.bfExists('myBloom', 'nonexist_{}'.format(x)) + res += (rv == x) + self.env.assertLess(res, 5) + + do_verify() + cmds = [] + cur = rb.bfScandump('myBloom', 0) + first = cur[0] + cmds.append(cur) + + while True: + cur = rb.bfScandump('myBloom', first) + first = cur[0] + if first == 0: + break + else: + cmds.append(cur) + prev_info = rb.execute_command('bf.debug', 'myBloom') + + # Remove the filter + rb.execute_command('del', 'myBloom') + + # Now, load all the commands: + for cmd in cmds: + rb.bfLoadChunk('myBloom', *cmd) + + cur_info = rb.execute_command('bf.debug', 'myBloom') + self.env.assertEqual(prev_info, cur_info) + do_verify() + + rb.execute_command('del', 'myBloom') + rb.bfCreate('myBloom', '0.0001', '10000000') + + ################### Test Cuckoo Filter ################### + def testCFAddInsert(self): + self.env.cmd("flushall") + rb = self.rb + self.env.assertTrue(rb.cfCreate('cuckoo', 1000)) + self.env.assertTrue(rb.cfAdd('cuckoo', 'filter')) + self.env.assertFalse(rb.cfAddNX('cuckoo', 'filter')) + self.env.assertEqual(1, rb.cfAddNX('cuckoo', 'newItem')) + self.env.assertEqual([1], rb.cfInsert('captest', ['foo'])) + self.env.assertEqual([1], rb.cfInsert('captest', ['foo'], capacity=1000)) + self.env.assertEqual([1], rb.cfInsertNX('captest', ['bar'])) + self.env.assertEqual([0, 0, 1], rb.cfInsertNX('captest', ['foo', 'bar', 'baz'])) + self.env.assertEqual([0], rb.cfInsertNX('captest', ['bar'], capacity=1000)) + self.env.assertEqual([1], rb.cfInsert('empty1', ['foo'], capacity=1000)) + self.env.assertEqual([1], rb.cfInsertNX('empty2', ['bar'], capacity=1000)) + + def testCFExistsDel(self): + self.env.cmd("flushall") + rb = self.rb + self.env.assertTrue(rb.cfCreate('cuckoo', 1000)) + self.env.assertTrue(rb.cfAdd('cuckoo', 'filter')) + self.env.assertTrue(rb.cfExists('cuckoo', 'filter')) + self.env.assertFalse(rb.cfExists('cuckoo', 'notexist')) + self.env.assertEqual(1, rb.cfCount('cuckoo', 'filter')) + self.env.assertEqual(0, rb.cfCount('cuckoo', 'notexist')) + self.env.assertTrue(rb.cfDel('cuckoo', 'filter')) + self.env.assertEqual(0, rb.cfCount('cuckoo', 'filter')) + + ################### Test Count-Min Sketch ################### + def testCMS(self): + self.env.cmd("flushall") + rb = self.rb + self.env.assertTrue(rb.cmsInitByDim('dim', 1000, 5)) + self.env.assertTrue(rb.cmsInitByProb('prob', 0.01, 0.01)) + self.env.assertTrue(rb.cmsIncrBy('dim', ['foo'], [5])) + self.env.assertEqual([0], rb.cmsQuery('dim', 'notexist')) + self.env.assertEqual([5], rb.cmsQuery('dim', 'foo')) + self.env.assertTrue(rb.cmsIncrBy('dim', ['foo', 'bar'], [5, 15])) + self.env.assertEqual([10, 15], rb.cmsQuery('dim', 'foo', 'bar')) + info = rb.cmsInfo('dim') + self.env.assertEqual(1000, info.width) + self.env.assertEqual(5, info.depth) + self.env.assertEqual(25, info.count) + + def testCMSMerge(self): + self.env.cmd("flushall") + rb = self.rb + self.env.assertTrue(rb.cmsInitByDim('A', 1000, 5)) + self.env.assertTrue(rb.cmsInitByDim('B', 1000, 5)) + self.env.assertTrue(rb.cmsInitByDim('C', 1000, 5)) + self.env.assertTrue(rb.cmsIncrBy('A', ['foo', 'bar', 'baz'], [5, 3, 9])) + self.env.assertTrue(rb.cmsIncrBy('B', ['foo', 'bar', 'baz'], [2, 3, 1])) + self.env.assertEqual([5, 3, 9], rb.cmsQuery('A', 'foo', 'bar', 'baz')) + self.env.assertEqual([2, 3, 1], rb.cmsQuery('B', 'foo', 'bar', 'baz')) + self.env.assertTrue(rb.cmsMerge('C', 2, ['A', 'B'])) + self.env.assertEqual([7, 6, 10], rb.cmsQuery('C', 'foo', 'bar', 'baz')) + self.env.assertTrue(rb.cmsMerge('C', 2, ['A', 'B'], ['1', '2'])) + self.env.assertEqual([9, 9, 11], rb.cmsQuery('C', 'foo', 'bar', 'baz')) + self.env.assertTrue(rb.cmsMerge('C', 2, ['A', 'B'], ['2', '3'])) + self.env.assertEqual([16, 15, 21], rb.cmsQuery('C', 'foo', 'bar', 'baz')) + + ################### Test Top-K ################### + def testTopK(self): + self.env.cmd("flushall") + rb = self.rb + # test list with empty buckets + self.env.assertTrue(rb.topkReserve('topk', 10, 50, 3, 0.9)) + self.env.assertTrue(rb.topkAdd('topk', 'A', 'B', 'C', 'D', 'E', 'A', 'A', 'B', 'C', + 'G', 'D', 'B', 'D', 'A', 'E', 'E')) + self.env.assertEqual([1, 1, 1, 1, 1, 0, 1], + rb.topkQuery('topk', 'A', 'B', 'C', 'D', 'E', 'F', 'G')) + self.env.assertEqual([4, 3, 2, 3, 3, 0, 1], + rb.topkCount('topk', 'A', 'B', 'C', 'D', 'E', 'F', 'G')) + + # test full list + self.env.assertTrue(rb.topkReserve('topklist', 3, 50, 3, 0.9)) + self.env.assertTrue(rb.topkAdd('topklist', 'A', 'B', 'C', 'D', 'E','A', 'A', 'B', 'C', + 'G', 'D', 'B', 'D', 'A', 'E', 'E')) + self.env.assertEqual(['D', 'A', 'B'], rb.topkList('topklist')) \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..224a779 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md \ No newline at end of file From b0838d7ee533b81f234bd995f9524a79020c6174 Mon Sep 17 00:00:00 2001 From: Ariel Date: Thu, 6 Jun 2019 14:34:44 +0300 Subject: [PATCH 02/17] Some comment --- redisbloom/client.py | 65 ++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 38 deletions(-) diff --git a/redisbloom/client.py b/redisbloom/client.py index 1952adb..73b4d0a 100644 --- a/redisbloom/client.py +++ b/redisbloom/client.py @@ -5,25 +5,6 @@ from redis._compat import (long, nativestr) from redis.exceptions import DataError -''' -class TSInfo(object): - chunkCount = None - labels = [] - lastTimeStamp = None - maxSamplesPerChunk = None - retentionSecs = None - rules = [] - - def __init__(self, args): - self.chunkCount = args['chunkCount'] - self.labels = list_to_dict(args['labels']) - self.lastTimeStamp = args['lastTimestamp'] - self.maxSamplesPerChunk = args['maxSamplesPerChunk'] - self.retentionSecs = args['retentionSecs'] - self.rules = args['rules'] -''' - - class CMSInfo(object): width = None depth = None @@ -317,8 +298,7 @@ def bfLoadChunk(self, key, iter, data): def cfCreate(self, key, capacity): """ - Creates a new Cuckoo Filter ``key`` with desired probability of false - positives ``errorRate`` expected entries to be inserted as ``size``. + Creates a new Cuckoo Filter ``key`` an initial ``capacity`` items. """ params = [key, capacity] @@ -334,7 +314,7 @@ def cfAdd(self, key, item): def cfAddNX(self, key, item): """ - Adds an ``item`` to a Cuckoo Filter ``key`` if item does not yet exist. + Adds an ``item`` to a Cuckoo Filter ``key`` only if item does not yet exist. Command might be slower that ``cfAdd``. """ params = [key, item] @@ -343,7 +323,9 @@ def cfAddNX(self, key, item): def cfInsert(self, key, items, capacity=None, nocreate=None): """ - Adds multiple ``items`` to a Cuckoo Filter ``key``. + Adds multiple ``items`` to a Cuckoo Filter ``key``, allowing the filter to be + created with a custom ``capacity` if it does not yet exist. + ``items`` must be provided as a list. """ params = [key] self.appendCapacity(params, capacity) @@ -355,7 +337,9 @@ def cfInsert(self, key, items, capacity=None, nocreate=None): def cfInsertNX(self, key, items, capacity=None, nocreate=None): """ - + Adds multiple ``items`` to a Cuckoo Filter ``key`` only if they do not exist yet, + allowing the filter to be created with a custom ``capacity` if it does not yet exist. + ``items`` must be provided as a list. """ params = [key] self.appendCapacity(params, capacity) @@ -383,7 +367,7 @@ def cfDel(self, key, item): def cfCount(self, key, item): """ - Checks whether ``items`` exist in Cuckoo Filter ``key``. + Returns the number of times an ``item`` may be in the ``key``. """ params = [key, item] @@ -418,8 +402,8 @@ def cfLoadChunk(self, key, iter, data): def cmsInitByDim(self, key, width, depth): """ - Creates a new Cuckoo Filter ``key`` with desired probability of false - positives ``errorRate`` expected entries to be inserted as ``size``. + Initializes a Count-Min Sketch ``key`` to dimensions + (``width``, ``depth``) specified by user. """ params = [key, width, depth] @@ -427,8 +411,8 @@ def cmsInitByDim(self, key, width, depth): def cmsInitByProb(self, key, error, probability): """ - Creates a new Cuckoo Filter ``key`` with desired probability of false - positives ``errorRate`` expected entries to be inserted as ``size``. + Initializes a Count-Min Sketch ``key`` to characteristics + (``error``, ``probability``) specified by user. """ params = [key, error, probability] @@ -436,7 +420,9 @@ def cmsInitByProb(self, key, error, probability): def cmsIncrBy(self, key, items, increments): """ - Adds an ``item`` to a Cuckoo Filter ``key``. + Adds/increases ``items`` to a Count-Min Sketch ``key`` by ''increments''. + Both ``items`` and ``increments`` are lists. + Example - cmsIncrBy('A', ['foo'], [1]) """ params = [key] self.appendItemsAndIncrements(params, items, increments) @@ -445,8 +431,8 @@ def cmsIncrBy(self, key, items, increments): def cmsQuery(self, key, *items): """ - Adds an ``item`` to a Cuckoo Filter ``key`` if item does not yet exist. - Command might be slower that ``cmsAdd``. + Returns count for an ``item`` from ``key``. + Multiple items can be queried with one call. """ params = [key] params += items @@ -455,7 +441,10 @@ def cmsQuery(self, key, *items): def cmsMerge(self, destKey, numKeys, srcKeys, weights=[]): """ - Adds multiple ``items`` to a Cuckoo Filter ``key``. + Merges ``numKeys`` of sketches into ``destKey``. Sketches specified in ``srcKeys``. + All sketches must have identical width and depth. + ``Weights`` can be used to multiply certain sketches. Default weight is 1. + Both ``srcKeys`` and ``weights`` are lists. """ params = [destKey, numKeys] params += srcKeys @@ -465,7 +454,7 @@ def cmsMerge(self, destKey, numKeys, srcKeys, weights=[]): def cmsInfo(self, key): """ - + Returns width, depth and total count of the sketch. """ return self.execute_command(self.CMS_INFO, key) @@ -485,7 +474,7 @@ def topkReserve(self, key, k, width, depth, decay): def topkAdd(self, key, *items): """ - Adds an ``item`` to a Cuckoo Filter ``key``. + Adds one ``item`` or more to a Cuckoo Filter ``key``. """ params = [key] params += items @@ -494,7 +483,7 @@ def topkAdd(self, key, *items): def topkQuery(self, key, *items): """ - Checks whether an ``item`` is one of Top-K items at ``key``. + Checks whether one ``item`` or more is a Top-K item at ``key``. """ params = [key] params += items @@ -503,7 +492,7 @@ def topkQuery(self, key, *items): def topkCount(self, key, *items): """ - Returns count for an ``item`` from ``key``. + Returns count for one ``item`` or more from ``key``. """ params = [key] params += items @@ -512,7 +501,7 @@ def topkCount(self, key, *items): def topkList(self, key): """ - Return full list of items in Top K list of ``key```. + Return full list of items in Top-K list of ``key```. """ return self.execute_command(self.TOPK_LIST, key) From 7fa81e7062e406cba684661497b7045b0a8bd48f Mon Sep 17 00:00:00 2001 From: Ariel Date: Tue, 11 Jun 2019 10:24:28 +0300 Subject: [PATCH 03/17] circleci --- .circleci/config.yml | 99 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..3d8266b --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,99 @@ + +# Python CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/language-python/ for more details +# +version: 2 +jobs: + build: + docker: + - image: circleci/python:3.7.1 + - image: redislabs/rebloom:latest + + working_directory: ~/repo + + steps: + - checkout + + - restore_cache: # Download and cache dependencies + keys: + - v1-dependencies-{{ checksum "requirements.txt" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + + - run: + name: install dependencies + command: | + virtualenv venv + . venv/bin/activate + pip install -r requirements.txt + pip install codecov + + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum "requirements.txt" }} + + - run: + name: run tests + command: | + . venv/bin/activate + REDIS_PORT=6379 coverage run test_commands.py + codecov + + - store_artifacts: + path: test-reports + destination: test-reports + + build_nightly: + docker: + - image: circleci/python:3.7.1 + - image: redislabs/rebloom:latest + + working_directory: ~/repo + + steps: + - checkout + + - restore_cache: # Download and cache dependencies + keys: + - v1-dependencies-{{ checksum "requirements.txt" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + + - run: + name: install dependencies + command: | + virtualenv venv + . venv/bin/activate + pip install -r requirements.txt + pip install codecov + + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum "requirements.txt" }} + + - run: + name: run tests + command: | + . venv/bin/activate + REDIS_PORT=6379 python test_commands.py + + # no need for store_artifacts on nightly builds + +workflows: + version: 2 + commit: + jobs: + - build + nightly: + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - master + jobs: + - build_nightly From 0cac4fb0291598b65c0a544d94039837bb5ae086 Mon Sep 17 00:00:00 2001 From: Ariel Date: Tue, 11 Jun 2019 10:36:09 +0300 Subject: [PATCH 04/17] test --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d8266b..6723908 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,7 +78,7 @@ jobs: name: run tests command: | . venv/bin/activate - REDIS_PORT=6379 python test_commands.py + REDIS_PORT=6379 python RLTest -t rltest_commands.py --module ./../bloom/rebloom.so -s # no need for store_artifacts on nightly builds From 50207be1e038d284fa6a9eaf1de21ef4fd211fb0 Mon Sep 17 00:00:00 2001 From: Ariel Date: Tue, 11 Jun 2019 10:44:27 +0300 Subject: [PATCH 05/17] . --- .circleci/config.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6723908..9503695 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ jobs: build: docker: - image: circleci/python:3.7.1 - - image: redislabs/rebloom:latest + - image: redislabs/rebloom:edge working_directory: ~/repo @@ -48,7 +48,7 @@ jobs: build_nightly: docker: - image: circleci/python:3.7.1 - - image: redislabs/rebloom:latest + - image: redislabs/rebloom:edge working_directory: ~/repo @@ -68,7 +68,9 @@ jobs: . venv/bin/activate pip install -r requirements.txt pip install codecov - + pip install git+https://github.com/RedisLabsModules/RLTest.git@master + pip install redis-py-cluster + - save_cache: paths: - ./venv From c67d4e8dd078bd54eff035dfbad24ea5f66cbd6f Mon Sep 17 00:00:00 2001 From: Ariel Date: Tue, 11 Jun 2019 10:45:56 +0300 Subject: [PATCH 06/17] . --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9503695..473c3d9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,9 +37,9 @@ jobs: - run: name: run tests command: | - . venv/bin/activate - REDIS_PORT=6379 coverage run test_commands.py - codecov + # . venv/bin/activate + # REDIS_PORT=6379 coverage run test_commands.py + #codecov - store_artifacts: path: test-reports From 0bc9ff0eb6fd67bd333da4a41323728e80313cf8 Mon Sep 17 00:00:00 2001 From: Ariel Date: Tue, 11 Jun 2019 10:47:36 +0300 Subject: [PATCH 07/17] . --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 473c3d9..f9dcb89 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,12 +34,12 @@ jobs: - ./venv key: v1-dependencies-{{ checksum "requirements.txt" }} - - run: - name: run tests - command: | - # . venv/bin/activate - # REDIS_PORT=6379 coverage run test_commands.py - #codecov + # - run: + # name: run tests + # command: | + # . venv/bin/activate + # REDIS_PORT=6379 coverage run test_commands.py + # codecov - store_artifacts: path: test-reports From 8dd061b5fea3a8cae475648f4dbf13b2d750dee5 Mon Sep 17 00:00:00 2001 From: Ariel Date: Tue, 11 Jun 2019 13:15:28 +0300 Subject: [PATCH 08/17] . --- .circleci/config.yml | 18 +++-- test_commands.py | 166 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 10 deletions(-) create mode 100644 test_commands.py diff --git a/.circleci/config.yml b/.circleci/config.yml index f9dcb89..4f2d329 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,12 +34,12 @@ jobs: - ./venv key: v1-dependencies-{{ checksum "requirements.txt" }} - # - run: - # name: run tests - # command: | - # . venv/bin/activate - # REDIS_PORT=6379 coverage run test_commands.py - # codecov + - run: + name: run tests + command: | + . venv/bin/activate + REDIS_PORT=6379 coverage run test_commands.py + codecov - store_artifacts: path: test-reports @@ -68,9 +68,7 @@ jobs: . venv/bin/activate pip install -r requirements.txt pip install codecov - pip install git+https://github.com/RedisLabsModules/RLTest.git@master - pip install redis-py-cluster - + - save_cache: paths: - ./venv @@ -80,7 +78,7 @@ jobs: name: run tests command: | . venv/bin/activate - REDIS_PORT=6379 python RLTest -t rltest_commands.py --module ./../bloom/rebloom.so -s + REDIS_PORT=6379 python test_commands.py # no need for store_artifacts on nightly builds diff --git a/test_commands.py b/test_commands.py new file mode 100644 index 0000000..f45a6bf --- /dev/null +++ b/test_commands.py @@ -0,0 +1,166 @@ +import unittest + +from time import sleep +from unittest import TestCase +from redisbloom.client import Client as RedisBloom + +xrange = range +rb = None +port = 6379 + +i = lambda l: [int(v) for v in l] + +class TestRedisBloom(TestCase): + def setUp(self): + global rb + rb = RedisBloom(port=port) + rb.flushdb() + + def testCreate(self): + '''Test CREATE/RESERVE calls''' + self.assertTrue(rb.bfCreate('bloom', 0.01, 1000)) + self.assertTrue(rb.cfCreate('cuckoo', 1000)) + self.assertTrue(rb.cmsInitByDim('cmsDim', 100, 5)) + self.assertTrue(rb.cmsInitByProb('cmsProb', 0.01, 0.01)) + self.assertTrue(rb.topkReserve('topk', 5, 100, 5, 0.9)) + + ################### Test Bloom Filter ################### + def testBFAdd(self): + self.assertTrue(rb.bfCreate('bloom', 0.01, 1000)) + self.assertEqual(1, rb.bfAdd('bloom', 'foo')) + self.assertEqual(0, rb.bfAdd('bloom', 'foo')) + self.assertEqual([0], i(rb.bfMAdd('bloom', 'foo'))) + self.assertEqual([0, 1], rb.bfMAdd('bloom', 'foo', 'bar')) + self.assertEqual([0, 0, 1], rb.bfMAdd('bloom', 'foo', 'bar', 'baz')) + self.assertEqual(1, rb.bfExists('bloom', 'foo')) + self.assertEqual(0, rb.bfExists('bloom', 'noexist')) + self.assertEqual([1, 0], i(rb.bfMExists('bloom', 'foo', 'noexist'))) + + def testBFInsert(self): + self.assertTrue(rb.bfCreate('bloom', 0.01, 1000)) + self.assertEqual([1], i(rb.bfInsert('bloom', ['foo']))) + self.assertEqual([0, 1], i(rb.bfInsert('bloom', ['foo', 'bar']))) + self.assertEqual([1], i(rb.bfInsert('captest', ['foo'], capacity=1000))) + self.assertEqual([1], i(rb.bfInsert('errtest', ['foo'], error=0.01))) + self.assertEqual(1, rb.bfExists('bloom', 'foo')) + self.assertEqual(0, rb.bfExists('bloom', 'noexist')) + self.assertEqual([1, 0], i(rb.bfMExists('bloom', 'foo', 'noexist'))) + + def testBFDumpLoad(self): + # Store a filter + rb.bfCreate('myBloom', '0.0001', '1000') + + # test is probabilistic and might fail. It is OK to change variables if + # certain to not break anything + def do_verify(): + res = 0 + for x in xrange(1000): + rb.bfAdd('myBloom', x) + rv = rb.bfExists('myBloom', x) + self.assertTrue(rv) + rv = rb.bfExists('myBloom', 'nonexist_{}'.format(x)) + res += (rv == x) + self.assertLess(res, 5) + + do_verify() + cmds = [] + cur = rb.bfScandump('myBloom', 0) + first = cur[0] + cmds.append(cur) + + while True: + cur = rb.bfScandump('myBloom', first) + first = cur[0] + if first == 0: + break + else: + cmds.append(cur) + prev_info = rb.execute_command('bf.debug', 'myBloom') + + # Remove the filter + rb.execute_command('del', 'myBloom') + + # Now, load all the commands: + for cmd in cmds: + rb.bfLoadChunk('myBloom', *cmd) + + cur_info = rb.execute_command('bf.debug', 'myBloom') + self.assertEqual(prev_info, cur_info) + do_verify() + + rb.execute_command('del', 'myBloom') + rb.bfCreate('myBloom', '0.0001', '10000000') + + ################### Test Cuckoo Filter ################### + def testCFAddInsert(self): + self.assertTrue(rb.cfCreate('cuckoo', 1000)) + self.assertTrue(rb.cfAdd('cuckoo', 'filter')) + self.assertFalse(rb.cfAddNX('cuckoo', 'filter')) + self.assertEqual(1, rb.cfAddNX('cuckoo', 'newItem')) + self.assertEqual([1], rb.cfInsert('captest', ['foo'])) + self.assertEqual([1], rb.cfInsert('captest', ['foo'], capacity=1000)) + self.assertEqual([1], rb.cfInsertNX('captest', ['bar'])) + self.assertEqual([0, 0, 1], rb.cfInsertNX('captest', ['foo', 'bar', 'baz'])) + self.assertEqual([0], rb.cfInsertNX('captest', ['bar'], capacity=1000)) + self.assertEqual([1], rb.cfInsert('empty1', ['foo'], capacity=1000)) + self.assertEqual([1], rb.cfInsertNX('empty2', ['bar'], capacity=1000)) + + def testCFExistsDel(self): + self.assertTrue(rb.cfCreate('cuckoo', 1000)) + self.assertTrue(rb.cfAdd('cuckoo', 'filter')) + self.assertTrue(rb.cfExists('cuckoo', 'filter')) + self.assertFalse(rb.cfExists('cuckoo', 'notexist')) + self.assertEqual(1, rb.cfCount('cuckoo', 'filter')) + self.assertEqual(0, rb.cfCount('cuckoo', 'notexist')) + self.assertTrue(rb.cfDel('cuckoo', 'filter')) + self.assertEqual(0, rb.cfCount('cuckoo', 'filter')) + + ################### Test Count-Min Sketch ################### + def testCMS(self): + self.assertTrue(rb.cmsInitByDim('dim', 1000, 5)) + self.assertTrue(rb.cmsInitByProb('prob', 0.01, 0.01)) + self.assertTrue(rb.cmsIncrBy('dim', ['foo'], [5])) + self.assertEqual([0], rb.cmsQuery('dim', 'notexist')) + self.assertEqual([5], rb.cmsQuery('dim', 'foo')) + self.assertTrue(rb.cmsIncrBy('dim', ['foo', 'bar'], [5, 15])) + self.assertEqual([10, 15], rb.cmsQuery('dim', 'foo', 'bar')) + info = rb.cmsInfo('dim') + self.assertEqual(1000, info.width) + self.assertEqual(5, info.depth) + self.assertEqual(25, info.count) + + def testCMSMerge(self): + self.assertTrue(rb.cmsInitByDim('A', 1000, 5)) + self.assertTrue(rb.cmsInitByDim('B', 1000, 5)) + self.assertTrue(rb.cmsInitByDim('C', 1000, 5)) + self.assertTrue(rb.cmsIncrBy('A', ['foo', 'bar', 'baz'], [5, 3, 9])) + self.assertTrue(rb.cmsIncrBy('B', ['foo', 'bar', 'baz'], [2, 3, 1])) + self.assertEqual([5, 3, 9], rb.cmsQuery('A', 'foo', 'bar', 'baz')) + self.assertEqual([2, 3, 1], rb.cmsQuery('B', 'foo', 'bar', 'baz')) + self.assertTrue(rb.cmsMerge('C', 2, ['A', 'B'])) + self.assertEqual([7, 6, 10], rb.cmsQuery('C', 'foo', 'bar', 'baz')) + self.assertTrue(rb.cmsMerge('C', 2, ['A', 'B'], ['1', '2'])) + self.assertEqual([9, 9, 11], rb.cmsQuery('C', 'foo', 'bar', 'baz')) + self.assertTrue(rb.cmsMerge('C', 2, ['A', 'B'], ['2', '3'])) + self.assertEqual([16, 15, 21], rb.cmsQuery('C', 'foo', 'bar', 'baz')) + + ################### Test Top-K ################### + def testTopK(self): + # test list with empty buckets + self.assertTrue(rb.topkReserve('topk', 10, 50, 3, 0.9)) + self.assertTrue(rb.topkAdd('topk', 'A', 'B', 'C', 'D', 'E', 'A', 'A', 'B', 'C', + 'G', 'D', 'B', 'D', 'A', 'E', 'E')) + self.assertEqual([1, 1, 1, 1, 1, 0, 1], + rb.topkQuery('topk', 'A', 'B', 'C', 'D', 'E', 'F', 'G')) + self.assertEqual([4, 3, 2, 3, 3, 0, 1], + rb.topkCount('topk', 'A', 'B', 'C', 'D', 'E', 'F', 'G')) + + # test full list + self.assertTrue(rb.topkReserve('topklist', 3, 50, 3, 0.9)) + self.assertTrue(rb.topkAdd('topklist', 'A', 'B', 'C', 'D', 'E','A', 'A', 'B', 'C', + 'G', 'D', 'B', 'D', 'A', 'E', 'E')) + self.assertEqual(['D', 'A', 'B'], rb.topkList('topklist')) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 66dfdb35f03ef5a31278c4ff1c37604df6120b1b Mon Sep 17 00:00:00 2001 From: Ariel Date: Tue, 11 Jun 2019 13:24:40 +0300 Subject: [PATCH 09/17] improve covarge --- redisbloom/client.py | 26 -------------------------- test_commands.py | 6 +++++- 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/redisbloom/client.py b/redisbloom/client.py index 73b4d0a..0ae9f80 100644 --- a/redisbloom/client.py +++ b/redisbloom/client.py @@ -32,32 +32,6 @@ def __init__(self, args): def spaceHolder(response): return response -def list_to_dict(aList): - return {nativestr(aList[i][0]):nativestr(aList[i][1]) - for i in range(len(aList))} - -def parse_range(response): - return [tuple((l[0], l[1].decode())) for l in response] - -def parse_m_range(response): - res = [] - for item in response: - res.append({ nativestr(item[0]) : [list_to_dict(item[1]), - parse_range(item[2])]}) - return res - -def parse_m_get(response): - res = [] - for item in response: - res.append({ nativestr(item[0]) : [list_to_dict(item[1]), - item[2], nativestr(item[3])]}) - return res - -def parse_info(response): - res = dict(zip(map(nativestr, response[::2]), response[1::2])) - info = TopKInfo(res) - return info - def parseToList(response): res = [] for item in response: diff --git a/test_commands.py b/test_commands.py index f45a6bf..c5b7399 100644 --- a/test_commands.py +++ b/test_commands.py @@ -160,7 +160,11 @@ def testTopK(self): self.assertTrue(rb.topkAdd('topklist', 'A', 'B', 'C', 'D', 'E','A', 'A', 'B', 'C', 'G', 'D', 'B', 'D', 'A', 'E', 'E')) self.assertEqual(['D', 'A', 'B'], rb.topkList('topklist')) - + info = rb.topkInfo('topklist') + self.assertEqual(3, info.k) + self.assertEqual(50, info.width) + self.assertEqual(3, info.depth) + self.assertAlmostEqual(0.9, float(info.decay)) if __name__ == '__main__': unittest.main() \ No newline at end of file From d2c70f3171f366e0049a72b89d95725886ef4b39 Mon Sep 17 00:00:00 2001 From: Ariel Date: Tue, 11 Jun 2019 13:29:46 +0300 Subject: [PATCH 10/17] appenditems --- redisbloom/client.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/redisbloom/client.py b/redisbloom/client.py index 0ae9f80..f11572e 100644 --- a/redisbloom/client.py +++ b/redisbloom/client.py @@ -152,7 +152,8 @@ def __init__(self, *args, **kwargs): @staticmethod def appendItems(params, items): - params.extend(['ITEMS', items]) + params.extend(['ITEMS']) + params += items @staticmethod def appendError(params, error): @@ -221,8 +222,7 @@ def bfInsert(self, key, items, capacity=None, error=None, noCreate=None, ): self.appendCapacity(params, capacity) self.appendError(params, error) self.appendNoCreate(params, noCreate) - params.extend(['ITEMS']) - params += items + self.appendItems(params, items) return self.execute_command(self.BF_INSERT, *params) @@ -304,8 +304,7 @@ def cfInsert(self, key, items, capacity=None, nocreate=None): params = [key] self.appendCapacity(params, capacity) self.appendNoCreate(params, nocreate) - params.extend(['ITEMS']) - params += items + self.appendItems(params, items) return self.execute_command(self.CF_INSERT, *params) @@ -318,8 +317,7 @@ def cfInsertNX(self, key, items, capacity=None, nocreate=None): params = [key] self.appendCapacity(params, capacity) self.appendNoCreate(params, nocreate) - params.extend(['ITEMS']) - params += items + self.appendItems(params, items) return self.execute_command(self.CF_INSERTNX, *params) From c0dde42ab76520d026abc65b73ef398eaee270b4 Mon Sep 17 00:00:00 2001 From: Ariel Date: Tue, 11 Jun 2019 14:05:04 +0300 Subject: [PATCH 11/17] check nocreate --- test_commands.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test_commands.py b/test_commands.py index c5b7399..4fce130 100644 --- a/test_commands.py +++ b/test_commands.py @@ -3,6 +3,7 @@ from time import sleep from unittest import TestCase from redisbloom.client import Client as RedisBloom +from redis import ResponseError xrange = range rb = None @@ -10,6 +11,10 @@ i = lambda l: [int(v) for v in l] +# Can be used with assertRaises +def run_func(func, *args, **kwargs): + func(*args, **kwargs) + class TestRedisBloom(TestCase): def setUp(self): global rb @@ -100,10 +105,12 @@ def testCFAddInsert(self): self.assertEqual([1], rb.cfInsert('captest', ['foo'])) self.assertEqual([1], rb.cfInsert('captest', ['foo'], capacity=1000)) self.assertEqual([1], rb.cfInsertNX('captest', ['bar'])) + self.assertEqual([1], rb.cfInsertNX('captest', ['food'], nocreate='1')) self.assertEqual([0, 0, 1], rb.cfInsertNX('captest', ['foo', 'bar', 'baz'])) self.assertEqual([0], rb.cfInsertNX('captest', ['bar'], capacity=1000)) self.assertEqual([1], rb.cfInsert('empty1', ['foo'], capacity=1000)) self.assertEqual([1], rb.cfInsertNX('empty2', ['bar'], capacity=1000)) + self.assertRaises(ResponseError, run_func(rb.cfInsert, 'noexist', ['foo'])) def testCFExistsDel(self): self.assertTrue(rb.cfCreate('cuckoo', 1000)) From 684260799f46443ce95521190e27f756edf22569 Mon Sep 17 00:00:00 2001 From: Ariel Date: Thu, 13 Jun 2019 14:53:17 +0300 Subject: [PATCH 12/17] README.md --- README.md | 55 ++++++++++++++++++++++++++++++++++++++++++-- redisbloom/client.py | 17 +------------- 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 8ab2c88..7bd3a8e 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,59 @@ +# Python client for RedisBloom [![license](https://img.shields.io/github/license/RedisBloom/redisbloom-py.svg)](https://github.com/RedisBloom/redisbloom-py) [![PyPI version](https://badge.fury.io/py/redisbloom.svg)](https://badge.fury.io/py/redisbloom) [![CircleCI](https://circleci.com/gh/RedisBloom/redisbloom-py/tree/master.svg?style=svg)](https://circleci.com/gh/RedisBloom/redisbloom-py/tree/master) [![GitHub issues](https://img.shields.io/github/release/RedisBloom/redisbloom-py.svg)](https://github.com/RedisBloom/redisbloom-py/releases/latest) [![Codecov](https://codecov.io/gh/RedisBloom/redisbloom-py/branch/master/graph/badge.svg)](https://codecov.io/gh/RedisBloom/redisbloom-py) -# redisbloom-py -Python client for Redisbloom +redisbloom-py is a package that gives developers easy access to several probabilistic data structures. The package extends [redis-py](https://github.com/andymccurdy/redis-py)'s interface with RedisBloom's API. + +### Installation +``` +$ pip install redisbloom +``` + +### Usage example + +```sql +# Using Bloom Filter +from redisbloom import Client +rb = Client() +rb.bfCreate('bloom', 0.01, 1000) +rb.bfAdd('bloom', 'foo') # returns 1 +rb.bfAdd('bloom', 'foo') # returns 0 +rb.bfExists('bloom', 'foo') # returns 1 +rb.bfExists('bloom', 'noexist') # returns 0 + +# Using Cuckoo Filter +from redisbloom import Client +rb = Client() +rb.cfCreate('cuckoo', 1000) +rb.cfAdd('cuckoo', 'filter') # returns 1 +rb.cfAddNX('cuckoo', 'filter') # returns 0 +rb.cfExists('cuckoo', 'filter') # returns 1 +rb.cfExists('cuckoo', 'noexist') # returns 0 + +# Using Cuckoo Filter +from redisbloom import Client +rb = Client() +rb.cmsInitByDim('dim', 1000, 5) +rb.cmsIncrBy('dim', ['foo'], [5]) +rb.cmsIncrBy('dim', ['foo', 'bar'], [5, 15]) +rb.cmsQuery('dim', 'foo', 'bar') # returns [10, 15] + +# Using Cuckoo Filter +from redisbloom import Client +rb = Client() +rb.topkReserve('topk', 3, 20, 3, 0.9) +rb.topkAdd('topk', 'A', 'B', 'C', 'D', 'E', 'A', 'A', 'B', + 'C', 'G', 'D', 'B', 'D', 'A', 'E', 'E') +rb.topkQuery('topk', 'A', 'B', 'C', 'D') # returns [1, 1, 0, 1] +rb.topkCount('topk', 'A', 'B', 'C', 'D') # returns [4, 3, 2, 3] +rb.topkList('topk') # returns ['D', 'A', 'B'] +``` + +### API +For complete documentation about RedisBloom's commands, refer to [RedisBloom's website](http://redisbloom.io). + +### License +[BSD 3-Clause](https://github.com/RedisBloom/redisbloom-py/blob/master/LICENSE) diff --git a/redisbloom/client.py b/redisbloom/client.py index f11572e..2634245 100644 --- a/redisbloom/client.py +++ b/redisbloom/client.py @@ -35,7 +35,7 @@ def spaceHolder(response): def parseToList(response): res = [] for item in response: - res.append(nativestr(item)) + res.append(item) return res class Client(Redis): #changed from StrictRedis @@ -90,21 +90,6 @@ class Client(Redis): #changed from StrictRedis TOPK_LIST = 'TOPK.LIST' TOPK_INFO = 'TOPK.INFO' - - CREATE_CMD = 'TS.CREATE' - ALTER_CMD = 'TS.ALTER' - ADD_CMD = 'TS.ADD' - INCRBY_CMD = 'TS.INCRBY' - DECRBY_CMD = 'TS.DECRBY' - CREATERULE_CMD = 'TS.CREATERULE' - DELETERULE_CMD = 'TS.DELETERULE' - RANGE_CMD = 'TS.RANGE' - MRANGE_CMD = 'TS.MRANGE' - GET_CMD = 'TS.GET' - MGET_CMD = 'TS.MGET' - INFO_CMD = 'TS.INFO' - QUERYINDEX_CMD = 'TS.QUERYINDEX' - def __init__(self, *args, **kwargs): """ Creates a new RedisBloom client. From 348eb80acd428a9e32c23169baf902df49deae38 Mon Sep 17 00:00:00 2001 From: Ariel Date: Thu, 13 Jun 2019 14:57:46 +0300 Subject: [PATCH 13/17] . --- redisbloom/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redisbloom/client.py b/redisbloom/client.py index 2634245..465ba7a 100644 --- a/redisbloom/client.py +++ b/redisbloom/client.py @@ -35,7 +35,7 @@ def spaceHolder(response): def parseToList(response): res = [] for item in response: - res.append(item) + res.append(nativestr(item)) return res class Client(Redis): #changed from StrictRedis From 46ba7ced08c4c68330edb9dd9082cf90a9d52498 Mon Sep 17 00:00:00 2001 From: Ariel Date: Sat, 15 Jun 2019 17:12:54 +0300 Subject: [PATCH 14/17] update heavyhitter detection --- redisbloom/client.py | 2 +- test_commands.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/redisbloom/client.py b/redisbloom/client.py index 465ba7a..211b4c6 100644 --- a/redisbloom/client.py +++ b/redisbloom/client.py @@ -126,7 +126,7 @@ def __init__(self, *args, **kwargs): self.CMS_INFO : CMSInfo, self.TOPK_RESERVE : bool_ok, - self.TOPK_ADD : bool_ok, + self.TOPK_ADD : spaceHolder, self.TOPK_QUERY : spaceHolder, self.TOPK_COUNT : spaceHolder, self.TOPK_LIST : parseToList, diff --git a/test_commands.py b/test_commands.py index 4fce130..1a1306d 100644 --- a/test_commands.py +++ b/test_commands.py @@ -154,10 +154,12 @@ def testCMSMerge(self): ################### Test Top-K ################### def testTopK(self): # test list with empty buckets - self.assertTrue(rb.topkReserve('topk', 10, 50, 3, 0.9)) - self.assertTrue(rb.topkAdd('topk', 'A', 'B', 'C', 'D', 'E', 'A', 'A', 'B', 'C', + self.assertTrue(rb.topkReserve('topk', 3, 50, 4, 0.9)) + self.assertEqual([None, None, None, None, None, None, None, None, + None, None, None, None, 'C', None, None, None], + rb.topkAdd('topk', 'A', 'B', 'C', 'D', 'E', 'A', 'A', 'B', 'C', 'G', 'D', 'B', 'D', 'A', 'E', 'E')) - self.assertEqual([1, 1, 1, 1, 1, 0, 1], + self.assertEqual([1, 1, 0, 1, 0, 0, 0], rb.topkQuery('topk', 'A', 'B', 'C', 'D', 'E', 'F', 'G')) self.assertEqual([4, 3, 2, 3, 3, 0, 1], rb.topkCount('topk', 'A', 'B', 'C', 'D', 'E', 'F', 'G')) From 5e7a5192aada3f7d8cbf9ee9d6369d80c5f9643d Mon Sep 17 00:00:00 2001 From: Ariel Date: Thu, 20 Jun 2019 19:32:50 +0300 Subject: [PATCH 15/17] fix for add returns value of dropped --- redisbloom/client.py | 43 +++++++++++++++++++++++-------------------- test_commands.py | 6 +++--- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/redisbloom/client.py b/redisbloom/client.py index 211b4c6..56e8b21 100644 --- a/redisbloom/client.py +++ b/redisbloom/client.py @@ -35,7 +35,10 @@ def spaceHolder(response): def parseToList(response): res = [] for item in response: - res.append(nativestr(item)) + if item is not None: + res.append(nativestr(item)) + else: + res.append(None) return res class Client(Redis): #changed from StrictRedis @@ -101,35 +104,35 @@ def __init__(self, *args, **kwargs): self.BF_RESERVE : bool_ok, #self.BF_ADD : spaceHolder, #self.BF_MADD : spaceHolder, - self.BF_INSERT : spaceHolder, - self.BF_EXISTS : spaceHolder, - self.BF_MEXISTS : spaceHolder, - self.BF_SCANDUMP : spaceHolder, - self.BF_LOADCHUNK : spaceHolder, + #self.BF_INSERT : spaceHolder, + #self.BF_EXISTS : spaceHolder, + #self.BF_MEXISTS : spaceHolder, + #self.BF_SCANDUMP : spaceHolder, + #self.BF_LOADCHUNK : spaceHolder, self.CF_RESERVE : bool_ok, - self.CF_ADD : spaceHolder, - self.CF_ADDNX : spaceHolder, - self.CF_INSERT : spaceHolder, - self.CF_INSERTNX : spaceHolder, - self.CF_EXISTS : spaceHolder, - self.CF_DEL : spaceHolder, - self.CF_COUNT : spaceHolder, - self.CF_SCANDUMP : spaceHolder, - self.CF_LOADDUMP : spaceHolder, + #self.CF_ADD : spaceHolder, + #self.CF_ADDNX : spaceHolder, + #self.CF_INSERT : spaceHolder, + #self.CF_INSERTNX : spaceHolder, + #self.CF_EXISTS : spaceHolder, + #self.CF_DEL : spaceHolder, + #self.CF_COUNT : spaceHolder, + #self.CF_SCANDUMP : spaceHolder, + #self.CF_LOADDUMP : spaceHolder, self.CMS_INITBYDIM : bool_ok, self.CMS_INITBYPROB : bool_ok, self.CMS_INCRBY : bool_ok, - self.CMS_QUERY : spaceHolder, + #self.CMS_QUERY : spaceHolder, self.CMS_MERGE : bool_ok, self.CMS_INFO : CMSInfo, self.TOPK_RESERVE : bool_ok, - self.TOPK_ADD : spaceHolder, - self.TOPK_QUERY : spaceHolder, - self.TOPK_COUNT : spaceHolder, - self.TOPK_LIST : parseToList, + self.TOPK_ADD : parseToList, + #self.TOPK_QUERY : spaceHolder, + #self.TOPK_COUNT : spaceHolder, + #self.TOPK_LIST : spaceHolder, self.TOPK_INFO : TopKInfo, } for k, v in six.iteritems(MODULE_CALLBACKS): diff --git a/test_commands.py b/test_commands.py index 1a1306d..77e0794 100644 --- a/test_commands.py +++ b/test_commands.py @@ -156,9 +156,9 @@ def testTopK(self): # test list with empty buckets self.assertTrue(rb.topkReserve('topk', 3, 50, 4, 0.9)) self.assertEqual([None, None, None, None, None, None, None, None, - None, None, None, None, 'C', None, None, None], + None, None, None, None, b'C', None, None, None, None], rb.topkAdd('topk', 'A', 'B', 'C', 'D', 'E', 'A', 'A', 'B', 'C', - 'G', 'D', 'B', 'D', 'A', 'E', 'E')) + 'G', 'D', 'B', 'D', 'A', 'E', 'E', 1)) self.assertEqual([1, 1, 0, 1, 0, 0, 0], rb.topkQuery('topk', 'A', 'B', 'C', 'D', 'E', 'F', 'G')) self.assertEqual([4, 3, 2, 3, 3, 0, 1], @@ -168,7 +168,7 @@ def testTopK(self): self.assertTrue(rb.topkReserve('topklist', 3, 50, 3, 0.9)) self.assertTrue(rb.topkAdd('topklist', 'A', 'B', 'C', 'D', 'E','A', 'A', 'B', 'C', 'G', 'D', 'B', 'D', 'A', 'E', 'E')) - self.assertEqual(['D', 'A', 'B'], rb.topkList('topklist')) + self.assertEqual([b'D', b'A', b'B'], rb.topkList('topklist')) info = rb.topkInfo('topklist') self.assertEqual(3, info.k) self.assertEqual(50, info.width) From dd951d97d60a9fa810596cde5be7453a24cf876e Mon Sep 17 00:00:00 2001 From: ashtul <44112901+ashtul@users.noreply.github.com> Date: Thu, 20 Jun 2019 19:36:40 +0300 Subject: [PATCH 16/17] Update test_commands.py --- test_commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_commands.py b/test_commands.py index 77e0794..b57efc5 100644 --- a/test_commands.py +++ b/test_commands.py @@ -156,7 +156,7 @@ def testTopK(self): # test list with empty buckets self.assertTrue(rb.topkReserve('topk', 3, 50, 4, 0.9)) self.assertEqual([None, None, None, None, None, None, None, None, - None, None, None, None, b'C', None, None, None, None], + None, None, None, None, 'C', None, None, None, None], rb.topkAdd('topk', 'A', 'B', 'C', 'D', 'E', 'A', 'A', 'B', 'C', 'G', 'D', 'B', 'D', 'A', 'E', 'E', 1)) self.assertEqual([1, 1, 0, 1, 0, 0, 0], @@ -168,7 +168,7 @@ def testTopK(self): self.assertTrue(rb.topkReserve('topklist', 3, 50, 3, 0.9)) self.assertTrue(rb.topkAdd('topklist', 'A', 'B', 'C', 'D', 'E','A', 'A', 'B', 'C', 'G', 'D', 'B', 'D', 'A', 'E', 'E')) - self.assertEqual([b'D', b'A', b'B'], rb.topkList('topklist')) + self.assertEqual(['D', 'A', 'B'], rb.topkList('topklist')) info = rb.topkInfo('topklist') self.assertEqual(3, info.k) self.assertEqual(50, info.width) @@ -176,4 +176,4 @@ def testTopK(self): self.assertAlmostEqual(0.9, float(info.decay)) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 3969fd0a06c53da56e28b13ab7b56a6642dec926 Mon Sep 17 00:00:00 2001 From: ashtul <44112901+ashtul@users.noreply.github.com> Date: Thu, 20 Jun 2019 19:38:11 +0300 Subject: [PATCH 17/17] Update client.py --- redisbloom/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redisbloom/client.py b/redisbloom/client.py index 56e8b21..bd0955b 100644 --- a/redisbloom/client.py +++ b/redisbloom/client.py @@ -132,7 +132,7 @@ def __init__(self, *args, **kwargs): self.TOPK_ADD : parseToList, #self.TOPK_QUERY : spaceHolder, #self.TOPK_COUNT : spaceHolder, - #self.TOPK_LIST : spaceHolder, + self.TOPK_LIST : parseToList, self.TOPK_INFO : TopKInfo, } for k, v in six.iteritems(MODULE_CALLBACKS): @@ -469,4 +469,4 @@ def topkInfo(self, key): """ Returns k, width, depth and decay values of ``key``. """ - return self.execute_command(self.TOPK_INFO, key) \ No newline at end of file + return self.execute_command(self.TOPK_INFO, key)