Skip to content
This repository has been archived by the owner on Sep 11, 2019. It is now read-only.

Support StrictRedis.eval for Lua scripts #9

Merged
merged 57 commits into from
Feb 13, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
b8d997d
Limited Lua support
blfoster Feb 5, 2018
da8a820
Bump patch version
blfoster Feb 5, 2018
0845484
Remove nightly Python build
blfoster Feb 5, 2018
7ea39cd
Return result from Lua
blfoster Feb 5, 2018
74389b7
Fix table handling
blfoster Feb 6, 2018
58d0763
Fix unit tests
blfoster Feb 6, 2018
77685dc
Fix test
blfoster Feb 6, 2018
0dd661f
Decode non-byte strings
blfoster Feb 6, 2018
507a125
Add eval
blfoster Feb 6, 2018
06e02b5
Restore nightly build
blfoster Feb 6, 2018
0fd83e6
Revert "Restore nightly build"
blfoster Feb 6, 2018
f2086fd
Update readme
blfoster Feb 6, 2018
6e0c1b3
Remove eval from unimplemented functions
blfoster Feb 6, 2018
73cc1d7
Code review fallout
blfoster Feb 7, 2018
87c2686
pep8:
blfoster Feb 7, 2018
e0e7718
Install lupa for CI
blfoster Feb 7, 2018
121de8c
Code review fallout
blfoster Feb 7, 2018
9de8754
pep8:
blfoster Feb 7, 2018
9e6b32f
Install lupa for CI
blfoster Feb 7, 2018
15232fa
Merge branch 'master' of ../fakeredis
blfoster Feb 7, 2018
9bf4173
More fallout
blfoster Feb 8, 2018
bfa38c9
Remove lua from .travis.yml
blfoster Feb 8, 2018
63949ba
More tests
blfoster Feb 8, 2018
975f88c
pcall
blfoster Feb 8, 2018
71a2c1e
Test nested ok response
blfoster Feb 8, 2018
1def833
pep8
blfoster Feb 8, 2018
dd23fb1
Fix broken tst
blfoster Feb 8, 2018
c91f0b1
More fallout
blfoster Feb 8, 2018
ee53358
Remove lua from .travis.yml
blfoster Feb 8, 2018
a30fdef
More tests
blfoster Feb 8, 2018
b426f0f
pcall
blfoster Feb 8, 2018
6153787
Test nested ok response
blfoster Feb 8, 2018
7fb69f4
pep8
blfoster Feb 8, 2018
49cf337
Merge branch 'master' of ../fakeredis
blfoster Feb 8, 2018
392206c
Remove extra braces
blfoster Feb 8, 2018
d5e4ce2
Merge branch 'master' of ../fakeredis
blfoster Feb 8, 2018
fe9065d
Fix type
blfoster Feb 8, 2018
296c2ff
Fix decoding
blfoster Feb 8, 2018
95762e2
Merge branch 'master' of ../fakeredis
blfoster Feb 8, 2018
fde2b75
Mess with string types some more
blfoster Feb 8, 2018
773f0f4
Merge branch 'master' of ../fakeredis
blfoster Feb 8, 2018
b1a0402
Fix test
blfoster Feb 8, 2018
856a031
Merge branch 'master' of ../fakeredis
blfoster Feb 8, 2018
acc44b1
Test for nil in table
blfoster Feb 8, 2018
23a43af
More tests
blfoster Feb 9, 2018
525a29b
Fix test
blfoster Feb 9, 2018
267e338
Remove unnecessary exception handler
blfoster Feb 9, 2018
6377669
WIP: fallout
blfoster Feb 12, 2018
244056b
Remove docstring
blfoster Feb 12, 2018
8f869b4
More fallout
blfoster Feb 12, 2018
b2b6786
Catch LuaError
blfoster Feb 12, 2018
fa252bc
Check number of keys
blfoster Feb 12, 2018
f917498
More fallout
blfoster Feb 12, 2018
82e7e65
More edge cases for numkeys
blfoster Feb 12, 2018
fdf9ecb
Remove braces
blfoster Feb 12, 2018
a6a7cd6
flake8
blfoster Feb 12, 2018
9f4d910
to_bytes
blfoster Feb 12, 2018
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 4 additions & 1 deletion README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,6 @@ scripting
* script kill
* script load
* evalsha
* eval
* script exists


Expand Down Expand Up @@ -266,6 +265,10 @@ they have all been tagged as 'slow' so you can skip them by running::
Revision history
================

Development version
-----
- `#9 <https://github.com/ska-sa/fakenewsredis/pull/9>`_ Add support for StrictRedis.eval for Lua scripts

0.9.4
-----
This is a minor bugfix and optimization release:
Expand Down
143 changes: 143 additions & 0 deletions fakenewsredis.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import types
import re
import functools
from itertools import count

import redis
from redis.exceptions import ResponseError
Expand Down Expand Up @@ -701,6 +702,148 @@ def sort(self, name, start=None, num=None, by=None, get=None, desc=False,
except KeyError:
return []

def eval(self, script, numkeys, *keys_and_args):
from lupa import LuaRuntime, LuaError

if any(
isinstance(numkeys, t) for t in (text_type, str, bytes)
):
try:
numkeys = int(numkeys)
except ValueError:
# Non-numeric string will be handled below.
pass
if not(isinstance(numkeys, int)):
raise ResponseError("value is not an integer or out of range")
elif numkeys > len(keys_and_args):
raise ResponseError("Number of keys can't be greater than number of args")
elif numkeys < 0:
raise ResponseError("Number of keys can't be negative")

keys_and_args = [to_bytes(v) for v in keys_and_args]
lua_runtime = LuaRuntime(unpack_returned_tuples=True)

set_globals = lua_runtime.eval(
"""
function(keys, argv, redis_call, redis_pcall)
redis = {}
redis.call = redis_call
redis.pcall = redis_pcall
redis.error_reply = function(msg) return {err=msg} end
redis.status_reply = function(msg) return {ok=msg} end
KEYS = keys
ARGV = argv
end
"""
)
expected_globals = set()
set_globals(
[None] + keys_and_args[:numkeys],
[None] + keys_and_args[numkeys:],
functools.partial(self._lua_redis_call, lua_runtime, expected_globals),
functools.partial(self._lua_redis_pcall, lua_runtime, expected_globals)
)
expected_globals.update(lua_runtime.globals().keys())

try:
result = lua_runtime.execute(script)
except LuaError as ex:
raise ResponseError(ex)

self._check_for_lua_globals(lua_runtime, expected_globals)

return self._convert_lua_result(result, nested=False)

def _convert_redis_result(self, result):
if isinstance(result, dict):
return [
i
for item in result.items()
for i in item
]
return result

def _convert_lua_result(self, result, nested=True):
from lupa import lua_type
if lua_type(result) == 'table':
for key in ('ok', 'err'):
if key in result:
msg = self._convert_lua_result(result[key])
if not isinstance(msg, bytes):
raise ResponseError("wrong number or type of arguments")
if key == 'ok':
return msg
elif nested:
return ResponseError(msg)
else:
raise ResponseError(msg)
# Convert Lua tables into lists, starting from index 1, mimicking the behavior of StrictRedis.
result_list = []
for index in count(1):
if index not in result:
break
item = result[index]
result_list.append(self._convert_lua_result(item))
return result_list
elif isinstance(result, text_type):
return to_bytes(result)
elif isinstance(result, float):
return int(result)
elif isinstance(result, bool):
return 1 if result else None
return result

def _check_for_lua_globals(self, lua_runtime, expected_globals):
actual_globals = set(lua_runtime.globals().keys())
if actual_globals != expected_globals:
raise ResponseError(
"Script attempted to set a global variables: %s" % ", ".join(
actual_globals - expected_globals
)
)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this is a good approach to preventing global variables being created: it doesn't make the script itself error out, and it doesn't actually prevent the globals from polluting the namespace.

Here is the code in redis itself that implements the protection. It may need some tweaks to adapt it - unfortunately I've never programmed in Lua so I'm not sure.

See also the function above the code in that link - it appears to disable readfile and dofile, presumably for security reasons.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is actually safe, because if we try to set a global variable and error out, the LuaRuntime instance will never be used again; if we try to run another Lua script, we'll get a new LuaRuntime, which will not have that global variable set. Unless there's something I'm missing. I do think this way is a bit more readable, but perhaps that's because I don't know Lua. I could add a few more lines to the unit test asserting that trying to set a global variable can't have a side effect...

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I hadn't spotted that the runtime gets discarded each time, so it's safer than I thought. I think it will still be different to real redis if a script sets a global variable, then modifies the database: in real redis it would error out as soon as it tries to set the global, whereas in this implementation it will get to modify the database before the error. Another case that will probably behave differently is a script that creates a global and then deletes it again, before your check.

We're starting to get to the point of diminishing returns. If you've got the time and energy to test it and fix things up, then go for it, but I realise that I've made you do a lot more work than you probably expected when you started. If you're running out of steam then this is something that can be left for another day.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 I don't think it should be possible to modify the database either after setting a global, because _lua_redis_call calls _check_for_lua_globals before executing any command that could change the database.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Aha, I'd missed that subtlety too. So then probably the only case that'll behave differently is if the script creates a global and deletes it again.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep. I'll leave that alone for now if you're ok with it.


def _lua_redis_pcall(self, lua_runtime, expected_globals, op, *args):
try:
return self._lua_redis_call(lua_runtime, expected_globals, op, *args)
except Exception as ex:
return lua_runtime.table_from({"err": str(ex)})

def _lua_redis_call(self, lua_runtime, expected_globals, op, *args):
# Check if we've set any global variables before making any change.
self._check_for_lua_globals(lua_runtime, expected_globals)
# These commands aren't necessarily all implemented, but if op is not one of these commands, we expect
# a ResponseError for consistency with Redis
commands = [
'append', 'auth', 'bitcount', 'bitfield', 'bitop', 'bitpos', 'blpop', 'brpop', 'brpoplpush',
'decr', 'decrby', 'del', 'dump', 'echo', 'eval', 'evalsha', 'exists', 'expire', 'expireat',
'flushall', 'flushdb', 'geoadd', 'geodist', 'geohash', 'geopos', 'georadius', 'georadiusbymember',
'get', 'getbit', 'getrange', 'getset', 'hdel', 'hexists', 'hget', 'hgetall', 'hincrby',
'hincrbyfloat', 'hkeys', 'hlen', 'hmget', 'hmset', 'hscan', 'hset', 'hsetnx', 'hstrlen', 'hvals',
'incr', 'incrby', 'incrbyfloat', 'info', 'keys', 'lindex', 'linsert', 'llen', 'lpop', 'lpush',
'lpushx', 'lrange', 'lrem', 'lset', 'ltrim', 'mget', 'migrate', 'move', 'mset', 'msetnx',
'object', 'persist', 'pexpire', 'pexpireat', 'pfadd', 'pfcount', 'pfmerge', 'ping', 'psetex',
'psubscribe', 'pttl', 'publish', 'pubsub', 'punsubscribe', 'rename', 'renamenx', 'restore',
'rpop', 'rpoplpush', 'rpush', 'rpushx', 'sadd', 'scan', 'scard', 'sdiff', 'sdiffstore', 'select',
'set', 'setbit', 'setex', 'setnx', 'setrange', 'shutdown', 'sinter', 'sinterstore', 'sismember',
'slaveof', 'slowlog', 'smembers', 'smove', 'sort', 'spop', 'srandmember', 'srem', 'sscan',
'strlen', 'subscribe', 'sunion', 'sunionstore', 'swapdb', 'touch', 'ttl', 'type', 'unlink',
'unsubscribe', 'wait', 'watch', 'zadd', 'zcard', 'zcount', 'zincrby', 'zinterstore', 'zlexcount',
'zrange', 'zrangebylex', 'zrangebyscore', 'zrank', 'zrem', 'zremrangebylex', 'zremrangebyrank',
'zremrangebyscore', 'zrevrange', 'zrevrangebylex', 'zrevrangebyscore', 'zrevrank', 'zscan',
'zscore', 'zunionstore'
]

op = op.lower()
if op not in commands:
raise ResponseError("Unknown Redis command called from Lua script")
special_cases = {
'del': FakeStrictRedis.delete,
'decrby': FakeStrictRedis.decr,
'incrby': FakeStrictRedis.incr
}
func = special_cases[op] if op in special_cases else getattr(FakeStrictRedis, op)
return self._convert_redis_result(func(self, *args))

def _retrive_data_from_sort(self, data, get):
if get is not None:
if isinstance(get, string_types):
Expand Down
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@
flake8<3.0.0
nose==1.3.4
redis==2.10.6
lupa==1.6
5 changes: 4 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,8 @@
],
install_requires=[
'redis',
]
],
extras_require={
"lua": ['lupa']
}
)