Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

fixed lua global variables

  • Loading branch information...
commit d88c3d892533e8d1ff96762a9402b52cc8053a55 1 parent b070b08
quantmind authored
Showing with 215 additions and 154 deletions.
  1. +1 −1  CHANGELOG.rst
  2. +9 −0 docs/source/_templates/sidebarintro.html
  3. +3 −4 runtests.py
  4. +1 −1  stdnet/__init__.py
  5. +8 −5 stdnet/apps/columnts/__init__.py
  6. +3 −0  stdnet/apps/columnts/models.py
  7. +3 −1 stdnet/apps/columnts/npts.py
  8. +17 −12 stdnet/apps/columnts/redis.py
  9. +2 −2 stdnet/backends/redisb.py
  10. +2 −2 stdnet/conf.py
  11. +24 −1 stdnet/exceptions.py
  12. +1 −1  stdnet/lib/__init__.py
  13. +16 −16 stdnet/lib/lua/columnts/columnts.lua
  14. +3 −3 stdnet/lib/lua/columnts/stats.lua
  15. +4 −4 stdnet/lib/lua/commands/keyinfo.lua
  16. +1 −1  stdnet/lib/lua/commands/zdiffstore.lua
  17. +2 −2 stdnet/lib/lua/odm/build_query.lua
  18. +1 −0  stdnet/lib/lua/odm/load_query.lua
  19. +5 −4 stdnet/lib/lua/odm/numberarray.lua
  20. +27 −26 stdnet/lib/redis/client.py
  21. +20 −16 stdnet/lib/redis/connection.py
  22. +18 −18 stdnet/lib/redis/exceptions.py
  23. +28 −12 stdnet/lib/redis/scripts.py
  24. +4 −3 tests/regression/columnts/main.py
  25. +1 −2  tests/regression/columnts/npts.py
  26. +2 −2 tests/regression/me.py
  27. +0 −6 tests/regression/redis/base.py
  28. +2 −2 tests/regression/redis/commands.py
  29. +3 −3 tests/regression/redis/pipeline.py
  30. +2 −2 tests/regression/redis/pool.py
  31. +1 −1  tests/regression/redis/scripting.py
  32. +1 −1  tests/regression/sorting.py
View
2  CHANGELOG.rst
@@ -50,7 +50,7 @@ Ver. 0.7c3 - 2012 May 02
* Added :mod:`stdnet.utils.path`.
* Added a Lua test suite for testing stand alone scripts. Requires lunatest_.
* PEP 386-compliant version number.
-* **572 regression tests** with **90%** coverage.
+* **573 regression tests** with **90%** coverage.
.. _vers06:
View
9 docs/source/_templates/sidebarintro.html
@@ -4,9 +4,18 @@
Redis data-structure server.
It is designed to be fast, memory efficient and highly customizable.
</p>
+<g:plusone annotation="inline"></g:plusone>
+<script type="text/javascript">
+ (function() {
+ var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;
+ po.src = 'https://apis.google.com/js/plusone.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);
+ })();
+</script>
<h3>Useful Links</h3>
<ul>
<li><a href="http://pypi.python.org/pypi/python-stdnet">Stdnet @ PyPI</a></li>
<li><a href="https://github.com/lsbardel/python-stdnet">Stdnet @ github</a></li>
<li><a href="http://twitter.com/lsbardel">lsbardel @ twitter</a></li>
+ <li><a href="https://groups.google.com/forum/?fromgroups#!forum/python-stdnet">email list @ google</a></li>
</ul>
View
7 runtests.py
@@ -4,16 +4,15 @@
import sys
import os
-from stdnet.conf import settings
-from stdnet.utils import Path
-from stdnet import test, getdb
-
## This is for dev environment with pulsar and dynts.
## If not available, some tests won't run
+from stdnet.utils import Path
p = Path(__file__)
p.add2python('pulsar', up=1, down=('pulsar',), must_exist=False)
p.add2python('dynts', up=1, down=('dynts',), must_exist=False)
+from stdnet.conf import settings
+from stdnet import test, getdb
from stdnet.test import nose, pulsar
View
2  stdnet/__init__.py
@@ -29,4 +29,4 @@
'Topic :: Internet'
]
-sphinxtogithub = True
+sphinxtogithub = False
View
13 stdnet/apps/columnts/__init__.py
@@ -1,8 +1,11 @@
'''\
-A timeseries application where each field is stored in a redis string.
-This data-structure is composed by several redis structure:
+An application for managing multivariate timeseries_. It provides
+tools for performing aggregation and statistics via lua scripts.
-* A Timeseries for holding times in an ordered fashion.
+The redis implementation uses several redis structures for a given
+class:`ColumnTS` instance:
+
+* A zset for holding times in an ordered fashion.
* A redis *set* for holding *fields* names.
* A redis string for each *field* in the timeseries.
@@ -17,8 +20,7 @@
from datetime date
from stdnet.apps.columnts ColumnTS
- ts = ColumnTS(id = 'test')
-
+ ts = ColumnTS(id='test')
ts.add(date(2012,2,21), {'open': 603.87, 'close': 614.00})
API
@@ -28,6 +30,7 @@
:members:
:member-order: bysource
+.. _timeseries: http://en.wikipedia.org/wiki/Time_series
'''
from . import redis
from .encoders import *
View
3  stdnet/apps/columnts/models.py
@@ -9,6 +9,7 @@
__all__ = ['TimeseriesCache', 'ColumnTS', 'ColumnTSField']
+
class TimeseriesCache(object):
cache = None
def __init__(self):
@@ -30,6 +31,8 @@ def clear(self):
class ColumnTS(odm.TS):
+ '''A specialised timeseries structure for handling several fields and
+statistical calculations.'''
default_multi_stats = ['covariance']
cache_class = TimeseriesCache
View
4 stdnet/apps/columnts/npts.py
@@ -1,6 +1,8 @@
'''Experimental!
This is an experimental module for converting ColumnTS into
-dynts.timeseries.
+dynts.timeseries. It requires dynts_.
+
+.. _dynts: https://github.com/quantmind/dynts
'''
from . import models as columnts
View
29 stdnet/apps/columnts/redis.py
@@ -6,8 +6,8 @@
from stdnet.lib import redis
-class RedisColumnTS(redisb.TS):
-
+class RedisColumnTS(redisb.Zset):
+ '''Redis backend for :class:`ColumnTS`'''
@property
def fieldsid(self):
return self.id + ':fields'
@@ -25,6 +25,9 @@ def flush(self):
keys, args = cache.merged_series
return self.client.script_call('timeseries_merge', keys, *args)
+ def _iter(self):
+ return iter(self.irange(novalues=True))
+
def allkeys(self):
return self.client.keys(self.id + '*')
@@ -44,21 +47,21 @@ def numfields(self):
'''Number of fields'''
return self.client.scard(self.fieldsid)
- def irange(self, start = 0, end = -1, fields = None, novalues = False,
- delete = False):
+ def irange(self, start = 0, end = -1, fields=None, novalues=False,
+ delete=False, **kwargs):
noval = 1 if novalues else 0
fields = fields or ()
delete = 1 if delete else 0
return self.client.script_call(
- 'timeseries_query', self.id, 'tsrange',
+ 'timeseries_query', self.id, 'zrange',
start, end, noval, delete, len(fields),
- *fields, fields = fields, novalues = novalues)
+ *fields, fields = fields, novalues=novalues)
- def range(self, start, end, fields = None, novalues = False):
+ def range(self, start, end, fields=None, novalues=False, **kwargs):
noval = 1 if novalues else 0
fields = fields or ()
return self.client.script_call(
- 'timeseries_query', self.id,'tsrangebytime',
+ 'timeseries_query', self.id, 'zrangebyscore',
start, end, noval, 0, len(fields), *fields,
fields = fields, novalues = novalues)
@@ -97,18 +100,18 @@ def merge(self, series, fields):
def istats(self, start, end, fields = None):
fields = fields or ()
return self.client.script_call('timeseries_stats', self.id,
- 'tsrange', start, end, 'uni', len(fields), *fields)
+ 'zrange', start, end, 'uni', len(fields), *fields)
def stats(self, start, end, fields = None):
fields = fields or ()
return self.client.script_call('timeseries_stats', self.id,
- 'tsrangebytime', start, end, 'uni', len(fields), *fields)
+ 'zrangebyscore', start, end, 'uni', len(fields), *fields)
def imulti_stats(self, start, end, fields, series, stats):
- return self._multi_stats('tsrange', start, end, fields, series, stats)
+ return self._multi_stats('zrange', start, end, fields, series, stats)
def multi_stats(self, start, end, fields, series, stats):
- return self._multi_stats('tsrangebytime', start, end, fields, series,
+ return self._multi_stats('zrangebyscore', start, end, fields, series,
stats)
def _multi_stats(self, command, start, end, fields, series, stats):
@@ -136,6 +139,8 @@ def _multi_stats(self, command, start, end, fields, series, stats):
redisb.BackendDataServer.struct_map['columnts'] = RedisColumnTS
+############################################################## SCRIPTS
+
class timeseries_session(redis.RedisScript):
script = (redis.read_lua_file('tabletools'),
redis.read_lua_file('columnts.columnts'),
View
4 stdnet/backends/redisb.py
@@ -476,7 +476,7 @@ def flush(self):
return result
def get(self, score):
- r = self.range(score,score,withscores=False)
+ r = self.range(score, score, withscores=False)
if r:
if len(r) > 1:
return r
@@ -484,7 +484,7 @@ def get(self, score):
return r[0]
def _iter(self):
- return iter(self.irange(withscores = False))
+ return iter(self.irange(withscores=False))
def size(self):
return self.client.zcard(self.id)
View
4 stdnet/conf.py
@@ -66,13 +66,13 @@ def __init__(self):
def redis_status(self):
from stdnet import getdb
- from stdnet.lib.redis import ConnectionError
+ from stdnet.lib.redis import RedisConnectionError
db = getdb(self.DEFAULT_BACKEND)
status = "ok"
if db.name == 'redis':
status = db.client.redis_status()
if not status:
- raise ConnectionError('No connection available for server\
+ raise RedisConnectionError('No connection available for server\
at "{0}"'.format(self.DEFAULT_BACKEND))
os.environ['stdnet_backend_status'] = status
return status
View
25 stdnet/exceptions.py
@@ -3,63 +3,86 @@ class StdNetException(Exception):
'''A general StdNet exception'''
pass
+
+class ConnectionError(StdNetException):
+ pass
+
+
class SessionNotAvailable(StdNetException):
pass
+
class ModelNotAvailable(StdNetException):
pass
+
class ModelNotRegistered(SessionNotAvailable):
'''A :class:`StdNetException` raised when trying to save an instance of a :class:`stdnet.odm.StdModel` not yet
registered with a :class:`stdnet.backends.BackendDataServer`. Check :func:`stdnet.odm.register` for details.'''
pass
+
class InvalidTransaction(StdNetException):
'''A :class:`StdNetException` raised when trying to create a transaction
with models registered with different backends.'''
pass
-class CommitException(StdNetException):
+
+class ResponseError(StdNetException):
+ '''Raised when an invalid response is returned from the backebd server.'''
+ pass
+
+
+class CommitException(ResponseError):
'''A :class:`StdNetException` raised when trying to create a transaction
with models registered with different backends.'''
def __init__(self, msg, failures = 1):
self.failures = failures
super(CommitException,self).__init__(msg)
+
class AlreadyRegistered(StdNetException):
pass
+
class ObjectNotValidated(StdNetException):
'''A :class:`StdNetException` raised when an instance of a :class:`stdnet.odm.StdModel` fails to validate
(probably required :class:`stdnet.odm.Field` are missing from the instance).'''
pass
+
class ImproperlyConfigured(StdNetException):
"A :class:`stdnet.StdNetException` raised when stdnet is somehow improperly configured"
pass
+
class BadCacheDataStructure(StdNetException):
pass
+
class FieldError(StdNetException):
'''Generic Field error'''
pass
+
class StructureFieldError(StdNetException):
'''A :class:`stdnet.FieldError` for :class:stdnet.odm.StructureField`.'''
pass
+
class FieldValueError(FieldError):
'''A :class:`stdnet.FieldError` raised when passing a wrong
value to a field. This exception is cought during the model instance
validation algorithm in :meth:`stdnet.odm.base.Metaclass.is_valid`.'''
pass
+
class QuerySetError(StdNetException):
'''A :class:`stdnet.StdNetException` raised during a :class:`stdnet.odm.query.QuerySet`
evaluation.'''
pass
+
class ObjectNotFound(QuerySetError):
'''A :class:`QuerySetError` raised when an object is not found.'''
pass
View
2  stdnet/lib/__init__.py
@@ -1,4 +1,4 @@
-hasextensions = True
+hasextensions = False
hr = None
from .fallback import *
from . import fallback
View
32 stdnet/lib/lua/columnts/columnts.lua
@@ -4,7 +4,7 @@ local nan = 0/0
local nildata = string.char(0,0,0,0,0,0,0,0,0)
-- Column timeseries class
-columnts = {
+local columnts = {
--
-- Initialize
init = function (self, key)
@@ -29,7 +29,7 @@ columnts = {
--
-- a set of fields
fields_set = function(self)
- f = {}
+ local f = {}
for _,name in ipairs(self:fields()) do
f[name] = self:fieldkey(name)
end
@@ -38,7 +38,7 @@ columnts = {
--
-- Length of timeseries
length = function (self)
- return redis.call('tslen', self.key) + 0
+ return redis.call('zcard', self.key) + 0
end,
--
-- Delete timeseries
@@ -51,12 +51,12 @@ columnts = {
--
-- Return the ordered list of times
times = function (self)
- return redis.call('tsrange', self.key, 0, -1, 'novalues')
+ return redis.call('zrange', self.key, 0, -1)
end,
--
-- The rank of timestamp in the timeseries
rank = function (self, timestamp)
- return redis.call('tsrank', self.key, timestamp)
+ return redis.call('zrank', self.key, timestamp)
end,
--
-- Return the unpacked value of field at rank
@@ -101,14 +101,14 @@ columnts = {
--
-- Add a timeseries, multiplied by the given weight, to self
addserie = function(self, ts, weight, fields, tsmul)
- local range = ts:range('tsrange', 0, -1, fields)
+ local range = ts:range('zrange', 0, -1, fields)
local times, field_values = unpack(range)
return self:add(times, field_values, weight, tsmul)
end,
--
-- shortcut for returning the whole range of a timeserie
all = function(self, fields)
- return self:range('tsrange', 0, -1, fields)
+ return self:range('zrange', 0, -1, fields)
end,
--
-- remove a field and return true or false
@@ -118,7 +118,7 @@ columnts = {
--
-- remove a timestamp from timeseries and return it
poptime = function(self, timestamp)
- local rank = redis.call('tsrank', self.key, timestamp)
+ local rank = redis.call('zrank', self.key, timestamp)
if rank then
rank = rank + 0
for i,field in pairs(fields) do
@@ -127,7 +127,7 @@ columnts = {
if rank > 0 then
data = redis.call('getrange', fieldid, (rank-1)*9, rank*9) + data
end
- redis.call('set',fieldid,data)
+ redis.call('set', fieldid, data)
end
end
end,
@@ -135,7 +135,7 @@ columnts = {
-- return an array containg a range of the timeseries. The array
-- contains two elements, an array of times and a dictionary of fields.
range = function(self, command, start, stop, fields, unpack_values)
- local times = redis.call(command, self.key, start, stop, 'novalues')
+ local times = redis.call(command, self.key, start, stop)
local field_values = {}
local data = {times, field_values}
local len = # times
@@ -143,7 +143,7 @@ columnts = {
return data
end
-- get the start rank (Also when we use tsrange. Important)
- start = redis.call('tsrank', self.key, times[1])
+ start = redis.call('zrank', self.key, times[1])
stop = start + len
if not fields or # fields == 0 then
fields = self:fields()
@@ -224,8 +224,8 @@ columnts = {
available = self:rank(timestamp)
-- This is a new timestamp
if not available then
- redis.call('tsadd', self.key, timestamp, 1)
- rank = redis.call('tsrank', self.key, timestamp) + 0
+ redis.call('zadd', self.key, timestamp, timestamp)
+ rank = redis.call('zrank', self.key, timestamp) + 0
rank9 = 9*rank
tslen = self:length()
-- loop over all fields and append/insert new data to the field strings
@@ -285,10 +285,10 @@ columnts = {
end
end
- if weight and not new_table then
+ if weight then
for timestamp, avail in pairs(time_set) do
if not avail then
- rank9 = redis.call('tsrank', self.key, timestamp)*9
+ rank9 = redis.call('zrank', self.key, timestamp)*9
for field, fkey in pairs(fields) do
redis.call('setrange', fkey, rank9, nildata)
end
@@ -356,7 +356,7 @@ columnts = {
end
}
-columnts_meta = {}
+local columnts_meta = {}
-- Constructor
function columnts:new(key)
local result = {}
View
6 stdnet/lib/lua/columnts/stats.lua
@@ -8,9 +8,8 @@ end
local function add_field_names(key, field_values, serie_names)
local fields = {}
for field, values in pairs(field_values) do
- name = key .. ' @ ' .. field
table.insert(fields, field)
- table.insert(serie_names, name)
+ table.insert(serie_names, key .. ' @ ' .. field)
end
return fields
end
@@ -69,13 +68,14 @@ stats.univariate = function (serie)
end
local result = {start=times[1], stop=times[N], len=N, stats=sts}
for field, values in pairs(serie.field_values) do
+ local dv, dv2
local N = 0
local min_val = 1.e10
local max_val =-1.e10
local sum_val = 0
local sum2_val = 0
local dsum, dsum2, dsum3, dsum4 = 0, 0, 0, 0
- local p, dv = nan
+ local p = nan
for i,v in ipairs(values) do
if v == v then
min_val = math.min(min_val, v)
View
8 stdnet/lib/lua/commands/keyinfo.lua
@@ -1,6 +1,6 @@
-- Retrieve information about keys
-- A list of keys, sorted in alphabetical order, is returned
-local start, stop, keys
+local start, stop, keys, num
if # ARGV > 0 then -- If argv is provided, it is the pattern to search
keys = redis.call('KEYS', ARGV[1])
if # ARGV > 1 then
@@ -24,10 +24,10 @@ type_table['list'] = 'llen'
type_table['hash'] = 'hlen'
type_table['ts'] = 'tslen' -- stdnet branch
type_table['string'] = 'strlen'
+local typ, command, len, key, idletime
local stats = {}
-local typ, command, len, j, num_keys
-num_keys = # keys
-j = 0
+local num_keys = # keys
+local j = 0
while j < num and start+j <= num_keys do
key = keys[start+j]
j = j + 1
View
2  stdnet/lib/lua/commands/zdiffstore.lua
@@ -18,7 +18,7 @@ if withscores == 'withscores' then -- REMOVE ONLY IF SUBTRACTING SCORES IS EQ
i = i + 1
local j = 0
while j < # data do
- value, score = data[j+1], data[j+2]
+ local value, score = data[j+1], data[j+2]
j = j + 2
if redis.call('zscore', dest, value) then
redis.call('zincrby', dest, -score, value)
View
4 stdnet/lib/lua/odm/build_query.lua
@@ -27,7 +27,7 @@ local function add (val)
redis.call('sadd', rkey, val)
end
else
- score = redis.call('zscore', idset, val)
+ local score = redis.call('zscore', idset, val)
if score ~= false then
redis.call('zadd', rkey, score, val)
end
@@ -62,7 +62,7 @@ while i < # ARGV do
end
elseif unique == 'u' then
-- Unique field but not an id. These fields maps to ids in an hash table
- mapkey = bk .. ':uni:' .. name
+ local mapkey = bk .. ':uni:' .. name
if what == 'key' then
-- This lookup is quite rare
if s == 's' then
View
1  stdnet/lib/lua/odm/load_query.lua
@@ -49,6 +49,7 @@ if ordering == 'explicit' then
if nested > 0 then
-- generate a temporary key where to store the hash table holding
-- the values to sort with
+ local ion, key, name
local skey = redis_randomkey(bk)
for i,id in pairs(redis_members(rkey)) do
local value = redis.call('hget', bk .. ':obj:' .. id, field)
View
9 stdnet/lib/lua/odm/numberarray.lua
@@ -5,7 +5,7 @@ local nan = 0/0
-- 8 bytes string for nil data
local nildata = string.char(0,0,0,0,0,0,0,0)
-array = {
+local array = {
--
-- Initialize with key and optional initial size and value
init = function (self, key, size, value)
@@ -65,8 +65,9 @@ array = {
end,
--
all_raw = function(self)
- data = {}
- local i=0,start
+ local start
+ local data = {}
+ local i=0
while i < self:length() do
start = 8*i
i = i + 1
@@ -85,7 +86,7 @@ array = {
}
-columnts_meta = {
+local columnts_meta = {
__index = function(self,index)
return self:get(index)
end,
View
53 stdnet/lib/redis/client.py
@@ -23,8 +23,8 @@
from .connection import *
from .exceptions import *
-from .scripts import script_call_back, get_script, pairs_to_dict,\
- load_missing_scripts
+from .scripts import eval_command_callback, get_script, pairs_to_dict,\
+ load_missing_scripts, script_command_callback
redis_command = namedtuple('redis_command','command args options callbacks')
@@ -148,16 +148,6 @@ def bytes_to_string(request, response, args, **options):
return response.decode(request.client.encoding)
else:
return response
-
-
-def script_command(request, response, args, command = None, **options):
- if command in ('FLUSH', 'KILL'):
- return response == b'OK'
- elif command == 'LOAD':
- return response.decode(request.client.encoding)
- else:
- return [int(r) for r in response]
-
def config_callback(request, response, args, **options):
if args[0] == 'GET':
@@ -247,13 +237,18 @@ class Redis(object):
'TTL': lambda request, response, args, **options: \
response != -1 and response or None,
'ZRANK': int_or_none,
- 'EVALSHA': script_call_back,
- 'EVAL': script_call_back,
- 'SCRIPT': script_command,
+ 'EVALSHA': eval_command_callback,
+ 'EVAL': eval_command_callback,
+ 'SCRIPT': script_command_callback,
'CONFIG': config_callback,
'SLOWLOG': slowlog_callback
}
)
+
+ RESPONSE_ERRBACKS = {
+ 'EVALSHA': eval_command_callback,
+ 'EVAL': eval_command_callback
+ }
_STATUS = ''
@@ -276,6 +271,7 @@ def __init__(self, address = None,
self.connection_pool = connection_pool
self.encoding = self.connection_pool.encoding
self.response_callbacks = self.RESPONSE_CALLBACKS.copy()
+ self.response_errbacks = self.RESPONSE_ERRBACKS.copy()
if check_status:
rstatus = Redis(address=connection_pool.address,
password=password,
@@ -317,8 +313,10 @@ def execute_command(self, *args, **options):
return connection.execute_command(self, *args, **options)
def _parse_response(self, request, response, command_name, args, options):
- if command_name in self.response_callbacks:
- cbk = self.response_callbacks[command_name]
+ callbacks = self.response_errbacks if isinstance(response, Exception)\
+ else self.response_callbacks
+ if command_name in callbacks:
+ cbk = callbacks[command_name]
return cbk(request, response, args, **options)
return response
@@ -395,8 +393,8 @@ def shutdown(self):
"Shutdown the server"
try:
self.execute_command('SHUTDOWN')
- except ConnectionError:
- # a ConnectionError here is expected
+ except RedisConnectionError:
+ # a RedisConnectionError here is expected
return
raise RedisError("SHUTDOWN seems to have failed.")
@@ -915,8 +913,6 @@ def zpopbyscore(self, name, start, stop = None, withscores = False,
'score',
start, stop, int(desc), int(withscores),
**options)
-
-
def _zaggregate(self, command, dest, keys,
aggregate=None, withscores = None, **options):
@@ -949,9 +945,9 @@ def redis_status(self):
try:
self.execute_command('TSLEN', str(uuid4()))
return 'stdnet'
- except ConnectionError:
+ except RedisConnectionError:
return ''
- except ResponseError:
+ except RedisInvalidResponse:
return 'vanilla'
def tslen(self, name, **options):
@@ -1078,7 +1074,7 @@ def hvals(self, name):
############################################################################
def _eval(self, command, body, keys, *args, **options):
if keys:
- if not isinstance(keys,collection_list):
+ if not isinstance(keys, collection_list):
params = (keys,)
else:
params = tuple(keys)
@@ -1105,8 +1101,9 @@ def script_call(self, name, keys, *args, **options):
def script_flush(self):
return self.execute_command('SCRIPT', 'FLUSH', command = 'FLUSH')
- def script_load(self, script):
- return self.execute_command('SCRIPT', 'LOAD', script, command='LOAD')
+ def script_load(self, script, script_name=None):
+ return self.execute_command('SCRIPT', 'LOAD', script, command='LOAD',
+ script_name=script_name)
############################################################################
## Script commands
@@ -1138,6 +1135,10 @@ def response_callbacks(self):
return self.client.response_callbacks
@property
+ def response_errbacks(self):
+ return self.client.response_errbacks
+
+ @property
def encoding(self):
return self.client.encoding
View
36 stdnet/lib/redis/connection.py
@@ -75,9 +75,11 @@
redis_after_receive = Signal(providing_args=["request"])
-PyRedisReader = lambda : fallback.RedisReader(InvalidResponse, ResponseError)
+PyRedisReader = lambda : fallback.RedisReader(RedisProtocolError,
+ RedisInvalidResponse)
if hr:
- RedisReader = lambda : hr.RedisReader(InvalidResponse, ResponseError)
+ RedisReader = lambda : hr.RedisReader(RedisProtocolError,
+ RedisInvalidResponse)
else:
RedisReader = PyRedisReader
@@ -160,19 +162,21 @@ def _send(self):
_errno, errmsg = 'UNKNOWN', e.args[0]
else:
_errno, errmsg = e.args
- raise ConnectionError("Error %s while writing to socket. %s." % \
+ raise RedisConnectionError("Error %s while writing to socket. %s." % \
(_errno, errmsg))
def close(self):
- redis_after_receive.send(self.client.__class__, request = self)
+ redis_after_receive.send(self.client.__class__, request=self)
c = self.connection
try:
- if isinstance(self.response, ResponseError):
- if str(self.response) == NoScriptError.msg:
- self.response = NoScriptError()
- else:
- raise self.response
+ #if isinstance(self.response, ResponseError):
+ # if str(self.response) == NoScriptError.msg:
+ # self.response = NoScriptError()
+ # else:
+ # raise self.response
self._response = self.client.parse_response(self)
+ if isinstance(self._response, Exception):
+ raise self._response
except:
c.disconnect()
raise
@@ -208,7 +212,7 @@ def execute(self):
while self.tried < self.retry:
try:
return self._sendrecv()
- except ConnectionError as e:
+ except RedisConnectionError as e:
if e.retry:
self.connection.disconnect(release_connection = False)
self.tried += 1
@@ -226,10 +230,10 @@ def read_response(self):
try:
stream = sock.recv(io.DEFAULT_BUFFER_SIZE)
except (socket.error, socket.timeout) as e:
- raise ConnectionError("Error while reading from socket: %s" % \
+ raise RedisConnectionError("Error while reading from socket: %s" % \
(e.args,))
if not stream:
- raise ConnectionError("Socket closed on remote end", True)
+ raise RedisConnectionError("Socket closed on remote end", True)
self.parse(stream)
return self._response
@@ -332,7 +336,7 @@ def connect(self, request, counter = 1):
try:
return self._connect(request, counter)
except socket.error as e:
- raise ConnectionError(self._error_message(e))
+ raise RedisConnectionError(self._error_message(e))
def _connect(self, request, counter):
self.sock.settimeout(self.socket_timeout)
@@ -357,14 +361,14 @@ def on_connect(self, request, counter):
r = self.execute_command(client, 'AUTH', self.password,
release_connection = False)
if not r:
- raise ConnectionError('Invalid Password ({0})'.format(counter))
+ raise RedisConnectionError('Invalid Password ({0})'.format(counter))
# if a database is specified, switch to it
if self.db:
r = self.execute_command(client, 'SELECT', self.db,
release_connection = False)
if not r:
- raise ConnectionError('Invalid Database "{0}". ({1})'\
+ raise RedisConnectionError('Invalid Database "{0}". ({1})'\
.format(self.db, counter))
return request
@@ -492,7 +496,7 @@ def get_connection(self):
def make_connection(self):
"Create a new connection"
if self._created_connections >= self.max_connections:
- raise ConnectionError("Too many connections")
+ raise RedisConnectionError("Too many connections")
self._created_connections += 1
return self.connection_class(self, **self.connection_kwargs)
View
36 stdnet/lib/redis/exceptions.py
@@ -1,38 +1,38 @@
"Core exceptions raised by the Redis client"
+import stdnet
-class RedisError(Exception):
- pass
+class RedisConnectionError(stdnet.ConnectionError):
+ def __init__(self, msg, retry = False):
+ self.retry = retry
+ super(RedisConnectionError,self).__init__(msg)
-
-class AuthenticationError(RedisError):
+
+class RedisProtocolError(stdnet.ResponseError):
pass
-
-class ConnectionError(RedisError):
- def __init__(self, msg, retry = False):
- self.retry = retry
- super(ConnectionError,self).__init__(msg)
+
+class RedisInvalidResponse(stdnet.ResponseError):
+ pass
-class ResponseError(RedisError):
+class AuthenticationError(RedisInvalidResponse):
pass
-class NoScriptError(RedisError):
+class NoScriptError(RedisInvalidResponse):
msg = 'NOSCRIPT No matching script. Please use EVAL.'
def __repr__(self):
return self.msg
__str__ = __repr__
-class ScriptError(RedisError):
- pass
-
-class InvalidResponse(ResponseError):
- pass
+class ScriptError(RedisInvalidResponse):
+
+ def __init__(self, command, name, msg):
+ msg = 'Error while executing {0} command on "{1}" script. {2}'\
+ .format(command,name,msg)
+ super(ScriptError,self).__init__(msg)
-class InvalidData(RedisError):
- pass
View
40 stdnet/lib/redis/scripts.py
@@ -12,12 +12,37 @@
'pairs_to_dict',
'get_script',
'registered_scripts',
+ 'script_command_callback',
'read_lua_file']
p = os.path
DEFAULT_LUA_PATH = p.join(p.dirname(p.dirname(p.abspath(__file__))),'lua')
+
+def script_command_callback(request, response, args, command=None,
+ script_name=None, **options):
+ if isinstance(response, Exception):
+ if script_name:
+ command = ' ' + command if command else ''
+ response = ScriptError('SCRIPT'+command, script_name, response)
+ return response
+ elif command in ('FLUSH', 'KILL'):
+ return response == b'OK'
+ elif command == 'LOAD':
+ return response.decode(request.client.encoding)
+ else:
+ return [int(r) for r in response]
+
+
+def eval_command_callback(request, response, args, script_name=None, **options):
+ s = _scripts.get(script_name)
+ if not s:
+ return response
+ else:
+ return s.start_callback(request, response, args, **options)
+
+
def pairs_to_dict(response, encoding, value_encoder = 0):
"Create a dict given a list of key/value pairs"
if response:
@@ -110,7 +135,7 @@ def evalsha(self, client, keys, *args, **options):
def load(self, client, keys, *args, **options):
'''Load this :class:`RedisScript` to redis and runs it using evalsha.
It returns the result of the `evalsha` command.'''
- client.script_load(self.script)
+ client.script_load(self.script, script_name=self.name)
return self.evalsha(client, keys, *args, **options)
def start_callback(self, request, response, args, **options):
@@ -131,8 +156,7 @@ def start_callback(self, request, response, args, **options):
else:
return response
elif isinstance(response, Exception):
- raise ScriptError('Lua redis script "{0}" error. {1}'\
- .format(self,response))
+ raise ScriptError('EVALSHA', self, response)
else:
return self.callback(request, response, args, **options)
@@ -152,15 +176,7 @@ def callback(self, request, response, args, **options):
:parameter args: parameters of the redis script.
:parameter options: Additional options for the callback.
'''
- return response
-
-
-def script_call_back(request, response, args, script_name=None, **options):
- s = _scripts.get(script_name)
- if not s:
- return response
- else:
- return s.start_callback(request, response, args, **options)
+ return response
def load_missing_scripts(pipe, commands, results):
View
7 tests/regression/columnts/main.py
@@ -13,6 +13,7 @@
skipUnless = test.unittest.skipUnless
do_tests = os.environ.get('stdnet_backend_status') == 'stdnet'
+do_tests = True
nan = float('nan')
this_path = os.path.split(os.path.abspath(__file__))[0]
@@ -235,10 +236,10 @@ def testBadQuery(self):
client.rpush(id, 'bla')
client.rpush(id, 'foo')
self.assertEqual(client.llen(id), 2)
- self.assertRaises(CommitException, ts.add,
+ self.assertRaises(redis.ScriptError, ts.add,
date(2012,1,23), {'open':586})
- self.assertRaises(redis.ResponseError, ts.irange)
- self.assertRaises(redis.ResponseError, ts.size)
+ self.assertRaises(redis.ScriptError, ts.irange)
+ self.assertRaises(redis.RedisInvalidResponse, ts.size)
def testGet(self):
ts = self.makeGoogle()
View
3  tests/regression/columnts/npts.py
@@ -16,8 +16,7 @@ class ColumnTimeSeriesNumpy(odm.StdModel):
from . import main
-do_tests = os.environ.get('stdnet_backend_status') == 'stdnet'\
- and npts is not None
+do_tests = npts is not None
skipUnless = main.skipUnless
@skipUnless(do_tests, 'Requires stdnet-redis and dynts')
View
4 tests/regression/me.py
@@ -1,6 +1,6 @@
from stdnet import test
from stdnet.conf import settings
-from stdnet.lib.redis import ConnectionError
+from stdnet.lib import redis
import stdnet as me
@@ -23,6 +23,6 @@ def testSettings(self):
db = settings.DEFAULT_BACKEND
try:
settings.DEFAULT_BACKEND = 'redis://dksnkdcnskcnskcn:6379?db=7'
- self.assertRaises(ConnectionError, settings.redis_status)
+ self.assertRaises(redis.RedisConnectionError, settings.redis_status)
finally:
settings.DEFAULT_BACKEND = db
View
6 tests/regression/redis/base.py
@@ -3,12 +3,6 @@
from stdnet import test, getdb
from stdnet.utils import flatzset
-
-ResponseError = redis.ResponseError
-RedisError = redis.RedisError
-ConnectionError = redis.ConnectionError
-
-
def get_version(info):
if 'redis_version' in info:
return info['redis_version']
View
4 tests/regression/redis/commands.py
@@ -8,9 +8,9 @@
if not ispy3k:
chr = unichr
-from .base import TestCase, ResponseError, RedisError, get_version
-
+from .base import TestCase, redis, get_version
+ResponseError = redis.RedisInvalidResponse
to_charlist = lambda x: [x[c:c + 1] for c in range(len(x))]
binary_set = lambda x : set(to_charlist(x))
View
6 tests/regression/redis/pipeline.py
@@ -1,4 +1,4 @@
-from .base import TestCase, ResponseError
+from .base import TestCase, redis
class PipelineTestCase(TestCase):
@@ -34,8 +34,8 @@ def test_invalid_command_in_pipeline(self):
self.assertEquals(result[1], True)
self.assertEquals(self.client['b'], b'2')
# we can't lpush to a key that's a string value, so this should
- # be a ResponseError exception
- self.assert_(isinstance(result[2], ResponseError))
+ # be a redis.RedisInvalidResponse exception
+ self.assert_(isinstance(result[2], redis.RedisInvalidResponse))
self.assertEquals(self.client['c'], b'a')
self.assertEquals(result[3], True)
self.assertEquals(self.client['d'], b'4')
View
4 tests/regression/redis/pool.py
@@ -1,4 +1,4 @@
-from .base import TestCase, redis, ConnectionError
+from .base import TestCase, redis
class DummyConnection(object):
@@ -32,7 +32,7 @@ def test_max_connections(self):
pool = self.get_pool(max_connections=2)
c1 = pool.get_connection()
c2 = pool.get_connection()
- self.assertRaises(ConnectionError, pool.get_connection)
+ self.assertRaises(redis.RedisConnectionError, pool.get_connection)
def test_release(self):
pool = self.get_pool()
View
2  tests/regression/redis/scripting.py
@@ -16,7 +16,7 @@
class test_script(redis.RedisScript):
script = (redis.read_lua_file('commands.utils'),
'''\
-js = cjson.decode(ARGV[1])
+local js = cjson.decode(ARGV[1])
return cjson.encode(js)''')
def callback(self, request, result, args, **options):
View
2  tests/regression/sorting.py
@@ -121,7 +121,7 @@ def testSortByFK(self):
self.assertEqual(ordering.name,'group_id')
self.assertEqual(ordering.nested.name,'name')
self.assertEqual(ordering.model,qs.model)
- self.checkOrder(qs,'group__name')
+ self.checkOrder(qs, 'group__name')
class TestOrderingModel(TestSort):
Please sign in to comment.
Something went wrong with that request. Please try again.