Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[cache] Using the query as the basis of the cache key #4016

Merged
merged 1 commit into from Jan 12, 2018
Merged

[cache] Using the query as the basis of the cache key #4016

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Empty file added superset/assets/npm-debug.log.3344327073
View file
Open in desktop
Empty file.
115 changes: 61 additions & 54 deletions superset/viz.py
View file
Open in desktop
Expand Up @@ -214,8 +214,6 @@ def query_obj(self):

@property
def cache_timeout(self):
if self.form_data.get('cache_timeout'):
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Given that the cache is explorer view agnostic the cache timeout is only associated with the query and not the explorer form_data.

return int(self.form_data.get('cache_timeout'))
if self.datasource.cache_timeout:
return self.datasource.cache_timeout
if (
Expand All @@ -229,44 +227,50 @@ def get_json(self, force=False):
self.get_payload(force),
default=utils.json_int_dttm_ser, ignore_nan=True)

@property
def cache_key(self):
form_data = self.form_data.copy()
merge_extra_filters(form_data)
s = str([(k, form_data[k]) for k in sorted(form_data.keys())])
return hashlib.md5(s.encode('utf-8')).hexdigest()
def cache_key(self, query_obj):
"""
The cache key is the datasource/query string tuple associated with the
object which needs to be fully deterministic.
"""

return hashlib.md5(
json.dumps((
self.datasource.id,
self.datasource.get_query_str(query_obj),
)).encode('utf-8'),
).hexdigest()

def get_payload(self, force=False):
"""Handles caching around the json payload retrieval"""
cache_key = self.cache_key
payload = None
query_obj = self.query_obj()
cache_key = self.cache_key(query_obj)
cached_dttm = None
data = None
stacktrace = None
rowcount = None
if not force and cache:
payload = cache.get(cache_key)

if payload:
stats_logger.incr('loaded_from_cache')
is_cached = True
try:
cached_data = zlib.decompress(payload)
if PY3:
cached_data = cached_data.decode('utf-8')
payload = json.loads(cached_data)
except Exception as e:
logging.error('Error reading cache: ' +
utils.error_msg_from_exception(e))
payload = None
return []
logging.info('Serving from cache')
cache_value = cache.get(cache_key)
if cache_value:
stats_logger.incr('loaded_from_cache')
is_cached = True
try:
cache_value = zlib.decompress(cache_value)
if PY3:
cache_value = cache_value.decode('utf-8')
cache_value = json.loads(cache_value)
data = cache_value['data']
cached_dttm = cache_value['dttm']
except Exception as e:
logging.error('Error reading cache: ' +
utils.error_msg_from_exception(e))
data = None
logging.info('Serving from cache')

if not payload:
if not data:
stats_logger.incr('loaded_from_source')
data = None
is_cached = False
cache_timeout = self.cache_timeout
stacktrace = None
rowcount = None
try:
df = self.get_df()
df = self.get_df(query_obj)
if not self.error_message:
data = self.get_data(df)
rowcount = len(df.index) if df is not None else 0
Expand All @@ -277,37 +281,40 @@ def get_payload(self, force=False):
self.status = utils.QueryStatus.FAILED
data = None
stacktrace = traceback.format_exc()
payload = {
'cache_key': cache_key,
'cache_timeout': cache_timeout,
'data': data,
'error': self.error_message,
'form_data': self.form_data,
'query': self.query,
'status': self.status,
'stacktrace': stacktrace,
'rowcount': rowcount,
}
payload['cached_dttm'] = datetime.utcnow().isoformat().split('.')[0]
logging.info('Caching for the next {} seconds'.format(
cache_timeout))
data = self.json_dumps(payload)
if PY3:
data = bytes(data, 'utf-8')
if cache and self.status != utils.QueryStatus.FAILED:

if data and cache and self.status != utils.QueryStatus.FAILED:
cached_dttm = datetime.utcnow().isoformat().split('.')[0]
try:
cache_value = json.dumps({
'data': data,
'dttm': cached_dttm,
})
if PY3:
cache_value = bytes(cache_value, 'utf-8')
cache.set(
cache_key,
zlib.compress(data),
timeout=cache_timeout)
zlib.compress(cache_value),
timeout=self.cache_timeout)
except Exception as e:
# cache.set call can fail if the backend is down or if
# the key is too large or whatever other reasons
logging.warning('Could not cache key {}'.format(cache_key))
logging.exception(e)
cache.delete(cache_key)
payload['is_cached'] = is_cached
return payload

return {
'cache_key': cache_key,
'cached_dttm': cached_dttm,
'cache_timeout': self.cache_timeout,
'data': data,
'error': self.error_message,
'form_data': self.form_data,
'is_cached': is_cached,
'query': self.query,
'status': self.status,
'stacktrace': stacktrace,
'rowcount': rowcount,
}

def json_dumps(self, obj):
return json.dumps(obj, default=utils.json_int_dttm_ser, ignore_nan=True)
Expand Down
1 change: 0 additions & 1 deletion tests/core_tests.py
View file
Open in desktop
Expand Up @@ -340,7 +340,6 @@ def test_warm_up_cache(self):
slc = self.get_slice('Girls', db.session)
data = self.get_json_resp(
'/superset/warm_up_cache?slice_id={}'.format(slc.id))

assert data == [{'slice_id': slc.id, 'slice_name': slc.slice_name}]

data = self.get_json_resp(
Expand Down
5 changes: 1 addition & 4 deletions tests/viz_tests.py
View file
Open in desktop
Expand Up @@ -101,11 +101,8 @@ def test_get_df_handles_dttm_col(self):

def test_cache_timeout(self):
datasource = Mock()
form_data = {'cache_timeout': '10'}
test_viz = viz.BaseViz(datasource, form_data)
self.assertEqual(10, test_viz.cache_timeout)
del form_data['cache_timeout']
datasource.cache_timeout = 156
test_viz = viz.BaseViz(datasource, form_data={})
self.assertEqual(156, test_viz.cache_timeout)
datasource.cache_timeout = None
datasource.database = Mock()
Expand Down