Permalink
Browse files

Added support for key sampling where database is too large to list al…

…l the keys. Model fields sampling_threshold and sampling_size control when sampling is used and how many keys will be shown.
  • Loading branch information...
ionelmc committed Feb 23, 2012
1 parent e5ce9c3 commit 0d605f833a0dbe33d724dec6b6017e7a4f82ec90
View
@@ -1,4 +1,4 @@
-# -*- encoding: utf8 -*-
+# -*- encoding: utf-8 -*-
from setuptools import setup, find_packages
import os
@@ -0,0 +1,39 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+
+ # Adding field 'RedisServer.sampling_threshold'
+ db.add_column('redisboard_redisserver', 'sampling_threshold', self.gf('django.db.models.fields.IntegerField')(default=1000), keep_default=False)
+
+ # Adding field 'RedisServer.sampling_size'
+ db.add_column('redisboard_redisserver', 'sampling_size', self.gf('django.db.models.fields.IntegerField')(default=200), keep_default=False)
+
+
+ def backwards(self, orm):
+
+ # Deleting field 'RedisServer.sampling_threshold'
+ db.delete_column('redisboard_redisserver', 'sampling_threshold')
+
+ # Deleting field 'RedisServer.sampling_size'
+ db.delete_column('redisboard_redisserver', 'sampling_size')
+
+
+ models = {
+ 'redisboard.redisserver': {
+ 'Meta': {'unique_together': "(('hostname', 'port'),)", 'object_name': 'RedisServer'},
+ 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '250'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'password': ('django.db.models.fields.CharField', [], {'max_length': '250', 'null': 'True', 'blank': 'True'}),
+ 'port': ('django.db.models.fields.IntegerField', [], {'default': '6379', 'null': 'True', 'blank': 'True'}),
+ 'sampling_size': ('django.db.models.fields.IntegerField', [], {'default': '200'}),
+ 'sampling_threshold': ('django.db.models.fields.IntegerField', [], {'default': '1000'})
+ }
+ }
+
+ complete_apps = ['redisboard']
View
@@ -42,13 +42,29 @@ class Meta:
("can_inspect", "Can inspect redis servers"),
)
- hostname = models.CharField(_("Hostname"), max_length=250, help_text=_('this can also be the absolute path to a redis socket'))
+ hostname = models.CharField(
+ _("Hostname"),
+ max_length = 250,
+ help_text = _('This can also be the absolute path to a redis socket')
+ )
+
port = models.IntegerField(_("Port"), validators=[
MaxValueValidator(65535), MinValueValidator(1)
], default=6379, blank=True, null=True)
password = models.CharField(_("Password"), max_length=250,
null=True, blank=True)
+ sampling_threshold = models.IntegerField(
+ _("Sampling threshold"),
+ default = 1000,
+ help_text = _("Number of keys after which only a sample (of random keys) is shown on the inspect page.")
+ )
+ sampling_size = models.IntegerField(
+ _("Sampling size"),
+ default = 200,
+ help_text = _("Number of random keys shown when sampling is used. Note that each key translates to a RANDOMKEY call in redis.")
+ )
+
def clean(self):
if not self.hostname.startswith('/') and not self.port:
raise ValidationError(_('Please provide either a hostname AND a port or the path to a redis socket'))
@@ -66,10 +66,34 @@
{% for db, db_detail in databases.items %}
<fieldset class="module aligned database-details">
- <h2>{% trans "Database" %} {{ db }}</h2>
+ <h2>
+ {% trans "Database" %} {{ db }}:
+ {% if db_detail.sampling %}
+ {% with db_detail.size as total %}
+ {% blocktrans count original.sampling_threshold as size %}
+ {{ size }} random key our of {{ total }}
+ {% plural %}
+ {{ size }} random keys our of {{ total }}
+ {% endblocktrans %}
+ {% endwith %}
+ {% else %}
+ {% blocktrans count db_detail.size as size %}
+ {{ size }} key
+ {% plural %}
+ {{ size }} keys
+ {% endblocktrans %}
+ {% endif %}
+
+ {% if db_detail.active %}
+ &larr;
+ {% else %}
+ &ndash; <a href="{% url 'admin:redisboard_redisserver_inspect' original.id %}?db={{ db }}">{% trans "Show details" %}</a>
+ {% endif %}
+ </h2>
+ {% if db_detail.active %}
<table>
<tr>
- <th rowspan="2">{% trans "Keys" %} ({{ db_detail|length }})</th>
+ <th rowspan="2">{% trans "Keys" %} ({{ db_detail.keys|length }})</th>
<th colspan="7">{% trans "Details" %}</th>
</tr>
<tr>
@@ -83,7 +107,7 @@ <h2>{% trans "Database" %} {{ db }}</h2>
<th>{% trans "Other" %}</th>
</tr>
- {% for key, key_detail in db_detail.items %}
+ {% for key, key_detail in db_detail.keys.items %}
<tr>
<td><a href="{% url 'admin:redisboard_redisserver_inspect' original.id %}?key={{ key|escape }}&db={{ db }}">{{ key }}</td>
{% if key_detail.error %}
@@ -102,6 +126,7 @@ <h2>{% trans "Database" %} {{ db }}</h2>
{% endfor %}
</table>
+ {% endif %}
</fieldset>
{% endfor %}
{% if key_details %}
View
@@ -6,6 +6,7 @@
from django.utils.datastructures import SortedDict
from django.conf import settings
from django.utils.functional import curry
+from django.http import HttpResponseNotFound
from redis.exceptions import ResponseError
@@ -82,14 +83,31 @@ def _get_key_details(conn, db, key, page):
return details
+def _get_db_summary(server, db):
+ conn = server.connection
+ conn.execute_command('SELECT', db)
+ return dict(size=conn.dbsize())
-def _get_db_details(conn, db):
+def _get_db_details(server, db):
+ conn = server.connection
conn.execute_command('SELECT', db)
+ size = conn.dbsize()
keys = conn.keys()
key_details = {}
- for key in keys:
- key_details[key] = _get_key_info(conn, key)
- return key_details
+ if size > server.sampling_threshold:
+ sampling = True
+ for _ in xrange(server.sampling_size):
+ key = conn.randomkey()
+ key_details[key] = _get_key_info(conn, key)
+ else:
+ sampling = False
+ for key in keys:
+ key_details[key] = _get_key_info(conn, key)
+ return dict(
+ keys = key_details,
+ sampling = sampling,
+ )
+
def inspect(request, server):
@@ -106,11 +124,26 @@ def inspect(request, server):
key_details = _get_key_details(conn, db, key, page)
else:
databases = sorted(name[2:] for name in conn.info() if name.startswith('db'))
-
+ total_size = 0
for db in databases:
- database_details[db] = _get_db_details(conn, db)
-
-
+ database_details[db] = summary = _get_db_summary(server, db)
+ total_size += summary['size']
+ if total_size < server.sampling_threshold:
+ for db in databases:
+ database_details[db].update(
+ _get_db_details(server, db),
+ active = True,
+ )
+ else:
+ if 'db' in request.GET:
+ db = request.GET['db']
+ if db in database_details:
+ database_details[db].update(
+ _get_db_details(server, db),
+ active = True,
+ )
+ else:
+ return HttpResponseNotFound("Unknown database.")
return render(request, "redisboard/inspect.html", {
'databases': database_details,
'key_details': key_details,

0 comments on commit 0d605f8

Please sign in to comment.