Browse files

benchmarking and cache population management commands

  • Loading branch information...
1 parent 3923c4e commit 8e2e38854be24bc46dde8a2b1c53802ea27a69c9 @yourcelf committed Oct 16, 2010
Showing with 626 additions and 0 deletions.
  1. +577 −0 afg/management/commands/
  2. +49 −0 afg/management/commands/
577 afg/management/commands/
@@ -0,0 +1,577 @@
+# This command builds a set of search URLs to use for stress testing the full
+# stack. Invoke with a base URL and number of search URLs to generate.
+# An example using `palb` as a benchmarker with this management command. This
+# will run a benchmark with 10 concurrent users making 1000 requests, each to a
+# distinct random but valid search URL, so as to miss the cache often:
+# python build_benchmark 1000 | xargs palb -c 10 -n 1000
+import sys
+import urllib
+import random
+from django.db.models.fields import TextField
+from import BaseCommand
+import haystack
+from afg.models import DiaryEntry
+from afg.search_indexes import DiaryEntryIndex
+class Command(BaseCommand):
+ args = '<base url> <number of urls>'
+ help = """Construct a list of random URLs to use for benchmarking the server."""
+ def handle(self, *args, **kwargs):
+ try:
+ base_url = args[0]
+ number_of_urls = int(args[1])
+ except IndexError:
+ print self.args
+ print
+ sys.exit()
+ options = {}
+ for facet in DiaryEntryIndex.search_facet_display:
+ field = DiaryEntryIndex.fields[facet]
+ name = field.index_fieldname.rstrip('_')
+ if name == 'total_casualties':
+ #XXX domain specific....
+ nums = range(0, 191)
+ options[name + '__lte'] = nums
+ options[name + '__gte'] = nums
+ elif isinstance(field, haystack.fields.DateTimeField):
+ dates = list(d.strftime("%Y-%m-%d") for d in DiaryEntry.objects.all().dates(name, 'day'))
+ options[name + '__lte'] = dates
+ options[name + '__gte'] = dates
+ elif isinstance(field, haystack.fields.IntegerField):
+ ints = range(DiaryEntry.objects.order_by(name).values(name)[0][name],
+ DiaryEntry.objects.order_by('-' + name).values(name)[0][name])
+ options[name + '__lte'] = ints
+ options[name + '__gte'] = ints
+ elif isinstance(field, haystack.fields.CharField):
+ if not isinstance(DiaryEntry._meta.get_field(name), TextField):
+ options[name] = [a[name] for a in DiaryEntry.objects.all().values(name).distinct()]
+ else:
+ pass
+ #XXX domain specific, phrase search
+ in_db_phrases = []
+ texts = [a['summary'] for a in DiaryEntry.objects.all().order_by('?').values('summary')[0:number_of_urls]]
+ for text in texts:
+ in_db_phrases += text.split()
+ options['q'] = random.sample(RANDOM_WORDS, min(len(RANDOM_WORDS), number_of_urls))
+ options['q'] += random.sample(in_db_phrases, number_of_urls)
+ urls = []
+ for i in range(number_of_urls):
+ # Choose a number of facets to search on.
+ num_facets = max(1, int(random.expovariate(1)))
+ facets = []
+ # 50% of the time, search for text.
+ if random.random() > 0.5:
+ facets.append('q')
+ facets += random.sample(options.keys(), num_facets - len(facets))
+ params = dict((facet, random.choice(options[facet])) for facet in facets)
+ urls.append(base_url + '?' + urllib.urlencode(params))
+ for url in urls:
+ print url
+RANDOM_WORDS = ["sever",
+"pina colada",
+"large order",
+"Front door",
+"in straitened circumstances",
+"cash out",
+"Rural deanery",
+"genus Callorhinus",
+"load factor",
+"Death bell",
+"hejaz, el",
+"Cobaea scandens",
+"white mullein",
+"Phyodactylus gecko",
+"To clear for action",
+"big-bang theory",
+"dry fly",
+"Listera cordata",
+"Clematis tangutica",
+"Indian banyan",
+"thackeray, william makepeace",
+"viral infection",
+"Mohammedan calendar",
+"happy medium",
+"combinative vs noncombinative",
+"Atrabiliary capsule",
+"group practice",
+"Kingdom of Norway",
+"barn owl",
+"cloud seeder",
+"yard goods",
+"Eleocharis acicularis",
+"wasp waist",
+"give suck",
+"herba impia",
+"capital of Kazakhstan",
+"proboscis monkey",
+"ghiberti, lorenzo",
+"cayenne pepper",
+"out to lunch",
+"Belgian franc",
+"party liner",
+"Snow partridge",
+"Carriage horse",
+"mach number",
+"Labrador tea",
+"weather strip",
+"loony bin",
+"dry-bulb thermometer",
+"family Struthionidae",
+"nonrapid eye movement",
+"wizard of the north",
+"Queensland tulipwood",
+"antianxiety agent",
+"solid-state physics",
+"addison, joseph",
+"respiratory acidosis",
+"Mullus surmulletus",
+"hair dye",
+"pectoralis major",
+"whit leather",
+"paton, john gibson",
+"large-leaved cucumber tree",
+"Mekong River",
+"Curtain lecture",
+"family Tetraodontidae",
+"Branchial clefts",
+"act of God",
+"welted thistle",
+"Safety touchdown",
+"Fire bug",
+"metabolic acidosis",
+"alkali poisoning",
+"corrosive sublimate",
+"million instructions per second",
+"sheath knife",
+"cinnabar chanterelle",
+"iodinated protein",
+"Cassia javonica",
+"walking ticket",
+"stratford de redcliffe, sir stafford canning, first viscount",
+"caesarean section",
+"flt. sgt.",
+"Piston rod",
+"Blue ruin",
+"Etropus rimosus",
+"To sit at meat",
+"Mickey Finn",
+"malacca cane",
+"rock hyrax",
+"butter stamp",
+"narcissistic personality",
+"drinking chocolate",
+"sigmoid colon",
+"Walling wax",
+"Glover's stitch",
+"superiority complex",
+"Yerba mansa",
+"skillet fish",
+"Curtail dog",
+"wiggle nail",
+"new albany",
+"Abacus harmonicus",
+"run out",
+"Tribonyx Mortierii",
+"creusot, le",
+"place aux dames",
+"spenser, edmund",
+"Parus ater",
+"stand watch",
+"Biblical Latin",
+"tillamook bay",
+"sum total",
+"Poterium Sanguisorba",
+"blackwell, alexander",
+"d'urfey, tom",
+"To make up",
+"squirreltail grass",
+"outside clinch",
+"A atricapillus",
+"grindal, edmund",
+"Jean Francois Champollion",
+"beta iron",
+"senior citizen",
+"order Actinomyxidia",
+"Tilt roof",
+"station break",
+"Fiber optics",
+"Triangular compasses",
+"Verbatim et literatim",
+"American cheese",
+"Horned horse",
+"Guevina avellana",
+"Indian file",
+"Cycloidal engine",
+"Charles Wesley",
+"boxing glove",
+"blue streak",
+"read/write memory",
+"Nelumbo nucifera",
+"fragonard, jean honore",
49 afg/management/commands/
@@ -0,0 +1,49 @@
+from __future__ import print_function
+import time
+import sys
+import urllib
+from import BaseCommand
+from afg.models import DiaryEntry
+class StatusPrinter(object):
+ def __init__(self, c=0, n=0):
+ self.c = c
+ self.n = n
+ self.previous = ""
+ def inc(self):
+ self.c += 1
+ def print(self):
+ print("\b" * len(self.previous), end="")
+ self.previous = "{0} / {1}".format(self.c, self.n)
+ print(self.previous, end="")
+ def end(self):
+ print()
+class Command(BaseCommand):
+ args = '<base url> [delay=0]'
+ help = """Initialize the cache of the given server with all diary entries."""
+ def handle(self, *args, **kwargs):
+ try:
+ base_url = args[0].rstrip('/')
+ try:
+ delay = int(args[1])
+ except IndexError:
+ delay = 0
+ except IndexError:
+ print(args)
+ print(help)
+ sys.exit()
+ sp = StatusPrinter(0, DiaryEntry.objects.count())
+ for entry in DiaryEntry.objects.all().values('report_key'):
+ urllib.urlopen("%s/id/%s/" % (base_url, entry['report_key']))
+ time.sleep(delay)
+ sp.print()
+ sp.end()

0 comments on commit 8e2e388

Please sign in to comment.