Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

restructure test code; better simplequeue tests

  • Loading branch information...
commit b8ac9a9ad1b2321c547c6d8d01a821f16e618bef 1 parent 8a0735d
@jehiah jehiah authored
View
46 simpleleveldb/test_shunt.py → shared_tests/test_shunt.py
@@ -5,19 +5,40 @@
import subprocess
import signal
import time
+import simplejson as json
+import urllib
import tornado.httpclient
logging.basicConfig(stream=sys.stdout, level=logging.INFO,
format='%(asctime)s %(process)d %(filename)s %(lineno)d %(levelname)s #| %(message)s',
datefmt='%H:%M:%S')
+def http_fetch_json(endpoint, params=None, status_code=200, status_txt="OK", body=None):
+ body = http_fetch(endpoint, params, 200, body)
+ data = json.loads(body)
+ assert data['status_code'] == status_code
+ assert data['status_txt'] == status_txt
+ return data['data']
+
+def http_fetch(endpoint, params=None, response_code=200, body=None):
+ http_client = tornado.httpclient.HTTPClient()
+ url = 'http://127.0.0.1:8080' + endpoint
+ if params:
+ url += '?' + urllib.urlencode(params, doseq=1)
+ method = "POST" if body else "GET"
+ try:
+ res = http_client.fetch(url, method=method, body=body)
+ except tornado.httpclient.HTTPError, e:
+ logging.info(e)
+ res = e.response
+ assert res.code == response_code
+ return res.body
+
def valgrind_cmd(cmd, *options):
assert isinstance(options, (list, tuple))
- dirname = os.path.dirname(__file__)
- if not cmd.startswith("/"):
- cmd = os.path.join(dirname, cmd)
- test_output_dir = os.path.join(dirname, "test_output")
+ assert cmd.startswith("/"), "valgrind_cmd must take a fully qualified executible path not %s" % cmd
+ test_output_dir = os.path.join(os.path.dirname(cmd), "test_output")
return [
'valgrind',
'-v',
@@ -57,13 +78,16 @@ def check_valgrind_output(filename):
class SubprocessTest(unittest.TestCase):
process_options = []
+ binary_name = ""
+ working_dir = None
def setUp(self):
"""setup method that starts up mongod instances using `self.mongo_options`"""
self.temp_dirs = []
self.processes = []
- dirname = os.path.dirname(__file__)
+ assert self.binary_name, "you must override self.binary_name"
+ assert self.working_dir, "set workign dir to os.path.dirname(__file__)"
- exe = os.path.join(dirname, 'simpleleveldb')
+ exe = os.path.join(self.working_dir, self.binary_name)
if os.path.exists(exe):
logging.info('removing old %s' % exe)
os.unlink(exe)
@@ -74,7 +98,7 @@ def setUp(self):
assert os.path.exists(exe), "compile failed"
- test_output_dir = os.path.join(dirname, "test_output")
+ test_output_dir = os.path.join(self.working_dir, "test_output")
if os.path.exists(test_output_dir):
logging.info('removing %s' % test_output_dir)
pipe = subprocess.Popen(['rm', '-rf', test_output_dir])
@@ -92,7 +116,7 @@ def setUp(self):
self.processes.append(pipe)
logging.debug('started process %s' % pipe.pid)
- self.wait_for('http://127.0.0.1:8080/', 5)
+ self.wait_for('http://127.0.0.1:8080/', max_time=5)
def wait_for(self, url, max_time):
# check up to 15 times till the endpoint specified is available waiting max_time
@@ -108,7 +132,11 @@ def wait_for(self, url, max_time):
time.sleep(step)
def graceful_shutdown(self):
- pass
+ try:
+ http_fetch('/exit', dict())
+ except:
+ # we never get a reply if this works correctly
+ time.sleep(1)
def tearDown(self):
"""teardown method that cleans up child mongod instances, and removes their temporary data files"""
View
1  simpleleveldb/simpleleveldb.c
@@ -515,6 +515,7 @@ void dump_csv_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
}
evhttp_clear_headers(&args);
+ json_object_put(jsobj);
evhttp_send_reply_start(req, 200, "OK");
/* run the first dump loop */
View
42 simpleleveldb/test_simpleleveldb.py
@@ -1,41 +1,15 @@
import os
-import simplejson as json
-import urllib
-import logging
-from test_shunt import valgrind_cmd, SubprocessTest
-import tornado.httpclient
-import time
+import sys
+sys.path.append(os.path.join(os.path.dirname(__file__), "../shared_tests"))
-def http_fetch_json(endpoint, params, status_code=200, status_txt="OK", body=None):
- body = http_fetch(endpoint, params, 200, body)
- data = json.loads(body)
- assert data['status_code'] == status_code
- assert data['status_txt'] == status_txt
- return data['data']
+import logging
+from test_shunt import valgrind_cmd, SubprocessTest, http_fetch, http_fetch_json
-def http_fetch(endpoint, params, response_code=200, body=None):
- http_client = tornado.httpclient.HTTPClient()
- url = 'http://127.0.0.1:8080' + endpoint
- if params:
- url += '?' + urllib.urlencode(params, doseq=1)
- method = "POST" if body else "GET"
- try:
- res = http_client.fetch(url, method=method, body=body)
- except tornado.httpclient.HTTPError, e:
- logging.info(e)
- res = e.response
- assert res.code == response_code
- return res.body
class SimpleLeveldbTest(SubprocessTest):
- process_options = [valgrind_cmd('simpleleveldb', '--db-file=%s/db' % os.path.join(os.path.dirname(__file__), "test_output"), '--enable-logging')]
-
- def graceful_shutdown(self):
- try:
- http_fetch('/exit', dict())
- except:
- # we never get a reply if this works correctly
- time.sleep(1)
+ binary_name = "simpleleveldb"
+ process_options = [valgrind_cmd(os.path.abspath('simpleleveldb'), '--db-file=%s/db' % os.path.join(os.path.dirname(__file__), "test_output"), '--enable-logging')]
+ working_dir = os.path.dirname(__file__)
def test_basic(self):
data = http_fetch_json('/put', dict(key='test', value='12345'))
@@ -85,7 +59,7 @@ def test_basic(self):
for x in range(505):
http_fetch_json('/put', dict(key='dump.%d' % x, value='dump.value.%d' % x))
- data = http_fetch('/dump_csv', {})
+ data = http_fetch('/dump_csv')
assert data.startswith("dump.0,dump.value.0\n")
assert data.endswith("test2,asdf2\n")
assert data.count("\n") > 505
View
134 simplequeue/SimpleQueue.py
@@ -1,134 +0,0 @@
-import sys
-import traceback
-import pycurl
-import cStringIO
-import urllib
-import logging
-
-class SimpleQueue:
- def __init__(self, address='127.0.0.1', port=8080, debug=False):
- self.address = address
- self.port = port
- self.debug = debug
-
- def get_request(self, url, timeout_ms=500):
- if self.debug:
- print url
- buffer = cStringIO.StringIO()
- curl = pycurl.Curl()
- curl.setopt(pycurl.URL, url)
- curl.setopt(pycurl.TIMEOUT_MS, timeout_ms)
- curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
- curl.setopt(pycurl.NOSIGNAL, 1)
- curl.perform()
- result = buffer.getvalue()
- buffer.close()
- curl.close()
-
- return result
-
- def post_request(self, url, data, timeout_ms=500):
- if self.debug:
- print url
- buffer = cStringIO.StringIO()
- curl = pycurl.Curl()
- curl.setopt(pycurl.URL, url)
- curl.setopt(pycurl.TIMEOUT_MS, timeout_ms)
- curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
- curl.setopt(pycurl.NOSIGNAL, 1)
- curl.setopt(pycurl.POST, 1)
- curl.setopt(pycurl.POSTFIELDS, data)
- curl.perform()
- result = buffer.getvalue()
- buffer.close()
- curl.close()
-
- return result
-
- def put(self, data, timeout_ms=500, post=False):
-
- try:
- if post:
- url = "http://%s:%d/put" % (self.address, self.port)
- self.post_request(url, data, timeout_ms)
- else:
- url = "http://%s:%d/put?data=%s" % (self.address, self.port, urllib.quote(data))
- self.get_request(url, timeout_ms)
- return True
- except:
- traceback.print_tb(sys.exc_info()[2])
- return False
-
- def mput(self, data, timeout_ms=500, separator="", post=True):
- url = "http://%s:%d/mput" % (self.address, self.port)
- if separator:
- url += "?separator=" + separator
-
- try:
- if post:
- url = "http://%s:%d/mput" % (self.address, self.port)
- if separator:
- url += "?separator=" + separator
- self.post_request(url, data, timeout_ms)
- else:
- url = "http://%s:%d/mput?data=%s" % (self.address, self.port, urllib.quote(data))
- if separator:
- url += "&separator=" + separator
- self.get_request(url, timeout_ms)
- return True
- except:
- traceback.print_tb(sys.exc_info()[2])
- return False
-
- def get(self, timeout_ms=500):
- url = "http://%s:%d/get" % (self.address, self.port)
-
- try:
- return self.get_request(url, timeout_ms)
- except:
- traceback.print_tb(sys.exc_info()[2])
- return False
-
- def mget(self, timeout_ms=500, num_items=1, separator=None):
- url = "http://%s:%d/mget?items=%d" % (self.address, self.port, num_items)
- if separator:
- url += "&separator=" + separator
-
- try:
- return self.get_request(url, timeout_ms)
- except:
- traceback.print_tb(sys.exc_info()[2])
- return False
-
- def dump(self, timeout_ms=500):
- url = "http://%s:%d/dump" % (self.address, self.port)
-
- try:
- return self.get_request(url, timeout_ms).strip().split('\n')
- except:
- traceback.print_tb(sys.exc_info()[2])
- return False
-
- def stats(self, timeout_ms=500, reset=False):
- url = "http://%s:%d/stats" % (self.address, self.port)
- stats = {}
-
- if reset:
- url = "%s?reset=1" % url
-
- try:
- response = self.get_request(url, timeout_ms)
-
- if reset:
- return True
-
- for line in response.split('\n'):
- parts = line.split(':')
- key = parts[0]
- value = parts[1]
- stats[key] = value
-
- return stats
- except:
- traceback.print_tb(sys.exc_info()[2])
- return False
View
11 simplequeue/simplequeue.c
@@ -8,6 +8,8 @@
#define VERSION "1.3.1"
+void exit_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx);
+
struct queue_entry {
TAILQ_ENTRY(queue_entry) entries;
size_t bytes;
@@ -290,6 +292,12 @@ void mput(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
evhttp_clear_headers(&args);
}
+void exit_cb(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
+{
+ fprintf(stdout, "/exit request recieved\n");
+ event_loopbreak();
+}
+
void dump(struct evhttp_request *req, struct evbuffer *evb, void *ctx)
{
struct queue_entry *entry;
@@ -354,9 +362,10 @@ int main(int argc, char **argv)
simplehttp_set_cb("/mput*", mput, NULL);
simplehttp_set_cb("/dump*", dump, NULL);
simplehttp_set_cb("/stats*", stats, NULL);
+ simplehttp_set_cb("/exit*", exit_cb, NULL);
simplehttp_main();
free_options();
-
+
if (overflow_log_fp) {
while (depth) {
overflow_one();
View
214 simplequeue/test_simplequeue.py
@@ -1,164 +1,60 @@
-# -*- coding: UTF-8 -*-
-from SimpleQueue import SimpleQueue
-RANDOM_STRINGS = [
-# the following line is a R-L language
-'زشك:یقینحاصلکردمکه'
-,'Iñtërnâtiôn�lizætiøn',
-'http://flash-mini.com/car2hand/http://flash-mini.com/car2hand/337358/1985_BMW,_316_(4Dr)_38,000_\xe0\xb8\x9a\xe0\xb8\xb2\xe0\xb8\x97_\xe0\xb8\x82\xe0\xb8\xb2\xe0\xb8\xa2_BMW_E30_(\xe0\xb9\x80\xe0\xb8\xad\xe0\xb8\xb2\xe0\xb9\x84\xe0\xb8\xa7\xe0\xb9\x89\xe0\xb8\x82\xe0\xb8\xb1\xe0\xb8\x9a\xe0\xb9\x80\xe0\xb8\xa5\xe0\xb9\x88\xe0\xb8\x99\xe0\xb9\x86\xe0\xb8\x8b\xe0\xb8\xb1\xe0\xb8\x81\xe0\xb8\x84\xe0\xb8\xb1\xe0\xb8\x99)/\xe0\xb9\x80\xe0\xb8\x84\xe0\xb8\xa3\xe0\xb8\xb7\xe0\xb9\x88\xe0\xb8\xad\xe0\xb8\x87Rb-20Turbo_<br>Inter_Rb-26<br>TurboRb-25_<.htm',
-'asdf',
-'!@#$%^&*()+}{":>?"} +%20',
-'string with\r +*',
-'string with newline\n +*',
-'string with \x01**',
-#'string with \x00**', # this will fail
-]
-
-PORT=8080
-
-def pytest_generate_tests(metafunc):
- if metafunc.function in [test_put_get]:
- for data in RANDOM_STRINGS:
- metafunc.addcall(funcargs=dict(data=data))
-
-def test_put_get(data):
- c = SimpleQueue(port=PORT, debug=True)
- c.put(data)
- d = c.get()
- assert d == data
-
-def test_put_mget():
- c = SimpleQueue(port=PORT, debug=True)
- c.put('test1')
- c.put('test2')
- items = c.mget(num_items=10).splitlines()
- assert len(items) == 2
- assert items[0] == 'test1'
- assert items[1] == 'test2'
- c.put('test3')
- c.put('test4')
- items = c.mget().splitlines()
- assert len(items) == 1
- assert items[0] == 'test3'
- assert c.get() == 'test4'
-
-def test_mget_badindex():
- c = SimpleQueue(port=PORT, debug=True)
- msg = c.mget(num_items=-2)
- assert msg == 'number of items must be > 0\n'
- msg2 = c.mget(num_items=0)
- assert msg2 == 'number of items must be > 0\n'
- c.put('test1')
- items = c.mget().splitlines()
- assert len(items) == 1
- assert items[0] == 'test1'
-
-def test_mput_get():
- c = SimpleQueue(port=PORT, debug=True)
-
- # default separator
- c.mput('test1\ntest2\ntest3')
- assert c.get() == 'test1'
- assert c.get() == 'test2'
- assert c.get() == 'test3'
-
- # separator specified
- c.mput(data='test4|test5|test6', separator='|')
- assert c.get() == 'test4'
- assert c.get() == 'test5'
- assert c.get() == 'test6'
-
- # default separator with get request instead of post
- c.mput('test1\ntest2\ntest3', post=False)
- assert c.get() == 'test1'
- assert c.get() == 'test2'
- assert c.get() == 'test3'
-
- # separator specified with get request instead of post
- c.mput(data='test4|test5|test6', separator='|', post=False)
- assert c.get() == 'test4'
- assert c.get() == 'test5'
- assert c.get() == 'test6'
-
- # yes this looks like double, actually checking nothing on queue == ''
- # instead of None
- assert c.get() == ''
- assert c.get() == ''
-
-def test_mput_mget():
- c = SimpleQueue(port=PORT, debug=True)
-
- # default separator
- c.mput('test1\ntest2\ntest3')
- items = c.mget(num_items=10).splitlines()
- assert len(items) == 3
- assert items[0] == 'test1'
- assert items[1] == 'test2'
- assert items[2] == 'test3'
-
- # separator specified
- c.mput(data='test4|test5|test6', separator='|')
- items = c.mget(num_items=10).splitlines()
- assert len(items) == 3
- assert items[0] == 'test4'
- assert items[1] == 'test5'
- assert items[2] == 'test6'
-
- # multichar separator specified, and at end
- c.mput(data='test7SEPtest8SEPtest9SEP', separator='SEP')
- items = c.mget(num_items=10).splitlines()
- assert len(items) == 3
- assert items[0] == 'test7'
- assert items[1] == 'test8'
- assert items[2] == 'test9'
-
- # default separator with get request instead of post
- c.mput('test1\ntest2\ntest3', post=False)
- items = c.mget(num_items=10).splitlines()
- assert len(items) == 3
- assert items[0] == 'test1'
- assert items[1] == 'test2'
- assert items[2] == 'test3'
-
- # separator specified with get request instead of post
- c.mput(data='test4|test5|test6', separator='|', post=False)
- items = c.mget(num_items=10).splitlines()
- assert len(items) == 3
- assert items[0] == 'test4'
- assert items[1] == 'test5'
- assert items[2] == 'test6'
-
- # multichar separator specified, and at end with get request instead of post
- c.mput(data='test7SEPtest8SEPtest9SEP', separator='SEP', post=False)
- items = c.mget(num_items=10).splitlines()
- assert len(items) == 3
- assert items[0] == 'test7'
- assert items[1] == 'test8'
- assert items[2] == 'test9'
+import os
+import sys
+sys.path.append(os.path.join(os.path.dirname(__file__), "../shared_tests"))
+
+import logging
+import simplejson as json
+from test_shunt import valgrind_cmd, SubprocessTest, http_fetch, http_fetch_json
+
+class SimplequeueTest(SubprocessTest):
+ binary_name = "simplequeue"
+ process_options = [valgrind_cmd(os.path.abspath('simplequeue'), '--enable-logging')]
+ working_dir = os.path.dirname(__file__)
+
+ def test_basic(self):
+ # put/get in order
+ http_fetch('/put', dict(data='12345'))
+ http_fetch('/put', dict(data='23456'))
+ data = json.loads(http_fetch('/stats', dict(format="json")))
+ assert data['depth'] == 2
+ assert data['bytes'] == 10
+ data = http_fetch('/get')
+ assert data == "12345"
+ data = http_fetch('/get')
+ assert data == "23456"
+ data = http_fetch('/get')
+ assert data == ''
+
+ data = json.loads(http_fetch('/stats', dict(format="json")))
+ assert data['depth'] == 0
+ assert data['bytes'] == 0
- # yes this looks like double, actually checking nothing on queue == ''
- # instead of None
- assert c.get() == ''
- assert c.get() == ''
+ # mget
+ http_fetch('/put', dict(data='12345'))
+ http_fetch('/put', dict(data='23456'))
+ http_fetch('/put', dict(data='34567'))
+ http_fetch('/put', dict(data='45678'))
+ data = http_fetch('/mget', dict(items=5))
+ assert data == '12345\n23456\n34567\n45678\n'
-def test_order():
- # first thing put in, should be first thing out
- c = SimpleQueue(port=PORT, debug=True)
- c.put('test1')
- c.put('test2')
- d1 = c.get()
- d2 = c.get()
- assert d1 == 'test1'
- assert d2 == 'test2'
+ # mput GET req
+ http_fetch('/mput', dict(data='test1\ntest2\ntest3'))
+ data = http_fetch('/get')
+ assert data == 'test1'
+ data = http_fetch('/get')
+ assert data == 'test2'
+ data = http_fetch('/get')
+ assert data == 'test3'
+
+ # mput POST req
+ http_fetch('/mput', body='test1\ntest2\ntest3')
+ data = http_fetch('/get')
+ assert data == 'test1'
+ data = http_fetch('/get')
+ assert data == 'test2'
+ data = http_fetch('/get')
+ assert data == 'test3'
-def test_dump():
- c = SimpleQueue(port=PORT, debug=True)
- c.put('test1')
- c.put('test2')
- d = c.dump()
- c.get()
- c.get()
- assert d == ['test1', 'test2']
if __name__ == "__main__":
- print "tests against port 5150"
- print "usage: py.test test_simplequeue.py"
+ print "usage: py.test"
Please sign in to comment.
Something went wrong with that request. Please try again.