add initial multimechanize loadtest module

.gitignore

@@ -19,3 +19,5 @@ startover.sh
 celery.db
 *.sql
 /resource_versions.py
+/loadtest/results/*
+/loadtest/localsettings.py

loadtest/README.mkd

@@ -0,0 +1,29 @@
+##CommCare HQ loadtest
+multi-mechanize load tests for CommCare HQ
+
+Work in progress
+
+###Running
+Requires [multi-mechanize](http://testutils.org/multi-mechanize/).  
+
+From the root commcare-hq directory run 
+
+    $ multimech-run loadtest
+
+Currently has the following user profiles:
+
+* login - logs a user in and grabs an empty report page
+* submit_form - submits a simple form with a single case
+* ota_restore - ota restores a mobile user
+* public_landingpage - hits the public HQ landing page
+
+Can tweak server, domain and user credentials by adding a localsettings.py file and overriding the following:
+
+    BASE_URL = 'https://staging.commcarehq.org'
+    DOMAIN = "demo"
+    USERNAME = "changeme@dimagi.com"
+    PASSWORD = "***"
+    MOBILE_USERNAME = "user@demo.commcarehq.org"
+    MOBILE_PASSWORD = "***"
+
+Edit config.cfg to tweak number of threads per user profile and length of test. See multimechanize docs for more info.

loadtest/config.cfg

@@ -0,0 +1,19 @@
+[global]
+run_time = 300
+rampup = 200
+results_ts_interval = 10
+progress_bar = on
+console_logging = off
+xml_report = off
+
+[user_group-1]
+threads: 5
+script: login.py
+
+[user_group-2]
+threads: 10
+script: submit_form.py
+
+[user_group-3]
+threads: 5
+script: ota_restore.py

loadtest/test_scripts/hq_settings.py

@@ -0,0 +1,75 @@
+import mechanize
+import cookielib
+
+
+BASE_URL = 'https://staging.commcarehq.org'
+DOMAIN = "demo"
+USERNAME = "changeme@dimagi.com"
+PASSWORD = "***"
+MOBILE_USERNAME = "user@demo.commcarehq.org"
+MOBILE_PASSWORD = "***"
+
+try:
+    from localsettings import *
+except ImportError:
+    pass
+
+def login_url():
+    return "%s%s" % (BASE_URL, "/accounts/login/")
+
+# this is necessary to make the runner happy
+class Transaction(object):
+    def run(self): return
+
+class HQTransaction(object):
+    """
+    Stick some shared stuff in here so we can use it across tests
+    and keep most of the config in one place.
+    """
+
+    def __init__(self):
+        self.custom_timers = {}
+        self.base_url = BASE_URL
+        self.domain = DOMAIN
+        self.username = USERNAME
+        self.password = PASSWORD
+        self.mobile_username = MOBILE_USERNAME
+        self.mobile_password = MOBILE_PASSWORD
+
+class User(object):
+    def __init__(self, username, password, browser):
+        self.username = username
+        self.password = password
+        self.browser = browser
+        self.logged_in = False
+
+    def ensure_logged_in(self):
+        if not self.logged_in:
+            _login(self.browser, self.username, self.password)
+            self.logged_in = True
+
+    def __str__(self):
+        return "User<user=%s,logged_in=%s>" % (self.username, self.logged_in)
+
+# utility functions
+def init_browser():
+    """Returns an initialized browser and associated cookie jar."""
+    br = mechanize.Browser()
+    cj = cookielib.LWPCookieJar()
+    br.set_cookiejar(cj)
+
+    br.set_handle_equiv(True)
+    br.set_handle_gzip(True)
+    br.set_handle_redirect(True)
+    br.set_handle_referer(True)
+    br.set_handle_robots(False)
+
+    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
+    return br
+
+def _login(browser, username, password):
+    _ = browser.open(login_url())
+    browser.select_form(name="form")
+    browser.form['username'] = username
+    browser.form['password'] = password
+    browser.submit()

loadtest/test_scripts/login.py

@@ -0,0 +1,24 @@
+import mechanize
+import time
+from hq_settings import init_browser, User, HQTransaction
+
+class Transaction(HQTransaction):
+
+    def run(self):
+        br = init_browser()
+        start_timer = time.time()
+        user = User(self.username, self.password, br)
+        user.ensure_logged_in()
+        latency = time.time() - start_timer
+        self.custom_timers['Login'] = latency  
+
+        resp = br.open('%s/a/%s/reports/' % (self.base_url, self.domain))
+        body = resp.read()
+        assert resp.code == 200, 'Bad HTTP Response'
+        assert "Case Activity" in body, "Couldn't find report list"
+
+
+if __name__ == '__main__':
+    trans = Transaction()
+    trans.run()
+    print trans.custom_timers

loadtest/test_scripts/ota_restore.py

@@ -0,0 +1,25 @@
+import mechanize
+import time
+from hq_settings import init_browser
+import hq_settings
+from hq_settings import HQTransaction
+
+class Transaction(HQTransaction):
+
+    def run(self):
+        br = init_browser()
+        url = "%s/a/%s/phone/restore/" % (self.base_url, self.domain)
+        start_timer = time.time()
+        br.add_password(url, self.mobile_username, self.mobile_password)
+        resp = br.open(url)
+        latency = time.time() - start_timer
+        self.custom_timers['ota-restore'] = latency  
+        body = resp.read()
+        assert resp.code == 200, 'Bad HTTP Response'
+        assert "Successfully restored" in body
+
+
+if __name__ == '__main__':
+    trans = Transaction()
+    trans.run()
+    print trans.custom_timers

loadtest/test_scripts/public_landingpage.py

@@ -0,0 +1,21 @@
+import mechanize
+import time
+from hq_settings import HQTransaction
+
+class Transaction(HQTransaction):
+
+    def run(self):
+        br = mechanize.Browser()
+        br.set_handle_robots(False)
+        start_timer = time.time()
+        resp = br.open(self.base_url + '/home/')
+        resp.read()
+        latency = time.time() - start_timer
+        self.custom_timers['Public_Landing_Page'] = latency  
+        assert (resp.code == 200), 'Bad HTTP Response'
+
+
+if __name__ == '__main__':
+    trans = Transaction()
+    trans.run()
+    print trans.custom_timers

loadtest/test_scripts/submit_form.py

@@ -0,0 +1,77 @@
+import mechanize
+import time
+from hq_settings import HQTransaction
+from datetime import datetime
+from urlparse import urlparse
+import httplib
+import uuid
+
+# ghetto
+SUBMIT_TEMPLATE = """<?xml version='1.0'?>
+<data xmlns:jrm="http://dev.commcarehq.org/jr/xforms" xmlns="http://www.commcarehq.org/loadtest">
+    <meta>
+        <deviceID>multimechanize</deviceID>
+        <timeStart>%(timestart)s</timeStart>
+        <timeEnd>%(timeend)s</timeEnd>
+        <username>multimechanize</username>
+        <userID>multimechanize</userID>
+        <instanceID>%(instanceid)s</instanceID>
+    </meta>
+    %(extras)s
+</data>"""
+
+CASE_TEMPLATE = """
+<case xmlns="http://commcarehq.org/case/transaction/v2" case_id="%(caseid)s"
+        date_modified="%(moddate)s" user_id="multimechanize">
+    <create>
+        <case_type_id>loadtest</case_type_id>
+        <case_name>load test case</case_name>
+    </create>
+    <update>
+        <prop1>val1</prop1>
+        <prop2>val2</prop2>
+    </update>
+</case>
+"""
+
+ISO_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+def _format_datetime(time):
+    return time.strftime(ISO_FORMAT)
+
+def _submission(extras=""):
+    return SUBMIT_TEMPLATE % {"timestart": _format_datetime(datetime.utcnow()),
+                              "timeend": _format_datetime(datetime.utcnow()),
+                              "instanceid": uuid.uuid4().hex,
+                              "extras": extras }
+def _case_submission():
+    caseblock = CASE_TEMPLATE % {"moddate": _format_datetime(datetime.utcnow()),
+                                 "caseid": uuid.uuid4().hex }
+    return _submission(extras=caseblock)
+
+def _post(data, url, content_type="text/xml"):
+    headers = {"content-type": content_type,
+               "content-length": len(data),
+               }
+
+    up = urlparse(url)
+    conn = httplib.HTTPSConnection(up.netloc) if url.startswith("https") else httplib.HTTPConnection(up.netloc) 
+    conn.request('POST', up.path, data, headers)
+    return conn.getresponse()
+
+class Transaction(HQTransaction):
+
+    def run(self):
+        submit = _case_submission()
+        start_timer = time.time()
+        url = "%s%s" % (self.base_url, "/a/cory/receiver") 
+        resp = _post(submit, url)
+        latency = time.time() - start_timer
+        self.custom_timers['submission'] = latency  
+        responsetext = resp.read()
+        assert resp.status == 201, 'Bad HTTP Response'
+        assert "Thanks for submitting" in responsetext, "Bad response text"
+
+if __name__ == '__main__':
+    trans = Transaction()
+    trans.run()
+    print trans.custom_timers