Browse files

bug 730704: Changes to kuma to use kumascript service

* Document view performs a request to the kumascript service to evaluate
  macros before rendering response

* Tweak to CKEditor setup, skips setting it up on Template:* pages in
  advance of more work for bug 731651

* Tests for kumascript usage in document view

* Constance config setting to enforce kumascript request timeout (or to
  disable entirely, which is the default)

* Tweaks to document view and model to support HTTP ETag and
  Last-Modified headers, in advance of caching support for
  bug 730714 and bug 730715

* Add kumascript service as a submodule

* Add node.js and kumascript config to vagrant/puppet

* Added tmux to vagrant install, because it's very handy

* Updates to .gitignore

* Bugfix to 404 on js/libs/jqueryui-min.js
  • Loading branch information...
1 parent 413b5b7 commit 238e14a4daec574cca4799f1f0ce235057fa6c39 @lmorchard lmorchard committed Feb 28, 2012
View
3 .gitignore
@@ -24,3 +24,6 @@ humans.txt
apps/humans/tmp
tmp
webroot/.htaccess
+kumascript.log
+kumascript_settings_local.json
+lib/product_details_json
View
3 .gitmodules
@@ -10,3 +10,6 @@
[submodule "vendor"]
path = vendor
url = git://github.com/mozilla/kuma-lib.git
+[submodule "kumascript"]
+ path = kumascript
+ url = git://github.com/lmorchard/kumascript.git
View
14 apps/wiki/models.py
@@ -2,6 +2,8 @@
from datetime import datetime
from itertools import chain
from urlparse import urlparse
+import hashlib
+import time
from pyquery import PyQuery
from tower import ugettext_lazy as _lazy, ugettext as _
@@ -13,6 +15,7 @@
from django.core.urlresolvers import resolve
from django.db import models
from django.http import Http404
+from django.utils.http import http_date
from south.modelsinspector import add_introspection_rules
@@ -495,6 +498,17 @@ def language(self):
operating_systems = _inherited('operating_systems', 'operating_system_set')
@property
+ def etag(self):
+ """Generate an ETag based on document content hash, suitable for an
+ HTTP header"""
+ return hashlib.md5(self.html.encode('utf8')).hexdigest()
+
+ @property
+ def last_modified(self):
+ """Generate a Last-Modified string suitable for an HTTP header"""
+ return http_date(time.mktime(self.modified.timetuple()))
+
+ @property
def full_path(self):
"""The full path of a document consists of {locale}/{slug}"""
return '%s/%s' % (self.locale, self.slug)
View
80 apps/wiki/tests/test_views.py
@@ -10,6 +10,8 @@
from nose.plugins.attrib import attr
from pyquery import PyQuery as pq
+import constance.config
+
import waffle
from waffle.models import Flag, Sample, Switch
@@ -113,6 +115,7 @@ def test_ui_locale(self):
self.assertEqual(response.status_code, 302)
assert ('/%s/docs/' % en) in response['Location']
+
class ViewTests(TestCaseBase):
fixtures = ['test_users.json', 'wiki/documents.json']
@@ -131,6 +134,83 @@ def test_json_view(self):
eq_('an article title', data['title'])
+class KumascriptIntegrationTests(TestCaseBase):
+ """Tests for usage of the kumascript service.
+
+ Note that these tests really just check whether or not the service was
+ used, and are not integration tests meant to exercise the real service.
+ """
+
+ fixtures = ['test_users.json']
+
+ def setUp(self):
+ super(KumascriptIntegrationTests, self).setUp()
+
+ self.d, self.r = doc_rev()
+ self.url = reverse('wiki.document',
+ args=['%s/%s' % (self.d.locale, self.d.slug)],
+ locale=settings.WIKI_DEFAULT_LANGUAGE)
+
+ # NOTE: We could do this instead of using the @patch decorator over and
+ # over, but it requires an upgrade of mock to 0.8.0
+
+ # self.mock_perform_kumascript_request = (
+ # mock.patch('wiki.views._perform_kumascript_request'))
+ # self.mock_perform_kumascript_request.return_value = self.d.html
+
+ def tearDown(self):
+ super(KumascriptIntegrationTests, self).tearDown()
+
+ constance.config.KUMASCRIPT_TIMEOUT = 0.0
+
+ # NOTE: We could do this instead of using the @patch decorator over and
+ # over, but it requires an upgrade of mock to 0.8.0
+
+ # self.mock_perform_kumascript_request.stop()
+
+ @mock.patch('wiki.views._perform_kumascript_request')
+ def test_basic_view(self, mock_perform_kumascript_request):
+ """When kumascript timeout is non-zero, the service should be used"""
+ mock_perform_kumascript_request.return_value = self.d.html
+ constance.config.KUMASCRIPT_TIMEOUT = 1.0
+ response = self.client.get(self.url, follow=False)
+ ok_(mock_perform_kumascript_request.called,
+ "kumascript should have been used")
+
+ @mock.patch('wiki.views._perform_kumascript_request')
+ def test_disabled(self, mock_perform_kumascript_request):
+ """When disabled, the kumascript service should not be used"""
+ mock_perform_kumascript_request.return_value = self.d.html
+ constance.config.KUMASCRIPT_TIMEOUT = 0.0
+ response = self.client.get(self.url, follow=False)
+ ok_(not mock_perform_kumascript_request.called,
+ "kumascript not should have been used")
+
+ @mock.patch('wiki.views._perform_kumascript_request')
+ def test_nomacros(self, mock_perform_kumascript_request):
+ mock_perform_kumascript_request.return_value = self.d.html
+ constance.config.KUMASCRIPT_TIMEOUT = 1.0
+ response = self.client.get('%s?nomacros' % self.url, follow=False)
+ ok_(not mock_perform_kumascript_request.called,
+ "kumascript should not have been used")
+
+ @mock.patch('wiki.views._perform_kumascript_request')
+ def test_raw(self, mock_perform_kumascript_request):
+ mock_perform_kumascript_request.return_value = self.d.html
+ constance.config.KUMASCRIPT_TIMEOUT = 1.0
+ response = self.client.get('%s?raw' % self.url, follow=False)
+ ok_(not mock_perform_kumascript_request.called,
+ "kumascript should not have been used")
+
+ @mock.patch('wiki.views._perform_kumascript_request')
+ def test_raw_macros(self, mock_perform_kumascript_request):
+ mock_perform_kumascript_request.return_value = self.d.html
+ constance.config.KUMASCRIPT_TIMEOUT = 1.0
+ response = self.client.get('%s?raw&macros' % self.url, follow=False)
+ ok_(mock_perform_kumascript_request.called,
+ "kumascript should have been used")
+
+
class DocumentEditingTests(TestCaseBase):
"""Tests for the document-editing view"""
View
75 apps/wiki/views.py
@@ -4,6 +4,8 @@
from urllib import urlencode
from string import ascii_letters
+import requests
+
try:
from functools import wraps
except ImportError:
@@ -19,6 +21,8 @@
from django.views.decorators.http import (require_GET, require_POST,
require_http_methods)
+import constance.config
+
from waffle.decorators import waffle_flag
import jingo
@@ -185,6 +189,14 @@ def document(request, document_slug, document_locale):
except Document.DoesNotExist:
pass
+ # Utility to set common headers used by all response exit points
+ def set_common_headers(r):
+ r['ETag'] = doc.etag
+ r['Last-Modified'] = doc.last_modified
+ if doc.current_revision:
+ r['x-kuma-revision'] = doc.current_revision.id
+ return r
+
related = doc.related_documents.order_by('-related_to__in_common')[0:5]
# Get the contributors. (To avoid this query, we could render the
@@ -201,11 +213,30 @@ def document(request, document_slug, document_locale):
# Grab some parameters that affect output
section_id = request.GET.get('section', None)
- show_raw = request.GET.get('raw', False)
- need_edit_links = request.GET.get('edit_links', False)
+ show_raw = request.GET.get('raw', False) is not False
+ no_macros = request.GET.get('nomacros', False) is not False
+ force_macros = request.GET.get('macros', False) is not False
+ need_edit_links = request.GET.get('edit_links', False) is not False
+
+ # Grab the document HTML as a fallback, then attempt to use kumascript:
+ doc_html = doc.html
+ if (constance.config.KUMASCRIPT_TIMEOUT > 0 and
+ (force_macros or (not no_macros and not show_raw))):
+ # We'll make a request to kumascript for macro evaluation only if:
+ # * The service isn't disabled with a timeout of 0
+ # * The request has *not* asked for raw source
+ # (eg. ?raw)
+ # * The request has *not* asked for no macro evaluation
+ # (eg. ?nomacros)
+ # * The request *has* asked for macro evaluation
+ # (eg. ?raw&macros)
+ resp_body = _perform_kumascript_request(document_locale,
+ document_slug)
+ if resp_body:
+ doc_html = resp_body
# Start applying some filters to the document HTML
- tool = wiki.content.parse(doc.html)
+ tool = wiki.content.parse(doc_html)
# If a section ID is specified, extract that section.
if section_id:
@@ -221,16 +252,48 @@ def document(request, document_slug, document_locale):
# iframe inclusion
if show_raw:
response = HttpResponse(tool.serialize())
- response['x-kuma-revision'] = doc.current_revision.id
response['x-frame-options'] = 'Allow'
- return response
+ return set_common_headers(response)
data = {'document': doc, 'document_html': tool.serialize(),
'redirected_from': redirected_from,
'related': related, 'contributors': contributors,
'fallback_reason': fallback_reason}
data.update(SHOWFOR_DATA)
- return jingo.render(request, 'wiki/document.html', data)
+
+ response = jingo.render(request, 'wiki/document.html', data)
+ # FIXME: For some reason, the ETag isn't coming through here.
+ return set_common_headers(response)
+
+
+def _perform_kumascript_request(document_locale, document_slug):
+ """Perform a kumascript GET request for a document locale and slug.
+
+ This is broken out into its own utility function, both to make the view
+ method simpler and to make it easy to mock out in testing.
+ """
+ resp_body = None
+
+ try:
+ url_tmpl = settings.KUMASCRIPT_URL_TEMPLATE
+ resp = requests.get(
+ url_tmpl.format(path='%s/%s' % (document_locale,
+ document_slug)),
+ timeout=constance.config.KUMASCRIPT_TIMEOUT)
+ if resp.status_code == 200:
+ # HACK: Assume we're getting UTF-8, which we should be.
+ # TODO: Better solution would be to upgrade the requests module
+ # in vendor from 0.6.1 to at least 0.10.6, and use resp.text,
+ # which does auto-detection. But, that will break things.
+ resp_body = resp.read().decode('utf8')
+
+ except Exception, e:
+ # Do nothing, if the kumascript service fails in some way.
+ # TODO: Log the failure more usefully here.
+ logging.debug("KS FAILED %s" % e)
+ pass
+
+ return resp_body
@waffle_flag('kumawiki')
1 kumascript
@@ -0,0 +1 @@
+Subproject commit c08d7505515866cd73940e15592037dd3cfca12d
View
16 kumascript_settings_local.json-dist
@@ -0,0 +1,16 @@
+{
+ "log": {
+ "console": true,
+ "file": {
+ "filename": "./kumascript.log",
+ "maxsize": 500000
+ }
+ },
+ "server": {
+ "port": 9080,
+ "numWorkers": 4,
+ "workerTimeout": 10000,
+ "document_url_template": "https://developer.mozilla.org/en-US/docs/{path}",
+ "template_url_template": "https://developer.mozilla.org/en-US/docs/en-US/Template:{path}"
+ }
+}
View
2 media/js/main.js
@@ -96,7 +96,7 @@ k = {};
* This lazy loads our jQueryUI script.
*/
function lazyLoadScripts() {
- var scripts = ['js/libs/jqueryui-min.js'],
+ var scripts = ['js/libs/jqueryui.min.js'],
styles = [], // was: ['css/jqueryui/jqueryui-min.css']
// turns out this messes with search
i;
View
28 media/js/wiki_ckeditor.js
@@ -1,5 +1,7 @@
-jQuery("#id_content").ckeditor(function() {
+(function () {
+
// Callback functions after CKE is ready
+ var setup_ckeditor = function () {
var $head = $("#article-head");
var $tools = $(".cke_toolbox");
@@ -38,6 +40,24 @@ jQuery("#id_content").ckeditor(function() {
// remove the id_content required attribute
$('#id_content').removeAttr("required");
- }, {
- customConfig : '/docs/ckeditor_config.js'
-});
+ };
+
+ jQuery("#id_content").each(function () {
+
+ var el = jQuery(this),
+ doc_slug = $('#id_slug').val();
+
+ if (doc_slug.toLowerCase().indexOf('template:') === 0) {
+ // HACK: If we're on a Template:* page, abort setting up CKEditor
+ // because it doesn't work well at all for these pages.
+ // See bug 731651 for further work down this path.
+ return;
+ }
+
+ el.ckeditor(setup_ckeditor, {
+ customConfig : '/docs/ckeditor_config.js'
+ });
+
+ });
+
+})();
View
7 puppet/files/etc/motd
@@ -1,5 +1,10 @@
Welcome to Kuma (developer-dev.mozilla.org)
-Run this to start the Django app:
+To update this dev environment:
+ sudo puppet apply /vagrant/puppet/manifests/dev-vagrant.pp
+
+To start the Django app:
./manage.py runserver_plus 0.0.0.0:8000
+To start the kumascript service:
+ node kumascript/run.js
View
1 puppet/files/etc/sysconfig/iptables
@@ -19,6 +19,7 @@
-A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 80 -j ACCEPT
-A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 3306 -j ACCEPT
-A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 8000 -j ACCEPT
+-A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 9080 -j ACCEPT
-A RH-Firewall-1-INPUT -p udp -m udp --dport 137 -j ACCEPT
-A RH-Firewall-1-INPUT -p udp -m udp --dport 138 -j ACCEPT
-A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 139 -j ACCEPT
View
7 puppet/files/home/vagrant/bash_profile
@@ -6,10 +6,11 @@ if [ -f ~/.bashrc ]; then
fi
# User specific environment and startup programs
-
-PATH=$PATH:$HOME/bin
-
+PATH=$HOME/bin:/vagrant/kumascript/node_modules/.bin:$PATH
export PATH
+# Ensure the virtualenv is in effect
. $HOME/kuma-venv/bin/activate
+
+# More useful to start in the project directory than in user home
cd /vagrant
View
16 puppet/files/vagrant/kumascript_settings_local.json
@@ -0,0 +1,16 @@
+{
+ "log": {
+ "console": true,
+ "file": {
+ "filename": "/home/vagrant/logs/kumascript.log",
+ "maxsize": 500000
+ }
+ },
+ "server": {
+ "port": 9080,
+ "numWorkers": 4,
+ "workerTimeout": 10000,
+ "document_url_template": "http://localhost/en-US/docs/{path}?raw=1",
+ "template_url_template": "http://localhost/en-US/docs/en-US/Template:{name}?raw=1"
+ }
+}
View
2 puppet/files/vagrant/settings_local.py
@@ -112,3 +112,5 @@
LOGIN_REDIRECT_URL = '/'
LOGIN_REDIRECT_URL_FAILURE = '/'
+
+KUMASCRIPT_URL_TEMPLATE = 'http://localhost:9080/docs/{path}'
View
16 puppet/manifests/classes/dev-hacks.pp
@@ -3,7 +3,7 @@
package {
[ "gcc-c++", "git", "subversion-devel", "mercurial", "vim-enhanced",
"man", "man-pages", "nfs-utils", "nfs-utils-lib", "telnet", "nc",
- "rsync", "samba", "java-1.6.0-openjdk"]:
+ "rsync", "samba", "java-1.6.0-openjdk", "tmux"]:
ensure => installed;
}
}
@@ -28,6 +28,11 @@
source => "$PROJ_DIR/puppet/files/vagrant/settings_local.py";
}
+ file { "$PROJ_DIR/kumascript_settings_local.json":
+ ensure => file,
+ source => "$PROJ_DIR/puppet/files/vagrant/kumascript_settings_local.json";
+ }
+
case $operatingsystem {
centos: {
@@ -106,6 +111,15 @@
# Last few things that need doing...
class dev_hacks_post {
+
+ # This bash_profile auto-activates the virtualenv on login, adds some
+ # useful things to the $PATH, etc.
+ file {
+ "/home/vagrant/.bash_profile":
+ source => "$PROJ_DIR/puppet/files/home/vagrant/bash_profile",
+ owner => "vagrant", group => "vagrant", mode => 0664;
+ }
+
case $operatingsystem {
centos: {
# Sync a yum cache up to host machine from VM
View
37 puppet/manifests/classes/nodejs.pp
@@ -0,0 +1,37 @@
+# Get node.js and npm installed under CentOS
+
+class node_repos {
+ exec {
+ "node_repo_download":
+ cwd => "$PROJ_DIR/puppet/cache",
+ command => "/usr/bin/wget http://nodejs.tchol.org/repocfg/el/nodejs-stable-release.noarch.rpm",
+ creates => "$PROJ_DIR/puppet/cache/nodejs-stable-release.noarch.rpm";
+ "node_repo_install":
+ cwd => "$PROJ_DIR/puppet/cache",
+ command => "/usr/bin/yum localinstall -y --nogpgcheck $PROJ_DIR/puppet/cache/nodejs-stable-release.noarch.rpm",
+ creates => "/etc/yum.repos.d/nodejs-stable.repo",
+ require => Exec['node_repo_download'];
+ }
+}
+
+class node_install {
+ package {
+ [ "nodejs", "nodejs-devel", "npm" ]:
+ ensure => installed;
+ }
+ file { "/usr/bin/node":
+ target => "/usr/bin/nodejs",
+ ensure => link,
+ require => Package['nodejs']
+ }
+ file { "/usr/include/node":
+ target => "/usr/include/nodejs",
+ ensure => link,
+ require => Package['nodejs-devel']
+ }
+}
+
+class nodejs {
+ include node_repos, node_install
+ Class['node_repos'] -> Class['node_install']
+}
View
7 puppet/manifests/classes/python.pp
@@ -33,13 +33,6 @@
command => "/usr/bin/virtualenv --no-site-packages /home/vagrant/kuma-venv",
creates => "/home/vagrant/kuma-venv"
}
- # This bash_profile auto-activates the virtualenv on login.
- file {
- "/home/vagrant/.bash_profile":
- source => "$PROJ_DIR/puppet/files/home/vagrant/bash_profile",
- owner => "vagrant", group => "vagrant", mode => 0664,
- require => Exec['virtualenv-create'];
- }
}
class python_modules {
View
1 puppet/manifests/dev-vagrant.pp
@@ -38,6 +38,7 @@
python: stage => langs;
php: stage => langs;
+ nodejs: stage => langs;
dekiwiki: stage => vendors;
View
1 requirements/dev.txt
@@ -10,6 +10,7 @@ mock==0.6.0
pyquery==0.5
translate-toolkit==1.6.0
pylint==0.20.0
+pygments==1.4
django-devserver
View
10 settings.py
@@ -940,6 +940,14 @@ def read_only_mode(env):
'We typically multiply this value by the retry number so, e.g., '
'the 4th retry waits 4*.5 = 2 seconds.'
),
+
+ KUMASCRIPT_TIMEOUT = (
+ 0.0,
+ 'Maximum seconds to wait for a response from the kumascript service. '
+ 'On timeout, the document gets served up as-is and without macro '
+ 'evaluation as an attempt at graceful failure. NOTE: a value of 0 '
+ 'disables kumascript altogether.'
+ ),
)
BROWSERID_VERIFICATION_URL = 'https://browserid.org/verify'
@@ -953,3 +961,5 @@ def read_only_mode(env):
BASKET_URL = 'https://basket.mozilla.com'
BASKET_APPS_NEWSLETTER = 'app-dev'
+
+KUMASCRIPT_URL_TEMPLATE = 'http://developer.mozilla.org:9080/docs/{path}'

0 comments on commit 238e14a

Please sign in to comment.