Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge branch 'master' of github.com:mozilla/kuma

  • Loading branch information...
commit 9153fb8c5854072e2d63df093bc6f01b5060034b 2 parents 593e407 + aac3a6d
Les Orchard lmorchard authored
Showing with 21,324 additions and 1,342 deletions.
  1. +6 −0 .gitignore
  2. +3 −0  .gitmodules
  3. +1 −4 README-vagrant.md
  4. +2 −4 Vagrantfile
  5. +41 −16 apps/dekicompat/backends.py
  6. 0  apps/dekicompat/management/__init__.py
  7. 0  apps/dekicompat/management/commands/__init__.py
  8. +833 −0 apps/dekicompat/management/commands/migrate_to_kuma_wiki.py
  9. +22 −1 apps/dekicompat/tests.py
  10. +153 −78 apps/demos/__init__.py
  11. +2 −1  apps/demos/feeds.py
  12. +1 −0  apps/demos/forms.py
  13. +125 −75 apps/demos/helpers.py
  14. +1 −1  apps/demos/models.py
  15. +2 −0  apps/demos/templates/demos/detail.html
  16. +35 −2 apps/demos/templates/demos/devderby_landing.html
  17. +9 −1 apps/demos/templates/demos/submit_noauth.html
  18. +19 −0 apps/demos/tests/test_helpers.py
  19. +30 −0 apps/demos/tests/test_templates.py
  20. +4 −2 apps/demos/tests/test_views.py
  21. +3 −3 apps/devmo/fixtures/bad_date.csv
  22. +2 −2 apps/devmo/fixtures/xss.csv
  23. +2 −25 apps/devmo/forms.py
  24. +92 −57 apps/devmo/models.py
  25. +62 −5 apps/devmo/templates/devmo/profile.html
  26. +1 −1  apps/devmo/templates/devmo/profile_edit.html
  27. +37 −0 apps/devmo/tests/test_forms.py
  28. +28 −13 apps/devmo/tests/test_models.py
  29. +81 −0 apps/devmo/tests/test_views.py
  30. +3 −0  apps/devmo/urls.py
  31. +35 −16 apps/devmo/views.py
  32. +2 −2 apps/docs/templates/docs/docs.html
  33. +4 −4 apps/landing/templates/landing/addons.html
  34. +72 −158 apps/landing/templates/landing/apps.html
  35. +59 −0 apps/landing/templates/landing/apps_subscribe.html
  36. +14 −1 apps/landing/templates/landing/home.html
  37. +325 −0 apps/landing/templates/landing/learn_html5.html
  38. +4 −4 apps/landing/templates/landing/mobile.html
  39. +3 −3 apps/landing/templates/landing/mozilla.html
  40. +4 −4 apps/landing/templates/landing/web.html
  41. +39 −7 apps/landing/test_views.py
  42. +1 −0  apps/landing/urls.py
  43. +42 −11 apps/landing/views.py
  44. +26 −21 apps/users/forms.py
  45. +4 −2 apps/users/helpers.py
  46. +6 −2 apps/users/models.py
  47. +7 −0 apps/users/templates/users/browserid_explanation.html
  48. +17 −0 apps/users/templates/users/browserid_header_signin.html
  49. +25 −1 apps/users/templates/users/browserid_register.html
  50. +9 −0 apps/users/templates/users/browserid_signin.html
  51. +8 −7 apps/users/templates/users/change_email.html
  52. +7 −0 apps/users/templates/users/email/reminder.ltxt
  53. +13 −17 apps/users/templates/users/login.html
  54. +10 −1 apps/users/templates/users/register.html
  55. +21 −0 apps/users/templates/users/send_email_reminder_done.html
  56. +1 −0  apps/users/tests/__init__.py
  57. +11 −1 apps/users/tests/test_forms.py
  58. +2 −2 apps/users/tests/test_helpers.py
  59. +47 −6 apps/users/tests/test_templates.py
  60. +165 −14 apps/users/tests/test_views.py
  61. +8 −1 apps/users/urls.py
  62. +18 −2 apps/users/utils.py
  63. +151 −63 apps/users/views.py
  64. +3 −2 apps/wiki/admin.py
  65. +93 −6 apps/wiki/content.py
  66. +4 −4 apps/wiki/events.py
  67. +11 −4 apps/wiki/feeds.py
  68. +31 −35 apps/wiki/forms.py
  69. +166 −0 apps/wiki/migrations/0007_auto__add_field_revision_mindtouch_old_id.py
  70. +174 −0 apps/wiki/migrations/0008_auto__add_field_document_mindtouch_page_id__add_field_document_modifie.py
  71. +175 −0 apps/wiki/migrations/0009_auto__add_field_revision_is_mindtouch_migration__add_unique_revision_m.py
  72. +165 −0 apps/wiki/migrations/0010_add_locale_slug_uniqueness.py
  73. +169 −0 apps/wiki/migrations/0011_auto__add_field_revision_tags.py
  74. +185 −0 apps/wiki/migrations/0012_auto__add_documenttag__add_taggeddocument.py
  75. +159 −23 apps/wiki/models.py
  76. +2 −2 apps/wiki/parser.py
  77. +3 −3 apps/wiki/tasks.py
  78. +51 −15 apps/wiki/templates/wiki/document.html
  79. +12 −14 apps/wiki/templates/wiki/document_revisions.html
  80. +22 −2 apps/wiki/templates/wiki/edit_document.html
  81. +1 −1  apps/wiki/templates/wiki/includes/page_buttons.html
  82. +7 −0 apps/wiki/templates/wiki/includes/revision_diff.html
  83. +3 −1 apps/wiki/tests/__init__.py
  84. +137 −1 apps/wiki/tests/test_content.py
  85. +0 −1  apps/wiki/tests/test_forms.py
  86. +53 −15 apps/wiki/tests/test_models.py
  87. +24 −23 apps/wiki/tests/test_templates.py
  88. +402 −52 apps/wiki/tests/test_views.py
  89. +3 −1 apps/wiki/urls.py
  90. +349 −77 apps/wiki/views.py
  91. +42 −24 configs/htaccess
  92. +1 −0  kumascript
  93. +17 −0 kumascript_settings_local.json-dist
  94. +0 −53 lib/responsys.py
  95. +10 −0 media/ace/ace.js
  96. +1 −0  media/ace/mode-javascript.js
  97. +1 −0  media/ace/theme-dreamweaver.js
  98. +9,753 −0 media/ace/worker-javascript.js
  99. +7 −6 media/ckeditor/skins/kuma/editor.css
  100. +149 −14 media/css/mdn-screen.css
  101. +48 −201 media/css/wiki-screen.css
  102. BIN  media/img/bg-orange-collage.jpg
  103. BIN  media/img/bg-starburst-five.png
  104. BIN  media/img/bg-starburst.png
  105. BIN  media/img/blank.gif
  106. BIN  media/img/devderby/judges/chriscoyier.jpg
  107. BIN  media/img/devderby/judges/chrisheilmann.png
  108. BIN  media/img/devderby/judges/remysharp.jpg
  109. BIN  media/img/html5-icons.png
  110. BIN  media/img/html5-logo-big.png
  111. +71 −0 media/js/apps-newsletter.js
  112. +1 −1  media/js/main.js
  113. +37 −20 media/js/mdn/init.js
  114. +25 −3 media/js/wiki.js
  115. +22 −7 media/js/wiki_ckeditor.js
  116. +165 −0 media/syntaxhighlighter/LGPL-LICENSE
  117. +20 −0 media/syntaxhighlighter/MIT-LICENSE
  118. +1 −0  media/syntaxhighlighter/VERSION
  119. +17 −0 media/syntaxhighlighter/scripts/shAutoloader.js
  120. +59 −0 media/syntaxhighlighter/scripts/shBrushAS3.js
  121. +75 −0 media/syntaxhighlighter/scripts/shBrushAppleScript.js
  122. +59 −0 media/syntaxhighlighter/scripts/shBrushBash.js
  123. +65 −0 media/syntaxhighlighter/scripts/shBrushCSharp.js
  124. +100 −0 media/syntaxhighlighter/scripts/shBrushColdFusion.js
  125. +97 −0 media/syntaxhighlighter/scripts/shBrushCpp.js
  126. +91 −0 media/syntaxhighlighter/scripts/shBrushCss.js
  127. +55 −0 media/syntaxhighlighter/scripts/shBrushDelphi.js
  128. +41 −0 media/syntaxhighlighter/scripts/shBrushDiff.js
  129. +52 −0 media/syntaxhighlighter/scripts/shBrushErlang.js
  130. +67 −0 media/syntaxhighlighter/scripts/shBrushGroovy.js
  131. +52 −0 media/syntaxhighlighter/scripts/shBrushJScript.js
  132. +57 −0 media/syntaxhighlighter/scripts/shBrushJava.js
  133. +58 −0 media/syntaxhighlighter/scripts/shBrushJavaFX.js
  134. +72 −0 media/syntaxhighlighter/scripts/shBrushPerl.js
  135. +88 −0 media/syntaxhighlighter/scripts/shBrushPhp.js
  136. +33 −0 media/syntaxhighlighter/scripts/shBrushPlain.js
  137. +74 −0 media/syntaxhighlighter/scripts/shBrushPowerShell.js
  138. +64 −0 media/syntaxhighlighter/scripts/shBrushPython.js
  139. +55 −0 media/syntaxhighlighter/scripts/shBrushRuby.js
  140. +94 −0 media/syntaxhighlighter/scripts/shBrushSass.js
  141. +51 −0 media/syntaxhighlighter/scripts/shBrushScala.js
  142. +66 −0 media/syntaxhighlighter/scripts/shBrushSql.js
  143. +56 −0 media/syntaxhighlighter/scripts/shBrushVb.js
  144. +69 −0 media/syntaxhighlighter/scripts/shBrushXml.js
  145. +17 −0 media/syntaxhighlighter/scripts/shCore.js
  146. +17 −0 media/syntaxhighlighter/scripts/shLegacy.js
  147. +226 −0 media/syntaxhighlighter/styles/shCore.css
  148. +328 −0 media/syntaxhighlighter/styles/shCoreDefault.css
  149. +331 −0 media/syntaxhighlighter/styles/shCoreDjango.css
  150. +339 −0 media/syntaxhighlighter/styles/shCoreEclipse.css
  151. +324 −0 media/syntaxhighlighter/styles/shCoreEmacs.css
  152. +328 −0 media/syntaxhighlighter/styles/shCoreFadeToGrey.css
  153. +324 −0 media/syntaxhighlighter/styles/shCoreMDUltra.css
  154. +324 −0 media/syntaxhighlighter/styles/shCoreMidnight.css
  155. +324 −0 media/syntaxhighlighter/styles/shCoreRDark.css
  156. +117 −0 media/syntaxhighlighter/styles/shThemeDefault.css
  157. +120 −0 media/syntaxhighlighter/styles/shThemeDjango.css
  158. +128 −0 media/syntaxhighlighter/styles/shThemeEclipse.css
  159. +113 −0 media/syntaxhighlighter/styles/shThemeEmacs.css
  160. +117 −0 media/syntaxhighlighter/styles/shThemeFadeToGrey.css
  161. +113 −0 media/syntaxhighlighter/styles/shThemeMDUltra.css
  162. +113 −0 media/syntaxhighlighter/styles/shThemeMidnight.css
  163. +113 −0 media/syntaxhighlighter/styles/shThemeRDark.css
  164. +1 −1  migrations/01-initial.sql
  165. +7 −2 puppet/files/etc/motd
  166. +1 −0  puppet/files/etc/sysconfig/iptables
  167. +4 −3 puppet/files/home/vagrant/bash_profile
  168. +47 −15 puppet/files/tmp/postimport.sql
  169. +17 −0 puppet/files/vagrant/kumascript_settings_local.json
  170. +13 −1 puppet/files/vagrant/settings_local.py
  171. +17 −2 puppet/manifests/classes/dev-hacks.pp
  172. +37 −0 puppet/manifests/classes/nodejs.pp
  173. +0 −7 puppet/manifests/classes/python.pp
  174. +1 −0  puppet/manifests/dev-vagrant.pp
  175. +1 −0  requirements/dev.txt
  176. +5 −0 scripts/migrate_all.sh
  177. +5 −0 scripts/migrate_recent.sh
  178. +5 −0 scripts/migrate_top.sh
  179. +9 −11 scripts/update_site.py
  180. +116 −5 settings.py
  181. +12 −5 templates/500.html
  182. +1 −1  templates/base.html
  183. +6 −36 templates/includes/login.html
  184. +1 −1  vendor
6 .gitignore
View
@@ -2,6 +2,8 @@
*.pyo
*.sw?
.vagrant
+vagrantconfig_local.yaml
+kuma.box
settings_local.py
pip-log.txt
.coverage
@@ -21,3 +23,7 @@ puppet/cache/*
humans.txt
apps/humans/tmp
tmp
+webroot/.htaccess
+kumascript.log
+kumascript_settings_local.json
+lib/product_details_json
3  .gitmodules
View
@@ -10,3 +10,6 @@
[submodule "vendor"]
path = vendor
url = git://github.com/mozilla/kuma-lib.git
+[submodule "kumascript"]
+ path = kumascript
+ url = git://github.com/mozilla/kumascript.git
5 README-vagrant.md
View
@@ -16,10 +16,8 @@ reasons.
# Open a terminal window.
# Install vagrant, see vagrantup.com
- # NOTE: Currently, vagrant v0.8.8 appears to fail with this setup, so we
- # need to revert to 0.8.7
sudo gem update
- sudo gem install vagrant --version=0.8.7
+ sudo gem install vagrant
# Clone a Kuma repo, switch to "mdn" branch (for now)
git clone git://github.com/mozilla/kuma.git
@@ -44,7 +42,6 @@ reasons.
# production site. This can take a long while, since there's over 500MB
vagrant ssh
sudo puppet apply /vagrant/puppet/manifests/dev-vagrant-mdn-import.pp
- mysql -uroot < /vagrant/puppet/files/tmp/postimport.sql
sudo puppet apply /vagrant/puppet/manifests/dev-vagrant.pp
# Edit files as usual on your host machine; the current directory is
6 Vagrantfile
View
@@ -29,16 +29,14 @@ Vagrant::Config.run do |config|
end
# This thing can be a little hungry for memory
- config.vm.customize do |vm|
- vm.memory_size = CONF['memory_size']
- end
+ config.vm.customize ["modifyvm", :id, "--memory", CONF['memory_size']]
# uncomment to enable VM GUI console, mainly for troubleshooting
if CONF['gui'] == true
config.vm.boot_mode = :gui
end
- config.vm.network(CONF['ip_address'])
+ config.vm.network :hostonly, CONF['ip_address']
# Increase vagrant's patience during hang-y CentOS bootup
# see: https://github.com/jedi4ever/veewee/issues/14
57 apps/dekicompat/backends.py
View
@@ -1,6 +1,5 @@
-import logging
-
from datetime import datetime
+import time
from urllib import urlencode
from urllib2 import HTTPError
import urlparse
@@ -9,11 +8,12 @@
from xml.sax.saxutils import escape as xml_escape
from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist
from django.contrib.auth.models import User
-
from pyquery import PyQuery as pq
import commonware
+import constance.config
from devmo.models import UserProfile
@@ -39,6 +39,8 @@
</permissions.user>
</user>"""
+class MindTouchAPIError(Exception):
+ pass
class DekiUserBackend(object):
"""
@@ -144,7 +146,10 @@ def get_user(self, user_id):
return None
@staticmethod
- def get_or_create_user(deki_user):
+ def get_or_create_user(deki_user, sync_attrs=('is_superuser',
+ 'is_staff',
+ 'is_active',
+ 'email')):
"""
Grab the User via their UserProfile and deki_user_id.
If non exists, create both.
@@ -157,21 +162,28 @@ def get_or_create_user(deki_user):
profile = UserProfile.objects.get(deki_user_id=deki_user.id)
user = profile.user
- except UserProfile.DoesNotExist:
+ except ObjectDoesNotExist:
# No existing profile, so try creating a new profile and user
+
+ # HACK: Usernames in Kuma are limited to 30 characters. There are
+ # around 0.1% of MindTouch users in production (circa 2011) whose
+ # names exceed this length. They're mostly the product of spammers
+ # and security tests but it will still throw MySQL-level errors
+ # during migration.
+ username = deki_user.username[:30]
+
user, created = (User.objects
- .get_or_create(username=deki_user.username))
- user.username = deki_user.username
+ .get_or_create(username=username))
+ user.username = username
user.email = deki_user.email
user.set_unusable_password()
user.save()
profile = UserProfile(deki_user_id=deki_user.id, user=user)
- profile.save()
+ profile.save(skip_mindtouch_put=True)
user.deki_user = deki_user
# Sync these attributes from Deki -> Django (for now)
- sync_attrs = ('is_superuser', 'is_staff', 'is_active', 'email')
needs_save = False
for sa in sync_attrs:
deki_val = getattr(deki_user, sa, None)
@@ -211,9 +223,9 @@ def generate_mindtouch_user_xml(user):
role = 'Contributor'
if user.is_staff and user.is_superuser:
role = 'Admin'
- user_xml = MINDTOUCH_USER_XML % {'username': user.username,
- 'email': user.email,
- 'fullname': user.get_profile().fullname,
+ user_xml = MINDTOUCH_USER_XML % {'username': user.username.encode('utf-8'),
+ 'email': user.email.encode('utf-8'),
+ 'fullname': user.get_profile().fullname.encode('utf-8'),
'status': 'active',
'language': user.get_profile().mindtouch_language,
'timezone': user.get_profile().mindtouch_timezone,
@@ -221,6 +233,10 @@ def generate_mindtouch_user_xml(user):
return user_xml
@staticmethod
+ def _perform_post_mindtouch_user(url, data, headers):
+ return requests.post(url, data=data, headers=headers)
+
+ @staticmethod
def post_mindtouch_user(user):
# post a new mindtouch user
user_url = '%s/@api/deki/users?apikey=%s' % (
@@ -228,10 +244,19 @@ def post_mindtouch_user(user):
settings.DEKIWIKI_APIKEY)
user_xml = DekiUserBackend.generate_mindtouch_user_xml(user)
headers = {'Content-Type': 'application/xml'}
- resp = requests.post(user_url, data=user_xml, headers=headers)
+ resp = DekiUserBackend._perform_post_mindtouch_user(user_url,
+ user_xml,
+ headers)
if resp.status_code is not 200:
- # TODO: decide WTF to do here
- pass
+ # HACK: MindTouch fails intermittently, so retry a few times
+ for i in range(constance.config.DEKIWIKI_POST_RETRIES):
+ resp = DekiUserBackend._perform_post_mindtouch_user(
+ user_url, data=user_xml, headers=headers)
+ if resp.status_code is 200:
+ break
+ time.sleep(constance.config.DEKIWIKI_API_RETRY_WAIT * i)
+ if resp.status_code is not 200:
+ raise MindTouchAPIError("post_mindtouch_user failed")
return DekiUser.parse_user_info(resp.content)
@staticmethod
@@ -264,7 +289,7 @@ def __init__(self, id, username, fullname, email, gravatar,
self.email = email
self.gravatar = gravatar
self.profile_url = profile_url
- self.is_active = False
+ self.is_active = True
self.is_staff = False
self.is_superuser = False
0  apps/dekicompat/management/__init__.py
View
No changes.
0  apps/dekicompat/management/commands/__init__.py
View
No changes.
833 apps/dekicompat/management/commands/migrate_to_kuma_wiki.py
View
@@ -0,0 +1,833 @@
+"""
+Migration tool that copies pages from a MindTouch database to the Kuma wiki.
+
+Should be idempotent - ie. running this repeatedly should result only in
+updates, and not duplicate documents or repeated revisions.
+
+TODO
+* https://bugzilla.mozilla.org/show_bug.cgi?id=710713
+* https://bugzilla.mozilla.org/showdependencytree.cgi?id=710713&hide_resolved=1
+"""
+import sys
+import re
+import time
+import datetime
+import itertools
+import hashlib
+from optparse import make_option
+
+from BeautifulSoup import BeautifulSoup
+from pyquery import PyQuery as pq
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.core.management.base import (BaseCommand, NoArgsCommand,
+ CommandError)
+import django.db
+from django.db import connections, connection, transaction, IntegrityError
+from django.utils import encoding, hashcompat
+
+import commonware.log
+
+from sumo.urlresolvers import reverse
+
+from wiki.models import (Document, Revision, CATEGORIES, SIGNIFICANCES)
+
+from wiki.models import REDIRECT_CONTENT
+import wiki.content
+from wiki.content import (ContentSectionTool, CodeSyntaxFilter,
+ DekiscriptMacroFilter)
+
+from dekicompat.backends import DekiUser, DekiUserBackend
+
+
+log = commonware.log.getLogger('kuma.migration')
+
+# Regular expression to match and extract page title from MindTouch redirects
+# eg. #REDIRECT [[en/DOM/Foo]], #REDIRECT[[foo_bar_baz]]
+MT_REDIR_PAT = re.compile(r"""^#REDIRECT ?\[\[([^\]]+)\]\]""")
+
+
+# See also: https://github.com/mozilla/kuma/blob/mdn/apps/devmo/models.py#L327
+# I'd just import from there, but wanted to do this a little differently
+MT_NAMESPACES = (
+ ('', 0),
+ ('Talk:', 1),
+ ('User:', 2),
+ ('User_talk:', 3),
+ ('Project:', 4),
+ ('Project_talk:', 5),
+
+ ('Help:', 12),
+ ('Help_talk:', 13),
+ ('Special:', -1),
+ ('Template:', 10),
+ ('Template_talk:', 11),
+)
+MT_NS_NAME_TO_ID = dict(MT_NAMESPACES)
+MT_NS_ID_TO_NAME = dict((x[1], x[0]) for x in MT_NAMESPACES)
+MT_MIGRATED_NS_IDS = (MT_NS_NAME_TO_ID[x] for x in (
+ '', 'Talk:', 'User:', 'User_talk:', 'Project:', 'Project_talk:'
+))
+
+# NOTE: These are MD5 hashes of garbage User page content. The criteria is that
+# the content was found to repeat more than 3 times, and was hand-reviewed by
+# lorchard who determined they were MindTouch default content. Blame him if
+# it's an overenthusiastic list.
+#
+# See also these SQL queries:
+# https://bugzilla.mozilla.org/show_bug.cgi?id=710753#c3
+# https://bugzilla.mozilla.org/show_bug.cgi?id=710753#c4
+#
+# And, see also this MySQL transcript listing the content involved:
+# https://bugzilla.mozilla.org/attachment.cgi?id=590867
+
+USER_NS_EXCLUDED_CONTENT_HASHES = """
+7479e8f30d5ab0e9202195a1bddec69d
+698141d0c92776d60d884ebce6d64d82
+ca0c3622cdb213281cf2dc698b15c357
+ce33312f48b8ce8a68c587173e276f3a
+9ba3b75ba5e3ba82cfad83a50186ab35
+e931344938b19ea93865568712c2b2de
+a40f1d06233eef791bcf8b55df46cace
+14d2e3e51d704084503f67eaaf47dc72
+d41d8cd98f00b204e9800998ecf8427e
+74ced08578951e424aff4e7a90f2b48b
+55abb153d6e5d1bc22dae9938074f38d
+43d1c34c5556ebf12e9d0601863eb752
+f53c0981035e2378c8e8692a1e7f9649
+68b329da9893e34099c7d8ad5cb9c940
+8766b3552715bed94c106f6824efb535
+7dbb4512068edc202eda2b853c415cb7
+63f484aade7cfab43340bd001370c132
+f71abdf1a61d4fbcf7a96c484f602434
+baf848927342e7fa737b14277fa566f8
+83c7ff527035fe0dd78c2330e08d6747
+b43e15a05b457a6b79a5c553e0fbd9a7
+3c9a42e7646f29f7c983fd0a8be88ecd
+c2346672e9d426b4b8cac99507220a14
+42c76681cb99f161fecccd2c1e56b4b0
+3e31c2cafadd3ec47a88d0fc446bb929
+0356162b5f5fc96b8d96222a839d05ec
+""".split("\n")
+
+# List of MindTouch locales mapped to Kuma locales.
+MT_TO_KUMA_LOCALE_MAP = getattr(settings, 'MT_TO_KUMA_LOCALE_MAP')
+
+
+class Command(BaseCommand):
+ help = """Migrate content from MindTouch to Kuma"""
+
+ option_list = BaseCommand.option_list + (
+
+ make_option('--wipe', action="store_true", dest="wipe", default=False,
+ help="Wipe all documents before migration"),
+
+ make_option('--all', action="store_true", dest="all", default=False,
+ help="Migrate all documents"),
+ make_option('--slug', dest="slug", default=None,
+ help="Migrate specific page by slug"),
+ make_option('--revisions', dest="revisions", type="int", default=999,
+ help="Limit revisions migrated per document"),
+ make_option('--viewed', dest="most_viewed", type="int", default=0,
+ help="Migrate # of most viewed documents"),
+ make_option('--recent', dest="recent", type="int", default=0,
+ help="Migrate # of recently modified documents"),
+ make_option('--longest', dest="longest", type="int", default=0,
+ help="Migrate # of longest documents"),
+ make_option('--redirects', dest="redirects", type="int", default=0,
+ help="Migrate # of documents containing redirects"),
+ make_option('--nonen', dest="nonen", type="int", default=0,
+ help="Migrate # of documents in locales other than en-US"),
+ make_option('--withsyntax', dest="withsyntax", type="int", default=0,
+ help="Migrate # of documents with syntax blocks"),
+ make_option('--withscripts', dest="withscripts", type="int", default=0,
+ help="Migrate # of documents that use scripts"),
+ make_option('--syntax-metrics', action="store_true",
+ dest="syntax_metrics", default=False,
+ help="Measure syntax highlighter usage, skip migration"),
+
+ make_option('--limit', dest="limit", type="int", default=99999,
+ help="Stop after a migrating a number of documents"),
+ make_option('--skip', dest="skip", type="int", default=0,
+ help="Skip a number of documents for migration"),
+
+ make_option('--maxlength', dest="maxlength", type="int",
+ default=1000000,
+ help="Maximum character length for page content"),
+
+ make_option('--update-revisions', action="store_true",
+ dest="update_revisions", default=False,
+ help="Force update to existing revisions"),
+ make_option('--update-documents', action="store_true",
+ dest="update_documents", default=False,
+ help="Force update to existing documents"),
+
+ make_option('--template-metrics', action="store_true",
+ dest="template_metrics", default=False,
+ help="Measure template usage, skip migration"),
+ make_option('--list-full-template', action="store_true",
+ dest="list_full_template", default=False,
+ help="Print the full template call, rather than"
+ " just the method used"),
+
+ make_option('--failfast', action='store_true', dest='failfast',
+ help="Do not trap exceptions; raise and exit errors"),
+ make_option('--verbose', action='store_true', dest='verbose',
+ help="Produce verbose output"),)
+
+ def handle(self, *args, **options):
+ """Main driver for the command"""
+ self.init(options)
+
+ if self.options['wipe']:
+ self.wipe_documents()
+
+ rows = self.gather_pages()
+
+ if options['template_metrics']:
+ self.handle_template_metrics(rows)
+ elif options['syntax_metrics']:
+ self.handle_syntax_metrics(rows)
+ else:
+ self.handle_migration(rows)
+
+ def init(self, options):
+ """Set up connections and options"""
+
+ settings.DATABASES.update(settings.MIGRATION_DATABASES)
+ settings.CACHE_BACKEND = 'dummy://'
+
+ self.options = options
+ self.admin_role_ids = (4,)
+ self.user_ids = {}
+
+ self.wikidb = connections['wikidb']
+ self.cur = self.wikidb.cursor()
+
+ self.kumadb = connections['default']
+
+ def handle_migration(self, rows):
+ self.docs_migrated = self.index_migrated_docs()
+ log.info("Found %s docs already migrated" %
+ len(self.docs_migrated.values()))
+
+ start_ts = ts_now = time.time()
+
+ self.rev_ct = 0
+ ct, skip_ct, error_ct = 0, 0, 0
+
+ for r in rows:
+ try:
+ if ct < self.options['skip']:
+ # Skip rows until past the option value
+ continue
+ if self.update_document(r):
+ # Something was actually updated and not skipped
+ ct += 1
+ else:
+ # This was a skip.
+ skip_ct += 1
+
+ # Free memory in query cache after each document.
+ django.db.reset_queries()
+
+ if ct >= self.options['limit']:
+ log.info("Reached limit of %s documents migrated" %
+ self.options['limit'])
+ break
+
+ except Exception, e:
+ if self.options['failfast']:
+ # If the option is set, then just bail out.
+ raise
+ else:
+ # Note: This traps *all* errors, so that the migration can get
+ # through what it can. This should really produce a problem
+ # documents report, though.
+ log.error('\t\tPROBLEM %s' % type(e))
+ error_ct += 1
+
+ ts_now = time.time()
+ duration = ts_now - start_ts
+ total_ct = ct + skip_ct + error_ct
+ if (total_ct % 10) == 0:
+ log.info("Rate: %s docs/sec, %s secs/doc, "
+ "%s total in %s seconds" %
+ ((total_ct + 1) / (duration + 1),
+ (duration + 1) / (total_ct + 1),
+ total_ct, duration))
+ log.info("Rate: %s revs/sec, %s total in %s seconds" %
+ ((self.rev_ct + 1) / (duration + 1),
+ self.rev_ct, duration))
+
+ log.info("Migration finished: %s seconds, %s migrated, "
+ "%s skipped, %s errors" %
+ ((time.time() - start_ts), ct, skip_ct, error_ct))
+
+ def handle_template_metrics(self, rows):
+ """Parse out DekiScript template calls from pages"""
+ # This regex seems to catch all the DekiScript calls
+ fn_pat = re.compile('^([0-9a-zA-Z_\.]+)')
+ wt_pat = re.compile(r"""^wiki.template\(["']([^'"]+)['"]""")
+
+ # PROCESS ALL THE PAGES!
+ for r in rows:
+
+ if not r['page_text'].strip():
+ # Page empty, so skip it.
+ continue
+
+ doc = pq(r['page_text'])
+ spans = doc.find('span.script')
+ for span in spans:
+ src = unicode(span.text).strip()
+ if self.options['list_full_template']:
+ print src.encode('utf-8')
+ else:
+ if src.startswith('wiki.template'):
+ pat = wt_pat
+ m = pat.match(src)
+ if not m:
+ continue
+ print (u"Template:%s" % m.group(1)).encode('utf-8')
+ else:
+ pat = fn_pat
+ m = pat.match(src)
+ if not m:
+ continue
+ out = m.group(1)
+ if out.startswith('template.'):
+ out = out.replace('template.', 'Template:')
+ if out.startswith('Template.'):
+ out = out.replace('Template.', 'Template:')
+ if '.' not in out and 'Template:' not in out:
+ out = u'Template:%s' % out
+ print out.encode('utf-8')
+
+ def handle_syntax_metrics(self, rows):
+ """Discover the languages used in syntax highlighting"""
+ for r in rows:
+ pt = r['page_text']
+ soup = BeautifulSoup(pt)
+ blocks = soup.findAll()
+ for block in blocks:
+ for attr in block.attrs:
+ if attr[0] == 'function':
+ print (u"%s\t%s\t%s" % (r['page_title'], block.name,
+ attr[1])).encode('utf-8')
+
+ @transaction.commit_on_success
+ def wipe_documents(self):
+ """Delete all documents"""
+ log.info("Wiping all Kuma documents and revisions")
+ kc = self.kumadb.cursor()
+ kc.execute("""
+ SET FOREIGN_KEY_CHECKS = 0;
+ TRUNCATE wiki_taggeddocument;
+ TRUNCATE wiki_documenttag;
+ TRUNCATE wiki_revision;
+ TRUNCATE wiki_document;
+ """)
+
+ def index_migrated_docs(self):
+ """Build an index of Kuma docs already migrated, mapping Mindtouch page
+ ID to document last-modified."""
+ kc = self.kumadb.cursor()
+ kc.execute("""
+ SELECT mindtouch_page_id, id, modified
+ FROM wiki_document
+ WHERE mindtouch_page_id IS NOT NULL
+ """)
+ return dict((r[0], (r[1], r[2])) for r in kc)
+
+ def gather_pages(self):
+ """Gather rows for pages using the current options"""
+ iters = []
+ ns_list = '(%s)' % (', '.join(str(x) for x in MT_MIGRATED_NS_IDS))
+
+ # TODO: Migrate pages from namespaces other than 0
+
+ if self.options['all']:
+ # Migrating all pages trumps any other criteria
+ where = """
+ WHERE page_namespace IN %s
+ ORDER BY page_timestamp DESC
+ """ % (ns_list)
+ self.cur.execute("SELECT count(*) FROM pages %s" % where)
+ log.info("Gathering ALL %s pages..." %
+ self.cur.fetchone()[0])
+ iters.append(self._query("SELECT * FROM pages %s" % where))
+
+ elif self.options['slug']:
+ # Use the slug in namespace 0, or parse apart slug and namespace ID
+ # if a colon is present.
+ ns, slug = 0, self.options['slug']
+ if ':' in slug:
+ ns_name, slug = slug.split(':', 1)
+ ns = MT_NS_NAME_TO_ID.get('%s:' % ns_name, 0)
+
+ # Migrating a single page...
+ log.info("Searching for %s" % self.options['slug'])
+ iters.append(self._query("""
+ SELECT *
+ FROM pages
+ WHERE
+ page_namespace = %s AND
+ page_title = %s
+ ORDER BY page_timestamp DESC
+ """, ns, slug))
+
+ else:
+ # TODO: Refactor these copypasta queries into something DRYer?
+
+ if self.options['most_viewed'] > 0:
+ # Grab the most viewed pages
+ log.info("Gathering %s most viewed pages..." %
+ self.options['most_viewed'])
+ iters.append(self._query("""
+ SELECT p.*, pc.*
+ FROM pages AS p, page_viewcount AS pc
+ WHERE
+ pc.page_id=p.page_id AND
+ page_namespace IN %s
+ ORDER BY pc.page_counter DESC
+ LIMIT %s
+ """ % (ns_list, '%s'), self.options['most_viewed']))
+
+ if self.options['recent'] > 0:
+ # Grab the most recently modified
+ log.info("Gathering %s recently modified pages..." %
+ self.options['recent'])
+ iters.append(self._query("""
+ SELECT *
+ FROM pages
+ WHERE page_namespace IN %s
+ ORDER BY page_timestamp DESC
+ LIMIT %s
+ """ % (ns_list, '%s'), self.options['recent']))
+
+ if self.options['longest'] > 0:
+ # Grab the longest pages
+ log.info("Gathering %s longest pages..." %
+ self.options['longest'])
+ iters.append(self._query("""
+ SELECT *
+ FROM pages
+ WHERE page_namespace IN %s
+ ORDER BY length(page_text) DESC
+ LIMIT %s
+ """ % (ns_list, '%s'), self.options['longest']))
+
+ if self.options['redirects'] > 0:
+ # Grab the redirect pages
+ log.info("Gathering %s redirects from MindTouch..." %
+ self.options['redirects'])
+ # HACK: Need to use "%%%%" here, just to get one "%". It's
+ # stinky, but it's because this string goes twice through %
+ # formatting - once for page namespace list, and once for SQL
+ # escaping in Django.
+ iters.append(self._query("""
+ SELECT * FROM pages
+ WHERE
+ page_namespace IN %s AND
+ page_text LIKE '#REDIRECT%%%%'
+ ORDER BY page_timestamp DESC
+ LIMIT %s
+ """ % (ns_list, '%s'), self.options['redirects']))
+
+ if self.options['nonen'] > 0:
+ # Grab non-en pages. Might catch a few pages with "en/" in the
+ # title, but not in the page_language.
+ log.info("Gathering %s pages in locales other than en-US..." %
+ self.options['nonen'])
+ iters.append(self._query("""
+ SELECT *
+ FROM pages
+ WHERE page_namespace IN %s AND
+ page_language <> 'en'
+ ORDER BY page_timestamp DESC
+ LIMIT %s
+ """ % (ns_list, '%s'), self.options['nonen']))
+
+ if self.options['withsyntax'] > 0:
+ log.info("Gathering %s pages with syntax highlighting" %
+ self.options['withsyntax'])
+ iters.append(self._query("""
+ SELECT *
+ FROM pages
+ WHERE page_namespace IN %s AND
+ page_text like '%%%%function="syntax.%%%%'
+ ORDER BY page_timestamp DESC
+ LIMIT %s
+ """ % (ns_list, '%s'), self.options['withsyntax']))
+
+ if self.options['withscripts'] > 0:
+ log.info("Gathering %s pages that use scripts" %
+ self.options['withscripts'])
+ iters.append(self._query("""
+ SELECT *
+ FROM pages
+ WHERE page_namespace IN %s AND
+ page_text like '%%%%span class="script"%%%%'
+ ORDER BY page_timestamp DESC
+ LIMIT %s
+ """ % (ns_list, '%s'), self.options['withscripts']))
+
+ return itertools.chain(*iters)
+
+ @transaction.commit_on_success
+ def update_document(self, r):
+ """Update Kuma document from given MindTouch page record"""
+ # Build the page slug from namespace + title or display name
+ locale, slug = self.get_kuma_locale_and_slug_for_page(r)
+
+ # Skip this document, if it has a blank timestamp.
+ # The only pages in production that have no timestamp are either in the
+ # Special: namespace (not migrated), or a couple of untitled and empty
+ # pages under the Template: or User: namespaces.
+ if not r['page_timestamp']:
+ log.debug("\t%s/%s (%s) skipped, no timestamp" %
+ (locale, slug, r['page_display_name']))
+ return False
+
+ # Check to see if this doc has already been migrated, and if the
+ # exising is doc is up to date.
+ page_ts = self.parse_timestamp(r['page_timestamp'])
+ last_mod = self.docs_migrated.get(r['page_id'], (None, None))[1]
+ if (not self.options['update_documents'] and last_mod is not None
+ and last_mod >= page_ts):
+ log.debug("\t%s/%s (%s) up to date" %
+ (locale, slug, r['page_display_name']))
+ return False
+
+ # Check to see if this doc's content hash falls in the list of User:
+ # namespace content we want to exclude.
+ if r['page_namespace'] == MT_NS_NAME_TO_ID['User:']:
+ content_hash = (hashlib.md5(r['page_text'].encode('utf-8'))
+ .hexdigest())
+ if content_hash in USER_NS_EXCLUDED_CONTENT_HASHES:
+ log.debug("\t%s/%s (%s) matched User: content exclusion list" %
+ (locale, slug, r['page_display_name']))
+ return False
+
+ # Check to see if this page's content is too long, skip if so.
+ if len(r['page_text']) > self.options['maxlength']:
+ log.debug("\t%s/%s (%s) skipped, page too long (%s > %s max)" %
+ (locale, slug, r['page_display_name'],
+ len(r['page_text']), self.options['maxlength']))
+ return False
+
+ log.info("\t%s/%s (%s)" % (locale, slug, r['page_display_name']))
+
+ # Ensure that the document exists, and has the MindTouch page ID
+ doc, created = Document.objects.get_or_create(
+ locale=locale, slug=slug,
+ title=r['page_display_name'], defaults=dict(
+ category=CATEGORIES[0][0],
+ ))
+ doc.mindtouch_page_id = r['page_id']
+
+ if created:
+ log.info("\t\tNew document created. (ID=%s)" % doc.pk)
+ else:
+ log.info("\t\tDocument already exists. (ID=%s)" % doc.pk)
+
+ tags = self.get_tags_for_page(r)
+
+ self.update_past_revisions(r, doc, tags)
+ self.update_current_revision(r, doc, tags)
+
+ return True
+
+ def update_past_revisions(self, r_page, doc, tags):
+ """Update past revisions for the given page row and document"""
+ ct_saved, ct_skipped, ct_error = 0, 0, 0
+
+ wc = self.wikidb.cursor()
+ kc = self.kumadb.cursor()
+
+ # Grab all the past revisions from MindTouch
+ wc.execute("""
+ SELECT * FROM old as o
+ WHERE o.old_page_id = %s
+ ORDER BY o.old_revision DESC
+ LIMIT %s
+ """, (r_page['page_id'], self.options['revisions'],))
+ old_rows = sorted(self._query_dicts(wc),
+ key=lambda r: r['old_revision'], reverse=True)
+
+ # Grab all the MindTouch old_ids from Kuma doc revisions
+ kc.execute("""
+ SELECT mindtouch_old_id, id
+ FROM wiki_revision
+ WHERE document_id=%s
+ """, (doc.pk,))
+ existing_old_ids = dict((r[0], r[1]) for r in kc)
+
+ # Process all the past revisions...
+ revs = []
+ for r in old_rows:
+
+ # Check if this already exists.
+ existing_id = None
+ if r['old_id'] in existing_old_ids:
+ existing_id = existing_old_ids[r['old_id']]
+ if not self.options['update_revisions']:
+ # If this revision has already been migrated, skip update.
+ ct_skipped += 1
+ continue
+
+ # Check to see if this revision's content is too long, skip if so.
+ if len(r['old_text']) > self.options['maxlength']:
+ ct_skipped += 1
+ continue
+
+ # Build up a dict of the row for the revision
+ ts = self.parse_timestamp(r['old_timestamp'])
+ rev_data = dict(
+ document_id=doc.pk,
+ mindtouch_old_id=r['old_id'],
+ is_mindtouch_migration=1,
+ slug=doc.slug,
+ title=doc.title,
+ tags=tags,
+ is_approved=True,
+ significance=SIGNIFICANCES[0][0],
+ summary='',
+ keywords='',
+ content=self.convert_page_text(r['old_text']),
+ comment=r['old_comment'],
+ created=ts,
+ creator_id=self.get_django_user_id_for_deki_id(r['old_user']),
+ reviewed=ts,
+ reviewer_id=self.get_superuser_id()
+ )
+ revs.append(rev_data)
+
+ ct_saved += 1
+
+ if len(revs):
+
+ # Build REPLACE INTO style SQL placeholders for the revisions. eg.:
+ # (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s),
+ # (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s),
+ # (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+ col_names = revs[0].keys()
+ one_row = '(%s)' % ', '.join('%s' for x in col_names)
+ placeholders = ",\n".join(one_row for x in revs)
+
+ # Flatten list of revisions data in chronological order, so that we
+ # get roughly time-sequential IDs and a flat list to fill the
+ # placeholders.
+ revs_flat = [rev[name]
+ for rev in sorted(revs, key=lambda x: x['created'])
+ for name in col_names]
+
+ # Build and execute a giant query to save all the revisions.
+ sql = ("REPLACE INTO wiki_revision (%s) VALUES %s" %
+ (', '.join(col_names), placeholders))
+ kc.execute(sql, revs_flat)
+
+ self.rev_ct += ct_saved + ct_skipped + ct_error
+ log.info("\t\tPast revisions: %s saved, %s skipped, %s errors" %
+ (ct_saved, ct_skipped, ct_error))
+
+ def update_current_revision(self, r, doc, tags):
+ """Update the current revision associated with a doc and MT page row"""
+ # HACK: Using old_id of None to indicate current MindTouch revision.
+ # All revisions of a Kuma document have revision records, whereas
+ # MindTouch only tracks "old" revisions.
+ p_id = r['page_user_id']
+ rev, created = Revision.objects.get_or_create(document=doc,
+ is_mindtouch_migration=True, mindtouch_old_id=None, defaults=dict(
+ creator_id=self.get_django_user_id_for_deki_id(p_id),
+ is_approved=True,
+ significance=SIGNIFICANCES[0][0],))
+
+ # Check to see if the current revision is up to date, in which case we
+ # can skip the update and save a little time.
+ page_ts = self.parse_timestamp(r['page_timestamp'])
+ if (not self.options['update_documents'] and not created and
+ page_ts <= rev.created):
+ log.info("\t\tCurrent revision up to date. (ID=%s)" % rev.pk)
+ return
+
+ rev.created = rev.reviewed = page_ts
+ rev.slug = doc.slug
+ rev.title = doc.title
+ rev.tags = tags
+ rev.content = self.convert_page_text(r['page_text'])
+
+ # HACK: Some comments end up being too long, but just truncate.
+ rev.comment = r['page_comment'][:255]
+
+ # Save, and force to current revision.
+ rev.save()
+ rev.make_current()
+
+ if created:
+ log.info("\t\tCurrent revision created. (ID=%s)" % rev.pk)
+ else:
+ log.info("\t\tCurrent revision updated. (ID=%s)" % rev.pk)
+
+ def convert_page_text(self, pt):
+ """Given a page row from MindTouch, do whatever needs doing to convert
+ the page content for Kuma."""
+
+ if pt.startswith('#REDIRECT'):
+ pt = self.convert_redirect(pt)
+
+ pt = self.convert_code_blocks(pt)
+ pt = self.convert_dekiscript_template_calls(pt)
+ # TODO: bug 710726 - Convert intra-wiki links?
+
+ return pt
+
+ def convert_redirect(self, pt):
+ """Convert MindTouch-style page redirects to Kuma-style"""
+ m = MT_REDIR_PAT.match(pt)
+ if m:
+ # TODO: Do we need a convert_title function for locale parsing (eg.
+ # as part of bug 710724?)
+ title = m.group(1)
+ href = reverse('wiki.document', args=[title])
+ pt = REDIRECT_CONTENT % dict(href=href, title=title)
+ return pt
+
+ def convert_code_blocks(self, pt):
+ pt = ContentSectionTool(pt).filter(CodeSyntaxFilter).serialize()
+ return pt
+
+ def convert_dekiscript_template_calls(self, pt):
+ return (wiki.content.parse(pt).filter(DekiscriptMacroFilter)
+ .serialize())
+
+ def get_tags_for_page(self, r):
+ """For a given page row, get the list of tags from MindTouch and build
+ a string representation for Kuma revisions."""
+ wc = self.wikidb.cursor()
+ wc.execute("""
+ SELECT t.tag_name
+ FROM tag_map AS tm, tags AS t, pages AS p
+ WHERE
+ t.tag_id=tm.tagmap_tag_id AND
+ p.page_id=tm.tagmap_page_id AND
+ p.page_id=%s
+ """, (r['page_id'],))
+
+ # HACK: To prevent MySQL truncation warnings, constrain the imported
+ # tags to 100 chars. Who wants tags that long, anyway?
+ mt_tags = [row[0][:100] for row in wc]
+
+ # To build a string representation, we need to quote or not quote based
+ # on the presence of commas or spaces in the tag name.
+ quoted = []
+ if len(mt_tags):
+ for tag in mt_tags:
+ if u',' in tag or u' ' in tag:
+ quoted.append('"%s"' % tag)
+ else:
+ quoted.append(tag)
+
+ return u', '.join(quoted)
+
+ def get_kuma_locale_and_slug_for_page(self, r):
+ """Given a MindTouch page row, derive the Kuma locale and slug."""
+
+ # Come up with a complete slug, along with MT namespace mapped to name.
+ ns_name = MT_NS_ID_TO_NAME.get(r['page_namespace'], '')
+ slug = '%s%s' % (ns_name, r['page_title'] or r['page_display_name'])
+
+ # Start from the default language
+ mt_language = ''
+
+ # If the page is not in a namespace, and it has paths in the slug...
+ if ns_name == '' and '/' in slug:
+ # Treat the first part of the slug path as locale and snip it off.
+ mt_language, new_slug = slug.split('/', 1)
+ if mt_language in MT_TO_KUMA_LOCALE_MAP:
+ # If it's a known language, then use the new slug
+ slug = new_slug
+ else:
+ # Otherwise, we'll preserve the slug and tack the default
+ # locale onto it. (eg. ServerJS/FAQ, CommonJS/FAQ)
+ mt_language = ''
+
+ if mt_language == '':
+ # Finally, fall back to the explicit page language
+ mt_language = r['page_language']
+
+ # Map from MT language to Kuma locale.
+ locale = MT_TO_KUMA_LOCALE_MAP.get(mt_language.lower(), '')
+
+ return (locale, slug)
+
+ def get_django_user_id_for_deki_id(self, deki_user_id):
+ """Given a Deki user ID, come up with a Django user object whether we
+ need to migrate it first or just fetch it."""
+ # If we don't already have this Deki user cached, look up or migrate
+ if deki_user_id not in self.user_ids:
+
+ # Look up the user straight from the database
+ self.cur.execute("SELECT * FROM users AS u WHERE u.user_id = %s",
+ (deki_user_id,))
+ r = list(self._query_dicts(self.cur))
+
+ if not len(r):
+ # HACK: If, for some reason the user is missing from MindTouch,
+ # just put and use the superuser. Seems to happen mainly for
+ # user #0, which is probably superuser anyway.
+ return self.get_superuser_id()
+
+ # Build a DekiUser object from the database record, and make sure
+ # it's active.
+ user = r[0]
+ deki_user = DekiUser(id=user['user_id'],
+ username=user['user_name'],
+ fullname=user['user_real_name'],
+ email=user['user_email'],
+ gravatar='',)
+ deki_user.is_active = True
+
+ # Scan user grants for admin roles to set Django flags.
+ self.cur.execute("""SELECT * FROM user_grants AS ug
+ WHERE user_id = %s""",
+ (deki_user_id,))
+ is_admin = False
+ for rg in self._query_dicts(self.cur):
+ if rg['role_id'] in self.admin_role_ids:
+ is_admin = True
+ deki_user.is_superuser = deki_user.is_staff = is_admin
+
+ # Finally get/create Django user and cache it.
+ user = DekiUserBackend.get_or_create_user(deki_user,
+ sync_attrs=[])
+ self.user_ids[deki_user_id] = user.pk
+
+ return self.user_ids[deki_user_id]
+
+ SUPERUSER_ID = None
+
+ def get_superuser_id(self):
+ """Get the first superuser from Django we can find."""
+ if not self.SUPERUSER_ID:
+ self.SUPERUSER_ID = User.objects.filter(is_superuser=1)[0].pk
+ return self.SUPERUSER_ID
+
+ def parse_timestamp(self, ts):
+ """HACK: Convert a MindTouch timestamp into something pythonic"""
+ # TODO: Timezone necessary here?
+ dt = datetime.datetime.fromtimestamp(
+ time.mktime(time.strptime(ts, "%Y%m%d%H%M%S")))
+ return dt
+
+ def _query(self, sql, *params):
+ self.cur.execute(sql, params)
+ return self._query_dicts(self.cur)
+
+ def _query_dicts(self, cursor):
+ """Wrapper for cursor.fetchall() that uses the cursor.description to
+ convert each row's list of columns to a dict."""
+ names = [x[0] for x in cursor.description]
+ return (dict(zip(names, row)) for row in cursor)
23 apps/dekicompat/tests.py
View
@@ -50,6 +50,26 @@ def test_new(self, post_mindtouch_user):
return test
+def mock_perform_post_mindtouch_user(test):
+ bad_resp = Response()
+ bad_resp.status_code = 500
+ bad_resp.content = "<FAIL><failure/></FAIL>"
+
+ if settings.DEKIWIKI_MOCK:
+ @mock.patch('dekicompat.backends.DekiUserBackend.'
+ '_perform_post_mindtouch_user')
+ def test_new(self, _perform_post_mindtouch_user):
+ _perform_post_mindtouch_user.return_value = bad_resp
+ try:
+ test(self)
+ except AttributeError:
+ pass
+ eq_(7, _perform_post_mindtouch_user.call_count)
+ return test_new
+ else:
+ return test
+
+
def mock_put_mindtouch_user(test):
if settings.DEKIWIKI_MOCK:
@mock.patch('dekicompat.backends.DekiUserBackend.put_mindtouch_user')
@@ -92,7 +112,8 @@ def test_new(self, get_deki_user_by_email):
def mock_missing_get_deki_user_by_email(test):
if settings.DEKIWIKI_MOCK:
- @mock.patch('dekicompat.backends.DekiUserBackend.get_deki_user_by_email')
+ @mock.patch('dekicompat.backends.DekiUserBackend'
+ '.get_deki_user_by_email')
def test_new(self, get_deki_user):
get_deki_user.return_value = None
test(self)
231 apps/demos/__init__.py
View
@@ -14,9 +14,14 @@
DEMOS_CACHE_NS_KEY = getattr(settings, 'DEMOS_CACHE_NS_KEY', 'demos_listing')
-# HACK: For easier L10N, define tag descriptions in code instead of as a DB model
-TAG_DESCRIPTIONS = dict( (x['tag_name'], x) for x in getattr(settings, 'TAG_DESCRIPTIONS', (
-
+# For easier L10N, define tag descriptions in code instead of as a DB model
+TAG_DESCRIPTIONS = dict((x['tag_name'], x) for x in getattr(
+ settings, 'TAG_DESCRIPTIONS', (
+ {
+ "tag_name": "challenge:none",
+ "title": _("None"),
+ "description": _("Removed from Derby"),
+ },
{
"tag_name": "challenge:2011:june",
"title": _("June 2011 Dev Derby Challenge - CSS3 Animations"),
@@ -136,13 +141,84 @@
"summary": _("CSS 3D Transforms extends CSS Transforms to allow elements rendered by CSS to be transformed in three-dimensional space."),
"description": _("CSS 3D Transforms extends CSS Transforms to allow elements rendered by CSS to be transformed in three-dimensional space."),
"learn_more": [],
- "tab_copy": _(""),
+ "tab_copy": _("Three-D interfaces have always been a fascination for as long as we have used computers. CSS 3D transforms allows you to add depth to effects and makes it easier to add more content into the same screen space by stacking it. By moving and rotating content in the X, Y and Z axis you can create beautiful transitions and interfaces without having to learn a new language."),
+ },
+ {
+ "tag_name": "challenge:2012:april",
+ "title": _("April 2012 Dev Derby Challenge - Audio"),
+ "short_title": _("Audio"),
+ "dateline": _("April 2012"),
+ "short_dateline": _("April"),
+ "tagline": _("Can you hear me now?"),
+ "summary": _("The HTML5 audio element lets you embed sound in webpages without requiring your users to rely on plug-ins."),
+ "description": _("The HTML5 audio element lets you embed sound in webpages without requiring your users to rely on plug-ins."),
+ "learn_more": [],
+ "tab_copy": _("<p>The HTML5 &lt;audio&gt; element lets you embed sound in Web pages. More importantly, it lets you do so without requiring your users to rely on plug-ins. This means sound for everyone, everywhere, in the most open way possible. In particular, you can play sounds in games with very low latency, making for a responsive, immersive game experience.</p><p>What else can you do with the audio element? Show us by submitting to the Dev Derby today.</p>"),
+ },
+ {
+ "tag_name": "challenge:2012:may",
+ "title": _("May 2012 Dev Derby Challenge - WebSockets API"),
+ "short_title": _("WebSockets API"),
+ "dateline": _("May 2012"),
+ "short_dateline": _("May"),
+ "tagline": _("send() us your best"),
+ "summary": _("With the Websocket API and protocol, you can open a two-way channel between the browser and a server, for scalable and real-time data flow. No more server polling!"),
+ "description": _("With the Websocket API and protocol, you can open a two-way channel between the browser and a server, for scalable and real-time data flow. No more server polling!"),
+ "learn_more": [],
+ "tab_copy": _("??"),
+ },
+ {
+ "tag_name": "challenge:2012:june",
+ "title": _("June 2012 Dev Derby Challenge - WebGL"),
+ "short_title": _("WebGL"),
+ "dateline": _("June 2012"),
+ "short_dateline": _("June"),
+ "tagline": _("The Web: Now in amazing 3D!"),
+ "summary": _("WebGL brings the power of OpenGL, for creating interactive 3D graphics, to the Web, with no plug-ins required."),
+ "description": _("WebGL brings the power of OpenGL, for creating interactive 3D graphics, to the Web, with no plug-ins required."),
+ "learn_more": [],
+ "tab_copy": _("??"),
+ },
+ {
+ "tag_name": "challenge:2012:july",
+ "title": _("July 2012 Dev Derby Challenge - ??"),
+ "short_title": _("??"),
+ "dateline": _("July 2012"),
+ "short_dateline": _("July"),
+ "tagline": _("??"),
+ "summary": _("??"),
+ "description": _("??"),
+ "learn_more": [],
+ "tab_copy": _("??"),
+ },
+ {
+ "tag_name": "challenge:2012:august",
+ "title": _("August 2012 Dev Derby Challenge - ??"),
+ "short_title": _("??"),
+ "dateline": _("August 2012"),
+ "short_dateline": _("August"),
+ "tagline": _("??"),
+ "summary": _("??"),
+ "description": _("??"),
+ "learn_more": [],
+ "tab_copy": _("??"),
+ },
+ {
+ "tag_name": "challenge:2012:september",
+ "title": _("September 2012 Dev Derby Challenge - ??"),
+ "short_title": _("??"),
+ "dateline": _("September 2012"),
+ "short_dateline": _("September"),
+ "tagline": _("??"),
+ "summary": _("??"),
+ "description": _("??"),
+ "learn_more": [],
+ "tab_copy": _("??"),
},
-
- {
- "tag_name": "tech:audio",
- "title": _("Audio"),
+ {
+ "tag_name": "tech:audio",
+ "title": _("Audio"),
"description": _("Mozilla's Audio Data API extends the current HTML5 API and allows web developers to read and write raw audio data."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/Introducing_the_Audio_API_Extension')),
@@ -150,8 +226,8 @@
(_('W3C Spec'), _('http://www.w3.org/TR/html5/video.html#audio')),
),
},
- {
- "tag_name": "tech:canvas",
+ {
+ "tag_name": "tech:canvas",
"title": _("Canvas"),
"description": _("The HTML5 canvas element allows you to display scriptable renderings of 2D shapes and bitmap images."),
"learn_more": (
@@ -160,9 +236,9 @@
(_('W3C Spec'), _('http://www.w3.org/TR/html5/the-canvas-element.html')),
),
},
- {
- "tag_name": "tech:css3",
- "title": _("CSS3"),
+ {
+ "tag_name": "tech:css3",
+ "title": _("CSS3"),
"description": _("Cascading Style Sheets level 3 (CSS3) provide serveral new features and properties to enhance the formatting and look of documents written in different kinds of markup languages like HTML or XML."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/CSS')),
@@ -170,27 +246,27 @@
(_('W3C Spec'), _('http://www.w3.org/TR/css3-roadmap/')),
),
},
- {
- "tag_name": "tech:device",
- "title": _("Device"),
+ {
+ "tag_name": "tech:device",
+ "title": _("Device"),
"description": _("Media queries and orientation events let authors adjust their layout on hand-held devices such as mobile phones."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/Detecting_device_orientation')),
(_('W3C Spec'), _('http://www.w3.org/TR/css3-mediaqueries/')),
),
},
- {
- "tag_name": "tech:files",
- "title": _("Files"),
+ {
+ "tag_name": "tech:files",
+ "title": _("Files"),
"description": _("The File API allows web developers to use file objects in web applications, as well as selecting and accessing their data."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/using_files_from_web_applications')),
(_('W3C Spec'), _('http://www.w3.org/TR/FileAPI/')),
),
},
- {
- "tag_name": "tech:fonts",
- "title": _("Fonts & Type"),
+ {
+ "tag_name": "tech:fonts",
+ "title": _("Fonts & Type"),
"description": _("The CSS3-Font specification contains enhanced features for fonts and typography like embedding own fonts via @font-face or controlling OpenType font features directly via CSS."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/css/@font-face')),
@@ -198,9 +274,9 @@
(_('W3C Spec'), _('http://www.w3.org/TR/css3-fonts/')),
),
},
- {
- "tag_name": "tech:forms",
- "title": _("Forms"),
+ {
+ "tag_name": "tech:forms",
+ "title": _("Forms"),
"description": _("Form elements and attributes in HTML5 provide a greater degree of semantic mark-up than HTML4 and remove a great deal of the need for tedious scripting and styling that was required in HTML4."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/HTML/HTML5/Forms_in_HTML5')),
@@ -208,9 +284,9 @@
(_('W3C Spec'), _('http://www.w3.org/TR/html5/forms.html')),
),
},
- {
- "tag_name": "tech:geolocation",
- "title": _("Geolocation"),
+ {
+ "tag_name": "tech:geolocation",
+ "title": _("Geolocation"),
"description": _("The Geolocation API allows web applications to access the user's geographical location."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/En/Using_geolocation')),
@@ -218,9 +294,9 @@
(_('W3C Spec'), _('http://dev.w3.org/geo/api/spec-source.html')),
),
},
- {
+ {
"tag_name": "tech:javascript",
- "title": _("JavaScript"),
+ "title": _("JavaScript"),
"description": _("JavaScript is a lightweight, object-oriented programming language, commonly used for scripting interactive behavior on web pages and in web applications."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/javascript')),
@@ -228,9 +304,9 @@
(_('ECMA Spec'), _('http://www.ecma-international.org/publications/standards/Ecma-262.htm')),
),
},
- {
+ {
"tag_name": "tech:html5",
- "title": _("HTML5"),
+ "title": _("HTML5"),
"description": _("HTML5 is the newest version of the HTML standard, currently under development."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/HTML/HTML5')),
@@ -238,9 +314,9 @@
(_('W3C Spec'), _('http://dev.w3.org/html5/spec/Overview.html')),
),
},
- {
- "tag_name": "tech:indexeddb",
- "title": _("IndexedDB"),
+ {
+ "tag_name": "tech:indexeddb",
+ "title": _("IndexedDB"),
"description": _("IndexedDB is an API for client-side storage of significant amounts of structured data and for high performance searches on this data using indexes. "),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/IndexedDB')),
@@ -248,9 +324,9 @@
(_('W3C Spec'), _('http://www.w3.org/TR/IndexedDB/')),
),
},
- {
- "tag_name": "tech:dragndrop",
- "title": _("Drag and Drop"),
+ {
+ "tag_name": "tech:dragndrop",
+ "title": _("Drag and Drop"),
"description": _("Drag and Drop features allow the user to move elements on the screen using the mouse pointer."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/DragDrop/Drag_and_Drop')),
@@ -258,9 +334,9 @@
(_('W3C Spec'), _('http://www.w3.org/TR/html5/dnd.html')),
),
},
- {
+ {
"tag_name": "tech:mobile",
- "title": _("Mobile"),
+ "title": _("Mobile"),
"description": _("Firefox Mobile brings the true Web experience to mobile phones and other non-PC devices."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/En/Mobile')),
@@ -268,9 +344,9 @@
(_('W3C Spec'), _('http://www.w3.org/Mobile/')),
),
},
- {
- "tag_name": "tech:offlinesupport",
- "title": _("Offline Support"),
+ {
+ "tag_name": "tech:offlinesupport",
+ "title": _("Offline Support"),
"description": _("Offline caching of web applications' resources using the application cache and local storage."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/dom/storage#localStorage')),
@@ -278,9 +354,9 @@
(_('W3C Spec'), _('http://dev.w3.org/html5/webstorage/')),
),
},
- {
- "tag_name": "tech:svg",
- "title": _("SVG"),
+ {
+ "tag_name": "tech:svg",
+ "title": _("SVG"),
"description": _("Scalable Vector Graphics (SVG) is an XML based language for describing two-dimensional vector graphics."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/SVG')),
@@ -288,9 +364,9 @@
(_('W3C Spec'), _('http://www.w3.org/TR/SVG11/')),
),
},
- {
- "tag_name": "tech:video",
- "title": _("Video"),
+ {
+ "tag_name": "tech:video",
+ "title": _("Video"),
"description": _("The HTML5 video element provides integrated support for playing video media without requiring plug-ins."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/En/Using_audio_and_video_in_Firefox')),
@@ -298,9 +374,9 @@
(_('W3C Spec'), _('http://www.w3.org/TR/html5/video.html')),
),
},
- {
- "tag_name": "tech:webgl",
- "title": _("WebGL"),
+ {
+ "tag_name": "tech:webgl",
+ "title": _("WebGL"),
"description": _("In the context of the HTML canvas element WebGL provides an API for 3D graphics in the browser."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/WebGL')),
@@ -308,9 +384,9 @@
(_('Khronos Spec'), _('http://www.khronos.org/webgl/')),
),
},
- {
- "tag_name": "tech:websockets",
- "title": _("WebSockets"),
+ {
+ "tag_name": "tech:websockets",
+ "title": _("WebSockets"),
"description": _("WebSockets is a technology that makes it possible to open an interactive communication session between the user's browser and a server."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/WebSockets')),
@@ -318,9 +394,9 @@
(_('W3C Spec'), _('http://dev.w3.org/html5/websockets/')),
),
},
- {
- "tag_name": "tech:webworkers",
- "title": _("Web Workers"),
+ {
+ "tag_name": "tech:webworkers",
+ "title": _("Web Workers"),
"description": _("Web Workers provide a simple means for web content to run scripts in background threads."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/En/Using_web_workers')),
@@ -328,9 +404,9 @@
(_('W3C Spec'), _('http://www.w3.org/TR/workers/')),
),
},
- {
- "tag_name": "tech:xhr",
- "title": _("XMLHttpRequest"),
+ {
+ "tag_name": "tech:xhr",
+ "title": _("XMLHttpRequest"),
"description": _("XMLHttpRequest (XHR) is used to send HTTP requests directly to a webserver and load the response data directly back into the script."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/En/XMLHttpRequest/Using_XMLHttpRequest')),
@@ -338,9 +414,9 @@
(_('W3C Spec'), _('http://www.w3.org/TR/XMLHttpRequest/')),
),
},
- {
- "tag_name": "tech:multitouch",
- "title": _("Multi-touch"),
+ {
+ "tag_name": "tech:multitouch",
+ "title": _("Multi-touch"),
"description": _("Track the movement of the user's finger on a touch screen, monitoring the raw touch events generated by the system."),
"learn_more": (
(_('MDN Documentation'), _('https://developer.mozilla.org/en/DOM/Touch_events')),
@@ -351,33 +427,33 @@
)))
# HACK: For easier L10N, define license in code instead of as a DB model
-DEMO_LICENSES = dict( (x['name'], x) for x in getattr(settings, 'DEMO_LICENSES', (
- {
- 'name': "mpl",
+DEMO_LICENSES = dict((x['name'], x) for x in getattr(settings, 'DEMO_LICENSES', (
+ {
+ 'name': "mpl",
'title': _("MPL/GPL/LGPL"),
'link': _('http://www.mozilla.org/MPL/'),
'icon': '',
},
- {
- 'name': "gpl",
+ {
+ 'name': "gpl",
'title': _("GPL"),
'link': _('http://www.opensource.org/licenses/gpl-license.php'),
'icon': '',
},
- {
- 'name': "bsd",
+ {
+ 'name': "bsd",
'title': _("BSD"),
'link': _('http://www.opensource.org/licenses/bsd-license.php'),
'icon': '',
},
- {
- 'name': "apache",
+ {
+ 'name': "apache",
'title': _("Apache"),
'link': _('http://www.apache.org/licenses/'),
'icon': '',
},
- {
- 'name': "publicdomain",
+ {
+ 'name': "publicdomain",
'title': _("Public Domain (where applicable by law)"),
'link': _('http://creativecommons.org/publicdomain/zero/1.0/'),
'icon': '',
@@ -408,8 +484,8 @@ def scale_image(img_upload, img_max_size):
x_offset = 0
y_offset = int(float(src_height - crop_height) / 3)
- img = img.crop((x_offset, y_offset,
- x_offset+int(crop_width), y_offset+int(crop_height)))
+ img = img.crop((x_offset, y_offset,
+ x_offset + int(crop_width), y_offset + int(crop_height)))
img = img.resize((dst_width, dst_height), Image.ANTIALIAS)
if img.mode != "RGB":
@@ -419,4 +495,3 @@ def scale_image(img_upload, img_max_size):
img_data = new_img.getvalue()
return ContentFile(img_data)
-
3  apps/demos/feeds.py
View
@@ -20,7 +20,8 @@
from devmo.urlresolvers import reverse
from devmo.models import UserProfile
-from .models import Submission, TAG_DESCRIPTIONS
+from . import TAG_DESCRIPTIONS
+from .models import Submission
MAX_FEED_ITEMS = getattr(settings, 'MAX_FEED_ITEMS', 15)
1  apps/demos/forms.py
View
@@ -109,6 +109,7 @@ class Meta:
choices = (
(TAG_DESCRIPTIONS[x]['tag_name'], TAG_DESCRIPTIONS[x]['title'])
for x in parse_tags(
+ 'challenge:none %s' %
constance.config.DEMOS_DEVDERBY_CHALLENGE_CHOICE_TAGS,
sorted=False)
if x in TAG_DESCRIPTIONS
200 apps/demos/helpers.py
View
@@ -1,50 +1,34 @@
import datetime
-import urllib
-import logging
import functools
import hashlib
import random
from django.core.cache import cache
-#from django.utils.translation import ungettext, ugettext
-from tower import ugettext_lazy as _lazy, ungettext
-
from django.conf import settings
+from django.utils.tzinfo import LocalTimezone
+from babel import localedata
import jingo
+from jingo import register
import jinja2
-from jinja2 import evalcontextfilter, Markup, escape
-from jingo import register, env
from tower import ugettext as _
from tower import ugettext, ungettext
-from django.core import urlresolvers
-
-from babel import localedata
-from babel.dates import format_date, format_time, format_datetime
-from babel.numbers import format_decimal
-
-from pytz import timezone
-from django.utils.tzinfo import LocalTimezone
-
-from django.core.urlresolvers import reverse as django_reverse
-from devmo.urlresolvers import reverse
-
from taggit.models import TaggedItem
+from threadedcomments.models import ThreadedComment
+from threadedcomments.forms import ThreadedCommentForm
+from threadedcomments.templatetags import threadedcommentstags
+import threadedcomments.views
from .models import Submission, TAG_DESCRIPTIONS, DEMO_LICENSES
from . import DEMOS_CACHE_NS_KEY
-from threadedcomments.models import ThreadedComment, FreeThreadedComment
-from threadedcomments.forms import ThreadedCommentForm, FreeThreadedCommentForm
-from threadedcomments.templatetags import threadedcommentstags
-import threadedcomments.views
-
# Monkeypatch threadedcomments URL reverse() to use devmo's
from devmo.urlresolvers import reverse
threadedcommentstags.reverse = reverse
-TEMPLATE_INCLUDE_CACHE_EXPIRES = getattr(settings, 'TEMPLATE_INCLUDE_CACHE_EXPIRES', 300)
+TEMPLATE_INCLUDE_CACHE_EXPIRES = getattr(settings,
+ 'TEMPLATE_INCLUDE_CACHE_EXPIRES', 300)
def new_context(context, **kw):
@@ -52,10 +36,12 @@ def new_context(context, **kw):
c.update(kw)
return c
+
# TODO:liberate ?
-def register_cached_inclusion_tag(template, key_fn=None, expires=TEMPLATE_INCLUDE_CACHE_EXPIRES):
- """Decorator for inclusion tags with output caching.
-
+def register_cached_inclusion_tag(template, key_fn=None,
+ expires=TEMPLATE_INCLUDE_CACHE_EXPIRES):
+ """Decorator for inclusion tags with output caching.
+
Accepts a string or function to generate a cache key based on the incoming
parameters, along with an expiration time configurable as
INCLUDE_CACHE_EXPIRES or an explicit parameter"""
@@ -71,7 +57,7 @@ def wrapper(*args, **kw):
cache_key = key_fn
else:
cache_key = key_fn(*args, **kw)
-
+
out = cache.get(cache_key)
if out is None:
context = f(*args, **kw)
@@ -82,29 +68,41 @@ def wrapper(*args, **kw):
return register.function(wrapper)
return decorator
-
+
+
def submission_key(prefix):
"""Produce a cache key function with a prefix, which generates the rest of
the key based on a submission ID and last-modified timestamp."""
def k(*args, **kw):
submission = args[0]
- return 'submission:%s:%s:%s' % ( prefix, submission.id, submission.modified )
+ return 'submission:%s:%s:%s' % (prefix,
+ submission.id,
+ submission.modified)
return k
+
# TOOO: All of these inclusion tags could probably be generated & registered
# from a dict of function names and inclusion tag args, since the method bodies
# are all identical. Might be astronaut architecture, though.
@register.inclusion_tag('demos/elements/demos_head.html')
-def demos_head(request): return locals()
+def demos_head(request):
+ return locals()
+
@register.inclusion_tag('demos/elements/submission_creator.html')
-def submission_creator(submission): return locals()
+def submission_creator(submission):
+ return locals()
+
@register.inclusion_tag('demos/elements/profile_link.html')
-def profile_link(user, show_gravatar=False, gravatar_size=48, gravatar_default='mm'): return locals()
+def profile_link(user, show_gravatar=False, gravatar_size=48,
+ gravatar_default='mm'):
+ return locals()
+
@register.inclusion_tag('demos/elements/submission_thumb.html')
-def submission_thumb(submission,extra_class=None,thumb_width="200",thumb_height="150"):
+def submission_thumb(submission, extra_class=None, thumb_width="200",
+ thumb_height="150"):
vars = locals()
flags = submission.get_flags()
@@ -113,17 +111,17 @@ def submission_thumb(submission,extra_class=None,thumb_width="200",thumb_height=
# TODO: Move to a constant or DB table? Too much view stuff here?
flags_meta = {
# flag name thumb class flag description
- 'firstplace': ( 'first-place', _('First Place') ),
- 'secondplace': ( 'second-place', _('Second Place') ),
- 'thirdplace': ( 'third-place', _('Third Place') ),
- 'finalist': ( 'finalist', _('Finalist') ),
- 'featured': ( 'featured', _('Featured') ),
+ 'firstplace': ('first-place', _('First Place')),
+ 'secondplace': ('second-place', _('Second Place')),
+ 'thirdplace': ('third-place', _('Third Place')),
+ 'finalist': ('finalist', _('Finalist')),
+ 'featured': ('featured', _('Featured')),
}
# If there are any flags, pass them onto the template. Special treatment
# for the first flag, which takes priority over all others for display in
# the thumb.
- main_flag = ( len(flags) > 0 ) and flags[0] or None
+ main_flag = (len(flags) > 0) and flags[0] or None
vars['all_flags'] = flags
vars['main_flag'] = main_flag
if main_flag in flags_meta:
@@ -132,28 +130,43 @@ def submission_thumb(submission,extra_class=None,thumb_width="200",thumb_height=
return vars
+
def submission_listing_cache_key(*args, **kw):
ns_key = cache.get(DEMOS_CACHE_NS_KEY)
if ns_key is None:
- ns_key = random.randint(1,10000)
+ ns_key = random.randint(1, 10000)
cache.set(DEMOS_CACHE_NS_KEY, ns_key)
- return 'demos_%s:%s' % (ns_key, hashlib.md5(args[0].get_full_path()+args[0].user.username).hexdigest())
-
-@register_cached_inclusion_tag('demos/elements/submission_listing.html', submission_listing_cache_key)
-def submission_listing(request, submission_list, is_paginated, paginator, page_obj, feed_title, feed_url,
- cols_per_row=3, pagination_base_url='', show_sorts=True, show_submit=False):
+ full_path = args[0].get_full_path()
+ username = args[0].user.username
+ return 'demos_%s:%s' % (ns_key,
+ hashlib.md5(full_path + username).hexdigest())
+
+
+@register_cached_inclusion_tag('demos/elements/submission_listing.html',
+ submission_listing_cache_key)
+def submission_listing(request, submission_list, is_paginated, paginator,
+ page_obj, feed_title, feed_url,
+ cols_per_row=3, pagination_base_url='', show_sorts=True,
+ show_submit=False):
return locals()
+
@register.inclusion_tag('demos/elements/tech_tags_list.html')
-def tech_tags_list(): return locals()
+def tech_tags_list():
+ return locals()
+
-# Not cached, because it's small and changes based on current search query string
+# Not cached, because it's small and changes based on
+# current search query string
@register.inclusion_tag('demos/elements/search_form.html')
@jinja2.contextfunction
def search_form(context):
return new_context(**locals())
+
bitly_api = None
+
+
def _get_bitly_api():
"""Get an instance of the bit.ly API class"""
global bitly_api
@@ -164,6 +177,7 @@ def _get_bitly_api():
bitly_api = bitly.Api(login, apikey)
return bitly_api
+
@register.filter
def bitly_shorten(url):
"""Attempt to shorten a given URL through bit.ly / mzl.la"""
@@ -175,13 +189,17 @@ def bitly_shorten(url):
# configured, fall back to using the original URL.
return url
+
@register.function
def devderby_tag_to_date_url(tag):
"""Turn a devderby tag like challenge:2011:june into a date-based URL"""
# HACK: Not super happy with this, but it works for now
- if not tag: return ''
+ if not tag:
+ return ''
parts = tag.split(':')
- return reverse('demos.views.devderby_by_date', args=( parts[-2], parts[-1] ))
+ return reverse('demos.views.devderby_by_date',
+ args=(parts[-2], parts[-1]))
+
@register.function
def license_link(license_name):
@@ -190,6 +208,7 @@ def license_link(license_name):
else:
return license_name
+
@register.function
def license_title(license_name):
if license_name in DEMO_LICENSES:
@@ -197,53 +216,69 @@ def license_title(license_name):
else:
return license_name
+
@register.function
def tag_title(tag):
- if not tag: return ''
+ if not tag:
+ return ''
name = (isinstance(tag, basestring)) and tag or tag.name
if name in TAG_DESCRIPTIONS:
return TAG_DESCRIPTIONS[name]['title']
else:
return name
+
@register.function
def tag_description(tag):
- if not tag: return ''
+ if not tag:
+ return ''
name = (isinstance(tag, basestring)) and tag or tag.name
- if name in TAG_DESCRIPTIONS:
+ if name in TAG_DESCRIPTIONS and 'description' in TAG_DESCRIPTIONS[name]:
return TAG_DESCRIPTIONS[name]['description']
else:
return name
+
@register.function
def tag_learn_more(tag):
- if not tag: return ''
- if (tag.name in TAG_DESCRIPTIONS and
+ if not tag:
+ return ''
+ if (tag.name in TAG_DESCRIPTIONS and
'learn_more' in TAG_DESCRIPTIONS[tag.name]):
return TAG_DESCRIPTIONS[tag.name]['learn_more']
else:
return []
+
@register.function
def tag_meta(tag, other_name):
"""Get metadata for a tag or tag name."""
# TODO: Replace usage of tag_{title,description,learn_more}?
- if not tag: return ''
+ if not tag:
+ return ''
name = (isinstance(tag, basestring)) and tag or tag.name
if name in TAG_DESCRIPTIONS and other_name in TAG_DESCRIPTIONS[name]:
return TAG_DESCRIPTIONS[name][other_name]
else:
return ''
+
@register.function
def tags_for_object(obj):
tags = obj.taggit_tags.all()
return tags
+
+@register.function
+def tech_tags_for_object(obj):
+ return obj.taggit_tags.all_ns('tech')
+
+
@register.function
def tags_used_for_submissions():
return TaggedItem.tags_for(Submission)
+
@register.filter
def date_diff(timestamp, to=None):
if not timestamp:
@@ -251,46 +286,54 @@ def date_diff(timestamp, to=None):
compare_with = to or datetime.date.today()
delta = timestamp - compare_with
-
- if delta.days == 0: return u"today"
- elif delta.days == -1: return u"yesterday"
- elif delta.days == 1: return u"tomorrow"
-
+
+ if delta.days == 0:
+ return u"today"
+ elif delta.days == -1:
+ return u"yesterday"
+ elif delta.days == 1:
+ return u"tomorrow"
+
chunks = (
(365.0, lambda n: ungettext('year', 'years', n)),
(30.0, lambda n: ungettext('month', 'months', n)),
- (7.0, lambda n : ungettext('week', 'weeks', n)),
- (1.0, lambda n : ungettext('day', 'days', n)),
+ (7.0, lambda n: ungettext('week', 'weeks', n)),
+ (1.0, lambda n: ungettext('day', 'days', n)),
)
-
+
for i, (chunk, name) in enumerate(chunks):
if abs(delta.days) >= chunk:
count = abs(round(delta.days / chunk, 0))
break
- date_str = ugettext('%(number)d %(type)s') % {'number': count, 'type': name(count)}
-
- if delta.days > 0: return "in " + date_str
- else: return date_str + " ago"
+ date_str = (ugettext('%(number)d %(type)s') %
+ {'number': count, 'type': name(count)})
+
+ if delta.days > 0:
+ return "in " + date_str
+ else:
+ return date_str + " ago"
+
# TODO: Maybe just register the template tag functions in the jingo environment
# directly, rather than building adapter functions?
-
@register.function
def get_threaded_comment_flat(content_object, tree_root=0):
return ThreadedComment.public.get_tree(content_object, root=tree_root)
+
@register.function
def get_threaded_comment_tree(content_object, tree_root=0):
"""Convert the flat list with depth indices into a true tree structure for
recursive template display"""
- root = dict( children=[] )
- parent_stack = [ root, ]
-
+ root = dict(children=[])
+ parent_stack = [root, ]
+
flat = ThreadedComment.public.get_tree(content_object, root=tree_root)
for comment in flat:
c = dict(comment=comment, children=[])
- if comment.depth > len(parent_stack) - 1 and len(parent_stack[-1]['children']):
+ if (comment.depth > len(parent_stack) - 1 and
+ len(parent_stack[-1]['children'])):
parent_stack.appen