Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

tweaks to make themes update check a reality for AMO (bug 641640)

  • Loading branch information...
commit 73bea2652a691062d23a8d3eb69e4053e268439a 1 parent 6d3d643
@cvan cvan authored
View
2  apps/addons/fixtures/addons/persona.json
@@ -139,7 +139,7 @@
"pk": 559,
"model": "addons.persona",
"fields": {
- "display_username": "My Persona",
+ "display_username": "persona_author",
"popularity": 10,
"license": null,
"footer": "BCBG_Persona_footer2.png",
View
2  apps/addons/forms.py
@@ -29,7 +29,7 @@
from translations.forms import TranslationFormMixin
from translations.models import Translation
from translations.widgets import TranslationTextInput
-from versions.models import License, Version
+from versions.models import Version
log = commonware.log.getLogger('z.addons')
View
7 apps/addons/models.py
@@ -1646,6 +1646,9 @@ def _image_url(self, filename, ssl=True):
'file': filename,
}
+ def _image_path(self, filename):
+ return os.path.join(settings.ADDONS_PATH, str(self.addon.id), filename)
+
@amo.cached_property
def thumb_url(self):
"""URL to Persona's thumbnail preview."""
@@ -1663,6 +1666,10 @@ def icon_url(self):
return self._image_url('preview_small.jpg')
@amo.cached_property
+ def icon_path(self):
+ return self._image_path('icon.png')
+
+ @amo.cached_property
def preview_url(self):
"""URL to Persona's big, 680px, preview."""
if self.is_new():
View
120 apps/addons/tests/test_theme_update.py
@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+import json
+from StringIO import StringIO
+
+from django.db import connection
+
+import mock
+from nose.tools import eq_
+
+import amo.tests
+from addons.models import Addon
+from services import theme_update
+
+
+class TestWSGIApplication(amo.tests.TestCase):
+
+ def setUp(self):
+ self.environ = {'wsgi.input': StringIO()}
+ self.start_response = mock.Mock()
+
+ @mock.patch('services.theme_update.ThemeUpdate')
+ def test_wsgi_application_200(self, ThemeUpdate_mock):
+ urls = {
+ '/themes/update-check/5': ['en-US', 5, None],
+ '/en-US/themes/update-check/5': ['en-US', 5, None],
+ '/fr/themes/update-check/5': ['fr', 5, None]
+ }
+
+ # From AMO we consume the ID as the `addon_id`.
+ for path_info, call_args in urls.iteritems():
+ environ = dict(self.environ, **{'PATH_INFO': path_info})
+ theme_update.application(environ, self.start_response)
+ ThemeUpdate_mock.assert_called_with(*call_args)
+
+ # From getpersonas.com we append `?src=gp` so we know to consume
+ # the ID as the `persona_id`.
+ self.environ['QUERY_STRING'] = 'src=gp'
+ for path_info, call_args in urls.iteritems():
+ environ = dict(self.environ, **{'PATH_INFO': path_info})
+ theme_update.application(environ, self.start_response)
+ call_args[2] = 'src=gp'
+ ThemeUpdate_mock.assert_called_with(*call_args)
+ self.start_response.assert_called_with('200 OK', mock.ANY)
+
+ @mock.patch('services.theme_update.ThemeUpdate')
+ def test_wsgi_application_404(self, ThemeUpdate_mock):
+ urls = [
+ '/xxx',
+ '/themes/update-check/xxx',
+ '/en-US/themes/update-check/xxx',
+ '/fr/themes/update-check/xxx'
+ ]
+
+ for path_info in urls:
+ environ = dict(self.environ, **{'PATH_INFO': path_info})
+ theme_update.application(environ, self.start_response)
+ assert not ThemeUpdate_mock.called
+ self.start_response.assert_called_with('404 Not Found', [])
+
+
+class TestThemeUpdate(amo.tests.TestCase):
+ fixtures = ['addons/persona']
+ good = {
+ 'username': 'persona_author',
+ 'description': 'yolo',
+ 'detailURL': '/en-US/addon/a15663/',
+ 'accentcolor': '#8d8d97',
+ 'iconURL': '/uploads/themes/15663/preview_small.jpg',
+ 'previewURL': '/uploads/themes/15663/preview.jpg',
+ 'textcolor': '#ffffff',
+ 'id': 15663,
+ 'headerURL': '/uploads/themes/15663/BCBG_Persona_header2.png',
+ 'dataurl': '',
+ 'name': 'My Persona',
+ 'author': 'persona_author',
+ 'updateURL': '/en-US/update-check/themes/15663',
+ 'version': '1.0',
+ 'footerURL': '/uploads/themes/15663/BCBG_Persona_footer2.png'
+ }
+
+ def check_good(self, data):
+ for k, v in self.good.iteritems():
+ got = data[k]
+ if k.endswith('URL'):
+ if k in ('iconURL', 'footerURL', 'headerURL', 'previewURL'):
+ assert got.find('?') > -1, (
+ '"%s" must contain "?" for modified timestamp' % k)
+ else:
+ eq_(got.find('?'), -1, '"%s" should not contain "?"' % k)
+
+ # Strip `?<modified>` timestamps.
+ got = got.rsplit('?')[0]
+
+ assert got.endswith(v), (
+ 'Expected "%s" to end with "%s". Got "%s".' % (k, v, got))
+ else:
+ eq_(got, v,
+ 'Expected "%s" for "%s". Got "%s".' % (v, k, got))
+
+ def get_update(self, *args):
+ update = theme_update.ThemeUpdate(*args)
+ update.cursor = connection.cursor()
+ return update
+
+ def test_get_json_bad_ids(self):
+ eq_(self.get_update('en-US', 999).get_json(), None)
+ eq_(self.get_update('en-US', 813).get_json(), None)
+
+ def test_get_json_good_ids(self):
+ self.addon = Addon.objects.get()
+ self.addon.summary = 'yolo'
+ self.addon.save()
+
+ # Testing `addon_id` from AMO.
+ self.check_good(
+ json.loads(self.get_update('en-US', 15663).get_json()))
+
+ # Testing `persona_id` from GP.
+ self.check_good(
+ json.loads(self.get_update('en-US', 813, 'src=gp').get_json()))
View
2  apps/addons/tests/test_views.py
@@ -1082,7 +1082,7 @@ def _test_legacy_by(self):
waffle.models.Switch.objects.filter(
name='personas-migration-completed').update(active=False)
r = self.client.get(self.url)
- eq_(pq(r.content)('h4.author').text(), 'by My Persona')
+ eq_(pq(r.content)('h4.author').text(), 'by persona_author')
def test_legacy_by(self):
self._test_legacy_by()
View
2  apps/discovery/tests/test_views.py
@@ -468,7 +468,7 @@ def test_page(self):
def test_by(self):
"""Test that the `by ... <authors>` section works."""
r = self.client.get(self.url)
- assert pq(r.content)('h2.author').text().startswith('by My Persona')
+ assert pq(r.content)('h2.author').text().startswith('by persona_author')
class TestDownloadSources(amo.tests.TestCase):
View
158 services/persona_update.py
@@ -1,158 +0,0 @@
-import base64
-import json
-import re
-
-from constants import base
-
-from django.core.management import setup_environ
-
-from utils import format_date, log_configure, log_exception, mypool
-
-from services.utils import settings
-setup_environ(settings)
-
-# Configure the log.
-log_configure()
-
-from django_statsd.clients import statsd
-
-
-# TODO: Update these to their correct locations.
-PERSONA_HOST = PERSONA_CDN_HOST = 'http://getpersonas.com'
-
-
-class PersonaUpdate(object):
-
- def __init__(self, locale, id):
- self.conn, self.cursor = None, None
- self.data = dict(locale=locale, id=id)
- self.data.update(atype=base.ADDON_PERSONA)
- self.data['row'] = {}
-
- if not self.cursor:
- self.conn = mypool.connect()
- self.cursor = self.conn.cursor()
-
- def get_update(self):
-
- sql = """
- SELECT
- personas.persona_id, addons.id,
- t_name.localized_string AS name,
- t_desc.localized_string AS description,
- personas.author, personas.display_username, personas.header,
- personas.footer, personas.accentcolor, personas.textcolor,
- UNIX_TIMESTAMP(personas.approve) AS modified
- FROM addons
- LEFT JOIN personas ON personas.addon_id=addons.id
- LEFT JOIN translations AS t_name
- ON t_name.id=personas.name AND t_name.locale=%(locale)s
- LEFT JOIN translations AS t_desc
- ON t_desc.id=personas.description AND t_desc.locale=%(locale)s
- WHERE personas.persona_id=%(id)s AND addons.addontype_id=%(atype)s
- """
-
- self.cursor.execute(sql, self.data)
- result = self.cursor.fetchone()
-
- if result:
- row = dict(zip([
- 'id', 'addon_id', 'name', 'description', 'author', 'username',
- 'header', 'footer', 'accentcolor', 'textcolor', 'modified'],
- list(result)))
- self.data['row'] = row
- return True
-
- return False
-
- def get_json(self):
- if self.get_update():
- return self.compose_good_json()
- else: # Persona not found.
- return None
-
- def url_prefix(self, id):
- """Uses last 2 digits to add extra directories.
-
- For example:
- url_prefix(1234) => '4/3/1234'
- """
- return '%s/%s/%s' % (id % 10, (id // 10) % 10, id)
-
- def compose_good_json(self):
-
- id = self.data['id']
- accent = self.data.get('accentcolor')
- text = self.data.get('textcolor')
- base_url = '%s/static/%s/' % (PERSONA_CDN_HOST,
- self.url_prefix(id))
- row = self.data.get('row')
-
- data = {
- 'id': id,
- 'name': row.get('name'),
- 'description': row.get('description'),
- 'author': row.get('author'),
- 'username': row.get('username'),
- 'headerURL': '%s/%s?%s' % (base_url, row['header'],
- row['modified']),
- 'footerURL': '%s/%s?%s' % (base_url, row['footer'],
- row['modified']),
- 'detailURL': '%s/persona/%s' % (PERSONA_HOST, id),
- 'previewURL': '%s/preview.jpg?%s' % (base_url,
- row['modified']),
- 'iconURL': '%s/preview_small.jpg?%s' % (base_url,
- row['modified']),
- # TODO: dataurl requires a call to the filesystem to base64.
- # 'dataurl': self.base64_icon(id),
- 'dataurl': '',
- 'accentcolor': '#%s' % accent if accent else None,
- 'textcolor': '#%s' % text if text else None,
- 'updateUrl': '%s/%s/update_check/%s' % (PERSONA_HOST,
- self.data['locale'], id),
- 'version': row.get('modified'),
- }
-
- return json.dumps(data)
-
- def base64_icon(self, persona_id):
- path = '%s/%s/preview_icon.jpg' % (settings.PERSONAS_PATH,
- self.url_prefix(id))
- with open(path, 'r') as f:
- return base64.b64encode(f.read())
-
- def get_headers(self, length):
- return [('Content-Type', 'application/json'),
- ('Cache-Control', 'public, max-age=3600'),
- ('Last-Modified', format_date(0)),
- ('Expires', format_date(3600)),
- ('Content-Length', str(length))]
-
-
-url_re = re.compile('/(?P<locale>[^/]+)/personas/update_check/(?P<id>\d+)$')
-
-
-def application(environ, start_response):
- status = '200 OK'
- with statsd.timer('services.persona_update'):
-
- data = environ['wsgi.input'].read()
- try:
- locale, id = url_re.match(environ['PATH_INFO']).groups()
- id = int(id)
- except AttributeError: # URL path incorrect.
- start_response('404 Not Found', [])
- return ['']
-
- try:
- update = PersonaUpdate(locale, id)
- output = update.get_json()
- if not output:
- start_response('404 Not Found', [])
- return ['']
- start_response(status, update.get_headers(len(output)))
- except:
- log_exception(data)
- raise
-
- return [output]
View
199 services/theme_update.py
@@ -0,0 +1,199 @@
+import base64
+import json
+import os
+import re
+from time import time
+
+from django.core.management import setup_environ
+
+from django_statsd.clients import statsd
+from wsgiref.handlers import format_date_time
+
+from constants import base
+from services.utils import settings
+from utils import log_configure, log_exception, mypool
+
+setup_environ(settings)
+
+# Configure the log.
+log_configure()
+
+
+class ThemeUpdate(object):
+
+ def __init__(self, locale, id_, qs=None):
+ self.conn, self.cursor = None, None
+ self.data = {
+ 'locale': locale,
+ 'id': id_,
+ # If we came from getpersonas.com, then look up by `persona_id`.
+ # Otherwise, look up `addon_id`.
+ 'primary_key': 'persona_id' if qs == 'src=gp' else 'addon_id',
+ 'atype': base.ADDON_PERSONA,
+ 'row': {}
+ }
+ if not self.cursor:
+ self.conn = mypool.connect()
+ self.cursor = self.conn.cursor()
+
+ def base64_icon(self, addon_id):
+ path = os.path.join(settings.ADDONS_PATH, str(addon_id), 'icon.png')
+ try:
+ with open(path, 'r') as f:
+ return base64.b64encode(f.read())
+ except IOError, e:
+ if len(e.args) == 1:
+ log_exception('I/O error: {0}'.format(e[0]))
+ else:
+ log_exception('I/O error({0}): {1}'.format(e[0], e[1]))
+ return ''
+
+ def get_headers(self, length):
+ return [('Cache-Control', 'public, max-age=3600'),
+ ('Content-Length', str(length)),
+ ('Content-Type', 'application/json'),
+ ('Expires', format_date_time(time() + 3600)),
+ ('Last-Modified', format_date_time(time()))]
+
+ def get_update(self):
+ """
+ TODO:
+
+ * When themes have versions let's not use
+ `personas.approve` as a `modified` timestamp. Either set this
+ during theme approval, or let's keep a hash of the header and
+ footer.
+
+ * Do a join on `addons_users` to get the actual correct user.
+ We're cheating and setting `personas.display_username` during
+ submission/management heh. But `personas.author` and
+ `personas.display_username` are not what we want.
+
+ """
+
+ sql = """
+ SELECT p.persona_id, a.id, a.slug,
+ t_name.localized_string AS name,
+ t_desc.localized_string AS description,
+ p.display_username, p.header,
+ p.footer, p.accentcolor, p.textcolor,
+ UNIX_TIMESTAMP(a.modified) AS modified
+ FROM addons AS a
+ LEFT JOIN personas AS p ON p.addon_id=a.id
+ LEFT JOIN translations AS t_name
+ ON t_name.id=a.name AND t_name.locale=%(locale)s
+ LEFT JOIN translations AS t_desc
+ ON t_desc.id=a.summary AND t_desc.locale=%(locale)s
+ WHERE p.{primary_key}=%(id)s AND
+ a.addontype_id=%(atype)s
+ """.format(primary_key=self.data['primary_key'])
+
+ self.cursor.execute(sql, self.data)
+ row = self.cursor.fetchone()
+
+ row_to_dict = lambda row: dict(zip((
+ 'persona_id', 'addon_id', 'slug', 'name', 'description',
+ 'username', 'header', 'footer', 'accentcolor', 'textcolor',
+ 'modified'),
+ list(row)))
+
+ if row:
+ self.data['row'] = row_to_dict(row)
+
+ # Fall back to `en-US` if the name was null for our locale.
+ # TODO: Write smarter SQL and don't rerun the whole query.
+ if not self.data['row']['name']:
+ self.data['locale'] = 'en-US'
+ self.cursor.execute(sql, self.data)
+ row = self.cursor.fetchone()
+ if row:
+ self.data['row'] = row_to_dict(row)
+
+ return True
+
+ return False
+
+ # TODO: Cache on row['modified']
+ def get_json(self):
+ if not self.get_update():
+ # Persona not found.
+ return
+
+ row = self.data['row']
+ accent = row.get('accentcolor')
+ text = row.get('textcolor')
+ return json.dumps({
+ 'id': row['addon_id'],
+ 'name': row.get('name'),
+ 'description': row.get('description'),
+ # TODO: Change this to be `addons_users.user.username`.
+ 'author': row.get('username'),
+ # TODO: Change this to be `addons_users.user.display_name`.
+ 'username': row.get('username'),
+ 'headerURL': self.image_url(row['header']),
+ 'footerURL': self.image_url(row['footer']),
+ 'detailURL': self.url('/addon/%s/' % row['slug']),
+ 'previewURL': self.image_url('preview.png'),
+ 'iconURL': self.image_url('icon.png'),
+ 'dataurl': self.base64_icon(row['addon_id']),
+ 'accentcolor': '#%s' % accent if accent else None,
+ 'textcolor': '#%s' % text if text else None,
+ 'updateURL': self.url('/update-check/themes/%s' % row['addon_id']),
+ # TODO: Change this when we add versions (bug 851881).
+ 'version': '1.0'
+ })
+
+ def image_url(self, filename):
+ row = self.data['row']
+
+ # Special cased for non-AMO-uploaded themes imported from getpersonas.
+ if row['persona_id'] != 0:
+ if filename == 'preview.png':
+ filename = 'preview.jpg'
+ elif filename == 'icon.png':
+ filename = 'preview_small.jpg'
+
+ image_url = settings.NEW_PERSONAS_IMAGE_URL % {'id': row['addon_id'],
+ 'file': filename}
+ return '%s?%s' % (image_url, row['modified'])
+
+ def url(self, url):
+ return '%s/%s%s' % (
+ settings.SITE_URL, self.data.get('locale', 'en-US'), url)
+
+
+url_re = re.compile('(?P<locale>.+)?/themes/update-check/(?P<id>\d+)$')
+
+
+def application(environ, start_response):
+ """
+ Developing locally?
+
+ gunicorn -b 0.0.0.0:7000 -w 12 -k sync -t 90 --max-requests 5000 \
+ -n gunicorn-theme_update services.wsgi.theme_update:application
+
+ """
+
+ status = '200 OK'
+ with statsd.timer('services.theme_update'):
+ data = environ['wsgi.input'].read()
+ try:
+ locale, id_ = url_re.match(environ['PATH_INFO']).groups()
+ locale = (locale or 'en-US').lstrip('/')
+ id_ = int(id_)
+ except AttributeError: # URL path incorrect.
+ start_response('404 Not Found', [])
+ return ['']
+
+ try:
+ update = ThemeUpdate(locale, id_, environ.get('QUERY_STRING'))
+ output = update.get_json()
+ if not output:
+ start_response('404 Not Found', [])
+ return ['']
+ start_response(status, update.get_headers(len(output)))
+ except:
+ log_exception(data)
+ raise
+
+ return [output]
View
5 services/utils.py
@@ -66,8 +66,9 @@
(?P<alpha>[a|b]?) # alpha/beta
(?P<alpha_ver>\d*) # alpha/beta version
(?P<pre>pre)? # pre release
- (?P<pre_ver>\d)? # pre release version""",
- re.VERBOSE)
+ (?P<pre_ver>\d)? # pre release version
+ """,
+ re.VERBOSE)
def get_mirror(status, id, row):
View
17 services/wsgi/persona_update.wsgi
@@ -1,17 +0,0 @@
-import os
-import site
-
-wsgidir = os.path.dirname(__file__)
-for path in ['../', '../..',
- '../../vendor/src',
- '../../vendor/src/django',
- '../../vendor/src/nuggets',
- '../../vendor/src/commonware',
- '../../vendor/src/statsd',
- '../../vendor/src/tower',
- '../../lib',
- '../../vendor/lib/python',
- '../../apps']:
- site.addsitedir(os.path.abspath(os.path.join(wsgidir, path)))
-
-from persona_update import application
View
12 services/wsgi/theme_update.py
@@ -0,0 +1,12 @@
+import os
+import site
+
+wsgidir = os.path.dirname(__file__)
+for path in ['../',
+ '../..',
+ '../../apps',
+ '../../lib',
+ '../../vendor/lib/python']:
+ site.addsitedir(os.path.abspath(os.path.join(wsgidir, path)))
+
+from ..theme_update import application
Please sign in to comment.
Something went wrong with that request. Please try again.