From 6eeaa28d7ae59e36ec1c54ddf4c06f462e4cb4c3 Mon Sep 17 00:00:00 2001 From: Piotr Zalewa Date: Mon, 8 Aug 2011 18:43:43 +0100 Subject: [PATCH] AMOOAuth added, not working --- apps/amo/tasks.py | 27 +++ apps/amo/urls.py | 4 + apps/amo/views.py | 9 + apps/jetpack/models.py | 40 +++++ apps/jetpack/templates/addon_edit.html | 3 + settings.py | 7 + urls.py | 3 + utils/amo.py | 228 +++++++++++++++++++++++++ utils/helpers.py | 54 ++++++ 9 files changed, 375 insertions(+) create mode 100644 apps/amo/tasks.py create mode 100644 apps/amo/urls.py create mode 100644 apps/amo/views.py create mode 100644 utils/amo.py diff --git a/apps/amo/tasks.py b/apps/amo/tasks.py new file mode 100644 index 00000000..4ef79f4e --- /dev/null +++ b/apps/amo/tasks.py @@ -0,0 +1,27 @@ +import commonware.log +import time +from statsd import statsd + +from celery.decorators import task + +from jetpack.models import PackageRevision +from utils.helpers import get_random_string +from xpi import xpi_utils + + +log = commonware.log.getLogger('f.celery') + + +@task +def upload_to_amo(rev_pk): + """Build XPI and upload to AMO + Read result and save amo_id + """ + tstart = time.time() + hashtag = get_random_string(10) + revision = PackageRevision.objects.get(pk=rev_pk) + response = revision.build_xpi( + hashtag=hashtag, + tstart=tstart) + # use created XPI + revision.upload_to_amo(hashtag) diff --git a/apps/amo/urls.py b/apps/amo/urls.py new file mode 100644 index 00000000..c4a6005e --- /dev/null +++ b/apps/amo/urls.py @@ -0,0 +1,4 @@ +from django.conf.urls.defaults import url, patterns + +urlpatterns = patterns('amo.views', + url(r'^upload_to_amo/(?P\d+)/', 'upload_to_amo', name='amo_upload')) diff --git a/apps/amo/views.py b/apps/amo/views.py new file mode 100644 index 00000000..8ff164b2 --- /dev/null +++ b/apps/amo/views.py @@ -0,0 +1,9 @@ +from django.shortcuts import render_to_response + +from amo import tasks + +def upload_to_amo(request, pk): + """Upload a XPI to AMO + """ + tasks.upload_to_amo.delay(pk) + return HttpResponse('{"delayed": true}') diff --git a/apps/jetpack/models.py b/apps/jetpack/models.py index 56986de8..13641330 100644 --- a/apps/jetpack/models.py +++ b/apps/jetpack/models.py @@ -40,6 +40,7 @@ from utils.exceptions import SimpleException from utils.helpers import pathify, alphanum, alphanum_plus from utils.os_utils import make_path +from utils.amo import AMOOAuth from xpi import xpi_utils log = commonware.log.getLogger('f.jetpack') @@ -315,6 +316,14 @@ def get_download_xpi_url(self): 'jp_addon_revision_xpi', args=[self.package.id_number, self.revision_number]) + def get_upload_to_amo_url(self): + " returns URL to upload to AMO " + if self.package.type != 'a': + raise Exception('Only Add-ons might be uploaded to AMO') + return reverse( + 'amo_upload', + args=[self.pk]) + def get_copy_url(self): " returns URL to copy the package " return reverse( @@ -1340,6 +1349,37 @@ class Meta: objects = PackageManager() + ################## + # AMO Integration + + def upload_to_amo(self, hashtag): + """Uploads Package to AMO, updates or creates as a new Addon + """ + # open XPI File + xpi_file = open(os.path.join('%s.xpi' % hashtag)) + # upload + data = {'xpi': xpi_file, + 'builtin': 0, + 'name': 'FREEDOM', + 'text': 'This is FREE!', + 'platform': 'linux', + 'authenticate_as': 2} + amo = AMOOAuth(domain=AMOOAUTH_DOMAIN, port=AMOOAUTH_PORT, + protocol=AMOOAUTH_PROTOCOL) + amo.set_consumer(consumer_key=AMOOAUTH_CONSUMERKEY, + consumer_secret=AMOOAUTH_CONSUMERSECRET) + if self.amo_id: + # update addon on AMO + # update jetpack ID if needed + pass + else: + # create addon on AMO + response = amo.create_addon(data) + # set amo_id + # set jetpack ID + + print response + ################## # Methods diff --git a/apps/jetpack/templates/addon_edit.html b/apps/jetpack/templates/addon_edit.html index 73ad8f28..4f012724 100644 --- a/apps/jetpack/templates/addon_edit.html +++ b/apps/jetpack/templates/addon_edit.html @@ -43,6 +43,9 @@
  • +
    • +
  • +
  • {% endblock %} diff --git a/settings.py b/settings.py index 72ade43d..0209e9cf 100644 --- a/settings.py +++ b/settings.py @@ -140,6 +140,13 @@ DOMAIN = "builder.addons.mozilla.org" SITE_URL = "https://%s" % DOMAIN +# AMO OAUTH DATA +AMOOAUTH_DOMAIN = "addons.mozilla.org" +AMOOAUTH_PORT = 8043 +AMOOAUTH_PROTOCOL = "https" +AMOOAUTH_CONSUMERKEY = "key" +AMOOAUTH_CONSUMERSECRET = "secret" + URLOPEN_TIMEOUT = 4 # default timeout for urllib2.urlopen (seconds) # set it in settings_local.py if AMO auth should be used diff --git a/urls.py b/urls.py index 7c5607bb..e16b3263 100644 --- a/urls.py +++ b/urls.py @@ -24,6 +24,9 @@ # XPI build (r'^xpi/', include('xpi.urls')), + # AMO upload + (r'^amo/', include('amo.urls')), + # /docs are an Apache rewrite (r'^tutorial/', include('tutorial.urls')), diff --git a/utils/amo.py b/utils/amo.py new file mode 100644 index 00000000..ca954463 --- /dev/null +++ b/utils/amo.py @@ -0,0 +1,228 @@ +""" +A class to interact with AMO's api, using OAuth. +Ripped off from Daves test_oauth.py and some notes from python-oauth2 +""" +# Wherein import almost every http or urllib in Python +import urllib +import urllib2 +from urlparse import urlparse, urlunparse, parse_qsl +import httplib2 +import oauth2 as oauth +import os +import re +import time +import json +import mimetools + +from helpers import encode_multipart, data_keys + +# AMO Specific end points +urls = { + 'login': '/users/login', + 'request_token': '/oauth/request_token/', + 'access_token': '/oauth/access_token/', + 'authorize': '/oauth/authorize/', + 'user': '/api/2/user/', + 'addon': '/api/2/addons/', + 'update': '/api/2/update/', + 'perf': '/api/2/performance/', +} + +storage_file = os.path.join(os.path.expanduser('~'), '.amo-oauth') +boundary = mimetools.choose_boundary() + +old = httplib2.Http.__init__ + + +# Ouch, I'll go to hell for this. +def hack(self, **kw): + kw['disable_ssl_certificate_validation'] = True + return old(self, **kw) + +httplib2.Http.__init__ = hack + + +class AMOOAuth: + """ + A base class to authenticate and work with AMO OAuth. + """ + signature_method = oauth.SignatureMethod_HMAC_SHA1() + + def __init__(self, domain='addons.mozilla.org', protocol='https', + port=443, prefix='', three_legged=False): + self.data = self.read_storage() + self.domain = domain + self.protocol = protocol + self.port = port + self.prefix = prefix + self.three_legged = three_legged + + def set_consumer(self, consumer_key, consumer_secret, save_storage=True): + self.should_save_storage = save_storage + self.data['consumer_key'] = consumer_key + self.data['consumer_secret'] = consumer_secret + if self.should_save_storage: + self.save_storage() + + def get_consumer(self): + return oauth.Consumer(self.data['consumer_key'], + self.data['consumer_secret']) + + def get_access(self): + return oauth.Token(self.data['access_token']['oauth_token'], + self.data['access_token']['oauth_token_secret']) + + def has_access_token(self): + return not self.three_legged or 'access_token' in self.data + + def read_storage(self): + if self.should_save_storage and os.path.exists(storage_file): + try: + return json.load(open(storage_file, 'r')) + except ValueError: + pass + return {} + + def url(self, key): + return urlunparse((self.protocol, '%s:%s' % (self.domain, self.port), + '%s/en-US/firefox%s' % (self.prefix, urls[key]), + '', '', '')) + + def shorten(self, url): + return urlunparse(['', ''] + list(urlparse(url)[2:])) + + def save_storage(self): + json.dump(self.data, open(storage_file, 'w')) + + def get_csrf(self, content): + return re.search("name='csrfmiddlewaretoken' value='(.*?)'", + content).groups()[0] + + def _request(self, token, method, url, data={}, headers={}, **kw): + parameters = data_keys(data) + parameters.update(kw) + request = (oauth.Request + .from_consumer_and_token(self.get_consumer(), token, + method, url, parameters)) + request.sign_request(self.signature_method, self.get_consumer(), token) + client = httplib2.Http() + if data and method == 'POST': + data = encode_multipart(boundary, data) + headers.update({'Content-Type': + 'multipart/form-data; boundary=%s' % boundary}) + else: + data = urllib.urlencode(data) + return client.request(request.to_url(), method=method, + headers=headers, body=data) + + def authenticate(self, username=None, password=None): + """ + This is only for the more convoluted three legged approach. + 1. Login into AMO. + 2. Get a request token for the consumer. + 3. Approve the consumer. + 4. Get an access token. + """ + # First we need to login to AMO, this takes a few steps. + # If this was being done in a browser, this wouldn't matter. + # + # This callback is pretty academic, but required so we can get + # verification token. + callback = 'http://foo.com/' + + opener = urllib2.build_opener(urllib2.HTTPCookieProcessor()) + urllib2.install_opener(opener) + res = opener.open(self.url('login')) + assert res.code == 200 + + # get the CSRF middleware token + if password is None: + password = raw_input('Enter password: ') + + csrf = self.get_csrf(res.read()) + data = urllib.urlencode({'username': username, + 'password': password, + 'csrfmiddlewaretoken': csrf}) + res = opener.open(self.url('login'), data) + assert res.code == 200 + + # We need these headers to be able to post to the authorize method + cookies = {} + # Need to find a better way to find the handler, -2 is fragile. + for cookie in opener.handlers[-2].cookiejar: + if cookie.name == 'sessionid': + cookies = {'Cookie': '%s=%s' % (cookie.name, cookie.value)} + # Step 1 completed, we can now be logged in for any future requests + + # Step 2, get a request token. + resp, content = self._request(None, 'GET', self.url('request_token'), + oauth_callback=callback) + assert resp['status'] == '200', 'Status was: %s' % resp.status + + request_token = dict(parse_qsl(content)) + assert request_token + token = oauth.Token(request_token['oauth_token'], + request_token['oauth_token_secret']) + + # Step 3, authorize the access of this consumer for this user account. + resp, content = self._request(token, 'GET', self.url('authorize'), + headers=cookies) + csrf = self.get_csrf(content) + data = {'authorize_access': True, + 'csrfmiddlewaretoken': csrf, + 'oauth_token': token.key} + resp, content = self._request(token, 'POST', self.url('authorize'), + headers=cookies, data=data, + oauth_callback=callback) + + assert resp.status == 302, 'Status was: %s' % resp.status + qsl = parse_qsl(resp['location'][len(callback) + 1:]) + verifier = dict(qsl)['oauth_verifier'] + token.set_verifier(verifier) + + # We have now authorized the app for this user. + resp, content = self._request(token, 'GET', self.url('access_token')) + access_token = dict(parse_qsl(content)) + self.data['access_token'] = access_token + self.save_storage() + # Done. Wasn't that fun? + + def get_params(self): + return dict(oauth_consumer_key=self.data['consumer_key'], + oauth_nonce=oauth.generate_nonce(), + oauth_signature_method='HMAC-SHA1', + oauth_timestamp=int(time.time()), + oauth_version='1.0') + + def _send(self, url, method, data): + resp, content = self._request(None, method, url, + data=data) + if resp.status != 200: + raise ValueError('%s: %s' % (resp.status, content)) + try: + return json.loads(content) + except ValueError: + return content + + def get_user(self): + return self._send(self.url('user'), 'GET', {}) + + def create_addon(self, data): + return self._send(self.url('addon'), 'POST', data) + + def update_addon(self, data): + return self._send(self.url('addon'), 'PUT', data) + + def create_perf(self, data): + return self._send(self.url('perf'), 'POST', data) + + +if __name__ == '__main__': + username = 'amckay@mozilla.com' + amo = AMOOAuth(domain="addons.mozilla.local", port=8000, protocol='http') + amo.set_consumer(consumer_key='CmAn9KhXR8SD3xUSrf', + consumer_secret='4hPsAW9yCecr4KRSR4DVKanCkgpqDETm') + if not amo.has_access_token(): + # This is an example, don't get too excited. + amo.authenticate(username=username) + print amo.get_user() diff --git a/utils/helpers.py b/utils/helpers.py index 3e5a1352..5b3486c2 100644 --- a/utils/helpers.py +++ b/utils/helpers.py @@ -1,4 +1,6 @@ import re +import mimetypes +import os from random import choice @@ -43,3 +45,55 @@ def render_json(request, template_name, *args, **kwargs): """ return render(request, template_name, mimetype='application/json', *args, **kwargs) + + +def to_str(s): + if isinstance(s, unicode): + return s.encode('utf-8', 'strict') + else: + return str(s) + + +def data_keys(data): + _data = {} + for k, v in data.items(): + if is_file(v): + v = '' + _data[to_str(k)] = v + return _data + + +def is_file(thing): + return hasattr(thing, "read") and callable(thing.read) + + +def encode_multipart(boundary, data): + """Ripped from django.""" + lines = [] + + for key, value in data.items(): + if is_file(value): + content_type = mimetypes.guess_type(value.name)[0] + if content_type is None: + content_type = 'application/octet-stream' + lines.extend([ + '--' + boundary, + 'Content-Disposition: form-data; name="%s"; filename="%s"' \ + % (to_str(key), to_str(os.path.basename(value.name))), + 'Content-Type: %s' % content_type, + '', + value.read(), + ]) + else: + lines.extend([ + '--' + boundary, + 'Content-Disposition: form-data; name="%s"' % to_str(key), + '', + to_str(value), + ]) + + lines.extend([ + '--' + boundary + '--', + '', + ]) + return '\r\n'.join(lines)