Switch from XML-RPC to JSON API.

zopefoundation · Apr 9, 2014 · 892a012 · 892a012
1 parent 9ba932a
commit 892a012
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 34 deletions.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -1,6 +1,12 @@
 Changelog
 =========
 
+Unreleased
+----------
+
+- Access PyPI via the supported JSON API, rather than unsupported / broken
+  XML-RPC API.
+
 0.9 - 2013-03-02
 ----------------
 

diff --git a/setup.py b/setup.py
@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-__version__ = '0.9'
+__version__ = '0.10dev'
 
 setup(
     name='zope.z2release',

diff --git a/src/zope/z2release/cli.py b/src/zope/z2release/cli.py
@@ -7,7 +7,6 @@
 import os
 import sys
 import urllib
-from xmlrpclib import Server
 
 from zope.z2release.utils import CasePreservingConfigParser
 from zope.z2release.utils import write_index
@@ -24,23 +23,25 @@ def fetch_cfg(url, version_file):
         raise ValueError('Failed to fetch %s' % url)
 
 
-def write_versions(version_file, server, dirname):
+def write_versions(version_file, dirname):
     CP = CasePreservingConfigParser()
     CP.read(version_file)
     for package in CP.options('versions'):
         version = CP.get('versions', package)
         if '#' in version:
             version = version.split('#')[0].strip()
-        write_index(server, package, version, dirname)
+        write_index(package, version, dirname)
     return CP
 
 
-def build_version_file(name, dirname, url, server):
+def build_version_file(name, dirname, url):
     version_file = os.path.join(dirname, name)
     fetch_cfg(url, version_file)
-    return write_versions(version_file, server, dirname)
+    return write_versions(version_file, dirname)
 
 
+GITHUB_RAW = 'https://raw.github.com/zopefoundation/Zope' 
+
 def main():
     if len(sys.argv) != 3:
         print('Usage: z2_kgs <tag-name> <destination-dirname>')
@@ -54,17 +55,17 @@ def main():
         print('Creating index directory: %s' % dirname)
         os.makedirs(dirname)
 
-    server = Server('http://pypi.python.org/pypi')
-
-    url = 'https://raw.github.com/zopefoundation/Zope/%s/versions.cfg' % tag
-    CP = build_version_file('versions.cfg', dirname, url, server)
+    url = '%s/%s/versions.cfg' % (GITHUB_RAW, tag)
+    CP = build_version_file('versions.cfg', dirname, url)
 
     buildout = CP.options('buildout')
     if 'extends' in buildout:
-        url = CP.get('buildout', 'extends')
+        extends = CP.get('buildout', 'extends').split('\n')
         name = 'ztk-versions.cfg'
-        if 'http' in url and not '\n' in url and url.endswith(name):
-            build_version_file(name, dirname, url, server)
+        for extend in extends:
+            if name in extend:
+                url = '%s/%s/ztk-versions.cfg' % (GITHUB_RAW, tag)
+                build_version_file(name, dirname, url)
 
 
 if __name__ == '__main__':

diff --git a/src/zope/z2release/utils.py b/src/zope/z2release/utils.py
@@ -1,20 +1,36 @@
+import json
 import os
 import urlparse
+import urllib2
 
 from ConfigParser import RawConfigParser as ConfigParser
 
 
+def _pypi_release_dists(project, version):
+    pypi = 'https://pypi.python.org/pypi/%s/%s/json' % (project, version)
+    try:
+        response = urllib2.urlopen(pypi)
+    except urllib2.HTTPError:
+        if '-' in project:
+            project = project.replace('-', '_')
+            pypi = 'https://pypi.python.org/pypi/%s/%s/json' % (project,
+                                                                version)
+            response = urllib2.urlopen(pypi)
+        else:
+            raise
+
+    info = json.load(response)
+    for dist in info['urls']:
+        yield dist['filename'], dist['url']
+
+
 class CasePreservingConfigParser(ConfigParser):
 
     def optionxform(self, option):
         return option  # don't flatten case!
 
 
-def write_index(server, package, version, dirname):
-    if '-' in package:
-        exists = server.search(dict(name=package))
-        if not exists:
-            package = package.replace('-', '_')
+def write_index(package, version, dirname):
 
     print('Package %s==%s' % (package, version))
     dest_dir = os.path.join(dirname, package)
@@ -27,22 +43,8 @@ def write_index(server, package, version, dirname):
 
     with open(index_html, 'w') as fp:
         fp.write('<html><body>')
-        lst = server.release_urls(package, version)
-        if lst:
-            # package hosted on PyPI
-            for d in lst:
-                link = '<a href="%s">%s</a>' % (d['url'], d['filename'])
-                fp.write(link + '<br/>')
-        else:
-            # for externally hosted packages we need to rely on the
-            # download_url metadata
-            rel_data = server.release_data(package, version)
-            download_url = rel_data['download_url']
-            if download_url == 'UNKNOWN':
-                raise RuntimeError(
-                    'Incorrect download_url for package %s' % package)
-            filename = os.path.basename(urlparse.urlparse(download_url)[2])
-            link = '<a href="%s">%s</a>' % (download_url, filename)
+        for filename, url in _pypi_release_dists(package, version):
+            link = '<a href="%s">%s</a>' % (url, filename)
             fp.write(link + '<br/>')
 
         fp.write('</body></html>')