Add remote(basename) method to ExtensionVersion, to return the co…

…ntents of a file within the extension. Add `as_dict()` method to `Extension` and `ExtensionVersion`, to avoid returning private properties.

CHANGELOG.md

@@ -1,10 +1,15 @@
 # Changelog
 
+## 0.0.3 (2018-06-21)
+
+* Add `remote(basename)` method to `ExtensionVersion`, to return the contents of a file within the extension.
+* Add `as_dict()` method to `Extension` and `ExtensionVersion`, to avoid returning private properties.
+
 ## 0.0.2 (2018-06-12)
 
-* Make `ExtensionRegistry` iterable.
+* Add `get(**kwargs)` method to `ExtensionRegistry`, to get a specific extension version.
+* Make `ExtensionRegistry` iterable, to iterate over all extension versions.
 * Remove `all()` method from `ExtensionRegistry`.
-* Add `get(**kwargs)` method to `ExtensionRegistry`.
 * Add package-specific exceptions.
 
 ## 0.0.1 (2018-06-11)

ocdsextensionregistry/extension.py

@@ -6,3 +6,11 @@ def __init__(self, data):
         self.id = data['Id']
         self.category = data['Category']
         self.core = data['Core'] == 'true'
+
+    def as_dict(self):
+        """
+        Returns the object's properties as a dictionary.
+
+        This method is defined to match the method in `ExtensionVersion`.
+        """
+        return self.__dict__

ocdsextensionregistry/extension_version.py

@@ -1,5 +1,7 @@
 import re
+from io import BytesIO
 from urllib.parse import urlparse
+from zipfile import ZipFile
 
 import requests
 
@@ -14,20 +16,51 @@ def __init__(self, data):
         self.version = data['Version']
         self.base_url = data['Base URL']
         self.download_url = data['Download URL']
+        self._file_cache = {}
 
     def update(self, other):
         """
         Merges in the properties of another Extension or ExtensionVersion object.
         """
-        for k, v in other.__dict__.items():
+        for k, v in other.as_dict().items():
             setattr(self, k, v)
 
+    def as_dict(self):
+        """
+        Returns the object's public properties as a dictionary.
+        """
+        return {key: value for key, value in self.__dict__.items() if not key.startswith('_')}
+
+    def remote(self, basename):
+        """
+        Returns the contents of the file within the extension.
+
+        If the extension has a download URL, downloads the ZIP archive and caches all its files' contents. Otherwise,
+        downloads and caches the requested file's contents. Raises an HTTPError if a download fails, and a KeyError if
+        the requested file isn't in the ZIP archive.
+        """
+        if basename not in self._file_cache:
+            if not self.download_url:
+                response = requests.get(self.base_url + basename)
+                response.raise_for_status()
+                self._file_cache[basename] = response.text
+            elif not self._file_cache:
+                response = requests.get(self.download_url, allow_redirects=True)
+                response.raise_for_status()
+                zipfile = ZipFile(BytesIO(response.content))
+                names = zipfile.namelist()
+                start = len(names[0])
+                for name in names[1:]:
+                    self._file_cache[name[start:]] = str(zipfile.read(name))
+
+        return self._file_cache[basename]
+
     @property
     def metadata(self):
         """
         Retrieves and returns the extension's extension.json file as a dict.
         """
-        return requests.get('{}extension.json'.format(self.base_url)).json()
+        return requests.get(self.base_url + 'extension.json').json()
 
     @property
     def repository_full_name(self):
@@ -76,10 +109,10 @@ def _repository_name(self, parsed, config):
         return re.match(config['name:pattern'], parsed.path).group(1)
 
     def _repository_html_page(self, parsed, config):
-        return '{}{}'.format(config['html_page:prefix'], self._repository_full_name(parsed, config))
+        return config['html_page:prefix'] + self._repository_full_name(parsed, config)
 
     def _repository_url(self, parsed, config):
-        return '{}{}{}'.format(config['url:prefix'], self._repository_full_name(parsed, config), config['url:suffix'])
+        return config['url:prefix'] + self._repository_full_name(parsed, config) + config['url:suffix']
 
     def _repository_property(self, prop):
         parsed = urlparse(self.base_url)

tests/test_extension.py

@@ -11,12 +11,23 @@ def test_init():
 
 
 def test_init_non_core():
-    for core in ('TRUE', 'True', 'false', 'foo', ''):
+    for core in ('TRUE', 'True', 'false', 'foo', '', None):
         obj = Extension(arguments(Core=core))
 
         assert obj.core is False
 
 
+def test_as_dict():
+    args = arguments(Core='')
+    obj = Extension(args)
+
+    assert obj.as_dict() == {
+        'id': args['Id'],
+        'category': args['Category'],
+        'core': False,
+    }
+
+
 def arguments(**kwargs):
     data = {
         'Id': 'location',

tests/test_extension_registry.py

@@ -42,7 +42,7 @@ def test_init_with_data():
     obj = ExtensionRegistry(extension_versions_data, extensions_data)
 
     assert len(obj.versions) == 14
-    assert obj.versions[0].__dict__ == {
+    assert obj.versions[0].as_dict() == {
         'id': 'charges',
         'date': '',
         'version': 'master',
@@ -52,7 +52,7 @@ def test_init_with_data():
         'core': False,
     }
     # Assume intermediate data is correctly parsed.
-    assert obj.versions[-1].__dict__ == {
+    assert obj.versions[-1].as_dict() == {
         'id': 'lots',
         'date': '2018-01-30',
         'version': 'v1.1.3',
@@ -67,15 +67,15 @@ def test_init_with_versions_only():
     obj = ExtensionRegistry(extension_versions_data)
 
     assert len(obj.versions) == 14
-    assert obj.versions[0].__dict__ == {
+    assert obj.versions[0].as_dict() == {
         'id': 'charges',
         'date': '',
         'version': 'master',
         'base_url': 'https://raw.githubusercontent.com/open-contracting/ocds_charges_extension/master/',
         'download_url': 'https://github.com/open-contracting/ocds_charges_extension/archive/master.zip',
     }
     # Assume intermediate data is correctly parsed.
-    assert obj.versions[-1].__dict__ == {
+    assert obj.versions[-1].as_dict() == {
         'id': 'lots',
         'date': '2018-01-30',
         'version': 'v1.1.3',
@@ -89,7 +89,7 @@ def test_filter():
     result = obj.filter(core=True, version='v1.1.3', category='tender')
 
     assert len(result) == 2
-    assert result[0].__dict__ == {
+    assert result[0].as_dict() == {
         'id': 'enquiries',
         'date': '2018-02-01',
         'version': 'v1.1.3',
@@ -98,7 +98,7 @@ def test_filter():
         'category': 'tender',
         'core': True,
     }
-    assert result[1].__dict__ == {
+    assert result[1].as_dict() == {
         'id': 'lots',
         'date': '2018-01-30',
         'version': 'v1.1.3',
@@ -121,7 +121,7 @@ def test_get():
     obj = ExtensionRegistry(extension_versions_data)
     result = obj.get(id='lots', version='v1.1.3')
 
-    assert result.__dict__ == {
+    assert result.as_dict() == {
         'id': 'lots',
         'date': '2018-01-30',
         'version': 'v1.1.3',

tests/test_extension_version.py

@@ -1,3 +1,6 @@
+import pytest
+import requests
+
 from ocdsextensionregistry import Extension, ExtensionVersion
 
 
@@ -21,6 +24,62 @@ def test_update():
     assert obj.core is True
 
 
+def test_update_ignore_private_properties():
+    obj = ExtensionVersion(arguments())
+    other = ExtensionVersion(arguments())
+    other._file_cache['key'] = 'value'
+    obj.update(other)
+
+    assert obj._file_cache == {}
+
+
+def test_as_dict():
+    args = arguments()
+    obj = ExtensionVersion(args)
+
+    assert obj.as_dict() == {
+        'id': args['Id'],
+        'date': args['Date'],
+        'version': args['Version'],
+        'base_url': args['Base URL'],
+        'download_url': args['Download URL'],
+    }
+
+
+def test_remote():
+    obj = ExtensionVersion(arguments(**{'Download URL': None}))
+    data = obj.remote('LICENSE')
+    # Repeat requests should return the same result.
+    data = obj.remote('LICENSE')
+
+    assert 'http://www.apache.org/licenses/' in data
+
+
+def test_remote_download_url():
+    obj = ExtensionVersion(arguments())
+    data = obj.remote('LICENSE')
+    # Repeat requests should return the same result.
+    data = obj.remote('LICENSE')
+
+    assert 'http://www.apache.org/licenses/' in data
+
+
+def test_remote_nonexistent():
+    obj = ExtensionVersion(arguments(**{'Download URL': None}))
+    with pytest.raises(requests.exceptions.HTTPError) as excinfo:
+        obj.remote('nonexistent')
+
+    assert str(excinfo.value) == "404 Client Error: Not Found for url: https://raw.githubusercontent.com/open-contracting/ocds_location_extension/v1.1.3/nonexistent"  # noqa
+
+
+def test_remote_download_url_nonexistent():
+    obj = ExtensionVersion(arguments())
+    with pytest.raises(KeyError) as excinfo:
+        obj.remote('nonexistent')
+
+    assert str(excinfo.value) == "'nonexistent'"
+
+
 def test_metadata():
     obj = ExtensionVersion(arguments())
     result = obj.metadata