Merge 590fcbc into aef48e3

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.0.5
+
+* Add `Codelist` and `CodelistCode` classes.
+* Add `files` property to `ExtensionVersion`, to return the contents of all files within the extension.
+* Add `schemas` property to `ExtensionVersion`, to return the schemas.
+* Add `codelists` property to `ExtensionVersion`, to return the codelissts.
+* Add `docs` property to `ExtensionVersion`, to return the contents of documentation files within the extension.
+* The `metadata` property of `ExtensionVersion` normalizes the contents of `extension.json` to provide consistent access.
+
 ## 0.0.4 (2018-06-27)
 
 * The `metadata` property of `ExtensionVersion` is cached.

docs/api/codelist.rst

@@ -0,0 +1,6 @@
+Codelist
+========
+
+.. autoclass:: ocdsextensionregistry.codelist.Codelist
+    :special-members:
+    :exclude-members: __weakref__

docs/api/codelist_code.rst

@@ -0,0 +1,6 @@
+Codelist Code
+=============
+
+.. autoclass:: ocdsextensionregistry.codelist_code.CodelistCode
+    :special-members:
+    :exclude-members: __weakref__

docs/conf.py

@@ -27,9 +27,9 @@
 author = 'Open Contracting Partnership'
 
 # The short X.Y version
-version = '0.0.4'
+version = '0.0.5'
 # The full version, including alpha/beta/rc tags
-release = '0.0.4'
+release = '0.0.5'
 
 
 # -- General configuration ---------------------------------------------------

ocdsextensionregistry/__init__.py

@@ -1,3 +1,5 @@
+from .codelist import Codelist  # noqa: F401
+from .codelist_code import CodelistCode  # noqa: F401
 from .extension import Extension  # noqa: F401
 from .extension_version import ExtensionVersion  # noqa: F401
 from .extension_registry import ExtensionRegistry  # noqa: F401

ocdsextensionregistry/codelist.py

@@ -0,0 +1,90 @@
+from collections import OrderedDict
+
+from .codelist_code import CodelistCode
+
+
+class Codelist:
+    def __init__(self, name):
+        self.name = name
+        self.rows = []
+
+    def __getitem__(self, index):
+        return self.rows[index]
+
+    def __iter__(self):
+        for row in self.rows:
+            yield row
+
+    def __len__(self):
+        return len(self.rows)
+
+    def __repr__(self):
+        return 'Codelist(name={}, rows={})'.format(repr(self.name), repr(self.rows))
+
+    def extend(self, rows, extension_name=None):
+        """
+        Adds rows to the codelist.
+        """
+        for row in rows:
+            self.rows.append(CodelistCode(row, extension_name))
+
+    def add_extension_column(self, field_name):
+        """
+        Adds a column for the name of the extension from which codes originate.
+        """
+        for row in self.rows:
+            row[field_name] = row.extension_name
+
+    def remove_deprecated_codes(self):
+        """
+        Removes deprecated codes and the Deprecated column.
+        """
+        self.rows = [row for row in self.rows if not row.pop('Deprecated', None)]
+
+    @property
+    def codes(self):
+        """
+        Returns the codes in the codelist.
+        """
+        return [row['Code'] for row in self.rows]
+
+    @property
+    def fieldnames(self):
+        """
+        Returns all fieldnames used in any rows.
+        """
+        fieldnames = OrderedDict()
+        for row in self.rows:
+            for field in row:
+                fieldnames[field] = True
+        return list(fieldnames.keys())
+
+    @property
+    def basename(self):
+        """
+        If the codelist modifies another codelist, returns the latter's name. Otherwise, returns its own name.
+        """
+        if self.patch:
+            return self.name[1:]
+        return self.name
+
+    @property
+    def patch(self):
+        """
+        Returns whether the codelist modifies another codelist.
+        """
+        return self.name.startswith(('+', '-'))
+
+    @property
+    def addend(self):
+        """
+        Returns whether the codelist adds codes to another codelist.
+        """
+        return self.name.startswith('+')
+
+    @property
+    def subtrahend(self):
+        """
+        Returns whether the codelist removes codes from another codelist.
+        """
+        return self.name.startswith('-')

ocdsextensionregistry/codelist_code.py

@@ -0,0 +1,33 @@
+from collections import Mapping
+
+
+class CodelistCode(Mapping):
+    def __init__(self, data, extension_name=None):
+        self.data = data
+        self.extension_name = extension_name
+
+    def __eq__(self, other):
+        if isinstance(other, CodelistCode):
+            return self.data == other.data and self.extension_name == other.extension_name
+        return dict.__eq__(self.data, other)
+
+    def __getitem__(self, key):
+        return self.data[key]
+
+    def __setitem__(self, key, value):
+        self.data[key] = value
+
+    def __iter__(self):
+        return iter(self.data)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __repr__(self):
+        if self.extension_name:
+            return 'CodelistCode(data={}, extension_name={})'.format(repr(self.data), repr(self.extension_name))
+        else:
+            return 'CodelistCode(data={})'.format(repr(self.data))
+
+    def pop(self, *args):
+        return self.data.pop(*args)

ocdsextensionregistry/extension_version.py

@@ -1,11 +1,18 @@
+import csv
 import json
+import os.path
 import re
-from io import BytesIO
+from collections import OrderedDict
+from io import BytesIO, StringIO
 from urllib.parse import urlparse
 from zipfile import ZipFile
 
 import requests
 
+from ocdsextensionregistry import Codelist
+
+SCHEMAS = ('record-package-schema.json', 'release-package-schema.json', 'release-schema.json')
+
 
 class ExtensionVersion:
     def __init__(self, data):
@@ -17,7 +24,11 @@ def __init__(self, data):
         self.version = data['Version']
         self.base_url = data['Base URL']
         self.download_url = data['Download URL']
-        self._file_cache = {}
+        self._files = None
+        self._metadata = None
+        self._schemas = None
+        self._codelists = None
+        self._docs = None
 
     def update(self, other):
         """
@@ -40,28 +51,129 @@ def remote(self, basename):
         downloads and caches the requested file's contents. Raises an HTTPError if a download fails, and a KeyError if
         the requested file isn't in the ZIP archive.
         """
-        if basename not in self._file_cache:
+        if basename not in self.files:
             if not self.download_url:
                 response = requests.get(self.base_url + basename)
                 response.raise_for_status()
-                self._file_cache[basename] = response.text
-            elif not self._file_cache:
+                self._files[basename] = response.text
+
+        return self.files[basename]
+
+    @property
+    def files(self):
+        """
+        Returns the contents of all files within the extension. Decodes the contents of CSV, JSON and Markdown files.
+
+        If the extension has a download URL, downloads the ZIP archive and caches all its files' contents. Otherwise,
+        returns an empty dict. Raises an HTTPError if the download fails.
+        """
+        if self._files is None:
+            self._files = {}
+
+            if self.download_url:
                 response = requests.get(self.download_url, allow_redirects=True)
                 response.raise_for_status()
                 zipfile = ZipFile(BytesIO(response.content))
                 names = zipfile.namelist()
                 start = len(names[0])
                 for name in names[1:]:
-                    self._file_cache[name[start:]] = zipfile.read(name).decode('utf-8')
+                    if name[-1] != '/' and name[start:] != '.travis.yml':
+                        content = zipfile.read(name)
+                        if os.path.splitext(name)[1] in ('.csv', '.json', '.md'):
+                            content = content.decode('utf-8')
+                        self._files[name[start:]] = content
 
-        return self._file_cache[basename]
+        return self._files
 
     @property
     def metadata(self):
         """
         Retrieves and returns the extension's extension.json file as a dict.
         """
-        return json.loads(self.remote('extension.json'))
+        if self._metadata is None:
+            self._metadata = json.loads(self.remote('extension.json'), object_pairs_hook=OrderedDict)
+
+            for field in ('name', 'description', 'documentationUrl'):
+                # Add required fields.
+                if field not in self._metadata:
+                    self._metadata[field] = {}
+                # Add language maps.
+                if isinstance(self._metadata[field], str):
+                    self._metadata[field] = {'en': self._metadata[field]}
+
+            if 'compatibility' not in self._metadata or isinstance(self._metadata['compatibility'], str):
+                self._metadata['compatibility'] = ['1.1']
+
+        return self._metadata
+
+    @property
+    def schemas(self):
+        """
+        Retrieves and returns the extension's schemas.
+        """
+        if self._schemas is None:
+            self._schemas = {}
+
+            if 'schemas' in self.metadata:
+                names = self.metadata['schemas']
+            elif self.download_url:
+                names = [name for name in self.files if name in SCHEMAS]
+            else:
+                names = SCHEMAS
+
+            for name in names:
+                try:
+                    self._schemas[name] = json.loads(self.remote(name), object_pairs_hook=OrderedDict)
+                except requests.exceptions.HTTPError:
+                    if 'schemas' in self.metadata:
+                        raise
+
+        return self._schemas
+
+    @property
+    def codelists(self):
+        """
+        Retrieves and returns the extension's codelists.
+
+        If the extension has no download URL, and if no codelists are listed in extension.json, returns an empty dict.
+        """
+        if self._codelists is None:
+            self._codelists = {}
+
+            if 'codelists' in self.metadata:
+                names = self.metadata['codelists']
+            elif self.download_url:
+                names = [name[10:] for name in self.files if name.startswith('codelists/')]
+            else:
+                names = []
+
+            for name in names:
+                try:
+                    self._codelists[name] = Codelist(name)
+                    # Use universal newlines mode, to avoid parsing errors.
+                    io = StringIO(self.remote('codelists/' + name), newline='')
+                    self._codelists[name].extend(csv.DictReader(io))
+                except requests.exceptions.HTTPError:
+                    if 'codelists' in self.metadata:
+                        raise
+
+        return self._codelists
+
+    @property
+    def docs(self):
+        """
+        Retrieves and returns the contents of documentation files within the extension.
+
+        If the extension has no download URL, returns an empty dict.
+        """
+        if self._docs is None:
+            self._docs = {}
+
+            for name, text in self.files.items():
+                if name.startswith('docs/'):
+                    self._docs[name[5:]] = text
+
+        return self._docs
 
     @property
     def repository_full_name(self):

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name='ocdsextensionregistry',
-    version='0.0.4',
+    version='0.0.5',
     author='James McKinney',
     author_email='james@slashpoundbang.com',
     url='https://github.com/open-contracting/extension_registry.py',