Merge f0d53cd into fc56bfe

CHANGELOG.md

@@ -16,6 +16,10 @@ New options:
 * combine-release-packages: `--uri`, `--published-date`
 * compile: `--schema`, `--uri`, `--published-date`, `--linked-releases`
 
+New commands:
+
+* upgrade
+
 Removed commands:
 
 * measure

README.rst

@@ -77,6 +77,17 @@ Optional arguments:
 
     cat tests/fixtures/realdata/release-package-1.json | ocdskit compile > out.json
 
+upgrade
+~~~~~~~
+
+Upgrades packages and releases from an old version of OCDS to a new version.
+
+OCDS 1.0 `describes <http://standard.open-contracting.org/1.0/en/schema/reference/#identifier>`__ an organization's ``name``, ``identifier``, ``address`` and ``contactPoint`` as relevant to identifying it. OCDS 1.1 `moves <http://standard.open-contracting.org/1.1/en/schema/reference/#parties>`__ organization data into a ``parties`` array. To upgrade from OCDS 1.0 to 1.1, we create an ``id`` for each organization, based on those identifying fields. This can result in duplicates in the ``parties`` array, if the same organization has different or missing ``name``, ``identifier``, ``address`` or ``contactPoint`` values in different contexts. This can also lead to data loss if the same organization has different values for other fields across occurrences; the command prints warnings in such cases.
+
+::
+
+    cat tests/fixtures/realdata/release-package-1.json | ocdskit upgrade 1.0:1.1 > out.json
+
 package-releases
 ~~~~~~~~~~~~~~~~
 

ocdskit/cli/__main__.py

@@ -21,6 +21,7 @@
     'ocdskit.cli.commands.split_record_packages',
     'ocdskit.cli.commands.split_release_packages',
     'ocdskit.cli.commands.tabulate',
+    'ocdskit.cli.commands.upgrade',
     'ocdskit.cli.commands.validate',
 )
 

ocdskit/cli/commands/upgrade.py

@@ -0,0 +1,30 @@
+from ocdskit import upgrade
+from ocdskit.cli.commands.base import BaseCommand
+from ocdskit.exceptions import CommandError
+
+
+class Command(BaseCommand):
+    name = 'upgrade'
+    help = 'upgrades packages and releases from an old version of OCDS to a new version'
+
+    def add_arguments(self):
+        self.add_argument('versions', help='the colon-separated old and new versions')
+
+    def handle(self):
+        versions = self.args.versions
+
+        version_from, version_to = versions.split(':')
+        if version_from < version_to:
+            direction = 'up'
+        else:
+            direction = 'down'
+
+        try:
+            upgrade_method = getattr(upgrade, 'upgrade_{}'.format(versions.replace('.', '').replace(':', '_')))
+        except AttributeError:
+            raise CommandError('{}grade from {} is not supported'.format(direction, versions.replace(':', ' to ')))
+
+        for line in self.buffer():
+            data = self.json_loads(line)
+            upgrade_method(data)
+            self.print(data)

ocdskit/upgrade.py

@@ -0,0 +1,211 @@
+import json
+import logging
+from collections import OrderedDict
+from copy import deepcopy
+from hashlib import md5
+
+logger = logging.getLogger('ocdskit')
+
+# See http://standard.open-contracting.org/1.0/en/schema/reference/#identifier
+organization_identification_1_0 = (
+    (None, ('name',)),
+    ('identifier', ('scheme', 'id', 'legalName', 'uri')),
+    ('address', ('streetAddress', 'locality', 'region', 'postalCode', 'countryName')),
+    ('contactPoint', ('name', 'email', 'telephone', 'faxNumber', 'url')),
+)
+
+
+def _move_to_top(data, fields):
+    for field in reversed(fields):
+        if field in data:
+            data.move_to_end(field, last=False)
+
+
+def upgrade_10_10(data):
+    pass
+
+
+def upgrade_11_11(data):
+    pass
+
+
+def upgrade_10_11(data):
+    """
+    Upgrades a record package, release package or release from 1.0 to 1.1.
+
+    Retains the deprecated Amendment.changes, Budget.source and Milestone.documents fields.
+
+    Note: Versioned releases within a record package are not upgraded.
+    """
+    if 'records' in data or 'releases' in data:  # package
+        data['version'] = '1.1'
+        _move_to_top(data, ('uri', 'version'))
+
+    if 'records' in data:  # record package
+        for record in data['records']:
+            if 'releases' in record:
+                for release in record['releases']:
+                    upgrade_release_10_11(release)
+            if 'compiledRelease' in record:
+                upgrade_release_10_11(record['compiledRelease'])
+    elif 'releases' in data:  # release package
+        for release in data['releases']:
+            upgrade_release_10_11(release)
+    else:  # release
+        upgrade_release_10_11(data)
+
+
+def upgrade_release_10_11(release):
+    """
+    Applies upgrades for organization handling, amendment handling and transactions terminology.
+    """
+    upgrade_parties_10_to_11(release)
+    upgrade_amendments_10_11(release)
+    upgrade_transactions_10_11(release)
+
+
+def upgrade_parties_10_to_11(release):
+    """
+    Converts organizations to organization references and fills in the ``parties`` array.
+    """
+    parties = _get_parties(release)
+
+    if 'buyer' in release:
+        release['buyer'] = _add_party(parties, release['buyer'], 'buyer')
+
+    if 'tender' in release:
+        if 'procuringEntity' in release['tender']:
+            release['tender']['procuringEntity'] = _add_party(parties, release['tender']['procuringEntity'], 'procuringEntity')  # noqa: E501
+        if 'tenderers' in release['tender']:
+            for i, tenderer in enumerate(release['tender']['tenderers']):
+                release['tender']['tenderers'][i] = _add_party(parties, tenderer, 'tenderer')
+
+    if 'awards' in release:
+        for award in release['awards']:
+            if 'suppliers' in award:
+                for i, supplier in enumerate(award['suppliers']):
+                    award['suppliers'][i] = _add_party(parties, supplier, 'supplier')
+
+    if parties:
+        if 'parties' not in release:
+            release['parties'] = []
+            _move_to_top(release, ('ocid', 'id', 'date', 'tag', 'initiationType', 'parties'))
+
+        for party in parties.values():
+            if party not in release['parties']:
+                release['parties'].append(party)
+
+
+def _get_parties(release):
+    parties = OrderedDict()
+
+    if 'parties' in release:
+        for party in release['parties']:
+            parties[party['id']] = party
+
+    return parties
+
+
+def _add_party(parties, party, role):
+    """
+    Adds an ``id`` to the party, adds the party to the ``parties`` array, and returns an OrganizationReference.
+    """
+    party = deepcopy(party)
+
+    if 'id' not in party:
+        parts = []
+        for parent, fields in organization_identification_1_0:
+            if not parent:
+                for field in fields:
+                    parts.append(_get_bytes(party, field))
+            elif parent in party:
+                for field in fields:
+                    parts.append(_get_bytes(party[parent], field))
+
+        party['id'] = md5(b'-'.join(parts)).hexdigest()
+        _move_to_top(party, ('id'))
+
+    _id = party['id']
+
+    if _id not in parties:
+        parties[_id] = party
+    else:
+        # Warn about information loss.
+        other = deepcopy(parties[_id])
+        roles = other.pop('roles')
+        if dict(party) != dict(other):
+            logger.warning('party differs in "{}" role than in "{}" roles:\n{}\n{}'.format(
+                role, ', '.join(roles), json.dumps(party), json.dumps(other)))
+
+    if 'roles' not in parties[_id]:
+        parties[_id]['roles'] = []
+        _move_to_top(parties[_id], ('id', 'roles'))
+
+    if role not in parties[_id]['roles']:
+        # Update the `roles` of the party in the `parties` array.
+        parties[_id]['roles'].append(role)
+
+    # Create the OrganizationReference.
+    organization_reference = OrderedDict([
+        ('id', _id),
+    ])
+    if 'name' in party:
+        organization_reference['name'] = party['name']
+
+    return organization_reference
+
+
+def _get_bytes(obj, field):
+    return bytes(obj.get(field) or '', 'utf-8')
+
+
+def upgrade_amendments_10_11(release):
+    """
+    Renames ``amendment`` to ``amendments`` under ``tender``, ``awards`` and ``contracts``. If ``amendments`` already
+    exists, it appends the ``amendment`` value to the ``amendments`` array, unless it already contains it.
+    """
+    if 'tender' in release:
+        _upgrade_amendment_10_11(release['tender'])
+    for field in ('awards', 'contracts'):
+        if field in release:
+            for block in release[field]:
+                _upgrade_amendment_10_11(block)
+
+
+def _upgrade_amendment_10_11(block):
+    if 'amendment' in block:
+        if 'amendments' not in block:
+            block['amendments'] = []
+        if block['amendment'] not in block['amendments']:
+            block['amendments'].append(block['amendment'])
+        del block['amendment']
+
+
+def upgrade_transactions_10_11(release):
+    """
+    Renames ``providerOrganization`` to ``payer``, ``receiverOrganization`` to ``payee``, and ``amount`` to ``value``
+    under ``contracts.implementation.transactions``, unless they already exist.
+
+    Converts ``providerOrganization`` and ``receiverOrganization`` from an Identifier to an OrganizationReference and
+    fills in the ``parties`` array.
+    """
+    parties = _get_parties(release)
+
+    if 'contracts' in release:
+        for contract in release['contracts']:
+            if 'implementation' in contract and 'transactions' in contract['implementation']:
+                for transaction in contract['implementation']['transactions']:
+                    if 'value' not in transaction:
+                        transaction['value'] = transaction['amount']
+                        del transaction['amount']
+
+                    for old, new in (('providerOrganization', 'payer'), ('receiverOrganization', 'payee')):
+                        if old in transaction and new not in transaction:
+                            party = OrderedDict([
+                                ('identifier', transaction[old]),
+                            ])
+                            if 'legalName' in transaction[old]:
+                                party['name'] = transaction[old]['legalName']
+
+                            transaction[new] = _add_party(parties, party, new)
+                            del transaction[old]

tests/commands/test_compile.py

@@ -31,7 +31,7 @@ def test_command_versioned(monkeypatch):
     assert actual.getvalue() == read('realdata/versioned-release-1.json') + read('realdata/versioned-release-2.json')
 
 
-def test_command_package(monkeypatch, caplog):
+def test_command_package(monkeypatch):
     stdin = read('realdata/release-package-1.json', 'rb') + read('realdata/release-package-2.json', 'rb')
 
     with patch('sys.stdin', TextIOWrapper(BytesIO(stdin))), patch('sys.stdout', new_callable=StringIO) as actual:

tests/commands/test_upgrade.py

@@ -0,0 +1,81 @@
+import sys
+from io import StringIO, TextIOWrapper, BytesIO
+from unittest.mock import patch
+
+import pytest
+
+from ocdskit.cli.__main__ import main
+from tests import read
+
+
+def test_command_record_package(monkeypatch):
+    stdin = read('realdata/record-package_1.0.json', 'rb')
+
+    with patch('sys.stdin', TextIOWrapper(BytesIO(stdin))), patch('sys.stdout', new_callable=StringIO) as actual:
+        monkeypatch.setattr(sys, 'argv', ['ocdskit', 'upgrade', '1.0:1.1'])
+        main()
+
+    assert actual.getvalue() == read('realdata/record-package_1.1.json')
+
+
+def test_command_release_package_buyer_procuring_entity_suppliers(monkeypatch):
+    stdin = read('realdata/release-package_1.0-1.json', 'rb')
+
+    with patch('sys.stdin', TextIOWrapper(BytesIO(stdin))), patch('sys.stdout', new_callable=StringIO) as actual:
+        monkeypatch.setattr(sys, 'argv', ['ocdskit', 'upgrade', '1.0:1.1'])
+        main()
+
+    assert actual.getvalue() == read('realdata/release-package_1.1-1.json')
+
+
+def test_command_release_package_transactions(monkeypatch):
+    stdin = read('realdata/release-package_1.0-2.json', 'rb')
+
+    with patch('sys.stdin', TextIOWrapper(BytesIO(stdin))), patch('sys.stdout', new_callable=StringIO) as actual:
+        monkeypatch.setattr(sys, 'argv', ['ocdskit', 'upgrade', '1.0:1.1'])
+        main()
+
+    assert actual.getvalue() == read('realdata/release-package_1.1-2.json')
+
+
+def test_command_release_tenderers_amendment(monkeypatch, caplog):
+    stdin = read('release_1.0.json', 'rb')
+
+    with patch('sys.stdin', TextIOWrapper(BytesIO(stdin))), patch('sys.stdout', new_callable=StringIO) as actual:
+        monkeypatch.setattr(sys, 'argv', ['ocdskit', 'upgrade', '1.0:1.1'])
+        main()
+
+    assert actual.getvalue() == read('release_1.1.json')
+
+    assert len(caplog.records) == 1
+    assert caplog.records[0].levelname == 'WARNING'
+    assert caplog.records[0].message == 'party differs in "supplier" role than in "tenderer" roles:\n' \
+        '{"name": "Acme Inc.", "additionalIdentifiers": [{"id": 1}], "id": "6760c32d3e2e5499d51a709f563ed39a"}\n' \
+        '{"id": "6760c32d3e2e5499d51a709f563ed39a", "name": "Acme Inc."}'
+
+
+def test_command_identity(monkeypatch):
+    stdin = b'{}'
+
+    for versions in ('1.0:1.0', '1.1:1.1'):
+        with patch('sys.stdin', TextIOWrapper(BytesIO(stdin))), patch('sys.stdout', new_callable=StringIO) as actual:
+            monkeypatch.setattr(sys, 'argv', ['ocdskit', 'upgrade', versions])
+            main()
+
+    assert actual.getvalue() == '{}\n'
+
+
+def test_command_downgrade(monkeypatch, caplog):
+    stdin = b'{}'
+
+    with pytest.raises(SystemExit) as excinfo:
+        with patch('sys.stdin', TextIOWrapper(BytesIO(stdin))), patch('sys.stdout', new_callable=StringIO) as actual:
+            monkeypatch.setattr(sys, 'argv', ['ocdskit', 'upgrade', '1.1:1.0'])
+            main()
+
+    assert actual.getvalue() == ''
+
+    assert len(caplog.records) == 1
+    assert caplog.records[0].levelname == 'CRITICAL'
+    assert caplog.records[0].message == 'downgrade from 1.1 to 1.0 is not supported'
+    assert excinfo.value.code == 1

tests/fixtures/realdata/record-package_1.0.json

@@ -0,0 +1 @@
+{"uri":"https://www.contrataciones.gov.py/datos/record-package/193399.json","publishedDate":"2018-12-17T17:31:43Z","records":[{"ocid":"ocds-03ad3f-193399","releases":[{"date":"2013-03-05T08:24:59Z","tag":["planning"],"url":"https://www.contrataciones.gov.py/datos/id/planning/193399-adquisicion-scanner.json"}],"compiledRelease":{"language":"es","ocid":"ocds-03ad3f-193399","id":"193399-adquisicion-scanner","date":"2018-12-17T17:31:43Z","tag":["compiled"],"initiationType":"tender","tender":{"id":"193399-adquisicion-scanner","title":"Adquisición de Scanner","status":"complete","value":{"currency":"PYG","amount":null},"procuringEntity":{"name":"Dirección Nacional de Contrataciones Públicas (DNCP)","contactPoint":{"name":"Abog. Cynthia Leite de Lezcano","email":"uoc@contrataciones.gov.py","telephone":"415-4000"}},"tenderPeriod":{"endDate":null,"startDate":"2010-03-15T09:00:00Z"},"submissionMethod":["electronicAuction"],"url":"https://www.contrataciones.gov.py/datos/id/convocatorias/193399-adquisicion-scanner"},"buyer":{"name":"Dirección Nacional de Contrataciones Públicas (DNCP)","contactPoint":{"name":"Abog. Cynthia Leite de Lezcano","email":"uoc@contrataciones.gov.py","telephone":"415-4000"}},"awards":[{"id":"193399-adquisicion-scanner","title":"Adquisición de Scanner","status":"active","date":"2010-04-23T12:25:10Z","value":{"amount":135630400,"currency":"PYG"},"suppliers":[{"name":"MASTER SOFT SRL","identifier":{"id":"80007525-0","legalName":"MASTER SOFT SRL","scheme":"Registro Único de Contribuyente emitido por el Ministerio de Hacienda"},"address":{"streetAddress":"CHOFERES DEL CHACO Nº1956","postalCode":"","locality":"ASUNCION","countryName":"Paraguay","region":"Asunción"},"contactPoint":{"name":"LUIS MARIA OREGGIONI, GISELA SELMA WEIBERLEN DE OREGGIONI","faxNumber":"","telephone":"662831","email":"mastersoft@tigo.com.py","url":null}}],"url":"https://www.contrataciones.gov.py/datos/id/adjudicaciones/193399-adquisicion-scanner"}]}}]}

tests/fixtures/realdata/record-package_1.1.json

@@ -0,0 +1 @@
+{"uri":"https://www.contrataciones.gov.py/datos/record-package/193399.json","version":"1.1","publishedDate":"2018-12-17T17:31:43Z","records":[{"ocid":"ocds-03ad3f-193399","releases":[{"date":"2013-03-05T08:24:59Z","tag":["planning"],"url":"https://www.contrataciones.gov.py/datos/id/planning/193399-adquisicion-scanner.json"}],"compiledRelease":{"ocid":"ocds-03ad3f-193399","id":"193399-adquisicion-scanner","date":"2018-12-17T17:31:43Z","tag":["compiled"],"initiationType":"tender","parties":[{"id":"a02fc5eae413fa29686db8eff8b439b6","roles":["buyer","procuringEntity"],"name":"Dirección Nacional de Contrataciones Públicas (DNCP)","contactPoint":{"name":"Abog. Cynthia Leite de Lezcano","email":"uoc@contrataciones.gov.py","telephone":"415-4000"}},{"id":"d20b2b5c7a36a5350d9304793414b85c","roles":["supplier"],"name":"MASTER SOFT SRL","identifier":{"id":"80007525-0","legalName":"MASTER SOFT SRL","scheme":"Registro Único de Contribuyente emitido por el Ministerio de Hacienda"},"address":{"streetAddress":"CHOFERES DEL CHACO Nº1956","postalCode":"","locality":"ASUNCION","countryName":"Paraguay","region":"Asunción"},"contactPoint":{"name":"LUIS MARIA OREGGIONI, GISELA SELMA WEIBERLEN DE OREGGIONI","faxNumber":"","telephone":"662831","email":"mastersoft@tigo.com.py","url":null}}],"language":"es","tender":{"id":"193399-adquisicion-scanner","title":"Adquisición de Scanner","status":"complete","value":{"currency":"PYG","amount":null},"procuringEntity":{"id":"a02fc5eae413fa29686db8eff8b439b6","name":"Dirección Nacional de Contrataciones Públicas (DNCP)"},"tenderPeriod":{"endDate":null,"startDate":"2010-03-15T09:00:00Z"},"submissionMethod":["electronicAuction"],"url":"https://www.contrataciones.gov.py/datos/id/convocatorias/193399-adquisicion-scanner"},"buyer":{"id":"a02fc5eae413fa29686db8eff8b439b6","name":"Dirección Nacional de Contrataciones Públicas (DNCP)"},"awards":[{"id":"193399-adquisicion-scanner","title":"Adquisición de Scanner","status":"active","date":"2010-04-23T12:25:10Z","value":{"amount":135630400,"currency":"PYG"},"suppliers":[{"id":"d20b2b5c7a36a5350d9304793414b85c","name":"MASTER SOFT SRL"}],"url":"https://www.contrataciones.gov.py/datos/id/adjudicaciones/193399-adquisicion-scanner"}]}}]}

tests/fixtures/realdata/release-package_1.0-1.json

@@ -0,0 +1 @@
+{"uri":"https://www.contrataciones.gov.py/datos/record-package/193399.json","publishedDate":"2018-12-17T17:31:43Z","releases":[{"language":"es","ocid":"ocds-03ad3f-193399","id":"193399-adquisicion-scanner","date":"2018-12-17T17:31:43Z","tag":["compiled"],"initiationType":"tender","tender":{"id":"193399-adquisicion-scanner","title":"Adquisición de Scanner","status":"complete","value":{"currency":"PYG","amount":null},"procuringEntity":{"name":"Dirección Nacional de Contrataciones Públicas (DNCP)","contactPoint":{"name":"Abog. Cynthia Leite de Lezcano","email":"uoc@contrataciones.gov.py","telephone":"415-4000"}},"tenderPeriod":{"endDate":null,"startDate":"2010-03-15T09:00:00Z"},"submissionMethod":["electronicAuction"],"url":"https://www.contrataciones.gov.py/datos/id/convocatorias/193399-adquisicion-scanner"},"buyer":{"name":"Dirección Nacional de Contrataciones Públicas (DNCP)","contactPoint":{"name":"Abog. Cynthia Leite de Lezcano","email":"uoc@contrataciones.gov.py","telephone":"415-4000"}},"awards":[{"id":"193399-adquisicion-scanner","title":"Adquisición de Scanner","status":"active","date":"2010-04-23T12:25:10Z","value":{"amount":135630400,"currency":"PYG"},"suppliers":[{"name":"MASTER SOFT SRL","identifier":{"id":"80007525-0","legalName":"MASTER SOFT SRL","scheme":"Registro Único de Contribuyente emitido por el Ministerio de Hacienda"},"address":{"streetAddress":"CHOFERES DEL CHACO Nº1956","postalCode":"","locality":"ASUNCION","countryName":"Paraguay","region":"Asunción"},"contactPoint":{"name":"LUIS MARIA OREGGIONI, GISELA SELMA WEIBERLEN DE OREGGIONI","faxNumber":"","telephone":"662831","email":"mastersoft@tigo.com.py","url":null}}],"url":"https://www.contrataciones.gov.py/datos/id/adjudicaciones/193399-adquisicion-scanner"}]}]}