Merge pull request #127 from OcelotProject/market-groups-supply

Suppliers for market groups. Fixes #99.

docs/space.rst

@@ -53,6 +53,8 @@ Markets don't start with production volumes - instead, their production volume i
 
 Our ``add_suppliers_to_markets`` function has not allocated *between* the possible suppliers - it has only created the list ``suppliers`` which has these possible suppliers. We then choose *how much* of each possible supplier will contribute to each market, based on the supplier's production volumes.
 
+.. autofunction:: ocelot.transformations.locations.markets.allocate_all_market_suppliers
+
 .. autofunction:: ocelot.transformations.locations.markets.allocate_suppliers
 
 Finding consumers

ocelot/errors.py

@@ -90,3 +90,8 @@ class InvalidTransformationFunction(OcelotError):
 class MissingSupplier(OcelotError):
     """Input from global or RoW market is needed, but this market doesn't exist"""
     pass
+
+
+class MarketGroupError(OcelotError):
+    """Error with market group definition or suppliers"""
+    pass

ocelot/transformations/locations/__init__.py

@@ -18,12 +18,12 @@
     add_recycled_content_suppliers_to_markets,
     add_suppliers_to_markets,
     assign_fake_pv_to_confidential_datasets,
-    allocate_suppliers,
+    allocate_all_market_suppliers,
     delete_allowed_zero_pv_market_datsets,
     delete_suppliers_list,
     update_market_production_volumes,
 )
-from .market_groups import substitute_market_group_links
+from .market_groups import link_market_group_suppliers
 from .rest_of_world import relabel_global_to_row, drop_zero_pv_row_datasets
 from functools import partial
 
@@ -41,13 +41,13 @@
     partial(add_suppliers_to_markets, from_type="market activity",
                                       to_type="market group"),
     partial(update_market_production_volumes, kind='market group'),
-    allocate_suppliers,
+    allocate_all_market_suppliers,
+    link_market_group_suppliers,
     # delete_suppliers_list,
     # drop_zero_pv_row_datasets,
     link_consumers_to_regional_markets,
     link_consumers_to_recycled_content_activities,
     link_consumers_to_global_markets,
     add_reference_product_codes,
     log_and_delete_unlinked_exchanges,
-    # substitute_market_group_links,  # TODO: Need clever approach
 )

ocelot/transformations/locations/_topology.py

@@ -31,18 +31,75 @@ def __init__(self):
             self.data[old] = self.data[fixed]
 
     @functools.lru_cache(maxsize=512)
-    def contained(self, location):
+    def contained(self, location, exclude_self=False):
         if location in ('GLO', 'RoW'):
             return set()
         faces = self(location)
         return {key
                 for key, value in self.data.items()
-                if not value.difference(faces)}
+                if not value.difference(faces)
+                and not (key == location and exclude_self)}
 
     def contains(self, parent, child):
         """Return boolean of whether ``parent`` contains ``child``"""
         return child in self.contained(parent)
 
+    def tree(self, datasets):
+        """Construct a tree of containing geographic relationships.
+
+        ``datasets`` is a list of datasets with the ``locations parameter``.
+
+        Returns a list of nested dictionaries like:
+
+            .. code-block:: python
+
+            {
+                "Europe": {
+                    "Western Europe": {
+                        "France": {},
+                        "Belgium": {}
+                    }
+                }
+            }
+
+        ``GLO`` contains all other locations, including ``RoW``. However, ``RoW`` contains nothing.
+
+        Behavior is not defined if the provided locations make a "diamond" shape where "A" contains "B" and "C", which each contain "D".
+
+        """
+        locations = {x['location'] for x in datasets}
+        filtered = lambda lst: {x for x in lst if x in locations}
+        contained = {loc: filtered(self.contained(loc, True)) for loc in locations}
+
+        # Remove redundant links, e.g. A contains B contains C; don't need A -> C.
+        for parent, children in contained.items():
+            give_up = []
+            for brother in children:
+                for sister in children:
+                    if brother == sister:
+                        continue
+                    elif brother in contained[sister]:
+                        give_up.append(brother)
+            for child in give_up:
+                children.discard(child)
+
+        children = {elem for lst in contained.values() for elem in lst}
+        parents = locations.difference(children).difference({'GLO', 'RoW'})
+
+        # Depth first search
+        def add_children(keys): 
+            return {key: add_children(contained[key]) for key in keys}
+
+        tree = add_children(parents)
+        if 'GLO' in locations:
+            tree = {'GLO': tree}
+            if 'RoW' in locations:
+                tree['GLO']['RoW'] = {}
+        elif 'RoW' in locations:
+            tree['RoW'] = {}
+
+        return tree
+
     @functools.lru_cache(maxsize=512)
     def intersects(self, location):
         if location in ('GLO', 'RoW'):

ocelot/transformations/locations/market_groups.py

@@ -1,4 +1,115 @@
 # -*- coding: utf-8 -*-
+from . import topology
+from ... import toolz
+from ...errors import MarketGroupError
+from ..utils import get_single_reference_product
+from .markets import allocate_suppliers, annotate_exchange
+import copy
+import itertools
+import logging
 
-def substitute_market_group_links(data):
-    pass
+logger = logging.getLogger('ocelot')
+
+
+def link_market_group_suppliers(data):
+    """Link suppliers to market groups, and adjust production volumes."""
+    filter_func = lambda x: x['type'] == "market group"
+    market_groups = dict(toolz.groupby(
+        'reference product',
+        filter(filter_func, data)
+    ))
+
+    # Check to make sure names are consistent
+    for group in market_groups.values():
+        if not len({ds['name'] for ds in group}) == 1:
+            raise MarketGroupError("Inconsistent activity names in market group")
+
+    for ref_product, groups in market_groups.items():
+        suppliers = [ds for ds in data
+                     if ds['type'] == 'market activity'
+                     and ds['reference product'] == ref_product]
+
+        # Put groups second so that if there are duplicates, the group will be retrieved
+        location_lookup = {x['location']: x for x in suppliers}
+        supplier_lookup = copy.deepcopy(location_lookup)
+        location_lookup.update({x['location']: x for x in groups})
+
+        tree = topology.tree(itertools.chain(suppliers, groups))
+
+        if [1 for x in groups if x['location'] == 'RoW']:
+            # Handling RoW is a little tricky. The RoW market group can contain
+            # markets which are not covered by other market groups. So we have
+            # to resolve what RoW means in each context.
+            row_faces = topology('__all__').difference(
+                set.union(*[topology(x['location']) for x in groups])
+            )
+            # This will include RoW, if present, but not GLO
+            row_activities = [x for x in suppliers
+                              if not topology(x['location']).difference(row_faces)
+                              and x['location'] != 'GLO']
+
+            # RoW suppliers need to be removed from GLO suppliers
+            if 'GLO' in tree:
+                for obj in row_activities:
+                    if (obj['location'] != 'RoW'
+                        and obj['location'] in tree['GLO']):
+                        del tree['GLO'][obj['location']]
+        else:
+            row_activities = []
+
+        # Turn `tree` from nested dictionaries to flat list of key, values.
+        # Breadth first search
+        def unroll(lst, dct):
+            for key, value in dct.items():
+                lst.append((key, value))
+            for value in dct.values():
+                if value:
+                    lst = unroll(lst, value)
+            return lst
+
+        flat = unroll([], tree)
+
+        for loc, children in flat:
+            if children and not location_lookup[loc]['type'] == 'market group':
+                raise MarketGroupError
+
+        def translate(obj):
+            return annotate_exchange(get_single_reference_product(obj), obj)
+
+        for parent, children in flat[::-1]:
+            # Special case RoW
+            if parent == 'RoW':
+                obj = location_lookup[parent]
+                obj['suppliers'] = [translate(act) for act in row_activities]
+            else:
+                obj = location_lookup[parent]
+                obj['suppliers'] = [translate(location_lookup[child])
+                                    for child in children]
+
+            # Also add supplier if market and market group have same location
+            if (parent in supplier_lookup
+                and location_lookup[parent]['type'] == 'market group'
+                and parent != 'RoW'):
+                obj['suppliers'].append(translate(supplier_lookup[parent]))
+
+            # For consistency in testing
+            obj['suppliers'].sort(key=lambda x: x['code'])
+
+            for exc in obj['suppliers']:
+                logger.info({
+                    'type': 'table element',
+                    'data': (obj['name'], obj['location'], exc['location'])
+                })
+
+            if not obj['suppliers']:
+                del obj['suppliers']
+                continue
+
+            allocate_suppliers(obj)
+
+    return data
+
+link_market_group_suppliers.__table__ = {
+    'title': "Link and allocate suppliers for market groups. Suppliers can be market activities or other market groups.",
+    'columns': ["Name", "Location", "Supplier Location"]
+}

ocelot/transformations/locations/markets.py

@@ -125,50 +125,57 @@ def add_suppliers_to_markets(data, from_type="transforming activity",
 }
 
 
-def allocate_suppliers(data):
-    """Allocate suppliers to a market dataset and create input exchanges.
+def allocate_all_market_suppliers(data):
+    """Allocate all market activity suppliers.
 
-    Works on both market activities and market groups.
+    Uses the function ``allocate_suppliers``, which modifies data in place.
 
-    The sum of the suppliers inputs should add up to the production amount of the market (reference product exchange amount), minus any constrained market links. Constrained market exchanges should already be in the list of dataset exchanges, with the attribute ``constrained``."""
-    MARKETS = ("market activity", "market group")
-    for ds in (o for o in data if o['type'] in MARKETS):
-        rp = get_single_reference_product(ds)
-        scale_factor = rp['amount']
-        total_pv = sum(o['production volume']['amount']
-                       for o in ds['suppliers'])
+    """
+    for ds in (o for o in data if o['type'] == "market activity"):
+        allocate_suppliers(ds)
+    return data
 
-        if not total_pv:
-            # TODO: Raise error here
-            print("Skipping zero total PV with multiple inputs:\n\t{}/{} ({}, {} suppliers)".format(ds['name'], rp['name'], ds['location'], len(ds['suppliers'])))
-            continue
 
-        for supply_exc in ds['suppliers']:
-            amount = supply_exc['production volume']['amount'] / total_pv * scale_factor
-            if not amount:
-                continue
-            ds['exchanges'].append(remove_exchange_uncertainty({
-                'amount': amount,
-                'name': supply_exc['name'],
-                'unit': supply_exc['unit'],
-                'type': 'from technosphere',
-                'tag': 'intermediateExchange',
-                'code': supply_exc['code']
-            }))
-
-            message = "Create input exchange of {:.4g} {} for '{}' from '{}' ({})"
-            detailed.info({
-                'ds': ds,
-                'message': message.format(
-                    amount,
-                    supply_exc['unit'],
-                    rp['name'],
-                    supply_exc['name'],
-                    supply_exc['location']
-                ),
-                'function': 'allocate_suppliers'
-            })
-    return data
+def allocate_suppliers(dataset):
+    """Allocate suppliers to a market dataset and create input exchanges.
+
+    The sum of the suppliers inputs should add up to the production amount of the market (reference product exchange amount), minus any constrained market links. Constrained market exchanges should already be in the list of dataset exchanges, with the attribute ``constrained``."""
+    rp = get_single_reference_product(dataset)
+    scale_factor = rp['amount']
+    total_pv = sum(o['production volume']['amount']
+                   for o in dataset['suppliers'])
+
+    if not total_pv:
+        # TODO: Raise error here
+        print("Skipping zero total PV with multiple inputs:\n\t{}/{} ({}, {} suppliers)".format(dataset['name'], rp['name'], dataset['location'], len(dataset['suppliers'])))
+        return
+
+    for supply_exc in dataset['suppliers']:
+        amount = supply_exc['production volume']['amount'] / total_pv * scale_factor
+        if not amount:
+            continue
+        dataset['exchanges'].append(remove_exchange_uncertainty({
+            'amount': amount,
+            'name': supply_exc['name'],
+            'unit': supply_exc['unit'],
+            'type': 'from technosphere',
+            'tag': 'intermediateExchange',
+            'code': supply_exc['code']
+        }))
+
+        message = "Create input exchange of {:.4g} {} for '{}' from '{}' ({})"
+        detailed.info({
+            'ds': dataset,
+            'message': message.format(
+                amount,
+                supply_exc['unit'],
+                rp['name'],
+                supply_exc['name'],
+                supply_exc['location']
+            ),
+            'function': 'allocate_suppliers'
+        })
+    return dataset
 
 
 def update_market_production_volumes(data, kind="market activity"):

ocelot/transformations/locations/rest_of_world.py

@@ -13,6 +13,9 @@ def relabel_global_to_row(data):
     """Change ``GLO`` locations to ``RoW`` if there are region-specific datasets in the activity group."""
     processed = []
     for key, datasets in toolz.groupby(activity_grouper, data).items():
+        if key[0].startswith("market group"):
+            processed.extend(datasets)
+            continue
         if len(datasets) > 1:
             check_single_global_dataset(datasets)