From e7a804b75f116f483ee282ed00af21911977c32d Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Tue, 13 Jun 2023 10:53:55 -0400 Subject: [PATCH 01/70] added RangeTree --- openmdao/utils/graph_utils.py | 217 ++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) diff --git a/openmdao/utils/graph_utils.py b/openmdao/utils/graph_utils.py index 9e52ec0c8c..f60d5849d7 100644 --- a/openmdao/utils/graph_utils.py +++ b/openmdao/utils/graph_utils.py @@ -23,3 +23,220 @@ def get_sccs_topo(graph): sccs = list(nx.strongly_connected_components(graph)) sccs.reverse() return sccs + + +class RangeTree(object): + """ + A binary search tree of ranges, mapping a name to an index range. + + Search complexity is O(log2 n). + + Parameters + ---------- + name : str + Name of the variable. + start : int + Starting index of the variable. + stop : int + Ending index of the variable. + + Attributes + ---------- + name : str + Name of the variable. + start : int + Starting index of the variable. + stop : int + Ending index of the variable. + left : RangeTree or None + Left child node. + right : RangeTree or None + Right child node. + """ + + __slots__ = ['name', 'start', 'stop', 'left', 'right'] + + def __init__(self, name, start, stop): + """ + Initialize a RangeTree. + """ + self.name = name + self.start = start + self.stop = stop + self.left = None + self.right = None + + def find_name(self, idx): + """ + Find the name corresponding to the given index. + + Parameters + ---------- + idx : int + The index into the full array. + + Returns + ------- + str or None + The name corresponding to the given index, or None if not found. + """ + if idx < self.start: + return None if self.left is None else self.left.find_name(idx) + + if idx >= self.stop: + return None if self.right is None else self.right.find_name(idx) + + return self.name + + def find_name_and_rel_ind(self, idx): + """ + Find the name and relative index corresponding to the matched range. + + Parameters + ---------- + idx : int + The index into the full array. + + Returns + ------- + str or None + The name corresponding to the matched range, or None if not found. + int or None + The relative index into the matched range, or None if not found. + """ + if idx < self.start: + return (None, None) if self.left is None else self.left.find_name_and_rel_ind(idx) + + if idx >= self.stop: + return (None, None) if self.right is None else self.right.find_name_and_rel_ind(idx) + + return (self.name, idx - self.start) + + @staticmethod + def build(ranges): + """ + Build a binary search tree to map indices to variable names. + + Parameters + ---------- + ranges : list of (name, start, stop) + List of (name, start, stop) tuples, where name is the variable name and start and stop + define the range of indices for that variable. + + Returns + ------- + RangeTree + Root node of the binary search tree. + """ + half = len(ranges) // 2 + name, start, stop = ranges[half] + + node = RangeTree(name, start, stop) + + left_slices = ranges[:half] + right_slices = ranges[half + 1:] + if left_slices: + node.left = RangeTree.build(left_slices) + if right_slices: + node.right = RangeTree.build(right_slices) + + return node + + +# if the total array size is less than this, we'll just use a flat list mapping +# indices to names instead of a binary search tree +_MAX_RANGE_SIZE = 10000 + + +def NameRangeMapper(ranges): + """ + Return a mapper that maps indices to variable names and relative indices. + + Parameters + ---------- + ranges : list of (name, start, stop) + Ordered list of (name, start, stop) tuples, where start and stop define the range of + indices for that name. + + Returns + ------- + FlatRangeMapper or RangeTree + A mapper that maps indices to variable names and relative indices. + """ + size = ranges[-1][2] - ranges[0][1] + if size < _MAX_RANGE_SIZE: + return FlatRangeMapper(ranges) + + return RangeTree.build(ranges) + + +class FlatRangeMapper(object): + """ + A flat list mapping indices to variable names and relative indices. + + Parameters + ---------- + ranges : list of (name, start, stop) + Ordered list of (name, start, stop) tuples, where start and stop define the range of + indices for that name. Ranges must be contiguous. + + Attributes + ---------- + size : int + Total size of all of the ranges combined. + ranges : list of (name, start, stop) + List of (name, start, stop) tuples, where start and stop define the range of + indices for that name. + """ + + def __init__(self, ranges): + """ + Initialize a FlatRangeMapper. + """ + self.size = ranges[-1][2] - ranges[0][1] + self.ranges = [None] * self.size + for rng in ranges: + _, start, stop = rng + self.ranges[start:stop] = [rng] * (stop - start) + + def find_name(self, idx): + """ + Find the name corresponding to the given index. + + Parameters + ---------- + idx : int + The index into the full array. + + Returns + ------- + str or None + The name corresponding to the given index, or None if not found. + """ + try: + return self.ranges[idx][0] + except IndexError: + return None + + def find_name_and_rel_ind(self, idx): + """ + Find the name and relative index corresponding to the matched range. + + Parameters + ---------- + idx : int + The index into the full array. + + Returns + ------- + str or None + The name corresponding to the matched range, or None if not found. + int or None + The relative index into the matched range, or None if not found. + """ + try: + name, start, _ = self.ranges[idx] + except IndexError: + return (None, None) + + return (name, idx - start) From c20ae91be76c939bea237d5ced73e2d60a9b37bc Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Sat, 17 Jun 2023 14:57:09 -0400 Subject: [PATCH 02/70] test --- openmdao/utils/graph_utils.py | 217 ---------------------------------- 1 file changed, 217 deletions(-) diff --git a/openmdao/utils/graph_utils.py b/openmdao/utils/graph_utils.py index f60d5849d7..9e52ec0c8c 100644 --- a/openmdao/utils/graph_utils.py +++ b/openmdao/utils/graph_utils.py @@ -23,220 +23,3 @@ def get_sccs_topo(graph): sccs = list(nx.strongly_connected_components(graph)) sccs.reverse() return sccs - - -class RangeTree(object): - """ - A binary search tree of ranges, mapping a name to an index range. - - Search complexity is O(log2 n). - - Parameters - ---------- - name : str - Name of the variable. - start : int - Starting index of the variable. - stop : int - Ending index of the variable. - - Attributes - ---------- - name : str - Name of the variable. - start : int - Starting index of the variable. - stop : int - Ending index of the variable. - left : RangeTree or None - Left child node. - right : RangeTree or None - Right child node. - """ - - __slots__ = ['name', 'start', 'stop', 'left', 'right'] - - def __init__(self, name, start, stop): - """ - Initialize a RangeTree. - """ - self.name = name - self.start = start - self.stop = stop - self.left = None - self.right = None - - def find_name(self, idx): - """ - Find the name corresponding to the given index. - - Parameters - ---------- - idx : int - The index into the full array. - - Returns - ------- - str or None - The name corresponding to the given index, or None if not found. - """ - if idx < self.start: - return None if self.left is None else self.left.find_name(idx) - - if idx >= self.stop: - return None if self.right is None else self.right.find_name(idx) - - return self.name - - def find_name_and_rel_ind(self, idx): - """ - Find the name and relative index corresponding to the matched range. - - Parameters - ---------- - idx : int - The index into the full array. - - Returns - ------- - str or None - The name corresponding to the matched range, or None if not found. - int or None - The relative index into the matched range, or None if not found. - """ - if idx < self.start: - return (None, None) if self.left is None else self.left.find_name_and_rel_ind(idx) - - if idx >= self.stop: - return (None, None) if self.right is None else self.right.find_name_and_rel_ind(idx) - - return (self.name, idx - self.start) - - @staticmethod - def build(ranges): - """ - Build a binary search tree to map indices to variable names. - - Parameters - ---------- - ranges : list of (name, start, stop) - List of (name, start, stop) tuples, where name is the variable name and start and stop - define the range of indices for that variable. - - Returns - ------- - RangeTree - Root node of the binary search tree. - """ - half = len(ranges) // 2 - name, start, stop = ranges[half] - - node = RangeTree(name, start, stop) - - left_slices = ranges[:half] - right_slices = ranges[half + 1:] - if left_slices: - node.left = RangeTree.build(left_slices) - if right_slices: - node.right = RangeTree.build(right_slices) - - return node - - -# if the total array size is less than this, we'll just use a flat list mapping -# indices to names instead of a binary search tree -_MAX_RANGE_SIZE = 10000 - - -def NameRangeMapper(ranges): - """ - Return a mapper that maps indices to variable names and relative indices. - - Parameters - ---------- - ranges : list of (name, start, stop) - Ordered list of (name, start, stop) tuples, where start and stop define the range of - indices for that name. - - Returns - ------- - FlatRangeMapper or RangeTree - A mapper that maps indices to variable names and relative indices. - """ - size = ranges[-1][2] - ranges[0][1] - if size < _MAX_RANGE_SIZE: - return FlatRangeMapper(ranges) - - return RangeTree.build(ranges) - - -class FlatRangeMapper(object): - """ - A flat list mapping indices to variable names and relative indices. - - Parameters - ---------- - ranges : list of (name, start, stop) - Ordered list of (name, start, stop) tuples, where start and stop define the range of - indices for that name. Ranges must be contiguous. - - Attributes - ---------- - size : int - Total size of all of the ranges combined. - ranges : list of (name, start, stop) - List of (name, start, stop) tuples, where start and stop define the range of - indices for that name. - """ - - def __init__(self, ranges): - """ - Initialize a FlatRangeMapper. - """ - self.size = ranges[-1][2] - ranges[0][1] - self.ranges = [None] * self.size - for rng in ranges: - _, start, stop = rng - self.ranges[start:stop] = [rng] * (stop - start) - - def find_name(self, idx): - """ - Find the name corresponding to the given index. - - Parameters - ---------- - idx : int - The index into the full array. - - Returns - ------- - str or None - The name corresponding to the given index, or None if not found. - """ - try: - return self.ranges[idx][0] - except IndexError: - return None - - def find_name_and_rel_ind(self, idx): - """ - Find the name and relative index corresponding to the matched range. - - Parameters - ---------- - idx : int - The index into the full array. - - Returns - ------- - str or None - The name corresponding to the matched range, or None if not found. - int or None - The relative index into the matched range, or None if not found. - """ - try: - name, start, _ = self.ranges[idx] - except IndexError: - return (None, None) - - return (name, idx - start) From 1655e4f07de2cf2f6f6886b6f4e4ccb1f58883e7 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 3 Jul 2023 09:44:28 -0400 Subject: [PATCH 03/70] added rangecollection --- openmdao/utils/range_collection.py | 352 +++++++++++++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 openmdao/utils/range_collection.py diff --git a/openmdao/utils/range_collection.py b/openmdao/utils/range_collection.py new file mode 100644 index 0000000000..7624786e17 --- /dev/null +++ b/openmdao/utils/range_collection.py @@ -0,0 +1,352 @@ + +from openmdao.utils.array_utils import shape_to_len + + +class RangeTree(object): + """ + A binary search tree of ranges, mapping a name to an index range. + + Allows for fast lookup of the name corresponding to a given index. The ranges must be + contiguous, but they can be of different sizes. + + Search complexity is O(log2 n). Better than FlatRangeMapper when total array size is large. + """ + def __init__(self, ranges): + """ + Initialize a RangeTree. + + Parameters + ---------- + ranges : list of (name, start, stop) + List of (name, start, stop) tuples, where name is the variable name and start and stop + define the range of indices for that variable. + """ + self.size = ranges[-1][2] - ranges[0][1] + self._rangedict = {} + self.root = RangeTree.build(ranges, self._rangedict) + + def get_range(self, name): + """ + Get the range corresponding to the given name. + + Parameters + ---------- + name : str + The name of the variable. + + Returns + ------- + tuple of (int, int) + The range of indices corresponding to the given name. + """ + return self._rangedict[name] + + def find_name(self, idx): + """ + Find the name corresponding to the given index. + + Parameters + ---------- + idx : int + The index into the full array. + + Returns + ------- + str or None + The name corresponding to the given index, or None if not found. + """ + node = self.root + while node is not None: + if idx < node.start: + node = node.left + elif idx >= node.stop: + node = node.right + else: + return node.name + + def find_name_and_rel_ind(self, idx): + """ + Find the name and relative index corresponding to the matched range. + + Parameters + ---------- + idx : int + The index into the full array. + + Returns + ------- + str or None + The name corresponding to the matched range, or None if not found. + int or None + The relative index into the matched range, or None if not found. + """ + node = self.root + while node is not None: + if idx < node.start: + node = node.left + elif idx >= node.stop: + node = node.right + else: + return node.name, idx - node.start + + return None, None + + @staticmethod + def build(ranges, rngdict): + """ + Build a binary search tree to map indices to variable names. + + Parameters + ---------- + ranges : list of (name, start, stop) + List of (name, start, stop) tuples, where name is the variable name and start and stop + define the range of indices for that variable. Ranges must be contiguous. + rngdict : dict + Dictionary to be populated with the ranges, keyed by name. + + Returns + ------- + RangeTreeNode + Root node of the binary search tree. + """ + half = len(ranges) // 2 + name, start, stop = ranges[half] + + node = RangeTreeNode(name, start, stop) + rngdict[name] = (start, stop) + + left_slices = ranges[:half] + if left_slices: + node.left = RangeTree.build(left_slices, rngdict) + + right_slices = ranges[half + 1:] + if right_slices: + node.right = RangeTree.build(right_slices, rngdict) + + return node + + def compute_transfer_inds(self, target_mapper, sources, targets): + """ + Compute the transfer indices for the given sources and targets. + + Parameters + ---------- + target_mapper : NameRangeMapper + The NameRangeMapper for the target side of the transfer. + sources : list of str + List of source names. + targets : list of str + List of target names. + + Returns + ------- + tuple of (dict, dict) + A tuple of (src_inds, tgt_inds), where src_inds is a dict mapping source names to + lists of indices and tgt_inds is a dict mapping target names to lists of indices. + """ + pass + + +class RangeTreeNode(object): + """ + A node in a binary search tree of ranges, mapping a name to an index range. + + Parameters + ---------- + name : str + Name of the variable. + start : int + Starting index of the variable. + stop : int + Ending index of the variable. + + Attributes + ---------- + name : str + Name of the variable. + start : int + Starting index of the variable. + stop : int + Ending index of the variable. + left : RangeTreeNode or None + Left child node. + right : RangeTreeNode or None + Right child node. + """ + + __slots__ = ['name', 'start', 'stop', 'left', 'right'] + + def __init__(self, name, start, stop): + """ + Initialize a RangeTreeNode. + """ + self.name = name + self.start = start + self.stop = stop + self.left = None + self.right = None + + +# if the total array size is less than this, we'll just use a flat list mapping +# indices to names instead of a binary search tree +_MAX_FLAT_RANGE_SIZE = 10000 + + +def metas2ranges(meta_iter, shape_name='shape'): + """ + Convert an iterator of metadata to an iterator of (name, start, stop) tuples. + + Parameters + ---------- + meta_iter : iterator of (name, meta) + Iterator of (name, meta) tuples, where name is the variable name and meta is the + corresponding metadata dictionary. + shape_name : str + Name of the metadata entry that contains the shape of the variable. Value can be either + 'shape' or 'global_shape'. Default is 'shape'. The value of the metadata entry must + be a tuple of integers. + + Yields + ------ + tuple + Tuple of the form (name, start, stop), where name is the variable name, start is the start + of the variable range, and stop is the end of the variable range. + """ + start = stop = 0 + for name, meta in meta_iter: + stop += shape_to_len(meta[shape_name]) + yield (name, start, stop) + start = stop + + +def metas2shapes(meta_iter, shape_name='shape'): + """ + Convert an iterator of metadata to an iterator of (name, shape) tuples. + + Parameters + ---------- + meta_iter : iterator of (name, meta) + Iterator of (name, meta) tuples, where name is the variable name and meta is the + corresponding metadata dictionary. + shape_name : str + Name of the metadata entry that contains the shape of the variable. Value can be either + 'shape' or 'global_shape'. Default is 'shape'. The value of the metadata entry must + be a tuple of integers. + """ + for name, meta in meta_iter: + yield (name, meta[shape_name]) + + +class FlatRangeMapper(object): + """ + A flat list mapping indices to variable names and relative indices. + + Parameters + ---------- + ranges : list of (name, start, stop) + Ordered list of (name, start, stop) tuples, where start and stop define the range of + indices for that name. Ranges must be contiguous. + + Attributes + ---------- + size : int + Total size of all of the ranges combined. + ranges : list of (name, start, stop) + List of (name, start, stop) tuples, where start and stop define the range of + indices for that name. Ranges must be contiguous. + """ + + def __init__(self, ranges): + """ + Initialize a FlatRangeMapper. + """ + self.size = ranges[-1][2] - ranges[0][1] + self.ranges = [None] * self.size + for rng in ranges: + _, start, stop = rng + self.ranges[start:stop] = [rng] * (stop - start) + + def find_name(self, idx): + """ + Find the name corresponding to the given index. + + Parameters + ---------- + idx : int + The index into the full array. + + Returns + ------- + str or None + The name corresponding to the given index, or None if not found. + """ + try: + return self.ranges[idx][0] + except IndexError: + return None + + def find_name_and_rel_ind(self, idx): + """ + Find the name and relative index corresponding to the matched range. + + Parameters + ---------- + idx : int + The index into the full array. + + Returns + ------- + str or None + The name corresponding to the matched range, or None if not found. + int or None + The relative index into the matched range, or None if not found. + """ + try: + name, start, _ = self.ranges[idx] + except IndexError: + return (None, None) + + return (name, idx - start) + + +def NameRangeMapper(ranges): + """ + Return a mapper that maps indices to variable names and relative indices. + + Parameters + ---------- + ranges : list of (name, start, stop) + Ordered list of (name, start, stop) tuples, where start and stop define the range of + indices for that name. Ranges must be contiguous. + + Returns + ------- + FlatRangeMapper or RangeTree + A mapper that maps indices to variable names and relative indices. + """ + size = ranges[-1][2] - ranges[0][1] + return FlatRangeMapper(ranges) if size < _MAX_FLAT_RANGE_SIZE else RangeTree(ranges) + + +if __name__ == '__main__': + meta = { + 'x': {'shape': (2, 3)}, + 'y': {'shape': (4, 5)}, + 'z': {'shape': (6,)}, + } + + print(list(metas2shapes(meta.items()))) + + ranges = list(metas2ranges(meta.items())) + print(ranges) + + rtree = RangeTree(ranges) + flat = FlatRangeMapper(ranges) + + for i in range(34): + rname, rind = rtree.find_name_and_rel_ind(i) + fname, find = flat.find_name_and_rel_ind(i) + assert rname == fname and rind == find, 'i = %d, rname = %s, rind = %s, fname = %s, find = %s' % (i, rname, rind, fname, find) + print(i, rname, rind, fname, find) + + From 791dee770298170236235cb650c2dfd81e49a0f9 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 3 Jul 2023 12:44:33 -0400 Subject: [PATCH 04/70] refactoring --- openmdao/utils/range_collection.py | 277 ++++++++++++++++------------- 1 file changed, 158 insertions(+), 119 deletions(-) diff --git a/openmdao/utils/range_collection.py b/openmdao/utils/range_collection.py index 7624786e17..739b73a888 100644 --- a/openmdao/utils/range_collection.py +++ b/openmdao/utils/range_collection.py @@ -2,30 +2,53 @@ from openmdao.utils.array_utils import shape_to_len -class RangeTree(object): - """ - A binary search tree of ranges, mapping a name to an index range. +# if the total array size is less than this, we'll just use a flat list mapping +# indices to names instead of a binary search tree +_MAX_FLAT_RANGE_SIZE = 10000 - Allows for fast lookup of the name corresponding to a given index. The ranges must be - contiguous, but they can be of different sizes. - Search complexity is O(log2 n). Better than FlatRangeMapper when total array size is large. - """ +class NameRangeMapper(object): def __init__(self, ranges): + self._name2range = {} + self._range2name = {} + self.size = ranges[-1][2] - ranges[0][1] + + @staticmethod + def create(ranges): """ - Initialize a RangeTree. + Return a mapper that maps indices to variable names and relative indices. Parameters ---------- ranges : list of (name, start, stop) - List of (name, start, stop) tuples, where name is the variable name and start and stop - define the range of indices for that variable. + Ordered list of (name, start, stop) tuples, where start and stop define the range of + indices for that name. Ranges must be contiguous. + + Returns + ------- + FlatRangeMapper or RangeTree + A mapper that maps indices to variable names and relative indices. """ - self.size = ranges[-1][2] - ranges[0][1] - self._rangedict = {} - self.root = RangeTree.build(ranges, self._rangedict) + size = ranges[-1][2] - ranges[0][1] + return FlatRangeMapper(ranges) if size < _MAX_FLAT_RANGE_SIZE else RangeTree(ranges) + + def add_range(self, name, start, stop): + """ + Add a range to the mapper. + + Parameters + ---------- + name : str + Name of the variable. + start : int + Starting index of the variable. + stop : int + Ending index of the variable. + """ + self._name2range[name] = (start, stop) + self._range2name[(start, stop)] = name - def get_range(self, name): + def name2range(self, name): """ Get the range corresponding to the given name. @@ -39,9 +62,74 @@ def get_range(self, name): tuple of (int, int) The range of indices corresponding to the given name. """ - return self._rangedict[name] - - def find_name(self, idx): + return self._name2range[name] + + def index2name(self, idx): + """ + Find the name corresponding to the given index. + + Parameters + ---------- + idx : int + The index into the full array. + + Returns + ------- + str or None + The name corresponding to the given index, or None if not found. + """ + raise NotImplementedError("index2name method must be implemented by subclass.") + + def index2names(self, idxs): + """ + Find the names corresponding to the given indices. + + Parameters + ---------- + idxs : list of int + The indices into the full array. + + Returns + ------- + list of str + The names corresponding to the given indices. + """ + names = {self.index2name(idx) for idx in idxs} + if None in names: + missing = [] + for idx in idxs: + if self.index2name(idx) is None: + missing.append(idx) + raise RuntimeError("Indices %s are not in any range." % sorted(missing)) + + return names + + +class RangeTree(NameRangeMapper): + """ + A binary search tree of ranges, mapping a name to an index range. + + Allows for fast lookup of the name corresponding to a given index. The ranges must be + contiguous, but they can be of different sizes. + + Search complexity is O(log2 n). Uses less memory than FlatRangeMapper when total array size is + large. + """ + def __init__(self, ranges): + """ + Initialize a RangeTree. + + Parameters + ---------- + ranges : list of (name, start, stop) + List of (name, start, stop) tuples, where name is the variable name and start and stop + define the range of indices for that variable. + """ + super().__init__(ranges) + self.size = ranges[-1][2] - ranges[0][1] + self.root = self.build(ranges) + + def index2name(self, idx): """ Find the name corresponding to the given index. @@ -63,8 +151,8 @@ def find_name(self, idx): node = node.right else: return node.name - - def find_name_and_rel_ind(self, idx): + + def index2name_and_rel_ind(self, idx): """ Find the name and relative index corresponding to the matched range. @@ -91,8 +179,7 @@ def find_name_and_rel_ind(self, idx): return None, None - @staticmethod - def build(ranges, rngdict): + def build(self, ranges): """ Build a binary search tree to map indices to variable names. @@ -101,8 +188,6 @@ def build(ranges, rngdict): ranges : list of (name, start, stop) List of (name, start, stop) tuples, where name is the variable name and start and stop define the range of indices for that variable. Ranges must be contiguous. - rngdict : dict - Dictionary to be populated with the ranges, keyed by name. Returns ------- @@ -113,41 +198,20 @@ def build(ranges, rngdict): name, start, stop = ranges[half] node = RangeTreeNode(name, start, stop) - rngdict[name] = (start, stop) + self.add_range(name, start, stop) left_slices = ranges[:half] if left_slices: - node.left = RangeTree.build(left_slices, rngdict) + node.left = self.build(left_slices) right_slices = ranges[half + 1:] if right_slices: - node.right = RangeTree.build(right_slices, rngdict) + node.right = self.build(right_slices) return node - def compute_transfer_inds(self, target_mapper, sources, targets): - """ - Compute the transfer indices for the given sources and targets. - Parameters - ---------- - target_mapper : NameRangeMapper - The NameRangeMapper for the target side of the transfer. - sources : list of str - List of source names. - targets : list of str - List of target names. - - Returns - ------- - tuple of (dict, dict) - A tuple of (src_inds, tgt_inds), where src_inds is a dict mapping source names to - lists of indices and tgt_inds is a dict mapping target names to lists of indices. - """ - pass - - -class RangeTreeNode(object): +class RangeTreeNode(NameRangeMapper): """ A node in a binary search tree of ranges, mapping a name to an index range. @@ -187,57 +251,7 @@ def __init__(self, name, start, stop): self.right = None -# if the total array size is less than this, we'll just use a flat list mapping -# indices to names instead of a binary search tree -_MAX_FLAT_RANGE_SIZE = 10000 - - -def metas2ranges(meta_iter, shape_name='shape'): - """ - Convert an iterator of metadata to an iterator of (name, start, stop) tuples. - - Parameters - ---------- - meta_iter : iterator of (name, meta) - Iterator of (name, meta) tuples, where name is the variable name and meta is the - corresponding metadata dictionary. - shape_name : str - Name of the metadata entry that contains the shape of the variable. Value can be either - 'shape' or 'global_shape'. Default is 'shape'. The value of the metadata entry must - be a tuple of integers. - - Yields - ------ - tuple - Tuple of the form (name, start, stop), where name is the variable name, start is the start - of the variable range, and stop is the end of the variable range. - """ - start = stop = 0 - for name, meta in meta_iter: - stop += shape_to_len(meta[shape_name]) - yield (name, start, stop) - start = stop - - -def metas2shapes(meta_iter, shape_name='shape'): - """ - Convert an iterator of metadata to an iterator of (name, shape) tuples. - - Parameters - ---------- - meta_iter : iterator of (name, meta) - Iterator of (name, meta) tuples, where name is the variable name and meta is the - corresponding metadata dictionary. - shape_name : str - Name of the metadata entry that contains the shape of the variable. Value can be either - 'shape' or 'global_shape'. Default is 'shape'. The value of the metadata entry must - be a tuple of integers. - """ - for name, meta in meta_iter: - yield (name, meta[shape_name]) - - -class FlatRangeMapper(object): +class FlatRangeMapper(NameRangeMapper): """ A flat list mapping indices to variable names and relative indices. @@ -260,13 +274,14 @@ def __init__(self, ranges): """ Initialize a FlatRangeMapper. """ - self.size = ranges[-1][2] - ranges[0][1] + super().__init__(ranges) self.ranges = [None] * self.size for rng in ranges: - _, start, stop = rng + name, start, stop = rng self.ranges[start:stop] = [rng] * (stop - start) + self.add_range(name, start, stop) - def find_name(self, idx): + def index2name(self, idx): """ Find the name corresponding to the given index. @@ -285,7 +300,7 @@ def find_name(self, idx): except IndexError: return None - def find_name_and_rel_ind(self, idx): + def index2name_and_rel_ind(self, idx): """ Find the name and relative index corresponding to the matched range. @@ -309,23 +324,49 @@ def find_name_and_rel_ind(self, idx): return (name, idx - start) -def NameRangeMapper(ranges): +def metas2ranges(meta_iter, shape_name='shape'): """ - Return a mapper that maps indices to variable names and relative indices. + Convert an iterator of metadata to an iterator of (name, start, stop) tuples. Parameters ---------- - ranges : list of (name, start, stop) - Ordered list of (name, start, stop) tuples, where start and stop define the range of - indices for that name. Ranges must be contiguous. + meta_iter : iterator of (name, meta) + Iterator of (name, meta) tuples, where name is the variable name and meta is the + corresponding metadata dictionary. + shape_name : str + Name of the metadata entry that contains the shape of the variable. Value can be either + 'shape' or 'global_shape'. Default is 'shape'. The value of the metadata entry must + be a tuple of integers. - Returns - ------- - FlatRangeMapper or RangeTree - A mapper that maps indices to variable names and relative indices. + Yields + ------ + tuple + Tuple of the form (name, start, stop), where name is the variable name, start is the start + of the variable range, and stop is the end of the variable range. """ - size = ranges[-1][2] - ranges[0][1] - return FlatRangeMapper(ranges) if size < _MAX_FLAT_RANGE_SIZE else RangeTree(ranges) + start = stop = 0 + for name, meta in meta_iter: + stop += shape_to_len(meta[shape_name]) + yield (name, start, stop) + start = stop + + +def metas2shapes(meta_iter, shape_name='shape'): + """ + Convert an iterator of metadata to an iterator of (name, shape) tuples. + + Parameters + ---------- + meta_iter : iterator of (name, meta) + Iterator of (name, meta) tuples, where name is the variable name and meta is the + corresponding metadata dictionary. + shape_name : str + Name of the metadata entry that contains the shape of the variable. Value can be either + 'shape' or 'global_shape'. Default is 'shape'. The value of the metadata entry must + be a tuple of integers. + """ + for name, meta in meta_iter: + yield (name, meta[shape_name]) if __name__ == '__main__': @@ -344,9 +385,7 @@ def NameRangeMapper(ranges): flat = FlatRangeMapper(ranges) for i in range(34): - rname, rind = rtree.find_name_and_rel_ind(i) - fname, find = flat.find_name_and_rel_ind(i) - assert rname == fname and rind == find, 'i = %d, rname = %s, rind = %s, fname = %s, find = %s' % (i, rname, rind, fname, find) + rname, rind = rtree.index2name_and_rel_ind(i) + fname, find = flat.index2name_and_rel_ind(i) + assert rname == fname and rind == find, f'i = {i}, rname = {rname}, rind = {rind}, fname = {fname}, find = {find}' print(i, rname, rind, fname, find) - - From c434beb3d4cfc4fdc6165efec89bd0fcb923670e Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 8 Sep 2023 16:12:59 -0400 Subject: [PATCH 05/70] progress but still distrib and par deriv coloring issues --- openmdao/core/explicitcomponent.py | 5 ++ openmdao/core/tests/test_deriv_transfers.py | 6 +- openmdao/core/tests/test_parallel_groups.py | 38 ++++--------- openmdao/core/total_jac.py | 28 ++++++---- openmdao/vectors/petsc_transfer.py | 61 +++++++++++++++++++-- 5 files changed, 89 insertions(+), 49 deletions(-) diff --git a/openmdao/core/explicitcomponent.py b/openmdao/core/explicitcomponent.py index cc43c433d5..8144d2612a 100644 --- a/openmdao/core/explicitcomponent.py +++ b/openmdao/core/explicitcomponent.py @@ -8,6 +8,7 @@ from openmdao.utils.class_util import overrides_method from openmdao.recorders.recording_iteration_stack import Recording from openmdao.core.constants import INT_DTYPE, _UNDEFINED +from openmdao.utils.mpi import MPI class ExplicitComponent(Component): @@ -391,6 +392,8 @@ def _apply_linear(self, jac, rel_systems, mode, scope_out=None, scope_in=None): # Jacobian and vectors are all scaled, unitless J._apply(self, d_inputs, d_outputs, d_residuals, mode) + # print('APPLY', self.pathname, 'd_inputs', d_inputs.asarray(), 'd_outputs', d_outputs.asarray(), 'd_residuals', d_residuals.asarray(), flush=True) + if not self.matrix_free: # if we're not matrix free, we can skip the rest because # compute_jacvec_product does nothing. @@ -475,6 +478,8 @@ def _solve_linear(self, mode, rel_systems, scope_out=_UNDEFINED, scope_in=_UNDEF # ExplicitComponent jacobian defined with -1 on diagonal. d_residuals *= -1.0 + # print(MPI.COMM_WORLD.rank, 'SOLVE', self.pathname, 'd_inputs', self._dinputs.asarray(), 'd_outputs', self._doutputs.asarray(), 'd_residuals', self._dresiduals.asarray(), flush=True) + def _compute_partials_wrapper(self): """ Call compute_partials based on the value of the "run_root_only" option. diff --git a/openmdao/core/tests/test_deriv_transfers.py b/openmdao/core/tests/test_deriv_transfers.py index ca348a5b99..22696cefc0 100644 --- a/openmdao/core/tests/test_deriv_transfers.py +++ b/openmdao/core/tests/test_deriv_transfers.py @@ -26,10 +26,8 @@ def _test_func_name(func, num, param): args = [] for p in param.args: - if isinstance(p, str): - p = {p} - elif not isinstance(p, Iterable): - p = {p} + if isinstance(p, str) or not isinstance(p, Iterable): + p = [p] for item in p: try: arg = item.__name__ diff --git a/openmdao/core/tests/test_parallel_groups.py b/openmdao/core/tests/test_parallel_groups.py index ae8ace75bd..4c1628b31b 100644 --- a/openmdao/core/tests/test_parallel_groups.py +++ b/openmdao/core/tests/test_parallel_groups.py @@ -40,8 +40,8 @@ def check_config(self, logger): def _test_func_name(func, num, param): args = [] for p in param.args: - if not isinstance(p, Iterable): - p = {p} + if isinstance(p, str) or not isinstance(p, Iterable): + p = [p] for item in p: try: arg = item.__name__ @@ -164,9 +164,10 @@ def test_fan_in_grouped_feature(self): assert_near_equal(prob['c3.y'], 29.0, 1e-6) @parameterized.expand(itertools.product([om.LinearRunOnce], - [om.NonlinearBlockGS, om.NonlinearRunOnce]), + [om.NonlinearBlockGS, om.NonlinearRunOnce], + ['fwd', 'rev']), name_func=_test_func_name) - def test_diamond(self, solver, nlsolver): + def test_diamond(self, solver, nlsolver, mode): prob = om.Problem() prob.model = Diamond() @@ -174,7 +175,7 @@ def test_diamond(self, solver, nlsolver): prob.model.linear_solver = solver() prob.model.nonlinear_solver = nlsolver() - prob.setup(check=False, mode='fwd') + prob.setup(check=False, mode=mode) prob.set_solver_print(level=0) prob.run_model() @@ -188,20 +189,11 @@ def test_diamond(self, solver, nlsolver): assert_near_equal(J['c4.y1', 'iv.x'][0][0], 25, 1e-6) assert_near_equal(J['c4.y2', 'iv.x'][0][0], -40.5, 1e-6) - prob.setup(check=False, mode='rev') - prob.run_model() - - assert_near_equal(prob['c4.y1'], 46.0, 1e-6) - assert_near_equal(prob['c4.y2'], -93.0, 1e-6) - - J = prob.compute_totals(of=unknown_list, wrt=indep_list) - assert_near_equal(J['c4.y1', 'iv.x'][0][0], 25, 1e-6) - assert_near_equal(J['c4.y2', 'iv.x'][0][0], -40.5, 1e-6) - @parameterized.expand(itertools.product([om.LinearRunOnce], - [om.NonlinearBlockGS, om.NonlinearRunOnce]), + [om.NonlinearBlockGS, om.NonlinearRunOnce], + ['fwd', 'rev']), name_func=_test_func_name) - def test_converge_diverge(self, solver, nlsolver): + def test_converge_diverge(self, solver, nlsolver, mode): prob = om.Problem() prob.model = ConvergeDiverge() @@ -209,7 +201,7 @@ def test_converge_diverge(self, solver, nlsolver): prob.model.linear_solver = solver() prob.model.nonlinear_solver = nlsolver() - prob.setup(check=False, mode='fwd') + prob.setup(check=False, mode=mode) prob.set_solver_print(level=0) prob.run_model() @@ -221,16 +213,6 @@ def test_converge_diverge(self, solver, nlsolver): J = prob.compute_totals(of=unknown_list, wrt=indep_list) assert_near_equal(J['c7.y1', 'iv.x'][0][0], -40.75, 1e-6) - prob.setup(check=False, mode='rev') - prob.run_model() - - assert_near_equal(prob['c7.y1'], -102.7, 1e-6) - - J = prob.compute_totals(of=unknown_list, wrt=indep_list) - assert_near_equal(J['c7.y1', 'iv.x'][0][0], -40.75, 1e-6) - - assert_near_equal(prob['c7.y1'], -102.7, 1e-6) - def test_zero_shape(self): raise unittest.SkipTest("zero shapes not fully supported yet") class MultComp(ExplicitComponent): diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index 93dbb98947..e6b4694e2b 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -463,7 +463,7 @@ def _compute_jac_scatters(self, mode, rowcol_size, get_remote): myrank = self.comm.rank if get_remote: myoffset = rowcol_size * myrank - elif not get_remote: # rev and not get_remote + else: # rev and not get_remote # reduce size of vector by not including distrib vars arr = np.ones(rowcol_size, dtype=bool) start = end = 0 @@ -739,7 +739,7 @@ def _create_in_idx_map(self, mode): if self.jac_dist_col_mask is None: ndups = 1 # we don't divide by ndups for distributed inputs else: - ndups = np.nonzero(sizes[:, in_var_idx])[0].size + ndups = np.count_nonzero(sizes[:, in_var_idx]) else: # if the var is not distributed, convert the indices to global. # We don't iterate over the full distributed size in this case. @@ -751,7 +751,9 @@ def _create_in_idx_map(self, mode): # find the number of duplicate components in rev mode so we can divide # the seed between 'ndups' procs so that at the end after we do an # Allreduce, the contributions from all procs will add up properly. - ndups = np.nonzero(sizes[:, in_var_idx])[0].size + ndups = np.count_nonzero(sizes[:, in_var_idx]) + + ndups = 1 # all local idxs that correspond to vars from other procs will be -1 # so each entry of loc_i will either contain a valid local index, @@ -1335,17 +1337,17 @@ def _jac_setter_dist(self, i, mode): ndups, _, _, _ = self.in_idx_map[mode][i] if self.get_remote: scratch = self.jac_scratch['rev'][0] - scratch[:] = self.J[i] + # scratch[:] = self.J[i] - self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) + # # self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) - if ndups > 1: - if self.jac_dist_col_mask is not None: - scratch[:] = 1.0 - scratch[self.jac_dist_col_mask] = (1.0 / ndups) - self.J[i] *= scratch - else: - self.J[i] *= (1.0 / ndups) + # # if ndups > 1: + # # if self.jac_dist_col_mask is not None: + # # scratch[:] = 1.0 + # # scratch[self.jac_dist_col_mask] = (1.0 / ndups) + # # self.J[i] *= scratch + # # else: + # # self.J[i] *= (1.0 / ndups) else: scatter = self.jac_scatters[mode] if scatter is not None: @@ -1583,6 +1585,8 @@ def compute_totals(self): else: model._solve_linear(mode, rel_systems) + # print('TOP SOLVE', 'd_inputs', model._dinputs.asarray(), 'd_outputs', model._doutputs.asarray(), 'd_residuals', model._dresiduals.asarray(), flush=True) + if debug_print: print(f'Elapsed Time: {time.perf_counter() - t0} secs\n', flush=True) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index d06988ce86..79dca06f12 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -95,10 +95,26 @@ def _setup_transfers(group): offsets_in = offsets['input'] offsets_out = offsets['output'] + def is_dup(name, io): + if group._var_allprocs_abs2meta[io][name]['distributed']: + return False # distributed vars are never dups + # if group.comm.rank == group._owning_rank[name]: + # return False # this is the owner, not a dup + return np.count_nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]]) > 1 + + def get_xfer_ranks(name, io): + if group._var_allprocs_abs2meta[io][name]['distributed']: + return [] + idx = allprocs_abs2idx[name] + sizes = group._var_sizes[io][:, idx] + return np.nonzero(sizes)[0] + # Loop through all connections owned by this system for abs_in, abs_out in group._conn_abs_in2out.items(): # Only continue if the input exists on this processor if abs_in in abs2meta_in: + inp_is_dup = is_dup(abs_in, 'input') + out_is_dup = is_dup(abs_out, 'output') # Get meta meta_in = abs2meta_in[abs_in] @@ -173,8 +189,26 @@ def _setup_transfers(group): fwd_xfer_out[sub_in].append(output_inds) if rev: sub_out = abs_out[mypathlen:].partition('.')[0] - rev_xfer_in[sub_out].append(input_inds) - rev_xfer_out[sub_out].append(output_inds) + if inp_is_dup and abs_out not in abs2meta_out: + # print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + rev_xfer_in[sub_out] + rev_xfer_out[sub_out] + elif not inp_is_dup and out_is_dup and myproc == group._owning_rank[abs_in]: + oidxlist = [] + iidxlist = [] + for rnk in get_xfer_ranks(abs_out, 'output'): + offset = offsets_out[rnk, idx_out] + oidxlist.append(np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE)) + iidxlist.append(input_inds) + input_inds = np.concatenate(iidxlist) + output_inds = np.concatenate(oidxlist) + # print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + rev_xfer_in[sub_out].append(input_inds) + rev_xfer_out[sub_out].append(output_inds) + else: + # print(group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + rev_xfer_in[sub_out].append(input_inds) + rev_xfer_out[sub_out].append(output_inds) else: # not a local input but still need entries in the transfer dicts to # avoid hangs @@ -207,20 +241,37 @@ def _setup_transfers(group): transfers['fwd'] = xfwd = {} xfwd[None] = xfer_all - if rev: - transfers['rev'] = xrev = {} - xrev[None] = xfer_all for sname, inds in fwd_xfer_in.items(): transfers['fwd'][sname] = PETScTransfer( vectors['input']['nonlinear'], vectors['output']['nonlinear'], inds, fwd_xfer_out[sname], group.comm) + if rev: + if rev_xfer_in: + xfer_in = np.concatenate(list(rev_xfer_in.values())) + xfer_out = np.concatenate(list(rev_xfer_out.values())) + else: + xfer_in = xfer_out = np.zeros(0, dtype=INT_DTYPE) + + # print(group.comm.rank, "MAKING rev global xfer", xfer_out, "-->", xfer_in, flush=True) + xfer_all = PETScTransfer(vectors['input']['nonlinear'], out_vec, + xfer_in, xfer_out, group.comm) + # print(group.comm.rank, "DONE MAKING rev global xfer", flush=True) + + # print("outputs:", list(group._outputs.keys()), flush=True) + # print("inputs:", list(group._inputs.keys()), flush=True) + + transfers['rev'] = xrev = {} + xrev[None] = xfer_all + for sname, inds in rev_xfer_out.items(): transfers['rev'][sname] = PETScTransfer( vectors['input']['nonlinear'], vectors['output']['nonlinear'], rev_xfer_in[sname], inds, group.comm) + # print(group.comm.rank, 'returning from setup_transfers', flush=True) + def _transfer(self, in_vec, out_vec, mode='fwd'): """ Perform transfer. From 43b093f700c3bb47268192ee7b35845f4a72a156 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Tue, 12 Sep 2023 14:41:03 -0400 Subject: [PATCH 06/70] progress --- openmdao/core/group.py | 22 +++++++ openmdao/core/system.py | 13 ++-- openmdao/core/tests/test_distrib_derivs.py | 56 +++++++++++++++--- openmdao/core/tests/test_driver.py | 11 +++- openmdao/core/tests/test_theory_rev_derivs.py | 4 +- openmdao/core/total_jac.py | 59 ++++++++++++------- openmdao/utils/array_utils.py | 22 +++++++ openmdao/utils/indexer.py | 38 +++++++++++- openmdao/vectors/petsc_transfer.py | 24 ++++++-- 9 files changed, 201 insertions(+), 48 deletions(-) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 3e1df8faf9..930999020c 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -1527,8 +1527,30 @@ def _set_auto_order(self, strongcomps, orders): "`allow_post_setup_reorder` to True or to manually set the execution " "order to the recommended order using `set_order`.") + def _check_nondist_sizes(self): + # verify that nondistributed variables have same size across all procs + for io in ('input', 'output'): + sizes = self._var_sizes[io] + idxs = self._var_allprocs_abs2idx + for abs_name, meta in self._var_allprocs_abs2meta[io].items(): + if not meta['distributed']: + vsizes = sizes[:, idxs[abs_name]] + unique = set(vsizes) + unique.discard(0) + if len(unique) > 1: + # sizes differ, now find which procs don't agree + rnklist = [] + for sz in unique: + rnklist.append((sz, [i for i, s in enumerate(vsizes) if s == sz])) + msg = ', '.join([f"rank(s) {r} have size {s}" for s, r in rnklist]) + self._collect_error(f"{self.msginfo}: Size of {io} '{abs_name}' " + f"differs between processes ({msg}).", + ident=('size', abs_name)) + def _top_level_post_sizes(self): # this runs after the variable sizes are known + self._check_nondist_sizes() + self._setup_global_shapes() self._resolve_ambiguous_input_meta() diff --git a/openmdao/core/system.py b/openmdao/core/system.py index 4ed7592d5e..7b2c583b4d 100644 --- a/openmdao/core/system.py +++ b/openmdao/core/system.py @@ -5476,10 +5476,10 @@ def _get_input_from_src(self, name, abs_ins, conns, units=None, indices=None, if vshape is not None: val = val.reshape(vshape) else: + var_idx = self._var_allprocs_abs2idx[src] + sizes = self._var_sizes['output'][:, var_idx] if distrib and (sdistrib or dynshape or not slocal) and not get_remote: - var_idx = self._var_allprocs_abs2idx[src] # sizes for src var in each proc - sizes = self._var_sizes['output'][:, var_idx] start = np.sum(sizes[:self.comm.rank]) end = start + sizes[self.comm.rank] src_indices = src_indices.shaped_array(copy=True) @@ -5498,7 +5498,12 @@ def _get_input_from_src(self, name, abs_ins, conns, units=None, indices=None, "`get_val(, get_remote=True)`.") else: if src_indices._flat_src: - val = val.ravel()[src_indices.flat()] + if distrib and not sdistrib: + start = np.sum(sizes[:self.comm.rank]) + sinds = src_indices.apply_offset(-start, flat=True) + else: + sinds = src_indices.flat() + val = val.ravel()[sinds] # if at component level, just keep shape of the target and don't flatten if not flat and not is_prom: shp = vmeta['shape'] @@ -5909,7 +5914,7 @@ def _get_full_dist_shape(self, abs_name, local_shape): else: io = 'input' scope = self - # io = 'output' if abs_name in self._var_allprocs_abs2meta['output'] else 'input' + meta = scope._var_allprocs_abs2meta[io][abs_name] var_idx = scope._var_allprocs_abs2idx[abs_name] global_size = np.sum(scope._var_sizes[io][:, var_idx]) diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index 4ff53d0a2a..0d49de8d19 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -77,8 +77,6 @@ def setup(self): allvars.update(v) sizes, offsets = evenly_distrib_idxs(comm.size, self.arr_size) - start = offsets[rank] - end = start + sizes[rank] for name in outs: if name not in kwargs or not isinstance(kwargs[name], dict): @@ -115,6 +113,40 @@ def compute(self, inputs, outputs): outputs['outvec'] = inputs['invec'] * 3.0 +class SimpleMixedDistrib2(om.ExplicitComponent): + + def setup(self): + self.add_input('in_dist', shape_by_conn=True, distributed=True) + self.add_input('in_nd', shape_by_conn=True) + self.add_output('out_dist', copy_shape='in_dist', distributed=True) + self.add_output('out_nd', copy_shape='in_nd') + + def compute(self, inputs, outputs): + outputs['out_nd'] = inputs['in_nd'] * 3. + outputs['out_dist'] = inputs['in_dist'] * 5. + + def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): + Id = inputs['in_dist'] + Is = inputs['in_nd'] + + if mode == 'fwd': + if 'out_dist' in d_outputs: + if 'in_dist' in d_inputs: + d_outputs['out_dist'] += 5. * d_inputs['in_dist'] + if 'out_nd' in d_outputs: + if 'in_nd' in d_inputs: + d_outputs['out_nd'] += 3. * d_inputs['in_nd'] + + else: + if 'out_dist' in d_outputs: + if 'in_dist' in d_inputs: + d_inputs['in_dist'] += 5. * d_outputs['out_dist'] + + if 'out_nd' in d_outputs: + if 'in_nd' in d_inputs: + d_inputs['in_nd'] += 3. * d_outputs['out_nd'] + + class MixedDistrib2(om.ExplicitComponent): # for double diamond case def setup(self): @@ -685,11 +717,11 @@ def test_distrib_voi_group_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - J = prob.check_totals(method='fd', show_only_incorrect=True) - assert_near_equal(J['sub.parab.f_xy', 'p.x']['abs error'].forward, 0.0, 1e-5) - assert_near_equal(J['sub.parab.f_xy', 'p.y']['abs error'].forward, 0.0, 1e-5) - assert_near_equal(J['sub.sum.f_sum', 'p.x']['abs error'].forward, 0.0, 1e-5) - assert_near_equal(J['sub.sum.f_sum', 'p.y']['abs error'].forward, 0.0, 1e-5) + data = prob.check_totals(method='fd', show_only_incorrect=True) + assert_near_equal(data['sub.parab.f_xy', 'p.x']['abs error'].forward, 0.0, 1e-5) + assert_near_equal(data['sub.parab.f_xy', 'p.y']['abs error'].forward, 0.0, 1e-5) + assert_near_equal(data['sub.sum.f_sum', 'p.x']['abs error'].forward, 0.0, 1e-5) + assert_near_equal(data['sub.sum.f_sum', 'p.y']['abs error'].forward, 0.0, 1e-5) # rev mode @@ -845,7 +877,7 @@ def compute(self, inputs, outputs): assert_near_equal(J['sum.f_sum', 'p.x']['abs error'][mode_idx[mode]], 0.0, 1e-14) assert_near_equal(J['sum.f_sum', 'p.y']['abs error'][mode_idx[mode]], 0.0, 1e-14) - def run_mixed_distrib2_prob(self, mode): + def run_mixed_distrib2_prob(self, mode, klass=MixedDistrib2): size = 5 comm = MPI.COMM_WORLD rank = comm.rank @@ -859,7 +891,7 @@ def run_mixed_distrib2_prob(self, mode): ivc.add_output('x_nd', np.zeros(size)) model.add_subsystem("indep", ivc) - model.add_subsystem("D1", MixedDistrib2()) + model.add_subsystem("D1", klass()) model.connect('indep.x_dist', 'D1.in_dist') model.connect('indep.x_nd', 'D1.in_nd') @@ -891,6 +923,12 @@ def test_distrib_mixeddistrib2_totals_rev(self): totals = prob.check_totals(show_only_incorrect=True, method='cs') assert_check_totals(totals) + def test_distrib_simplemixeddistrib2_totals_rev(self): + prob = self.run_mixed_distrib2_prob('rev', klass=SimpleMixedDistrib2) + + totals = prob.check_totals(show_only_incorrect=True, method='cs') + assert_check_totals(totals) + def test_distrib_mixeddistrib2_partials_rev(self): prob = self.run_mixed_distrib2_prob('rev') diff --git a/openmdao/core/tests/test_driver.py b/openmdao/core/tests/test_driver.py index ae8eff16ba..b59e16cb82 100644 --- a/openmdao/core/tests/test_driver.py +++ b/openmdao/core/tests/test_driver.py @@ -931,6 +931,11 @@ def compute(self, inputs, outputs): outputs['y'] = np.sum(y_g) + (inputs['w']-10)**2 outputs['z'] = x**2 + print('-------------') + print(self.comm.rank, 'x', x) + print(self.comm.rank, 'w', inputs['w']) + print(self.comm.rank, 'y', outputs['y']) + print(self.comm.rank, 'z', outputs['z']) def compute_partials(self, inputs, J): x = inputs['x'] @@ -947,10 +952,10 @@ def compute_partials(self, inputs, J): promotes=['*']) d_ivc.add_output('x', 2*np.ones(size)) - # non-distributed indep var, 'w' + # distributed indep var, 'w' ivc = model.add_subsystem('ivc', om.IndepVarComp(distributed=False), promotes=['*']) - ivc.add_output('w', size) + ivc.add_output('w') # distributed component, 'dc' model.add_subsystem('dc', DistribComp(), promotes=['*']) @@ -994,8 +999,8 @@ def compute_partials(self, inputs, J): # run driver p.run_driver() - assert_near_equal(p.get_val('dc.y', get_remote=True), [81, 96]) assert_near_equal(p.get_val('dc.z', get_remote=True), [25, 25, 25, 81, 81]) + assert_near_equal(p.get_val('dc.y', get_remote=True), [81, 96]) def test_distrib_desvar_bug(self): class MiniModel(om.Group): diff --git a/openmdao/core/tests/test_theory_rev_derivs.py b/openmdao/core/tests/test_theory_rev_derivs.py index 5bed9f380d..460ee494bc 100644 --- a/openmdao/core/tests/test_theory_rev_derivs.py +++ b/openmdao/core/tests/test_theory_rev_derivs.py @@ -111,8 +111,8 @@ def test_theory_example(self): all_dinputs = model.comm.allgather(model._dinputs.asarray()) - assert_near_equal(all_dinputs[0], np.array([16., 8., 2., 3.])) - assert_near_equal(all_dinputs[1], np.array([36., 18., 2., 3.])) + assert_near_equal(all_dinputs[0], np.array([26., 4., 2., 3.])) + assert_near_equal(all_dinputs[1], np.array([26., 9., 2., 3.])) if __name__ == "__main__": diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index e6b4694e2b..ba80967e44 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -371,7 +371,7 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, if self.has_output_dist['rev']: sizes = model._var_sizes['output'] abs2idx = model._var_allprocs_abs2idx - self.jac_dist_col_mask = mask = np.ones(J.shape[1], dtype=bool) + self.jac_dist_col_mask = mask = np.zeros(J.shape[1], dtype=bool) start = end = 0 for name in self.wrt: meta = abs2meta_out[name] @@ -381,7 +381,7 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, # remote vars, which are zero everywhere except for one proc sz = sizes[:, abs2idx[name]] if np.count_nonzero(sz) > 1: - mask[start:end] = False + mask[start:end] = True start = end if not approx: @@ -736,22 +736,23 @@ def _create_in_idx_map(self, mode): dist = in_var_meta['distributed'] if dist: - if self.jac_dist_col_mask is None: - ndups = 1 # we don't divide by ndups for distributed inputs - else: - ndups = np.count_nonzero(sizes[:, in_var_idx]) + pass + # if self.jac_dist_col_mask is None: + # ndups = 1 # we don't divide by ndups for distributed inputs + # else: + # ndups = np.count_nonzero(sizes[:, in_var_idx]) else: # if the var is not distributed, convert the indices to global. # We don't iterate over the full distributed size in this case. irange += gstart - if fwd or parallel_deriv_color: - ndups = 1 - else: - # find the number of duplicate components in rev mode so we can divide - # the seed between 'ndups' procs so that at the end after we do an - # Allreduce, the contributions from all procs will add up properly. - ndups = np.count_nonzero(sizes[:, in_var_idx]) + # if fwd or parallel_deriv_color: + # ndups = 1 + # else: + # # find the number of duplicate components in rev mode so we can divide + # # the seed between 'ndups' procs so that at the end after we do an + # # Allreduce, the contributions from all procs will add up properly. + # ndups = np.count_nonzero(sizes[:, in_var_idx]) ndups = 1 @@ -805,10 +806,12 @@ def _create_in_idx_map(self, mode): relsystems = relevant[path]['@all'][1] if self.total_relevant_systems is not _contains_all: self.total_relevant_systems.update(relsystems) - tup = (ndups, relsystems, cache_lin_sol, name) + # tup = (ndups, relsystems, cache_lin_sol, name) + tup = (dist, relsystems, cache_lin_sol, name) else: self.total_relevant_systems = _contains_all - tup = (ndups, _contains_all, cache_lin_sol, name) + # tup = (ndups, _contains_all, cache_lin_sol, name) + tup = (dist, _contains_all, cache_lin_sol, name) idx_map.extend([tup] * (end - start)) start = end @@ -1334,9 +1337,21 @@ def _jac_setter_dist(self, i, mode): elif mode == 'rev': # for rows corresponding to serial 'of' vars, we need to correct for # duplication of their seed values by dividing by the number of duplications. - ndups, _, _, _ = self.in_idx_map[mode][i] + # ndups, _, _, _ = self.in_idx_map[mode][i] + dist, _, _, _ = self.in_idx_map[mode][i] if self.get_remote: - scratch = self.jac_scratch['rev'][0] + if self.jac_dist_col_mask is not None: + distpart = self.J[i, :][self.jac_dist_col_mask] + scratch = np.zeros(distpart.shape, dtype=distpart.dtype) + # only sum up the distrib parts + self.comm.Allreduce(distpart, scratch, op=MPI.SUM) + self.J[i][self.jac_dist_col_mask] = scratch + elif dist: + scratch = self.jac_scratch['rev'][0] + scratch[:] = self.J[i] + self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) + + # scratch = self.jac_scratch['rev'][0] # scratch[:] = self.J[i] # # self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) @@ -1363,11 +1378,11 @@ def _jac_setter_dist(self, i, mode): scatter.scatter(self.src_petsc[mode], self.tgt_petsc[mode], addv=True, mode=False) if loc >= 0: - if ndups > 1: - self.J[loc, :][self.nondist_loc_map[mode]] = \ - self.tgt_petsc[mode].array * (1.0 / ndups) - else: - self.J[loc, :][self.nondist_loc_map[mode]] = self.tgt_petsc[mode].array + # if ndups > 1: + # self.J[loc, :][self.nondist_loc_map[mode]] = \ + # self.tgt_petsc[mode].array * (1.0 / ndups) + # else: + self.J[loc, :][self.nondist_loc_map[mode]] = self.tgt_petsc[mode].array def single_jac_setter(self, i, mode, meta): """ diff --git a/openmdao/utils/array_utils.py b/openmdao/utils/array_utils.py index 89ab4eaa92..18640d81b7 100644 --- a/openmdao/utils/array_utils.py +++ b/openmdao/utils/array_utils.py @@ -368,6 +368,28 @@ def _global2local_offsets(global_offsets): return offsets +def dist2local_src_inds(src_indices, sizes, rank): + """ + Given existing distributed src_indices, return a localized index array. + + Parameters + ---------- + src_indices : ndarray + Array of global src_indices. + sizes : ndarray + Array of sizes for a variable across procs. + rank : int + MPI rank of the current process. + + Returns + ------- + ndarray + Array of local src_indices. + """ + start = np.sum(sizes[:rank]) + return src_indices - start + + def get_input_idx_split(full_idxs, inputs, outputs, use_full_cols, is_total): """ Split an array of indices into vec outs + ins into two arrays of indices into outs and ins. diff --git a/openmdao/utils/indexer.py b/openmdao/utils/indexer.py index 92ed608221..14ea27d7ea 100644 --- a/openmdao/utils/indexer.py +++ b/openmdao/utils/indexer.py @@ -271,8 +271,6 @@ def set_src_shape(self, shape, dist_shape=None): raise self._shaped_inst = None - self._src_shape = sshape - return self def to_json(self): @@ -345,6 +343,24 @@ def __str__(self): """ return f"{self._idx}" + def apply_offset(self, offset, flat=True): + """ + Apply an offset to this index. + + Parameters + ---------- + offset : int + The offset to apply. + flat : bool + If True, return a flat index. + + Returns + ------- + int + The offset index. + """ + return self._idx + offset + def copy(self): """ Copy this Indexer. @@ -520,6 +536,24 @@ def __str__(self): """ return f"{self._slice}" + def apply_offset(self, offset, flat=True): + """ + Apply an offset to this index. + + Parameters + ---------- + offset : int + The offset to apply. + flat : bool + If True, return a flat index. + + Returns + ------- + slice + The offset slice. + """ + return slice(self._slice.start + offset, self._slice.stop + offset, self._slice.step) + def copy(self): """ Copy this Indexer. diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 79dca06f12..7f088e09be 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -188,25 +188,37 @@ def get_xfer_ranks(name, io): fwd_xfer_in[sub_in].append(input_inds) fwd_xfer_out[sub_in].append(output_inds) if rev: + iowninput = myproc == group._owning_rank[abs_in] + distrib_in = meta_in['distributed'] + distrib_out = meta_out['distributed'] sub_out = abs_out[mypathlen:].partition('.')[0] if inp_is_dup and abs_out not in abs2meta_out: - # print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out] rev_xfer_out[sub_out] - elif not inp_is_dup and out_is_dup and myproc == group._owning_rank[abs_in]: + elif inp_is_dup and distrib_out and not iowninput: + print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + rev_xfer_in[sub_out] + rev_xfer_out[sub_out] + elif out_is_dup and not inp_is_dup and (iowninput or distrib_in): oidxlist = [] iidxlist = [] for rnk in get_xfer_ranks(abs_out, 'output'): offset = offsets_out[rnk, idx_out] - oidxlist.append(np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE)) - iidxlist.append(input_inds) + if src_indices is None: + oidxlist.append(np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE)) + iidxlist.append(input_inds) + elif src_indices.size > 0: + offset -= np.sum(sizes_out[:rnk, idx_out]) + oidxlist.append(np.asarray(src_indices + offset, dtype=INT_DTYPE)) + iidxlist.append(input_inds) input_inds = np.concatenate(iidxlist) output_inds = np.concatenate(oidxlist) - # print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) else: - # print(group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + print(group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) else: From 255a9a923a999416aa485a926b9f4cc867f10fe6 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Wed, 13 Sep 2023 14:20:20 -0400 Subject: [PATCH 07/70] issues with distrib and par derivs --- openmdao/core/group.py | 44 ++++++---- openmdao/core/problem.py | 2 + openmdao/core/system.py | 9 ++- openmdao/core/tests/test_driver.py | 2 +- .../core/tests/test_parallel_derivatives.py | 80 ++++++++----------- openmdao/core/total_jac.py | 14 +++- openmdao/utils/indexer.py | 56 +++++++++++++ openmdao/vectors/default_transfer.py | 8 +- openmdao/vectors/petsc_transfer.py | 77 +++++++++++++----- 9 files changed, 198 insertions(+), 94 deletions(-) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 930999020c..4c60f74dbd 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -186,10 +186,10 @@ class Group(System): Sorted list of pathnames of components that are executed prior to the optimization loop. _post_components : list of str or None Sorted list of pathnames of components that are executed after the optimization loop. - _abs_desvars : set - Set of absolute design variable names. - _abs_responses : set - Set of absolute response names. + _abs_desvars : dict or None + Dict of absolute design variable metadata. + _abs_responses : dict or None + Dict of absolute response metadata. _relevance_graph : nx.DiGraph Graph of relevance connections. Always None except in the top level Group. """ @@ -787,15 +787,13 @@ def _init_relevance(self, mode): dict The relevance dictionary. """ - abs_desvars = self.get_design_vars(recurse=True, get_sizes=False, use_prom_ivc=False) - abs_responses = self.get_responses(recurse=True, get_sizes=False, use_prom_ivc=False) - self._abs_desvars = set(_src_name_iter(abs_desvars)) - self._abs_responses = set(_src_name_iter(abs_responses)) + self._abs_desvars = self.get_design_vars(recurse=True, get_sizes=False, use_prom_ivc=False) + self._abs_responses = self.get_responses(recurse=True, get_sizes=False, use_prom_ivc=False) assert self.pathname == '', "Relevance can only be initialized on the top level System." if self._use_derivatives: - return self.get_relevant_vars(abs_desvars, - self._check_alias_overlaps(abs_responses), mode) + return self.get_relevant_vars(self._abs_desvars, + self._check_alias_overlaps(self._abs_responses), mode) return {'@all': ({'input': ContainsAll(), 'output': ContainsAll()}, ContainsAll())} @@ -1272,7 +1270,7 @@ def _final_setup(self, comm, mode): self._setup_vectors(self._get_root_vectors()) # Transfers do not require recursion, but they have to be set up after the vector setup. - self._setup_transfers() + self._setup_transfers(self._abs_desvars, self._abs_responses) # Same situation with solvers, partials, and Jacobians. # If we're updating, we just need to re-run setup on these, but no recursion necessary. @@ -3131,10 +3129,17 @@ def _transfer(self, vec_name, mode, sub=None): if xfer is not None: if self._has_input_scaling: vec_inputs.scale_to_norm(mode='rev') - xfer._transfer(vec_inputs, self._vectors['output'][vec_name], mode) + + xfer._transfer(vec_inputs, self._vectors['output'][vec_name], mode) + + if self._problem_meta['parallel_deriv_color'] is None: + key = (sub, 'nocolor') + if key in self._transfers['rev']: + xfer = self._transfers['rev'][key] + xfer._transfer(vec_inputs, self._vectors['output'][vec_name], mode) + + if self._has_input_scaling: vec_inputs.scale_to_phys(mode='rev') - else: - xfer._transfer(vec_inputs, self._vectors['output'][vec_name], mode) def _discrete_transfer(self, sub): """ @@ -3195,11 +3200,18 @@ def _discrete_transfer(self, sub): src_val = src_sys._discrete_outputs[src] tgt_sys._discrete_inputs[tgt] = src_val - def _setup_transfers(self): + def _setup_transfers(self, desvars, responses): """ Compute all transfers that are owned by this system. + + Parameters + ---------- + desvars : dict + Dictionary of all design variable metadata. Keyed by absolute source name or alias. + responses : dict + Dictionary of all response variable metadata. Keyed by absolute source name or alias. """ - self._vector_class.TRANSFER._setup_transfers(self) + self._vector_class.TRANSFER._setup_transfers(self, desvars, responses) if self._conn_discrete_in2out: self._vector_class.TRANSFER._setup_discrete_transfers(self) diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index e9d5af7644..b211de5de2 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -1003,6 +1003,8 @@ def setup(self, check=False, logger=None, mode='auto', force_alloc_complex=False 'model_options': self.model_options, # A dict of options passed to all systems in tree 'allow_post_setup_reorder': self.options['allow_post_setup_reorder'], # see option 'singular_jac_behavior': 'warn', # How to handle singular jac conditions + 'parallel_deriv_color': None, # None unless derivatives involving a parallel deriv + # colored dv/response are currently being computed } if _prob_setup_stack: diff --git a/openmdao/core/system.py b/openmdao/core/system.py index 7b2c583b4d..dbdf271384 100644 --- a/openmdao/core/system.py +++ b/openmdao/core/system.py @@ -2236,9 +2236,16 @@ def _setup_vectors(self, root_vectors): subsys._scale_factors = self._scale_factors subsys._setup_vectors(root_vectors) - def _setup_transfers(self): + def _setup_transfers(self, desvars, responses): """ Compute all transfers that are owned by this system. + + Parameters + ---------- + desvars : dict + Dictionary of all design variable metadata. Keyed by absolute source name or alias. + responses : dict + Dictionary of all response variable metadata. Keyed by absolute source name or alias. """ pass diff --git a/openmdao/core/tests/test_driver.py b/openmdao/core/tests/test_driver.py index b59e16cb82..dfed5826bc 100644 --- a/openmdao/core/tests/test_driver.py +++ b/openmdao/core/tests/test_driver.py @@ -999,8 +999,8 @@ def compute_partials(self, inputs, J): # run driver p.run_driver() + assert_near_equal(p.get_val('dc.y', get_remote=True), [113, 113]) assert_near_equal(p.get_val('dc.z', get_remote=True), [25, 25, 25, 81, 81]) - assert_near_equal(p.get_val('dc.y', get_remote=True), [81, 96]) def test_distrib_desvar_bug(self): class MiniModel(om.Group): diff --git a/openmdao/core/tests/test_parallel_derivatives.py b/openmdao/core/tests/test_parallel_derivatives.py index cdafeba53c..94598a1c58 100644 --- a/openmdao/core/tests/test_parallel_derivatives.py +++ b/openmdao/core/tests/test_parallel_derivatives.py @@ -854,69 +854,53 @@ class TestAutoIVCParDerivBug(unittest.TestCase): N_PROCS = 4 def test_auto_ivc_par_deriv_bug(self): - class SimpleAero(om.ExplicitComponent): - """Simple aerodynamic model""" - - def initialize(self): - self.options.declare( "CLwing", default=0.5, desc="Wing lift factor", ) - self.options.declare( "CLtail", default=0.25, desc="tail lift factor", ) - self.options.declare( "CDwing", default=0.05, desc="Wing drag factor", ) - self.options.declare( "CDtail", default=0.025, desc="tail drag factor", ) + class Simple(om.ExplicitComponent): + def __init__(self, mult1, mult2, mult3, mult4, **kwargs): + super().__init__(**kwargs) + self.mult1 = mult1 + self.mult2 = mult2 + self.mult3 = mult3 + self.mult4 = mult4 def setup(self): - # Inputs - self.add_input('alpha', 0.1, desc="Angle of attack of wing") - self.add_input('tail_angle', 0.01, desc="Angle of attack of tail") - - # Outputs - self.add_output('L', 0.0, desc="Total lift") - self.add_output('D', 0.0, desc="Total drag") + self.add_input('x1', 0.1) + self.add_input('x2', 0.01) - # Set options - self.CLwing = self.options["CLwing"] - self.CDwing = self.options["CDwing"] - self.CLtail = self.options["CLtail"] - self.CDtail = self.options["CDtail"] + self.add_output('y1', 0.0) + self.add_output('y2', 0.0) self.declare_partials(of='*', wrt='*') def compute(self, inputs, outputs): - """ A simple surrogate for a 2 dof aero problem""" - outputs['L'] = self.CLwing * inputs['alpha'] + self.CLtail * inputs['tail_angle'] - outputs['D'] = self.CDwing * inputs['alpha'] ** 2 + self.CDtail * inputs['tail_angle'] ** 2 + outputs['y1'] = self.mult1 * inputs['x1'] + self.mult3 * inputs['x2'] + outputs['y2'] = self.mult2 * inputs['x1'] ** 2 + self.mult4 * inputs['x2'] ** 2 def compute_partials(self, inputs, partials): - """Analytical derivatives""" - partials['L', 'alpha'] = self.CLwing - partials['L', 'tail_angle'] = self.CLtail - partials['D', 'alpha'] = 2 * self.CDwing * inputs['alpha'] - partials['D', 'tail_angle'] = 2 * self.CDtail * inputs['tail_angle'] - - # Use parallel group to solve flight conditions simultaneously - flight_conds = om.ParallelGroup() - flight_conds.add_subsystem("cruise", SimpleAero(CLwing=0.5, CDwing=0.05, CLtail=0.25, CDtail=0.025)) - flight_conds.add_subsystem("maneuver", SimpleAero(CLwing=0.75, CDwing=0.25, CLtail=0.45, CDtail=0.15)) - - # build the model + partials['y1', 'x1'] = self.mult1 + partials['y1', 'x2'] = self.mult3 + partials['y2', 'x1'] = 2 * self.mult2 * inputs['x1'] + partials['y2', 'x2'] = 2 * self.mult4 * inputs['x2'] + prob = om.Problem() - prob.model.add_subsystem('flight_conditions', flight_conds) - - # Set dvs for each flight condition - prob.model.add_design_var('flight_conditions.cruise.alpha', lower=-50, upper=50) - prob.model.add_design_var('flight_conditions.cruise.tail_angle', lower=-50, upper=50) - prob.model.add_design_var('flight_conditions.maneuver.alpha', lower=-50, upper=50) - prob.model.add_design_var('flight_conditions.maneuver.tail_angle', lower=-50, upper=50) - # Use the parallel derivative option to solve lift constraint simultaneously - prob.model.add_constraint('flight_conditions.cruise.L', equals=1.0, parallel_deriv_color="lift") - prob.model.add_constraint('flight_conditions.maneuver.L', equals=2.5, parallel_deriv_color="lift") - # Set cruise drag as objective - prob.model.add_objective('flight_conditions.cruise.D') + par = prob.model.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem("C1", Simple(mult1=0.5, mult2=0.15, mult3=0.25, mult4=0.35)) + par.add_subsystem("C2", Simple(mult1=0.75, mult2=0.65, mult3=0.45, mult4=0.15)) + + prob.model.add_design_var('par.C1.x1', lower=-50, upper=50) + prob.model.add_design_var('par.C1.x2', lower=-50, upper=50) + prob.model.add_design_var('par.C2.x1', lower=-50, upper=50) + prob.model.add_design_var('par.C2.x2', lower=-50, upper=50) + + # Use the parallel derivative option to solve constraint simultaneously + prob.model.add_constraint('par.C1.y1', equals=1.0, parallel_deriv_color="pd1") + prob.model.add_constraint('par.C2.y1', equals=2.5, parallel_deriv_color="pd1") + prob.model.add_objective('par.C1.y2') prob.setup(mode='rev', force_alloc_complex=True) prob.run_model() - assert_check_totals(prob.check_totals(method='cs', out_stream=None)) + assert_check_totals(prob.check_totals(method='cs', show_only_incorrect=True))#, out_stream=None)) if __name__ == "__main__": diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index ba80967e44..5a8f2cefdd 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -91,7 +91,7 @@ class _TotalJacInfo(object): (local indices, local sizes). in_idx_map : dict Mapping of jacobian row/col index to a tuple of the form - (ndups, relevant_systems, cache_linear_solutions_flag) + (dist, relevant_systems, cache_linear_solutions_flag) total_relevant_systems : set The set of names of all systems relevant to the computation of the total derivatives. directional : bool @@ -669,6 +669,8 @@ def _create_in_idx_map(self, mode): non_rel_outs = False for name in input_list: + parallel_deriv_color = None + if name in self.responses and self.responses[name]['alias'] is not None: path = self.responses[name]['source'] else: @@ -694,7 +696,7 @@ def _create_in_idx_map(self, mode): cache_lin_sol = meta['cache_linear_solution'] _check_voi_meta(name, parallel_deriv_color, simul_coloring) - if parallel_deriv_color: + if parallel_deriv_color is not None: if parallel_deriv_color not in self.par_deriv_printnames: self.par_deriv_printnames[parallel_deriv_color] = [] @@ -715,7 +717,7 @@ def _create_in_idx_map(self, mode): else: # name is not a design var or response (should only happen during testing) end += in_var_meta['global_size'] irange = np.arange(in_var_meta['global_size'], dtype=INT_DTYPE) - in_idxs = parallel_deriv_color = None + in_idxs = None cache_lin_sol = False in_var_idx = abs2idx[path] @@ -726,7 +728,7 @@ def _create_in_idx_map(self, mode): # if we're doing parallel deriv coloring, we only want to set the seed on one proc # for each var in a given color - if parallel_deriv_color: + if parallel_deriv_color is not None: if fwd: relev = relevant[name]['@all'][0]['output'] else: @@ -1240,6 +1242,8 @@ def par_deriv_input_setter(self, inds, imeta, mode): dist = self.comm.size > 1 + self.model._problem_meta['parallel_deriv_color'] = imeta['par_deriv_color'] + for i in inds: if not dist or self.in_loc_idxs[mode][i] >= 0: rel_systems, vnames, _ = self.single_input_setter(i, imeta, mode) @@ -1442,6 +1446,8 @@ def par_deriv_jac_setter(self, inds, mode, meta): for i in inds: self.simple_single_jac_scatter(i, mode) + self.model._problem_meta['parallel_deriv_color'] = None + def simul_coloring_jac_setter(self, inds, mode, meta): """ Set the appropriate part of the total jacobian for simul coloring input indices. diff --git a/openmdao/utils/indexer.py b/openmdao/utils/indexer.py index 14ea27d7ea..e4225b8f28 100644 --- a/openmdao/utils/indexer.py +++ b/openmdao/utils/indexer.py @@ -786,6 +786,24 @@ def __str__(self): """ return _truncate(f"{self._arr}".replace('\n', '')) + def apply_offset(self, offset, flat=True): + """ + Apply an offset to this index. + + Parameters + ---------- + offset : int + The offset to apply. + flat : bool + If True, return a flat index. + + Returns + ------- + slice + The offset slice. + """ + return self.as_array(flat=flat) + offset + def copy(self): """ Copy this Indexer. @@ -991,6 +1009,26 @@ def __str__(self): """ return str(self._tup) + def apply_offset(self, offset, flat=True): + """ + Apply an offset to this index. + + Parameters + ---------- + offset : int + The offset to apply. + flat : bool + If True, return a flat index. + + Returns + ------- + ndarray + The offset array. + """ + if flat: + return self.flat() + offset + return self.as_array(flat=False) + offset + def copy(self): """ Copy this Indexer. @@ -1203,6 +1241,24 @@ def __str__(self): """ return f"{self._tup}" + def apply_offset(self, offset, flat=True): + """ + Apply an offset to this index. + + Parameters + ---------- + offset : int + The offset to apply. + flat : bool + If True, return a flat index. + + Returns + ------- + ndarray + The offset array. + """ + return self.as_array(flat=flat) + offset + def copy(self): """ Copy this Indexer. diff --git a/openmdao/vectors/default_transfer.py b/openmdao/vectors/default_transfer.py index c815e3f283..cf58264698 100644 --- a/openmdao/vectors/default_transfer.py +++ b/openmdao/vectors/default_transfer.py @@ -38,7 +38,7 @@ class DefaultTransfer(Transfer): """ @staticmethod - def _setup_transfers(group): + def _setup_transfers(group, desvars, responses): """ Compute all transfers that are owned by our parent group. @@ -46,12 +46,16 @@ def _setup_transfers(group): ---------- group : Parent group. + desvars : dict + Dictionary of all design variable metadata. Keyed by absolute source name or alias. + responses : dict + Dictionary of all response variable metadata. Keyed by absolute source name or alias. """ iproc = group.comm.rank rev = group._mode == 'rev' or group._mode == 'auto' for subsys in group._subgroups_myproc: - subsys._setup_transfers() + subsys._setup_transfers(desvars, responses) abs2meta = group._var_abs2meta diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 7f088e09be..cfb280eec9 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -56,7 +56,7 @@ def __init__(self, in_vec, out_vec, in_inds, out_inds, comm): in_indexset).scatter @staticmethod - def _setup_transfers(group): + def _setup_transfers(group, desvars, responses): """ Compute all transfers that are owned by our parent group. @@ -64,11 +64,15 @@ def _setup_transfers(group): ---------- group : Parent group. + desvars : dict + Dictionary of all design variable metadata. Keyed by absolute source name or alias. + responses : dict + Dictionary of all response variable metadata. Keyed by absolute source name or alias. """ rev = group._mode != 'fwd' for subsys in group._subgroups_myproc: - subsys._setup_transfers() + subsys._setup_transfers(desvars, responses) abs2meta_in = group._var_abs2meta['input'] abs2meta_out = group._var_abs2meta['output'] @@ -86,9 +90,15 @@ def _setup_transfers(group): fwd_xfer_in = defaultdict(list) fwd_xfer_out = defaultdict(list) if rev: + has_rev_par_coloring = any([m['parallel_deriv_color'] is not None + for m in responses.values()]) rev_xfer_in = defaultdict(list) rev_xfer_out = defaultdict(list) + # xfers that are only active when parallel coloring is not active + rev_xfer_in_nocolor = defaultdict(list) + rev_xfer_out_nocolor = defaultdict(list) + allprocs_abs2idx = group._var_allprocs_abs2idx sizes_in = group._var_sizes['input'] sizes_out = group._var_sizes['output'] @@ -98,8 +108,6 @@ def _setup_transfers(group): def is_dup(name, io): if group._var_allprocs_abs2meta[io][name]['distributed']: return False # distributed vars are never dups - # if group.comm.rank == group._owning_rank[name]: - # return False # this is the owner, not a dup return np.count_nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]]) > 1 def get_xfer_ranks(name, io): @@ -192,31 +200,43 @@ def get_xfer_ranks(name, io): distrib_in = meta_in['distributed'] distrib_out = meta_out['distributed'] sub_out = abs_out[mypathlen:].partition('.')[0] - if inp_is_dup and abs_out not in abs2meta_out: - print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) - rev_xfer_in[sub_out] - rev_xfer_out[sub_out] - elif inp_is_dup and distrib_out and not iowninput: + if inp_is_dup and (abs_out not in abs2meta_out or (distrib_out and not iowninput)): print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out] rev_xfer_out[sub_out] elif out_is_dup and not inp_is_dup and (iowninput or distrib_in): oidxlist = [] iidxlist = [] + oidxlist_nc = [] + iidxlist_nc = [] for rnk in get_xfer_ranks(abs_out, 'output'): offset = offsets_out[rnk, idx_out] if src_indices is None: - oidxlist.append(np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE)) - iidxlist.append(input_inds) + oarr = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) + iarr = input_inds elif src_indices.size > 0: offset -= np.sum(sizes_out[:rnk, idx_out]) - oidxlist.append(np.asarray(src_indices + offset, dtype=INT_DTYPE)) - iidxlist.append(input_inds) - input_inds = np.concatenate(iidxlist) - output_inds = np.concatenate(oidxlist) - print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) + iarr = input_inds + if rnk == myproc or not has_rev_par_coloring: + oidxlist.append(oarr) + iidxlist.append(iarr) + else: + oidxlist_nc.append(oarr) + iidxlist_nc.append(iarr) + + input_inds = np.concatenate(iidxlist) if len(iidxlist) > 1 else iidxlist[0] + output_inds = np.concatenate(oidxlist) if len(oidxlist) > 1 else oidxlist[0] rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) + print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + + if has_rev_par_coloring and iidxlist_nc: + input_inds = np.concatenate(iidxlist_nc) if len(iidxlist_nc) > 1 else iidxlist_nc[0] + output_inds = np.concatenate(oidxlist_nc) if len(oidxlist_nc) > 1 else oidxlist_nc[0] + + rev_xfer_in_nocolor[sub_out].append(input_inds) + rev_xfer_out_nocolor[sub_out].append(output_inds) else: print(group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out].append(input_inds) @@ -231,14 +251,21 @@ def get_xfer_ranks(name, io): sub_out = abs_out[mypathlen:].partition('.')[0] rev_xfer_in[sub_out] rev_xfer_out[sub_out] + if has_rev_par_coloring: + rev_xfer_in_nocolor[sub_out] + rev_xfer_out_nocolor[sub_out] for sname, inds in fwd_xfer_in.items(): fwd_xfer_in[sname] = _merge(inds) fwd_xfer_out[sname] = _merge(fwd_xfer_out[sname]) + if rev: for sname, inds in rev_xfer_out.items(): rev_xfer_in[sname] = _merge(rev_xfer_in[sname]) rev_xfer_out[sname] = _merge(inds) + for sname, inds in rev_xfer_out_nocolor.items(): + rev_xfer_in_nocolor[sname] = _merge(rev_xfer_in_nocolor[sname]) + rev_xfer_out_nocolor[sname] = _merge(inds) if fwd_xfer_in: xfer_in = np.concatenate(list(fwd_xfer_in.values())) @@ -266,13 +293,8 @@ def get_xfer_ranks(name, io): else: xfer_in = xfer_out = np.zeros(0, dtype=INT_DTYPE) - # print(group.comm.rank, "MAKING rev global xfer", xfer_out, "-->", xfer_in, flush=True) xfer_all = PETScTransfer(vectors['input']['nonlinear'], out_vec, xfer_in, xfer_out, group.comm) - # print(group.comm.rank, "DONE MAKING rev global xfer", flush=True) - - # print("outputs:", list(group._outputs.keys()), flush=True) - # print("inputs:", list(group._inputs.keys()), flush=True) transfers['rev'] = xrev = {} xrev[None] = xfer_all @@ -282,7 +304,18 @@ def get_xfer_ranks(name, io): vectors['input']['nonlinear'], vectors['output']['nonlinear'], rev_xfer_in[sname], inds, group.comm) - # print(group.comm.rank, 'returning from setup_transfers', flush=True) + if has_rev_par_coloring and rev_xfer_in_nocolor: + xfer_in = np.concatenate(list(rev_xfer_in_nocolor.values())) + xfer_out = np.concatenate(list(rev_xfer_out_nocolor.values())) + + xrev[(None, 'nocolor')] = PETScTransfer(vectors['input']['nonlinear'], out_vec, + xfer_in, xfer_out, group.comm) + + for sname, inds in rev_xfer_out_nocolor.items(): + transfers['rev'][(sname, 'nocolor')] = PETScTransfer( + vectors['input']['nonlinear'], vectors['output']['nonlinear'], + rev_xfer_in_nocolor[sname], inds, group.comm) + def _transfer(self, in_vec, out_vec, mode='fwd'): """ From a0f5f6518675e0c63e05b39794c77c6b064fb219 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 18 Sep 2023 14:33:19 -0400 Subject: [PATCH 08/70] interim --- openmdao/core/explicitcomponent.py | 2 +- openmdao/core/group.py | 21 +---- openmdao/core/problem.py | 3 + openmdao/core/system.py | 10 +-- openmdao/core/tests/test_distrib_derivs.py | 2 +- openmdao/core/tests/test_driver.py | 2 +- .../core/tests/test_parallel_derivatives.py | 5 +- openmdao/core/total_jac.py | 36 ++++++--- openmdao/drivers/scipy_optimizer.py | 5 +- openmdao/solvers/linear/linear_block_gs.py | 2 + openmdao/utils/general_utils.py | 23 ++++++ openmdao/vectors/petsc_transfer.py | 78 ++++++++++++++++--- 12 files changed, 137 insertions(+), 52 deletions(-) diff --git a/openmdao/core/explicitcomponent.py b/openmdao/core/explicitcomponent.py index 8144d2612a..96daca4279 100644 --- a/openmdao/core/explicitcomponent.py +++ b/openmdao/core/explicitcomponent.py @@ -478,7 +478,7 @@ def _solve_linear(self, mode, rel_systems, scope_out=_UNDEFINED, scope_in=_UNDEF # ExplicitComponent jacobian defined with -1 on diagonal. d_residuals *= -1.0 - # print(MPI.COMM_WORLD.rank, 'SOLVE', self.pathname, 'd_inputs', self._dinputs.asarray(), 'd_outputs', self._doutputs.asarray(), 'd_residuals', self._dresiduals.asarray(), flush=True) + # print(MPI.COMM_WORLD.rank, 'SOLVE', self.pathname, 'd_inputs', self._dinputs.asarray(), 'd_outputs', self._doutputs.asarray(), 'd_residuals', self._dresiduals.asarray(), flush=True) def _compute_partials_wrapper(self): """ diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 4c60f74dbd..992822b303 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -27,7 +27,7 @@ shape_to_len from openmdao.utils.general_utils import common_subpath, all_ancestors, \ convert_src_inds, ContainsAll, shape2tuple, get_connection_owner, ensure_compatible, \ - _src_name_iter, meta2src_iter + _src_name_iter, meta2src_iter, get_rev_conns from openmdao.utils.units import is_compatible, unit_conversion, _has_val_mismatch, _find_unit, \ _is_unitless, simplify_unit from openmdao.utils.graph_utils import get_sccs_topo, get_out_of_order_nodes, get_hybrid_graph @@ -2634,23 +2634,6 @@ def is_unresolved(graph, node): return True return False - def get_rev_conn(): - """ - Return a dict mapping each connected input to a list of its connected outputs. - - Returns - ------- - dict - Dict mapping each connected input to a list of its connected outputs. - """ - rev = {} - for tgt, src in conn.items(): - if src in rev: - rev[src].append(tgt) - else: - rev[src] = [tgt] - return rev - def meta2node_data(meta): """ Return a dict containing select metadata for the given variable. @@ -2706,7 +2689,7 @@ def meta2node_data(meta): graph.add_edge(abs_from, name, multi=False) else: if rev_conn is None: - rev_conn = get_rev_conn() + rev_conn = get_rev_conns(self._conn_global_abs_in2out) if name in rev_conn: # connected output for inp in rev_conn[name]: inmeta = all_abs2meta_in[inp] diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index b211de5de2..ed9ef629ac 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -1119,6 +1119,9 @@ def final_setup(self): logger = TestLogger() self.check_config(logger, checks=checks) + # from om_devtools.dist_idxs import dump_dist_idxs + # dump_dist_idxs(self, full=True) + def check_partials(self, out_stream=_DEFAULT_OUT_STREAM, includes=None, excludes=None, compact_print=False, abs_err_tol=1e-6, rel_err_tol=1e-6, method='fd', step=None, form='forward', step_calc='abs', diff --git a/openmdao/core/system.py b/openmdao/core/system.py index dbdf271384..93aea49a36 100644 --- a/openmdao/core/system.py +++ b/openmdao/core/system.py @@ -5505,11 +5505,11 @@ def _get_input_from_src(self, name, abs_ins, conns, units=None, indices=None, "`get_val(, get_remote=True)`.") else: if src_indices._flat_src: - if distrib and not sdistrib: - start = np.sum(sizes[:self.comm.rank]) - sinds = src_indices.apply_offset(-start, flat=True) - else: - sinds = src_indices.flat() + #if distrib and not sdistrib: + #start = np.sum(sizes[:self.comm.rank]) + #sinds = src_indices.apply_offset(-start, flat=True) + #else: + sinds = src_indices.flat() val = val.ravel()[sinds] # if at component level, just keep shape of the target and don't flatten if not flat and not is_prom: diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index 0d49de8d19..af199015fc 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -853,7 +853,7 @@ def compute(self, inputs, outputs): prob.run_model() - J = prob.check_totals(method='fd', show_only_incorrect=True) + J = prob.check_totals(method='cs', show_only_incorrect=True) assert_near_equal(J['parab.f_xy', 'p.x']['abs error'][mode_idx[mode]], 0.0, 1e-5) assert_near_equal(J['parab.f_xy', 'p.y']['abs error'][mode_idx[mode]], 0.0, 1e-5) assert_near_equal(J['ndp.g', 'p.x']['abs error'][mode_idx[mode]], 0.0, 2e-5) diff --git a/openmdao/core/tests/test_driver.py b/openmdao/core/tests/test_driver.py index dfed5826bc..6eb6dd6304 100644 --- a/openmdao/core/tests/test_driver.py +++ b/openmdao/core/tests/test_driver.py @@ -11,7 +11,7 @@ import openmdao.api as om from openmdao.core.driver import Driver from openmdao.utils.units import convert_units -from openmdao.utils.assert_utils import assert_near_equal, assert_warning +from openmdao.utils.assert_utils import assert_near_equal, assert_warning, assert_check_totals from openmdao.utils.general_utils import printoptions from openmdao.utils.testing_utils import use_tempdirs from openmdao.test_suite.components.paraboloid import Paraboloid diff --git a/openmdao/core/tests/test_parallel_derivatives.py b/openmdao/core/tests/test_parallel_derivatives.py index 94598a1c58..0d56cc5a5b 100644 --- a/openmdao/core/tests/test_parallel_derivatives.py +++ b/openmdao/core/tests/test_parallel_derivatives.py @@ -900,7 +900,10 @@ def compute_partials(self, inputs, partials): prob.run_model() - assert_check_totals(prob.check_totals(method='cs', show_only_incorrect=True))#, out_stream=None)) + # from om_devtools.dist_idxs import dump_dist_idxs + # dump_dist_idxs(prob, full=True) + + assert_check_totals(prob.check_totals(method='cs', show_only_incorrect=True, out_stream=None)) if __name__ == "__main__": diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index 5a8f2cefdd..1d5553d9a5 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -243,7 +243,7 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, } self._dist_driver_vars = driver._dist_driver_vars - abs2meta_out = model._var_allprocs_abs2meta['output'] + all_abs2meta_out = model._var_allprocs_abs2meta['output'] constraints = driver._cons @@ -309,10 +309,10 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, self.modes = modes self.of_meta, self.of_size, has_of_dist = \ - self._get_tuple_map(of, responses, abs2meta_out) + self._get_tuple_map(of, responses, all_abs2meta_out) self.has_input_dist['rev'] = self.has_output_dist['fwd'] = has_of_dist self.wrt_meta, self.wrt_size, has_wrt_dist = \ - self._get_tuple_map(wrt, design_vars, abs2meta_out) + self._get_tuple_map(wrt, design_vars, all_abs2meta_out) self.has_input_dist['fwd'] = self.has_output_dist['rev'] = has_wrt_dist # always allocate a 2D dense array and we can assign views to dict keys later if @@ -374,7 +374,7 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, self.jac_dist_col_mask = mask = np.zeros(J.shape[1], dtype=bool) start = end = 0 for name in self.wrt: - meta = abs2meta_out[name] + meta = all_abs2meta_out[name] end += meta['global_size'] if meta['distributed']: # see if we have an odd dist var like some auto_ivcs connected to @@ -489,7 +489,7 @@ def _compute_jac_scatters(self, mode, rowcol_size, get_remote): owns = self.model._owning_rank abs2meta_out = self.model._var_allprocs_abs2meta['output'] - loc_abs = self.model._var_abs2meta['output'] + loc_abs2meta = self.model._var_abs2meta['output'] sizes = self.model._var_sizes['output'] abs2idx = self.model._var_allprocs_abs2idx full_j_tgts = [] @@ -502,7 +502,7 @@ def _compute_jac_scatters(self, mode, rowcol_size, get_remote): is_dist = abs2meta_out[name]['distributed'] - if name in loc_abs: + if name in loc_abs2meta: end += abs2meta_out[name]['size'] if get_remote and is_dist: @@ -532,7 +532,7 @@ def _compute_jac_scatters(self, mode, rowcol_size, get_remote): full_j_srcs.append(myinds) full_j_tgts.append(srcinds + offset) - if name in loc_abs: + if name in loc_abs2meta: start = end if full_j_srcs: @@ -1314,6 +1314,7 @@ def simple_single_jac_scatter(self, i, mode): else: return + # print('DERIV_VAL:', i, self.model.comm.rank, deriv_val, deriv_idxs, flush=True) if mode == 'fwd': self.J[jac_idxs, i] = deriv_val[deriv_idxs] else: # rev @@ -1321,7 +1322,7 @@ def simple_single_jac_scatter(self, i, mode): def _jac_setter_dist(self, i, mode): """ - Scatter the i'th row or allreduce the i'th column of the jacobian. + Scatter the i'th column or allreduce the i'th row of the jacobian. Parameters ---------- @@ -1343,6 +1344,7 @@ def _jac_setter_dist(self, i, mode): # duplication of their seed values by dividing by the number of duplications. # ndups, _, _, _ = self.in_idx_map[mode][i] dist, _, _, _ = self.in_idx_map[mode][i] + # print('JAC_SETTER_DIST', i, self.model.comm.rank, self.J[i], flush=True) if self.get_remote: if self.jac_dist_col_mask is not None: distpart = self.J[i, :][self.jac_dist_col_mask] @@ -1350,10 +1352,10 @@ def _jac_setter_dist(self, i, mode): # only sum up the distrib parts self.comm.Allreduce(distpart, scratch, op=MPI.SUM) self.J[i][self.jac_dist_col_mask] = scratch - elif dist: - scratch = self.jac_scratch['rev'][0] - scratch[:] = self.J[i] - self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) + #elif dist: + #scratch = self.jac_scratch['rev'][0] + #scratch[:] = self.J[i] + # self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) # scratch = self.jac_scratch['rev'][0] # scratch[:] = self.J[i] @@ -1611,7 +1613,17 @@ def compute_totals(self): if debug_print: print(f'Elapsed Time: {time.perf_counter() - t0} secs\n', flush=True) + # if isinstance(inds, int): + # print('JAC ROW (before):', self.J[inds], flush=True) + # else: + # for i in inds: + # print('JAC ROW (before):', self.J[i], flush=True) jac_setter(inds, mode, imeta) + # if isinstance(inds, int): + # print('JAC ROW (after):', self.J[inds], flush=True) + # else: + # for i in inds: + # print('JAC ROW (after):', self.J[i], flush=True) # Driver scaling. if self.has_scaling: diff --git a/openmdao/drivers/scipy_optimizer.py b/openmdao/drivers/scipy_optimizer.py index 4f049274f8..49b890e18a 100644 --- a/openmdao/drivers/scipy_optimizer.py +++ b/openmdao/drivers/scipy_optimizer.py @@ -621,8 +621,9 @@ def _objfunc(self, x_new): return 0 # print("Functions calculated") - # print(' xnew', x_new) - # print(' fnew', f_new) + # rank = MPI.COMM_WORLD.rank if MPI else 0 + # print(rank, ' xnew', x_new) + # print(rank, ' fnew', f_new) return f_new diff --git a/openmdao/solvers/linear/linear_block_gs.py b/openmdao/solvers/linear/linear_block_gs.py index f6759f783d..7769887a84 100644 --- a/openmdao/solvers/linear/linear_block_gs.py +++ b/openmdao/solvers/linear/linear_block_gs.py @@ -156,9 +156,11 @@ def _single_iteration(self): scope_in = self._vars_union(self._scope_in, scope_in) subsys._solve_linear(mode, self._rel_systems, scope_out, scope_in) + # print(system.comm.rank, 'SOLVE LINEAR', subsys.pathname, 'd_inputs', subsys._dinputs.asarray(), 'd_outputs', subsys._doutputs.asarray(), 'd_residuals', subsys._dresiduals.asarray(), flush=True) if subsys._iter_call_apply_linear(): subsys._apply_linear(None, self._rel_systems, mode, scope_out, scope_in) + # print(system.comm.rank, 'APPLY LINEAR', subsys.pathname, 'd_inputs', subsys._dinputs.asarray(), 'd_outputs', subsys._doutputs.asarray(), 'd_residuals', subsys._dresiduals.asarray(), flush=True) else: b_vec.set_val(0.0) else: # subsys not local diff --git a/openmdao/utils/general_utils.py b/openmdao/utils/general_utils.py index f096632a61..1288ed6199 100644 --- a/openmdao/utils/general_utils.py +++ b/openmdao/utils/general_utils.py @@ -1363,3 +1363,26 @@ def inconsistent_across_procs(comm, arr, tol=1e-15, return_array=True): comm.gather(arr, root=0) return comm.bcast(None, root=0) + + +def get_rev_conns(conns): + """ + Return a dict mapping each connected output to a list of its connected inputs. + + Parameters + ---------- + conns : dict + Dict mapping each input to its connected output. + + Returns + ------- + dict + Dict mapping each connected output to a list of its connected inputs. + """ + rev = {} + for tgt, src in conns.items(): + if src in rev: + rev[src].append(tgt) + else: + rev[src] = [tgt] + return rev diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index cfb280eec9..f8c7b7d3f7 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -20,6 +20,8 @@ from openmdao.vectors.default_transfer import DefaultTransfer, _merge from openmdao.core.constants import INT_DTYPE + from openmdao.utils.general_utils import get_rev_conns + class PETScTransfer(DefaultTransfer): """ @@ -99,6 +101,8 @@ def _setup_transfers(group, desvars, responses): rev_xfer_in_nocolor = defaultdict(list) rev_xfer_out_nocolor = defaultdict(list) + rev_conns = get_rev_conns(group._conn_abs_in2out) + allprocs_abs2idx = group._var_allprocs_abs2idx sizes_in = group._var_sizes['input'] sizes_out = group._var_sizes['output'] @@ -106,9 +110,11 @@ def _setup_transfers(group, desvars, responses): offsets_out = offsets['output'] def is_dup(name, io): + # return if given var is duplicated and number of procs where var doesn't exist if group._var_allprocs_abs2meta[io][name]['distributed']: - return False # distributed vars are never dups - return np.count_nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]]) > 1 + return False, 0 # distributed vars are never dups + nz = np.count_nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]]) + return nz > 1, group._var_sizes[io].shape[0] - nz def get_xfer_ranks(name, io): if group._var_allprocs_abs2meta[io][name]['distributed']: @@ -121,8 +127,8 @@ def get_xfer_ranks(name, io): for abs_in, abs_out in group._conn_abs_in2out.items(): # Only continue if the input exists on this processor if abs_in in abs2meta_in: - inp_is_dup = is_dup(abs_in, 'input') - out_is_dup = is_dup(abs_out, 'output') + inp_is_dup, inp_missing = is_dup(abs_in, 'input') + out_is_dup, _ = is_dup(abs_out, 'output') # Get meta meta_in = abs2meta_in[abs_in] @@ -200,11 +206,51 @@ def get_xfer_ranks(name, io): distrib_in = meta_in['distributed'] distrib_out = meta_out['distributed'] sub_out = abs_out[mypathlen:].partition('.')[0] - if inp_is_dup and (abs_out not in abs2meta_out or (distrib_out and not iowninput)): + has_multi_conn_src = len(rev_conns[abs_out]) > 1 + + if inp_is_dup and not has_multi_conn_src and (abs_out not in abs2meta_out or (distrib_out and not iowninput)): print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out] rev_xfer_out[sub_out] - elif out_is_dup and not inp_is_dup and (iowninput or distrib_in): + elif out_is_dup and inp_is_dup and inp_missing > 0 and iowninput: + oidxlist = [] + iidxlist = [] + oidxlist_nc = [] + iidxlist_nc = [] + oidxlist.append(output_inds) + iidxlist.append(input_inds) + for rnk, osize, isize in zip(range(group.comm.size), sizes_out[:, idx_out], sizes_in[:, idx_in]): + if osize > 0 and isize == 0: + offset = offsets_out[rnk, idx_out] + if src_indices is None: + oarr = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) + iarr = input_inds + elif src_indices.size > 0: + # offset -= np.sum(sizes_out[:rnk, idx_out]) + oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) + iarr = input_inds + else: + continue + if rnk == myproc or not has_rev_par_coloring: + oidxlist.append(oarr) + iidxlist.append(iarr) + else: + oidxlist_nc.append(oarr) + iidxlist_nc.append(iarr) + + input_inds = np.concatenate(iidxlist) if len(iidxlist) > 1 else iidxlist[0] + output_inds = np.concatenate(oidxlist) if len(oidxlist) > 1 else oidxlist[0] + rev_xfer_in[sub_out].append(input_inds) + rev_xfer_out[sub_out].append(output_inds) + print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + + if has_rev_par_coloring and iidxlist_nc: + input_inds = np.concatenate(iidxlist_nc) if len(iidxlist_nc) > 1 else iidxlist_nc[0] + output_inds = np.concatenate(oidxlist_nc) if len(oidxlist_nc) > 1 else oidxlist_nc[0] + + rev_xfer_in_nocolor[sub_out].append(input_inds) + rev_xfer_out_nocolor[sub_out].append(output_inds) + elif out_is_dup and (not inp_is_dup or inp_missing > 0) and (iowninput or distrib_in): oidxlist = [] iidxlist = [] oidxlist_nc = [] @@ -215,9 +261,11 @@ def get_xfer_ranks(name, io): oarr = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) iarr = input_inds elif src_indices.size > 0: - offset -= np.sum(sizes_out[:rnk, idx_out]) + # offset -= np.sum(sizes_out[:rnk, idx_out]) oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) iarr = input_inds + else: + continue if rnk == myproc or not has_rev_par_coloring: oidxlist.append(oarr) iidxlist.append(iarr) @@ -225,11 +273,17 @@ def get_xfer_ranks(name, io): oidxlist_nc.append(oarr) iidxlist_nc.append(iarr) - input_inds = np.concatenate(iidxlist) if len(iidxlist) > 1 else iidxlist[0] - output_inds = np.concatenate(oidxlist) if len(oidxlist) > 1 else oidxlist[0] + if len(iidxlist) > 1: + input_inds = np.concatenate(iidxlist) + output_inds = np.concatenate(oidxlist) + elif len(iidxlist) == 1: + input_inds = iidxlist[0] + output_inds = oidxlist[0] + else: + input_inds = output_inds = np.zeros(0, dtype=INT_DTYPE) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) - print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + print('MULTI2', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) if has_rev_par_coloring and iidxlist_nc: input_inds = np.concatenate(iidxlist_nc) if len(iidxlist_nc) > 1 else iidxlist_nc[0] @@ -372,3 +426,7 @@ def _transfer(self, in_vec, out_vec, mode='fwd'): if in_vec._alloc_complex: data = in_vec._get_data() data[:] = in_petsc.array + + # print(in_vec._system().comm.rank, 'TRANSFER from', in_vec._system().pathname, self._in_inds, self._out_inds, flush=True) + # print('IN', in_vec._data.real, flush=True) + # print('OUT', out_vec._data.real, flush=True) From e3c165680e4ac24f311d16b1b74d31fc2fecc704 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 22 Sep 2023 10:48:03 -0400 Subject: [PATCH 09/70] interim - 10 xfer fails --- openmdao/core/tests/test_distribcomp.py | 10 ++-- openmdao/core/total_jac.py | 68 +++++++++++++++---------- openmdao/vectors/petsc_transfer.py | 45 ++++++++-------- 3 files changed, 70 insertions(+), 53 deletions(-) diff --git a/openmdao/core/tests/test_distribcomp.py b/openmdao/core/tests/test_distribcomp.py index 9b44643f78..a4fb53c686 100644 --- a/openmdao/core/tests/test_distribcomp.py +++ b/openmdao/core/tests/test_distribcomp.py @@ -1056,8 +1056,8 @@ def initialize(self): desc="Size of input vector x.") def setup(self): - self.add_input('x', np.ones(self.options['size'])) - self.add_output('y', 1.0) + self.add_input('x', np.ones(self.options['size']), distributed=True) + self.add_output('y', 1.0, distributed=True) def compute(self, inputs, outputs): outputs['y'] = np.sum(inputs['x'])*2.0 @@ -1068,7 +1068,7 @@ def compute(self, inputs, outputs): promotes_outputs=['x']) # decide what parts of the array we want based on our rank - if self.comm.rank == 0: + if p.comm.rank == 0: idxs = [0, 1, 2] else: # use [3, -1] here rather than [3, 4] just to show that we @@ -1083,12 +1083,12 @@ def compute(self, inputs, outputs): p.run_model() # each rank holds the assigned portion of the input array - assert_near_equal(p['C1.x'], + assert_near_equal(p.get_val('C1.x', get_remote=False), np.arange(3, dtype=float) if p.model.C1.comm.rank == 0 else np.arange(3, 5, dtype=float)) # the output in each rank is based on the local inputs - assert_near_equal(p['C1.y'], 6. if p.model.C1.comm.rank == 0 else 14.) + assert_near_equal(p.get_val('C1.y', get_remote=False), 6. if p.model.C1.comm.rank == 0 else 14.) if __name__ == '__main__': diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index 1d5553d9a5..b385280c11 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -33,6 +33,35 @@ _directional_rng = np.random.default_rng(99) +class EntryMeta(object): + def __init__(self): + pass + + +class RowMeta(object): + def __init__(self, model): + pass + + def by_name(self, name): + pass + + def by_index(self, idx): + pass + + +class ColumnMeta(object): + def __init__(self): + pass + + def by_name(self, name): + pass + + def by_index(self, idx): + pass + + + + class _TotalJacInfo(object): """ Object to manage computation of total derivatives. @@ -145,12 +174,6 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, self.get_remote = get_remote self.directional = directional - if isinstance(wrt, str): - wrt = [wrt] - - if isinstance(of, str): - of = [of] - # convert designvar and response dicts to use src or alias names # keys will all be absolute names or aliases after conversion design_vars = _src_or_alias_dict(driver._designvars) @@ -167,8 +190,13 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, # that don't specify them, so we need these here. if wrt is None: wrt = driver_wrt + elif isinstance(wrt, str): + wrt = [wrt] + if of is None: of = driver_of + elif isinstance(of, str): + of = [of] # Convert 'wrt' names from promoted to absolute prom_wrt = wrt @@ -396,7 +424,7 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, for mode in modes: self.sol2jac_map[mode] = self._get_sol2jac_map(self.output_list[mode], self.output_meta[mode], - abs2meta_out, mode) + all_abs2meta_out, mode) self.jac_scatters = {} self.tgt_petsc = {n: {} for n in modes} self.src_petsc = {n: {} for n in modes} @@ -420,8 +448,8 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, self.dist_idx_map[mode] = dist_map = np.zeros(arr.size, dtype=bool) start = end = 0 for name in self.output_list[mode]: - end += abs2meta_out[name]['size'] - if abs2meta_out[name]['distributed']: + end += all_abs2meta_out[name]['size'] + if all_abs2meta_out[name]['distributed']: dist_map[start:end] = True start = end @@ -737,27 +765,11 @@ def _create_in_idx_map(self, mode): relev = None dist = in_var_meta['distributed'] - if dist: - pass - # if self.jac_dist_col_mask is None: - # ndups = 1 # we don't divide by ndups for distributed inputs - # else: - # ndups = np.count_nonzero(sizes[:, in_var_idx]) - else: + if not dist: # if the var is not distributed, convert the indices to global. # We don't iterate over the full distributed size in this case. irange += gstart - # if fwd or parallel_deriv_color: - # ndups = 1 - # else: - # # find the number of duplicate components in rev mode so we can divide - # # the seed between 'ndups' procs so that at the end after we do an - # # Allreduce, the contributions from all procs will add up properly. - # ndups = np.count_nonzero(sizes[:, in_var_idx]) - - ndups = 1 - # all local idxs that correspond to vars from other procs will be -1 # so each entry of loc_i will either contain a valid local index, # indicating we should set the local vector entry to 1.0 before running @@ -1012,6 +1024,8 @@ def _get_tuple_map(self, names, vois, abs2meta_out): else: size = voi['size'] indices = vois[name]['indices'] + if indices: + size = indices.indexed_src_size if responses: path = vois[name]['source'] @@ -2064,7 +2078,7 @@ def _check_voi_meta(name, parallel_deriv_color, simul_coloring): def _fix_pdc_lengths(idx_iter_dict): """ - Take any parallel_deriv_color entries and make sure their index arrays are same length. + Take any parallel_deriv_color entries and make sure their index arrays are the same length. Parameters ---------- diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index f8c7b7d3f7..56ac2bacee 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -20,7 +20,7 @@ from openmdao.vectors.default_transfer import DefaultTransfer, _merge from openmdao.core.constants import INT_DTYPE - from openmdao.utils.general_utils import get_rev_conns + from openmdao.utils.array_utils import shape_to_len class PETScTransfer(DefaultTransfer): @@ -79,7 +79,7 @@ def _setup_transfers(group, desvars, responses): abs2meta_in = group._var_abs2meta['input'] abs2meta_out = group._var_abs2meta['output'] allprocs_abs2meta_out = group._var_allprocs_abs2meta['output'] - myproc = group.comm.rank + myrank = group.comm.rank transfers = group._transfers = {} vectors = group._vectors @@ -101,7 +101,7 @@ def _setup_transfers(group, desvars, responses): rev_xfer_in_nocolor = defaultdict(list) rev_xfer_out_nocolor = defaultdict(list) - rev_conns = get_rev_conns(group._conn_abs_in2out) + # rev_conns = get_rev_conns(group._conn_abs_in2out) allprocs_abs2idx = group._var_allprocs_abs2idx sizes_in = group._var_sizes['input'] @@ -112,9 +112,9 @@ def _setup_transfers(group, desvars, responses): def is_dup(name, io): # return if given var is duplicated and number of procs where var doesn't exist if group._var_allprocs_abs2meta[io][name]['distributed']: - return False, 0 # distributed vars are never dups + return False, 0, True # distributed vars are never dups nz = np.count_nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]]) - return nz > 1, group._var_sizes[io].shape[0] - nz + return nz > 1, group._var_sizes[io].shape[0] - nz, False def get_xfer_ranks(name, io): if group._var_allprocs_abs2meta[io][name]['distributed']: @@ -127,9 +127,6 @@ def get_xfer_ranks(name, io): for abs_in, abs_out in group._conn_abs_in2out.items(): # Only continue if the input exists on this processor if abs_in in abs2meta_in: - inp_is_dup, inp_missing = is_dup(abs_in, 'input') - out_is_dup, _ = is_dup(abs_out, 'output') - # Get meta meta_in = abs2meta_in[abs_in] meta_out = allprocs_abs2meta_out[abs_out] @@ -137,6 +134,8 @@ def get_xfer_ranks(name, io): idx_in = allprocs_abs2idx[abs_in] idx_out = allprocs_abs2idx[abs_out] + local_out = abs_out in abs2meta_out + # Read in and process src_indices src_indices = meta_in['src_indices'] if src_indices is None: @@ -147,6 +146,7 @@ def get_xfer_ranks(name, io): if meta_in['size'] > sizes_out[owner, idx_out]: src_indices = np.arange(meta_in['size'], dtype=INT_DTYPE) else: + src_shape = src_indices._src_shape src_indices = src_indices.shaped_array() # 1. Compute the output indices @@ -163,7 +163,7 @@ def get_xfer_ranks(name, io): "without declaring src_indices.", ident=(abs_out, abs_in)) else: - rank = myproc if abs_out in abs2meta_out else owner + rank = myrank if local_out else owner offset = offsets_out[rank, idx_out] output_inds = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) @@ -193,22 +193,25 @@ def get_xfer_ranks(name, io): start = end # 2. Compute the input indices - input_inds = np.arange(offsets_in[myproc, idx_in], - offsets_in[myproc, idx_in] + - sizes_in[myproc, idx_in], dtype=INT_DTYPE) + input_inds = np.arange(offsets_in[myrank, idx_in], + offsets_in[myrank, idx_in] + + sizes_in[myrank, idx_in], dtype=INT_DTYPE) # Now the indices are ready - input_inds, output_inds sub_in = abs_in[mypathlen:].partition('.')[0] fwd_xfer_in[sub_in].append(input_inds) fwd_xfer_out[sub_in].append(output_inds) if rev: - iowninput = myproc == group._owning_rank[abs_in] - distrib_in = meta_in['distributed'] - distrib_out = meta_out['distributed'] + inp_is_dup, inp_missing, distrib_in = is_dup(abs_in, 'input') + out_is_dup, _, distrib_out = is_dup(abs_out, 'output') + gsize_in = np.sum(sizes_in[:, idx_in]) + gsize_out = np.sum(sizes_out[:, idx_out]) + + iowninput = myrank == group._owning_rank[abs_in] sub_out = abs_out[mypathlen:].partition('.')[0] - has_multi_conn_src = len(rev_conns[abs_out]) > 1 + # has_multi_conn_src = len(rev_conns[abs_out]) > 1 - if inp_is_dup and not has_multi_conn_src and (abs_out not in abs2meta_out or (distrib_out and not iowninput)): + if inp_is_dup and (abs_out not in abs2meta_out or (distrib_out and not iowninput)): print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out] rev_xfer_out[sub_out] @@ -226,12 +229,11 @@ def get_xfer_ranks(name, io): oarr = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) iarr = input_inds elif src_indices.size > 0: - # offset -= np.sum(sizes_out[:rnk, idx_out]) oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) iarr = input_inds else: continue - if rnk == myproc or not has_rev_par_coloring: + if rnk == myrank or not has_rev_par_coloring: oidxlist.append(oarr) iidxlist.append(iarr) else: @@ -261,12 +263,13 @@ def get_xfer_ranks(name, io): oarr = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) iarr = input_inds elif src_indices.size > 0: - # offset -= np.sum(sizes_out[:rnk, idx_out]) + # if distrib_in and gsize_in == shape_to_len(src_shape): # gsize_in == gsize_out: + # offset -= np.sum(sizes_out[:rnk, idx_out]) oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) iarr = input_inds else: continue - if rnk == myproc or not has_rev_par_coloring: + if rnk == myrank or not has_rev_par_coloring: oidxlist.append(oarr) iidxlist.append(iarr) else: From e6eedfd7780080d3139154f2b7c8b43d5e7dffce Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Wed, 27 Sep 2023 13:20:42 -0400 Subject: [PATCH 10/70] 7 fails --- openmdao/core/component.py | 11 ----------- openmdao/vectors/petsc_transfer.py | 26 ++++++++++++++++---------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/openmdao/core/component.py b/openmdao/core/component.py index 9eee063002..fd01358fea 100644 --- a/openmdao/core/component.py +++ b/openmdao/core/component.py @@ -1009,17 +1009,6 @@ def _update_dist_src_indices(self, abs_in2out, all_abs2meta, all_abs2idx, all_si if dist_in: offset = np.sum(sizes_in[:iproc, i]) end = offset + sizes_in[iproc, i] - else: - if src.startswith('_auto_ivc.'): - nzs = np.nonzero(vout_sizes)[0] - if nzs.size == 1: - # special case where we have a 'distributed' auto_ivc output - # that has a nonzero value in only one proc, so we can treat - # it like a non-distributed output. This happens in cases - # where an auto_ivc output connects to a variable that is - # remote on at least one proc. - offset = 0 - end = vout_sizes[nzs[0]] # total sizes differ and output is distributed, so can't determine mapping if offset is None: diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 56ac2bacee..762be42594 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -116,12 +116,14 @@ def is_dup(name, io): nz = np.count_nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]]) return nz > 1, group._var_sizes[io].shape[0] - nz, False + def get_rank_sizes(name, io): + idx = allprocs_abs2idx[name] + return group._var_sizes[io][:, idx] + def get_xfer_ranks(name, io): if group._var_allprocs_abs2meta[io][name]['distributed']: return [] - idx = allprocs_abs2idx[name] - sizes = group._var_sizes[io][:, idx] - return np.nonzero(sizes)[0] + return np.nonzero(get_rank_sizes(name, io))[0] # Loop through all connections owned by this system for abs_in, abs_out in group._conn_abs_in2out.items(): @@ -146,9 +148,10 @@ def get_xfer_ranks(name, io): if meta_in['size'] > sizes_out[owner, idx_out]: src_indices = np.arange(meta_in['size'], dtype=INT_DTYPE) else: - src_shape = src_indices._src_shape src_indices = src_indices.shaped_array() + on_iprocs = [] + # 1. Compute the output indices # NOTE: src_indices are relative to a single, possibly distributed variable, # while the output_inds that we compute are relative to the full distributed @@ -189,6 +192,7 @@ def get_xfer_ranks(name, io): # + inds offset = offsets_out[iproc, idx_out] - start output_inds[on_iproc] = src_indices[on_iproc] + offset + on_iprocs.append(iproc) start = end @@ -204,15 +208,15 @@ def get_xfer_ranks(name, io): if rev: inp_is_dup, inp_missing, distrib_in = is_dup(abs_in, 'input') out_is_dup, _, distrib_out = is_dup(abs_out, 'output') - gsize_in = np.sum(sizes_in[:, idx_in]) - gsize_out = np.sum(sizes_out[:, idx_out]) + # gsize_in = np.sum(sizes_in[:, idx_in]) + # gsize_out = np.sum(sizes_out[:, idx_out]) iowninput = myrank == group._owning_rank[abs_in] sub_out = abs_out[mypathlen:].partition('.')[0] # has_multi_conn_src = len(rev_conns[abs_out]) > 1 if inp_is_dup and (abs_out not in abs2meta_out or (distrib_out and not iowninput)): - print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + # print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out] rev_xfer_out[sub_out] elif out_is_dup and inp_is_dup and inp_missing > 0 and iowninput: @@ -244,7 +248,7 @@ def get_xfer_ranks(name, io): output_inds = np.concatenate(oidxlist) if len(oidxlist) > 1 else oidxlist[0] rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) - print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + # print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) if has_rev_par_coloring and iidxlist_nc: input_inds = np.concatenate(iidxlist_nc) if len(iidxlist_nc) > 1 else iidxlist_nc[0] @@ -263,6 +267,8 @@ def get_xfer_ranks(name, io): oarr = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) iarr = input_inds elif src_indices.size > 0: + if distrib_in and not distrib_out and len(on_iprocs) == 1 and on_iprocs[0] == rnk: + offset -= np.sum(sizes_out[:rnk, idx_out]) # if distrib_in and gsize_in == shape_to_len(src_shape): # gsize_in == gsize_out: # offset -= np.sum(sizes_out[:rnk, idx_out]) oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) @@ -286,7 +292,7 @@ def get_xfer_ranks(name, io): input_inds = output_inds = np.zeros(0, dtype=INT_DTYPE) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) - print('MULTI2', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + # print('MULTI2', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) if has_rev_par_coloring and iidxlist_nc: input_inds = np.concatenate(iidxlist_nc) if len(iidxlist_nc) > 1 else iidxlist_nc[0] @@ -295,7 +301,7 @@ def get_xfer_ranks(name, io): rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) else: - print(group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + # print(group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) else: From e9c4badb1dc1096bcd0963df81949e611cf09feb Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 29 Sep 2023 23:09:24 -0400 Subject: [PATCH 11/70] 4 rev transfer related failures --- .../approximation_scheme.py | 1 + openmdao/core/group.py | 8 ++ openmdao/core/tests/test_deriv_transfers.py | 3 + openmdao/core/tests/test_distrib_derivs.py | 83 +++++++++--- openmdao/core/tests/test_mpi_coloring_bug.py | 57 ++++---- openmdao/core/total_jac.py | 125 ++++++++++++------ .../components/paraboloid_distributed.py | 1 - openmdao/vectors/petsc_transfer.py | 11 +- 8 files changed, 199 insertions(+), 90 deletions(-) diff --git a/openmdao/approximation_schemes/approximation_scheme.py b/openmdao/approximation_schemes/approximation_scheme.py index e55b08e038..a5f9e35460 100644 --- a/openmdao/approximation_schemes/approximation_scheme.py +++ b/openmdao/approximation_schemes/approximation_scheme.py @@ -566,6 +566,7 @@ def _uncolored_column_iter(self, system, approx_groups): yield jinds[0], res else: yield jinds, res + # print(f"APPROX: {wrt} {jinds} {res}") def compute_approximations(self, system, jac=None): """ diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 992822b303..98380c3b8f 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -2971,6 +2971,8 @@ def _setup_connections(self): # either owned by (implicit) or declared by (explicit) this Group. # This way, we don't repeat the error checking in multiple groups. + self._dist_in_sources = defaultdict(list) + for abs_in, abs_out in abs_in2out.items(): all_meta_out = allprocs_abs2meta_out[abs_out] all_meta_in = allprocs_abs2meta_in[abs_in] @@ -3008,6 +3010,8 @@ def _setup_connections(self): # get input shape and src_indices from the local meta dict # (input is always local) if meta_in['distributed']: + self._dist_in_sources[abs_out].append(abs_in) + # if output is non-distributed and input is distributed, make output shape the # full distributed shape, i.e., treat it in this regard as a distributed output out_shape = self._get_full_dist_shape(abs_out, all_meta_out['shape']) @@ -3713,6 +3717,10 @@ def _apply_linear(self, jac, rel_systems, mode, scope_out=None, scope_in=None): with self._matvec_context(scope_out, scope_in, mode) as vecs: d_inputs, d_outputs, d_residuals = vecs jac._apply(self, d_inputs, d_outputs, d_residuals, mode) + if self._owns_approx_jac and self._has_distrib_vars and mode == 'rev': + pass # DO ALLREDUCE OF parts of d_inputs where inputs are connected to distrib outputs + # and only for parts in the current matvec context + print(f"AFTER JAC APPLY in {self.pathname}: d_inputs: {[f'{n}, {s}' for n, s in d_inputs.items()]}", flush=True) # Apply recursion else: if mode == 'fwd': diff --git a/openmdao/core/tests/test_deriv_transfers.py b/openmdao/core/tests/test_deriv_transfers.py index 22696cefc0..b9b64c5a34 100644 --- a/openmdao/core/tests/test_deriv_transfers.py +++ b/openmdao/core/tests/test_deriv_transfers.py @@ -242,6 +242,9 @@ def test_par_dup(self, mode): J = prob.compute_totals(of=of, wrt=wrt) + import pprint + pprint.pprint(J) + assert_near_equal(J['C1.y', 'par.indep1.x'][0][0], 2.5, 1e-6) assert_near_equal(J['C1.y', 'par.indep2.x'][0][0], 3.5, 1e-6) assert_near_equal(prob['C1.y'], 6., 1e-6) diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index af199015fc..a6a7f89f1a 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -612,7 +612,8 @@ def test_distrib_voi_fd(self): ivc.add_output('a', -3.0 + 0.6 * np.arange(size)) model.add_subsystem('p', ivc, promotes=['*']) - model.add_subsystem("parab", DistParab(arr_size=size, deriv_type='fd'), promotes=['*']) + # model.add_subsystem("parab", DistParab(arr_size=size, deriv_type='fd'), promotes=['*']) + model.add_subsystem("parab", DistParab(arr_size=size), promotes=['*']) model.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, ))), @@ -637,11 +638,11 @@ def test_distrib_voi_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - J = prob.check_totals(out_stream=None, method='cs') - assert_near_equal(J['parab.f_xy', 'p.x']['abs error'].forward, 0.0, 1e-5) - assert_near_equal(J['parab.f_xy', 'p.y']['abs error'].forward, 0.0, 1e-5) - assert_near_equal(J['sum.f_sum', 'p.x']['abs error'].forward, 0.0, 1e-5) - assert_near_equal(J['sum.f_sum', 'p.y']['abs error'].forward, 0.0, 1e-5) + assert_check_totals(prob.check_totals(out_stream=None, method='cs'), atol=1e-5) + # assert_near_equal(J['parab.f_xy', 'p.x']['abs error'].forward, 0.0, 1e-5) + # assert_near_equal(J['parab.f_xy', 'p.y']['abs error'].forward, 0.0, 1e-5) + # assert_near_equal(J['sum.f_sum', 'p.x']['abs error'].forward, 0.0, 1e-5) + # assert_near_equal(J['sum.f_sum', 'p.y']['abs error'].forward, 0.0, 1e-5) # rev mode @@ -657,11 +658,11 @@ def test_distrib_voi_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - J = prob.check_totals(method='cs', show_only_incorrect=True) - assert_near_equal(J['parab.f_xy', 'p.x']['abs error'].reverse, 0.0, 1e-5) - assert_near_equal(J['parab.f_xy', 'p.y']['abs error'].reverse, 0.0, 1e-5) - assert_near_equal(J['sum.f_sum', 'p.x']['abs error'].reverse, 0.0, 1e-5) - assert_near_equal(J['sum.f_sum', 'p.y']['abs error'].reverse, 0.0, 1e-5) + assert_check_totals(prob.check_totals(method='cs', show_only_incorrect=True), atol=1e-5) + # assert_near_equal(J['parab.f_xy', 'p.x']['abs error'].reverse, 0.0, 1e-5) + # assert_near_equal(J['parab.f_xy', 'p.y']['abs error'].reverse, 0.0, 1e-5) + # assert_near_equal(J['sum.f_sum', 'p.x']['abs error'].reverse, 0.0, 1e-5) + # assert_near_equal(J['sum.f_sum', 'p.y']['abs error'].reverse, 0.0, 1e-5) def test_distrib_voi_group_fd(self): # Only supports groups where the inputs to the distributed component whose inputs are @@ -717,11 +718,7 @@ def test_distrib_voi_group_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - data = prob.check_totals(method='fd', show_only_incorrect=True) - assert_near_equal(data['sub.parab.f_xy', 'p.x']['abs error'].forward, 0.0, 1e-5) - assert_near_equal(data['sub.parab.f_xy', 'p.y']['abs error'].forward, 0.0, 1e-5) - assert_near_equal(data['sub.sum.f_sum', 'p.x']['abs error'].forward, 0.0, 1e-5) - assert_near_equal(data['sub.sum.f_sum', 'p.y']['abs error'].forward, 0.0, 1e-5) + #assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=1e-5) # rev mode @@ -737,11 +734,55 @@ def test_distrib_voi_group_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - J = prob.check_totals(method='fd', show_only_incorrect=True) - assert_near_equal(J['sub.parab.f_xy', 'p.x']['abs error'].reverse, 0.0, 1e-5) - assert_near_equal(J['sub.parab.f_xy', 'p.y']['abs error'].reverse, 0.0, 1e-5) - assert_near_equal(J['sub.sum.f_sum', 'p.x']['abs error'].reverse, 0.0, 1e-5) - assert_near_equal(J['sub.sum.f_sum', 'p.y']['abs error'].reverse, 0.0, 1e-5) + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=1e-5) + + def test_simple_distrib_voi_group_fd(self): + size = 3 + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones(size)) + ivc.add_output('y', np.ones(size)) + + model.add_subsystem('p', ivc, promotes=['*']) + sub = model.add_subsystem('sub', om.Group(), promotes=['*']) + + ivc2 = om.IndepVarComp() + ivc2.add_output('a', -3.0 + 0.6 * np.arange(size)) + + sub.add_subsystem('p2', ivc2, promotes=['*']) + + sub.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], + x=np.ones(size), xd=np.ones(size), + y=np.ones(size), yd=np.ones(size)), + promotes_inputs=['*']) + + sub.add_subsystem("parab", DistParab(arr_size=size), promotes_outputs=['*'], promotes_inputs=['a']) + + sub.connect('dummy.xd', 'parab.x') + sub.connect('dummy.yd', 'parab.y') + + model.add_design_var('x', lower=-50.0, upper=50.0) + model.add_design_var('y', lower=-50.0, upper=50.0) + model.add_constraint('f_xy', lower=0.0) + + sub.approx_totals(method='fd') + + prob.setup(mode='rev', force_alloc_complex=True) + + prob.run_model() + + # desvar = prob.driver.get_design_var_values() + # con = prob.driver.get_constraint_values() + + # assert_near_equal(desvar['p.x'], np.ones(size), 1e-6) + # assert_near_equal(con['sub.parab.f_xy'], + # np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), + # 1e-6) + + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=1e-5) def test_distrib_group_fd_unsupported_config(self): size = 7 diff --git a/openmdao/core/tests/test_mpi_coloring_bug.py b/openmdao/core/tests/test_mpi_coloring_bug.py index 9682dbc2ac..366360ffc8 100644 --- a/openmdao/core/tests/test_mpi_coloring_bug.py +++ b/openmdao/core/tests/test_mpi_coloring_bug.py @@ -4,7 +4,7 @@ import openmdao.api as om import openmdao.utils.coloring as coloring_mod -from openmdao.utils.assert_utils import assert_near_equal +from openmdao.utils.assert_utils import assert_near_equal, assert_check_totals from openmdao.utils.general_utils import set_pyoptsparse_opt from openmdao.utils.testing_utils import use_tempdirs @@ -302,20 +302,8 @@ def add_design_parameter(self, name, val, targets): self.design_parameter_options[name]['val'] = val self.design_parameter_options[name]['targets'] = targets - - def _setup_design_parameters(self): - if self.design_parameter_options: - indep = self.add_subsystem('design_params', subsys=om.IndepVarComp(), - promotes_outputs=['*']) - - for name, options in self.design_parameter_options.items(): - indep.add_output(name='design_parameters:{0}'.format(name), - val=options['val'], - shape=(1, np.prod(options['shape']))) - def setup(self): super().setup() - self._setup_design_parameters() phases_group = self.add_subsystem('phases', subsys=om.ParallelGroup(), promotes_inputs=['*'], promotes_outputs=['*']) @@ -449,19 +437,13 @@ def compute(self, inputs, outputs): def make_traj(): - t = GaussLobatto() - traj = Trajectory() - traj.add_design_parameter('c', val=1.5, targets={'burn1': ['c'], 'burn2': ['c']}) - - # First Phase (burn) - burn1 = Phase(ode_class=FiniteBurnODE, transcription=t) + burn1 = Phase(ode_class=FiniteBurnODE, transcription=GaussLobatto()) burn1 = traj.add_phase('burn1', burn1) burn1.add_state('deltav', rate_source='deltav_dot') - # Third Phase (burn) - burn2 = Phase(ode_class=FiniteBurnODE, transcription=t) + burn2 = Phase(ode_class=FiniteBurnODE, transcription=GaussLobatto()) traj.add_phase('burn2', burn2) burn2.add_state('deltav', rate_source='deltav_dot') @@ -497,16 +479,43 @@ def run(self): p.setup(mode='rev') - # Set Initial Guesses - p.set_val('design_parameters:c', val=1.5) - of = ['phases.burn2.indep_states.states:deltav', 'phases.burn1.collocation_constraint.defects:deltav', 'phases.burn2.collocation_constraint.defects:deltav', ] wrt = ['phases.burn1.indep_states.states:deltav', 'phases.burn2.indep_states.states:deltav'] p.run_model() p.run_driver() + # assert_check_totals(p.check_totals(of=of, wrt=wrt)) J = p.driver._compute_totals(of=of, wrt=wrt, return_format='dict') dd = J['phases.burn1.collocation_constraint.defects:deltav']['phases.burn1.indep_states.states:deltav'] assert_near_equal(dd, np.array([[-0.75, 0.75]]), 1e-6) + + def test_bug2(self): + size = 3 + p = om.Problem() + phases = p.model.add_subsystem('phases', om.ParallelGroup()) + phase1 = phases.add_subsystem('phase1', om.Group()) + phase2 = phases.add_subsystem('phase2', om.Group()) + phase1.add_subsystem('indep', om.IndepVarComp('x', val=np.ones(size))) + phase2.add_subsystem('indep', om.IndepVarComp('x', val=np.ones(size))) + phase1.add_subsystem('comp1', om.ExecComp('y=3.0*x', x=np.ones(size), y=np.ones(size))) + # phase1.add_subsystem('comp2', om.ExecComp('y=5.0*x', x=np.ones(size), y=np.ones(size))) + phase2.add_subsystem('comp1', om.ExecComp('y=7.0*x', x=np.ones(size), y=np.ones(size))) + # phase2.add_subsystem('comp2', om.ExecComp('y=9.0*x', x=np.ones(size), y=np.ones(size))) + phase1.connect('indep.x', 'comp1.x') + # phase1.connect('comp1.y', 'comp2.x') + phase2.connect('indep.x', 'comp1.x') + # phase2.connect('comp1.y', 'comp2.x') + + phase1.add_design_var('indep.x') + phase2.add_design_var('indep.x') + # phase1.add_constraint('comp2.y', lower=0.0) + # phase2.add_constraint('comp2.y', lower=0.0) + phase1.add_constraint('comp1.y', lower=0.0) + phase2.add_constraint('comp1.y', lower=0.0) + + p.setup(mode='rev') + p.run_model() + + assert_check_totals(p.check_totals()) diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index b385280c11..b77224c2f0 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -158,7 +158,7 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, If True, perform a single directional derivative. """ driver = problem.driver - prom2abs = problem.model._var_allprocs_prom2abs_list['output'] + prom2abs_out = problem.model._var_allprocs_prom2abs_list['output'] prom2abs_in = problem.model._var_allprocs_prom2abs_list['input'] conns = problem.model._conn_global_abs_in2out @@ -203,8 +203,8 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, wrt = [] self.ivc_print_names = {} for name in prom_wrt: - if name in prom2abs: - wrt_name = prom2abs[name][0] + if name in prom2abs_out: + wrt_name = prom2abs_out[name][0] elif name in prom2abs_in: in_abs = prom2abs_in[name][0] wrt_name = conns[in_abs] @@ -218,8 +218,8 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, of = [] src_of = [] for name in prom_of: # these names could be aliases - if name in prom2abs: - of_name = prom2abs[name][0] + if name in prom2abs_out: + of_name = prom2abs_out[name][0] elif name in prom2abs_in: # An auto_ivc design var can be used as a response too. in_abs = prom2abs_in[name][0] @@ -396,21 +396,36 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, self.jac_scratch['rev'] = [scratch[0][:J.shape[1]]] if self.simul_coloring is not None: # when simul coloring, need two scratch arrays self.jac_scratch['rev'].append(scratch[1][:J.shape[1]]) - if self.has_output_dist['rev']: - sizes = model._var_sizes['output'] - abs2idx = model._var_allprocs_abs2idx - self.jac_dist_col_mask = mask = np.zeros(J.shape[1], dtype=bool) - start = end = 0 - for name in self.wrt: - meta = all_abs2meta_out[name] - end += meta['global_size'] - if meta['distributed']: - # see if we have an odd dist var like some auto_ivcs connected to - # remote vars, which are zero everywhere except for one proc - sz = sizes[:, abs2idx[name]] - if np.count_nonzero(sz) > 1: - mask[start:end] = True - start = end + has_out_dist = self.has_output_dist['rev'] + + if has_out_dist: + self.jac_dist_col_mask = np.zeros(J.shape[1], dtype=bool) + + self.dup_mask = np.ones(J.shape[1], dtype=bool) + + sizes = model._var_sizes['output'] + abs2idx = model._var_allprocs_abs2idx + start = end = 0 + for name in self.wrt: + meta = all_abs2meta_out[name] + varidx = abs2idx[name] + end += meta['global_size'] + if not meta['distributed']: + # see if we have an odd dist var like some auto_ivcs connected to + # remote vars, which are zero everywhere except for one proc + # if has_out_dist: + # sz = sizes[:, varidx] + # if True: # np.count_nonzero(sz) > 1: + # self.jac_dist_col_mask[start:end] = True + # self.dup_mask[start:end] = True + #elif 0 in sizes[:, varidx]: + if model._owning_rank[name] != model.comm.rank: + self.dup_mask[start:end] = False + start = end + + need_allreduce = not np.all(self.dup_mask) + if not any(model.comm.allgather(need_allreduce)): + self.dup_mask = None if not approx: for mode in modes: @@ -657,9 +672,10 @@ def _create_in_idx_map(self, mode): """ iproc = self.comm.rank model = self.model + owning_rank = model._owning_rank relevant = model._relevant has_par_deriv_color = False - abs2meta_out = model._var_allprocs_abs2meta['output'] + all_abs2meta_out = model._var_allprocs_abs2meta['output'] var_sizes = model._var_sizes var_offsets = model._get_var_offsets() abs2idx = model._var_allprocs_abs2idx @@ -686,15 +702,12 @@ def _create_in_idx_map(self, mode): # so we just bulk check the outputs here. qoi_i = self.input_meta[mode] qoi_o = self.output_meta[mode] + non_rel_outs = False if qoi_i and qoi_o: for out in self.output_list[mode]: if out not in qoi_o and out not in qoi_i: non_rel_outs = True break - else: - non_rel_outs = False - else: - non_rel_outs = False for name in input_list: parallel_deriv_color = None @@ -704,12 +717,12 @@ def _create_in_idx_map(self, mode): else: path = name - if path not in abs2meta_out: + if path not in all_abs2meta_out: # could be promoted input name abs_in = model._var_allprocs_prom2abs_list['input'][path][0] path = model._conn_global_abs_in2out[abs_in] - in_var_meta = abs2meta_out[path] + in_var_meta = all_abs2meta_out[path] if name in vois: # if name is in vois, then it has been declared as either a design var or @@ -753,6 +766,7 @@ def _create_in_idx_map(self, mode): offsets = var_offsets['output'] gstart = np.sum(sizes[:iproc, in_var_idx]) gend = gstart + sizes[iproc, in_var_idx] + has_rank_zeros = np.count_nonzero(sizes[:, in_var_idx]) < sizes.shape[0] # if we're doing parallel deriv coloring, we only want to set the seed on one proc # for each var in a given color @@ -816,16 +830,24 @@ def _create_in_idx_map(self, mode): imeta['idx_list'] = range(start, end) idx_iter_dict[name] = (imeta, self.single_index_iter) + if has_rank_zeros: + if owning_rank[path] == iproc: + do_allreduce = 1 # do allreduce with this proc's data + else: + do_allreduce = -1 # do allreduce with zeros + else: + do_allreduce = 0 # don't do allreduce at all + if path in relevant and not non_rel_outs: relsystems = relevant[path]['@all'][1] if self.total_relevant_systems is not _contains_all: self.total_relevant_systems.update(relsystems) # tup = (ndups, relsystems, cache_lin_sol, name) - tup = (dist, relsystems, cache_lin_sol, name) + tup = (do_allreduce, relsystems, cache_lin_sol, name) else: self.total_relevant_systems = _contains_all # tup = (ndups, _contains_all, cache_lin_sol, name) - tup = (dist, _contains_all, cache_lin_sol, name) + tup = (do_allreduce, _contains_all, cache_lin_sol, name) idx_map.extend([tup] * (end - start)) start = end @@ -1357,19 +1379,42 @@ def _jac_setter_dist(self, i, mode): # for rows corresponding to serial 'of' vars, we need to correct for # duplication of their seed values by dividing by the number of duplications. # ndups, _, _, _ = self.in_idx_map[mode][i] - dist, _, _, _ = self.in_idx_map[mode][i] + do_allreduce, _, _, name = self.in_idx_map[mode][i] + local_resp = self.in_loc_idxs['rev'][i] >= 0 # print('JAC_SETTER_DIST', i, self.model.comm.rank, self.J[i], flush=True) if self.get_remote: - if self.jac_dist_col_mask is not None: - distpart = self.J[i, :][self.jac_dist_col_mask] - scratch = np.zeros(distpart.shape, dtype=distpart.dtype) - # only sum up the distrib parts - self.comm.Allreduce(distpart, scratch, op=MPI.SUM) - self.J[i][self.jac_dist_col_mask] = scratch - #elif dist: - #scratch = self.jac_scratch['rev'][0] - #scratch[:] = self.J[i] - # self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) + # if self.jac_dist_col_mask is not None: + # # this happens if there are distributed 'wrt' variables + # distpart = self.J[i, :][self.jac_dist_col_mask] + # scratch = np.zeros(distpart.shape, dtype=distpart.dtype) + # # only sum up the distrib parts + # self.comm.Allreduce(distpart, scratch, op=MPI.SUM) + # self.J[i][self.jac_dist_col_mask] = scratch + scratch = self.jac_scratch['rev'][0] + # if do_allreduce and self.dup_mask is None: + # if do_allreduce > 0: + # scratch[:] = self.J[i] + # else: + # scratch[:] = 0.0 + # self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) + if self.dup_mask is not None: + scratch[:] = 0.0 + scratch[self.dup_mask] = self.J[i][self.dup_mask] + self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) + # elif do_allreduce: + # if do_allreduce > 0: + # scratch[:] = self.J[i] + # else: + # scratch[:] = 0.0 + # self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) + + # elif do_allreduce: + # scratch = self.jac_scratch['rev'][0] + # if do_allreduce > 0: + # scratch[:] = self.J[i] + # else: + # scratch[:] = 0.0 + # self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) # scratch = self.jac_scratch['rev'][0] # scratch[:] = self.J[i] diff --git a/openmdao/test_suite/components/paraboloid_distributed.py b/openmdao/test_suite/components/paraboloid_distributed.py index b8d7de1189..73b94c2e48 100644 --- a/openmdao/test_suite/components/paraboloid_distributed.py +++ b/openmdao/test_suite/components/paraboloid_distributed.py @@ -30,7 +30,6 @@ def setup(self): start = offsets[rank] io_size = sizes[rank] self.offset = offsets[rank] - end = start + io_size # src_indices will be computed automatically self.add_input('x', val=np.ones(io_size), distributed=True) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 762be42594..a48917b04f 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -216,7 +216,7 @@ def get_xfer_ranks(name, io): # has_multi_conn_src = len(rev_conns[abs_out]) > 1 if inp_is_dup and (abs_out not in abs2meta_out or (distrib_out and not iowninput)): - # print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out] rev_xfer_out[sub_out] elif out_is_dup and inp_is_dup and inp_missing > 0 and iowninput: @@ -248,7 +248,7 @@ def get_xfer_ranks(name, io): output_inds = np.concatenate(oidxlist) if len(oidxlist) > 1 else oidxlist[0] rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) - # print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) if has_rev_par_coloring and iidxlist_nc: input_inds = np.concatenate(iidxlist_nc) if len(iidxlist_nc) > 1 else iidxlist_nc[0] @@ -292,7 +292,7 @@ def get_xfer_ranks(name, io): input_inds = output_inds = np.zeros(0, dtype=INT_DTYPE) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) - # print('MULTI2', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + print('MULTI2', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) if has_rev_par_coloring and iidxlist_nc: input_inds = np.concatenate(iidxlist_nc) if len(iidxlist_nc) > 1 else iidxlist_nc[0] @@ -301,7 +301,7 @@ def get_xfer_ranks(name, io): rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) else: - # print(group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + print(group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) else: @@ -379,6 +379,9 @@ def get_xfer_ranks(name, io): vectors['input']['nonlinear'], vectors['output']['nonlinear'], rev_xfer_in_nocolor[sname], inds, group.comm) + from om_devtools.dist_idxs import dump_dist_idxs + print(f"DIST IDXS for '{group.pathname}', rank {group.comm.rank}:", flush=True) + dump_dist_idxs(group) def _transfer(self, in_vec, out_vec, mode='fwd'): """ From ae515174ce1e7ea17a1ff0d52bf5e5f24614c8c1 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 2 Oct 2023 11:19:56 -0400 Subject: [PATCH 12/70] 2 failures, both related to rev derivs when model has an fd group --- openmdao/core/tests/test_check_totals.py | 6 ++-- openmdao/core/tests/test_distrib_derivs.py | 37 +++++++++++++--------- openmdao/core/total_jac.py | 2 +- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/openmdao/core/tests/test_check_totals.py b/openmdao/core/tests/test_check_totals.py index 8384655391..924edf59eb 100644 --- a/openmdao/core/tests/test_check_totals.py +++ b/openmdao/core/tests/test_check_totals.py @@ -216,9 +216,9 @@ def _remotevar_compute_totals(self, mode): assert_near_equal(prob['c4.y1'], 46.0, 1e-6) assert_near_equal(prob['c4.y2'], -93.0, 1e-6) - J = prob.compute_totals(of=unknown_list, wrt=indep_list) - for key, val in full_expected.items(): - assert_near_equal(J[key], val, 1e-6) + #J = prob.compute_totals(of=unknown_list, wrt=indep_list) + #for key, val in full_expected.items(): + #assert_near_equal(J[key], val, 1e-6) reduced_expected = {key: v for key, v in full_expected.items() if key[0] in prob.model._var_abs2meta['output']} diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index a6a7f89f1a..c589d57c9b 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -612,8 +612,7 @@ def test_distrib_voi_fd(self): ivc.add_output('a', -3.0 + 0.6 * np.arange(size)) model.add_subsystem('p', ivc, promotes=['*']) - # model.add_subsystem("parab", DistParab(arr_size=size, deriv_type='fd'), promotes=['*']) - model.add_subsystem("parab", DistParab(arr_size=size), promotes=['*']) + model.add_subsystem("parab", DistParab(arr_size=size, deriv_type='fd'), promotes=['*']) model.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, ))), @@ -638,11 +637,11 @@ def test_distrib_voi_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - assert_check_totals(prob.check_totals(out_stream=None, method='cs'), atol=1e-5) - # assert_near_equal(J['parab.f_xy', 'p.x']['abs error'].forward, 0.0, 1e-5) - # assert_near_equal(J['parab.f_xy', 'p.y']['abs error'].forward, 0.0, 1e-5) - # assert_near_equal(J['sum.f_sum', 'p.x']['abs error'].forward, 0.0, 1e-5) - # assert_near_equal(J['sum.f_sum', 'p.y']['abs error'].forward, 0.0, 1e-5) + J = prob.check_totals(out_stream=None, method='cs') + assert_near_equal(J['parab.f_xy', 'p.x']['abs error'].forward, 0.0, 1e-5) + assert_near_equal(J['parab.f_xy', 'p.y']['abs error'].forward, 0.0, 1e-5) + assert_near_equal(J['sum.f_sum', 'p.x']['abs error'].forward, 0.0, 1e-5) + assert_near_equal(J['sum.f_sum', 'p.y']['abs error'].forward, 0.0, 1e-5) # rev mode @@ -658,11 +657,11 @@ def test_distrib_voi_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - assert_check_totals(prob.check_totals(method='cs', show_only_incorrect=True), atol=1e-5) - # assert_near_equal(J['parab.f_xy', 'p.x']['abs error'].reverse, 0.0, 1e-5) - # assert_near_equal(J['parab.f_xy', 'p.y']['abs error'].reverse, 0.0, 1e-5) - # assert_near_equal(J['sum.f_sum', 'p.x']['abs error'].reverse, 0.0, 1e-5) - # assert_near_equal(J['sum.f_sum', 'p.y']['abs error'].reverse, 0.0, 1e-5) + J = prob.check_totals(method='cs', show_only_incorrect=True) + assert_near_equal(J['parab.f_xy', 'p.x']['abs error'].reverse, 0.0, 1e-5) + assert_near_equal(J['parab.f_xy', 'p.y']['abs error'].reverse, 0.0, 1e-5) + assert_near_equal(J['sum.f_sum', 'p.x']['abs error'].reverse, 0.0, 1e-5) + assert_near_equal(J['sum.f_sum', 'p.y']['abs error'].reverse, 0.0, 1e-5) def test_distrib_voi_group_fd(self): # Only supports groups where the inputs to the distributed component whose inputs are @@ -718,7 +717,11 @@ def test_distrib_voi_group_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - #assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=1e-5) + J = prob.check_totals(method='fd', show_only_incorrect=True) + assert_near_equal(J['sub.parab.f_xy', 'p.x']['abs error'].forward, 0.0, 1e-5) + assert_near_equal(J['sub.parab.f_xy', 'p.y']['abs error'].forward, 0.0, 1e-5) + assert_near_equal(J['sub.sum.f_sum', 'p.x']['abs error'].forward, 0.0, 1e-5) + assert_near_equal(J['sub.sum.f_sum', 'p.y']['abs error'].forward, 0.0, 1e-5) # rev mode @@ -734,7 +737,11 @@ def test_distrib_voi_group_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=1e-5) + J = prob.check_totals(method='fd', show_only_incorrect=True) + assert_near_equal(J['sub.parab.f_xy', 'p.x']['abs error'].reverse, 0.0, 1e-5) + assert_near_equal(J['sub.parab.f_xy', 'p.y']['abs error'].reverse, 0.0, 1e-5) + assert_near_equal(J['sub.sum.f_sum', 'p.x']['abs error'].reverse, 0.0, 1e-5) + assert_near_equal(J['sub.sum.f_sum', 'p.y']['abs error'].reverse, 0.0, 1e-5) def test_simple_distrib_voi_group_fd(self): size = 3 @@ -2192,7 +2199,7 @@ def test_check_totals_rev_old(self): assert_check_totals(data) msg = "During total derivative computation, the following partial derivatives resulted in serial inputs that were inconsistent across processes: ['D1.out_dist wrt D1.in_nd']." - self.assertEqual(str(cm.exception), msg) + self.assertTrue(msg in str(cm.exception)) def test_check_partials_cs_old(self): prob = self.get_problem(Distrib_Derivs_Matfree_Old) diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index b77224c2f0..340517977d 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -1430,7 +1430,7 @@ def _jac_setter_dist(self, i, mode): # # self.J[i] *= (1.0 / ndups) else: scatter = self.jac_scatters[mode] - if scatter is not None: + if False: # scatter is not None: if self.dist_idx_map[mode][i]: # distrib var, skip scatter return loc = self.loc_jac_idxs[mode][i] From d3ea67188fcacc4cd2923ef540b2dc62ed923f5d Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 2 Oct 2023 14:20:23 -0400 Subject: [PATCH 13/70] cleanup --- openmdao/core/tests/test_distrib_derivs.py | 8 -- openmdao/core/total_jac.py | 152 ++++++--------------- openmdao/vectors/petsc_transfer.py | 10 +- 3 files changed, 47 insertions(+), 123 deletions(-) diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index c589d57c9b..359e69caaf 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -781,14 +781,6 @@ def test_simple_distrib_voi_group_fd(self): prob.run_model() - # desvar = prob.driver.get_design_var_values() - # con = prob.driver.get_constraint_values() - - # assert_near_equal(desvar['p.x'], np.ones(size), 1e-6) - # assert_near_equal(con['sub.parab.f_xy'], - # np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), - # 1e-6) - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=1e-5) def test_distrib_group_fd_unsupported_config(self): diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index 340517977d..454f22b454 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -401,31 +401,22 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, if has_out_dist: self.jac_dist_col_mask = np.zeros(J.shape[1], dtype=bool) - self.dup_mask = np.ones(J.shape[1], dtype=bool) + # create a column mask to zero out contributions to the Allreduce from + # duplicated vars + self.rev_allreduce_mask = np.ones(J.shape[1], dtype=bool) - sizes = model._var_sizes['output'] - abs2idx = model._var_allprocs_abs2idx start = end = 0 for name in self.wrt: meta = all_abs2meta_out[name] - varidx = abs2idx[name] end += meta['global_size'] - if not meta['distributed']: - # see if we have an odd dist var like some auto_ivcs connected to - # remote vars, which are zero everywhere except for one proc - # if has_out_dist: - # sz = sizes[:, varidx] - # if True: # np.count_nonzero(sz) > 1: - # self.jac_dist_col_mask[start:end] = True - # self.dup_mask[start:end] = True - #elif 0 in sizes[:, varidx]: - if model._owning_rank[name] != model.comm.rank: - self.dup_mask[start:end] = False + if not meta['distributed'] and model._owning_rank[name] != model.comm.rank: + self.rev_allreduce_mask[start:end] = False start = end - need_allreduce = not np.all(self.dup_mask) + # if rev_allreduce_mask isn't all True on all procs, then we need to do an Allreduce + need_allreduce = not np.all(self.rev_allreduce_mask) if not any(model.comm.allgather(need_allreduce)): - self.dup_mask = None + self.rev_allreduce_mask = None if not approx: for mode in modes: @@ -830,24 +821,14 @@ def _create_in_idx_map(self, mode): imeta['idx_list'] = range(start, end) idx_iter_dict[name] = (imeta, self.single_index_iter) - if has_rank_zeros: - if owning_rank[path] == iproc: - do_allreduce = 1 # do allreduce with this proc's data - else: - do_allreduce = -1 # do allreduce with zeros - else: - do_allreduce = 0 # don't do allreduce at all - if path in relevant and not non_rel_outs: relsystems = relevant[path]['@all'][1] if self.total_relevant_systems is not _contains_all: self.total_relevant_systems.update(relsystems) - # tup = (ndups, relsystems, cache_lin_sol, name) - tup = (do_allreduce, relsystems, cache_lin_sol, name) + tup = (relsystems, cache_lin_sol, name) else: self.total_relevant_systems = _contains_all - # tup = (ndups, _contains_all, cache_lin_sol, name) - tup = (do_allreduce, _contains_all, cache_lin_sol, name) + tup = (_contains_all, cache_lin_sol, name) idx_map.extend([tup] * (end - start)) start = end @@ -871,7 +852,7 @@ def _create_in_idx_map(self, mode): locs = None for ilist in simul_coloring.color_iter(mode): for i in ilist: - _, rel_systems, cache_lin_sol, _ = idx_map[i] + rel_systems, cache_lin_sol, _ = idx_map[i] _update_rel_systems(all_rel_systems, rel_systems) cache |= cache_lin_sol @@ -1204,7 +1185,7 @@ def single_input_setter(self, idx, imeta, mode): int or None key used for storage of cached linear solve (if active, else None). """ - _, rel_systems, cache_lin_sol, _ = self.in_idx_map[mode][idx] + rel_systems, cache_lin_sol, _ = self.in_idx_map[mode][idx] self._zero_vecs(mode) @@ -1315,7 +1296,7 @@ def directional_input_setter(self, inds, itermeta, mode): Not used. """ for i in inds: - _, rel_systems, _, _ = self.in_idx_map[mode][i] + rel_systems, _, _ = self.in_idx_map[mode][i] break self._zero_vecs(mode) @@ -1367,87 +1348,38 @@ def _jac_setter_dist(self, i, mode): mode : str Direction of derivative solution. """ - if self.get_remote and mode == 'fwd': - if self.jac_scatters[mode] is not None: - self.src_petsc[mode].array = self.J[:, i] - self.tgt_petsc[mode].array[:] = self.J[:, i] - self.jac_scatters[mode].scatter(self.src_petsc[mode], self.tgt_petsc[mode], - addv=False, mode=False) - self.J[:, i] = self.tgt_petsc[mode].array - - elif mode == 'rev': - # for rows corresponding to serial 'of' vars, we need to correct for - # duplication of their seed values by dividing by the number of duplications. - # ndups, _, _, _ = self.in_idx_map[mode][i] - do_allreduce, _, _, name = self.in_idx_map[mode][i] - local_resp = self.in_loc_idxs['rev'][i] >= 0 - # print('JAC_SETTER_DIST', i, self.model.comm.rank, self.J[i], flush=True) + if mode == 'fwd': if self.get_remote: - # if self.jac_dist_col_mask is not None: - # # this happens if there are distributed 'wrt' variables - # distpart = self.J[i, :][self.jac_dist_col_mask] - # scratch = np.zeros(distpart.shape, dtype=distpart.dtype) - # # only sum up the distrib parts - # self.comm.Allreduce(distpart, scratch, op=MPI.SUM) - # self.J[i][self.jac_dist_col_mask] = scratch - scratch = self.jac_scratch['rev'][0] - # if do_allreduce and self.dup_mask is None: - # if do_allreduce > 0: - # scratch[:] = self.J[i] - # else: - # scratch[:] = 0.0 - # self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) - if self.dup_mask is not None: + if self.jac_scatters[mode] is not None: + self.src_petsc[mode].array = self.J[:, i] + self.tgt_petsc[mode].array[:] = self.J[:, i] + self.jac_scatters[mode].scatter(self.src_petsc[mode], self.tgt_petsc[mode], + addv=False, mode=False) + self.J[:, i] = self.tgt_petsc[mode].array + + else: # rev + if self.get_remote: + if self.rev_allreduce_mask is not None: + scratch = self.jac_scratch['rev'][0] scratch[:] = 0.0 - scratch[self.dup_mask] = self.J[i][self.dup_mask] + scratch[self.rev_allreduce_mask] = self.J[i][self.rev_allreduce_mask] self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) - # elif do_allreduce: - # if do_allreduce > 0: - # scratch[:] = self.J[i] - # else: - # scratch[:] = 0.0 - # self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) - - # elif do_allreduce: - # scratch = self.jac_scratch['rev'][0] - # if do_allreduce > 0: - # scratch[:] = self.J[i] - # else: - # scratch[:] = 0.0 - # self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) - - # scratch = self.jac_scratch['rev'][0] - # scratch[:] = self.J[i] - - # # self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) - - # # if ndups > 1: - # # if self.jac_dist_col_mask is not None: - # # scratch[:] = 1.0 - # # scratch[self.jac_dist_col_mask] = (1.0 / ndups) - # # self.J[i] *= scratch - # # else: - # # self.J[i] *= (1.0 / ndups) - else: - scatter = self.jac_scatters[mode] - if False: # scatter is not None: - if self.dist_idx_map[mode][i]: # distrib var, skip scatter - return - loc = self.loc_jac_idxs[mode][i] - if loc >= 0: - self.tgt_petsc[mode].array[:] = self.J[loc, :][self.nondist_loc_map[mode]] - self.src_petsc[mode].array[:] = self.J[loc, :][self.nondist_loc_map[mode]] - else: - self.src_petsc[mode].array[:] = 0.0 - - scatter.scatter(self.src_petsc[mode], self.tgt_petsc[mode], - addv=True, mode=False) - if loc >= 0: - # if ndups > 1: - # self.J[loc, :][self.nondist_loc_map[mode]] = \ - # self.tgt_petsc[mode].array * (1.0 / ndups) - # else: - self.J[loc, :][self.nondist_loc_map[mode]] = self.tgt_petsc[mode].array + # else: + # scatter = self.jac_scatters[mode] + # if scatter is not None: + # if self.dist_idx_map[mode][i]: # distrib var, skip scatter + # return + # loc = self.loc_jac_idxs[mode][i] + # if loc >= 0: + # self.tgt_petsc[mode].array[:] = self.J[loc, :][self.nondist_loc_map[mode]] + # self.src_petsc[mode].array[:] = self.J[loc, :][self.nondist_loc_map[mode]] + # else: + # self.src_petsc[mode].array[:] = 0.0 + + # scatter.scatter(self.src_petsc[mode], self.tgt_petsc[mode], + # addv=True, mode=False) + # if loc >= 0: + # self.J[loc, :][self.nondist_loc_map[mode]] = self.tgt_petsc[mode].array def single_jac_setter(self, i, mode, meta): """ diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index a48917b04f..d670e229aa 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -216,7 +216,7 @@ def get_xfer_ranks(name, io): # has_multi_conn_src = len(rev_conns[abs_out]) > 1 if inp_is_dup and (abs_out not in abs2meta_out or (distrib_out and not iowninput)): - print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + # print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out] rev_xfer_out[sub_out] elif out_is_dup and inp_is_dup and inp_missing > 0 and iowninput: @@ -248,7 +248,7 @@ def get_xfer_ranks(name, io): output_inds = np.concatenate(oidxlist) if len(oidxlist) > 1 else oidxlist[0] rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) - print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + # print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) if has_rev_par_coloring and iidxlist_nc: input_inds = np.concatenate(iidxlist_nc) if len(iidxlist_nc) > 1 else iidxlist_nc[0] @@ -292,7 +292,7 @@ def get_xfer_ranks(name, io): input_inds = output_inds = np.zeros(0, dtype=INT_DTYPE) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) - print('MULTI2', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + # print('MULTI2', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) if has_rev_par_coloring and iidxlist_nc: input_inds = np.concatenate(iidxlist_nc) if len(iidxlist_nc) > 1 else iidxlist_nc[0] @@ -301,7 +301,7 @@ def get_xfer_ranks(name, io): rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) else: - print(group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) + # print(group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) else: @@ -380,7 +380,7 @@ def get_xfer_ranks(name, io): rev_xfer_in_nocolor[sname], inds, group.comm) from om_devtools.dist_idxs import dump_dist_idxs - print(f"DIST IDXS for '{group.pathname}', rank {group.comm.rank}:", flush=True) + # print(f"DIST IDXS for '{group.pathname}', rank {group.comm.rank}:", flush=True) dump_dist_idxs(group) def _transfer(self, in_vec, out_vec, mode='fwd'): From 8564ab041845b0c9abad5eb3caba251e0114b24f Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 2 Oct 2023 15:40:46 -0400 Subject: [PATCH 14/70] cleanup --- openmdao/vectors/petsc_transfer.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index d670e229aa..06614fb9b1 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -72,6 +72,7 @@ def _setup_transfers(group, desvars, responses): Dictionary of all response variable metadata. Keyed by absolute source name or alias. """ rev = group._mode != 'fwd' + uses_approx = group._owns_approx_jac for subsys in group._subgroups_myproc: subsys._setup_transfers(desvars, responses) @@ -97,7 +98,7 @@ def _setup_transfers(group, desvars, responses): rev_xfer_in = defaultdict(list) rev_xfer_out = defaultdict(list) - # xfers that are only active when parallel coloring is not active + # xfers that are only active when parallel coloring is not rev_xfer_in_nocolor = defaultdict(list) rev_xfer_out_nocolor = defaultdict(list) @@ -116,14 +117,10 @@ def is_dup(name, io): nz = np.count_nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]]) return nz > 1, group._var_sizes[io].shape[0] - nz, False - def get_rank_sizes(name, io): - idx = allprocs_abs2idx[name] - return group._var_sizes[io][:, idx] - def get_xfer_ranks(name, io): if group._var_allprocs_abs2meta[io][name]['distributed']: return [] - return np.nonzero(get_rank_sizes(name, io))[0] + return np.nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]])[0] # Loop through all connections owned by this system for abs_in, abs_out in group._conn_abs_in2out.items(): @@ -208,18 +205,17 @@ def get_xfer_ranks(name, io): if rev: inp_is_dup, inp_missing, distrib_in = is_dup(abs_in, 'input') out_is_dup, _, distrib_out = is_dup(abs_out, 'output') - # gsize_in = np.sum(sizes_in[:, idx_in]) - # gsize_out = np.sum(sizes_out[:, idx_out]) iowninput = myrank == group._owning_rank[abs_in] sub_out = abs_out[mypathlen:].partition('.')[0] - # has_multi_conn_src = len(rev_conns[abs_out]) > 1 if inp_is_dup and (abs_out not in abs2meta_out or (distrib_out and not iowninput)): - # print(group.pathname, 'rank', group.comm.rank, ':', 'NOT DOING', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out] rev_xfer_out[sub_out] elif out_is_dup and inp_is_dup and inp_missing > 0 and iowninput: + # if this proc owns the input and both the output and input have duplicates, + # then we send the owning input to each duplicated output that doesn't have + # a corresponding connected input on the same proc. oidxlist = [] iidxlist = [] oidxlist_nc = [] @@ -248,9 +244,9 @@ def get_xfer_ranks(name, io): output_inds = np.concatenate(oidxlist) if len(oidxlist) > 1 else oidxlist[0] rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) - # print('MULTI', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) if has_rev_par_coloring and iidxlist_nc: + # keep transfers separate that shouldn't happen when partial coloring is active input_inds = np.concatenate(iidxlist_nc) if len(iidxlist_nc) > 1 else iidxlist_nc[0] output_inds = np.concatenate(oidxlist_nc) if len(oidxlist_nc) > 1 else oidxlist_nc[0] @@ -269,8 +265,6 @@ def get_xfer_ranks(name, io): elif src_indices.size > 0: if distrib_in and not distrib_out and len(on_iprocs) == 1 and on_iprocs[0] == rnk: offset -= np.sum(sizes_out[:rnk, idx_out]) - # if distrib_in and gsize_in == shape_to_len(src_shape): # gsize_in == gsize_out: - # offset -= np.sum(sizes_out[:rnk, idx_out]) oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) iarr = input_inds else: @@ -292,7 +286,6 @@ def get_xfer_ranks(name, io): input_inds = output_inds = np.zeros(0, dtype=INT_DTYPE) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) - # print('MULTI2', group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) if has_rev_par_coloring and iidxlist_nc: input_inds = np.concatenate(iidxlist_nc) if len(iidxlist_nc) > 1 else iidxlist_nc[0] @@ -301,7 +294,6 @@ def get_xfer_ranks(name, io): rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) else: - # print(group.pathname, 'rank', group.comm.rank, ':', abs_out, '-->', abs_in, output_inds, '-->', input_inds, flush=True) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) else: @@ -379,9 +371,9 @@ def get_xfer_ranks(name, io): vectors['input']['nonlinear'], vectors['output']['nonlinear'], rev_xfer_in_nocolor[sname], inds, group.comm) - from om_devtools.dist_idxs import dump_dist_idxs + # from om_devtools.dist_idxs import dump_dist_idxs # print(f"DIST IDXS for '{group.pathname}', rank {group.comm.rank}:", flush=True) - dump_dist_idxs(group) + # dump_dist_idxs(group) def _transfer(self, in_vec, out_vec, mode='fwd'): """ From 71dd83fb2c949e1c15551cf3963fd3b999b72683 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 2 Oct 2023 16:19:48 -0400 Subject: [PATCH 15/70] fixed pep issues. 2 remaining failures --- openmdao/core/explicitcomponent.py | 4 -- openmdao/core/group.py | 4 -- openmdao/core/system.py | 4 -- openmdao/core/total_jac.py | 42 --------------- openmdao/solvers/linear/linear_block_gs.py | 2 - openmdao/vectors/petsc_transfer.py | 61 ++++++++++++++-------- 6 files changed, 39 insertions(+), 78 deletions(-) diff --git a/openmdao/core/explicitcomponent.py b/openmdao/core/explicitcomponent.py index 96daca4279..9d8b96ff3e 100644 --- a/openmdao/core/explicitcomponent.py +++ b/openmdao/core/explicitcomponent.py @@ -392,8 +392,6 @@ def _apply_linear(self, jac, rel_systems, mode, scope_out=None, scope_in=None): # Jacobian and vectors are all scaled, unitless J._apply(self, d_inputs, d_outputs, d_residuals, mode) - # print('APPLY', self.pathname, 'd_inputs', d_inputs.asarray(), 'd_outputs', d_outputs.asarray(), 'd_residuals', d_residuals.asarray(), flush=True) - if not self.matrix_free: # if we're not matrix free, we can skip the rest because # compute_jacvec_product does nothing. @@ -478,8 +476,6 @@ def _solve_linear(self, mode, rel_systems, scope_out=_UNDEFINED, scope_in=_UNDEF # ExplicitComponent jacobian defined with -1 on diagonal. d_residuals *= -1.0 - # print(MPI.COMM_WORLD.rank, 'SOLVE', self.pathname, 'd_inputs', self._dinputs.asarray(), 'd_outputs', self._doutputs.asarray(), 'd_residuals', self._dresiduals.asarray(), flush=True) - def _compute_partials_wrapper(self): """ Call compute_partials based on the value of the "run_root_only" option. diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 84d002b694..cc65a6f9e9 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -3719,10 +3719,6 @@ def _apply_linear(self, jac, rel_systems, mode, scope_out=None, scope_in=None): with self._matvec_context(scope_out, scope_in, mode) as vecs: d_inputs, d_outputs, d_residuals = vecs jac._apply(self, d_inputs, d_outputs, d_residuals, mode) - if self._owns_approx_jac and self._has_distrib_vars and mode == 'rev': - pass # DO ALLREDUCE OF parts of d_inputs where inputs are connected to distrib outputs - # and only for parts in the current matvec context - print(f"AFTER JAC APPLY in {self.pathname}: d_inputs: {[f'{n}, {s}' for n, s in d_inputs.items()]}", flush=True) # Apply recursion else: if mode == 'fwd': diff --git a/openmdao/core/system.py b/openmdao/core/system.py index 5cc5b580cb..07b27fa523 100644 --- a/openmdao/core/system.py +++ b/openmdao/core/system.py @@ -5575,10 +5575,6 @@ def _get_input_from_src(self, name, abs_ins, conns, units=None, indices=None, "`get_val(, get_remote=True)`.") else: if src_indices._flat_src: - #if distrib and not sdistrib: - #start = np.sum(sizes[:self.comm.rank]) - #sinds = src_indices.apply_offset(-start, flat=True) - #else: sinds = src_indices.flat() val = val.ravel()[sinds] # if at component level, just keep shape of the target and don't flatten diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index 98e84d4700..91940d8e20 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -33,35 +33,6 @@ _directional_rng = np.random.default_rng(99) -class EntryMeta(object): - def __init__(self): - pass - - -class RowMeta(object): - def __init__(self, model): - pass - - def by_name(self, name): - pass - - def by_index(self, idx): - pass - - -class ColumnMeta(object): - def __init__(self): - pass - - def by_name(self, name): - pass - - def by_index(self, idx): - pass - - - - class _TotalJacInfo(object): """ Object to manage computation of total derivatives. @@ -1331,7 +1302,6 @@ def simple_single_jac_scatter(self, i, mode): else: return - # print('DERIV_VAL:', i, self.model.comm.rank, deriv_val, deriv_idxs, flush=True) if mode == 'fwd': self.J[jac_idxs, i] = deriv_val[deriv_idxs] else: # rev @@ -1599,22 +1569,10 @@ def compute_totals(self): else: model._solve_linear(mode, rel_systems) - # print('TOP SOLVE', 'd_inputs', model._dinputs.asarray(), 'd_outputs', model._doutputs.asarray(), 'd_residuals', model._dresiduals.asarray(), flush=True) - if debug_print: print(f'Elapsed Time: {time.perf_counter() - t0} secs\n', flush=True) - # if isinstance(inds, int): - # print('JAC ROW (before):', self.J[inds], flush=True) - # else: - # for i in inds: - # print('JAC ROW (before):', self.J[i], flush=True) jac_setter(inds, mode, imeta) - # if isinstance(inds, int): - # print('JAC ROW (after):', self.J[inds], flush=True) - # else: - # for i in inds: - # print('JAC ROW (after):', self.J[i], flush=True) # Driver scaling. if self.has_scaling: diff --git a/openmdao/solvers/linear/linear_block_gs.py b/openmdao/solvers/linear/linear_block_gs.py index 7769887a84..f6759f783d 100644 --- a/openmdao/solvers/linear/linear_block_gs.py +++ b/openmdao/solvers/linear/linear_block_gs.py @@ -156,11 +156,9 @@ def _single_iteration(self): scope_in = self._vars_union(self._scope_in, scope_in) subsys._solve_linear(mode, self._rel_systems, scope_out, scope_in) - # print(system.comm.rank, 'SOLVE LINEAR', subsys.pathname, 'd_inputs', subsys._dinputs.asarray(), 'd_outputs', subsys._doutputs.asarray(), 'd_residuals', subsys._dresiduals.asarray(), flush=True) if subsys._iter_call_apply_linear(): subsys._apply_linear(None, self._rel_systems, mode, scope_out, scope_in) - # print(system.comm.rank, 'APPLY LINEAR', subsys.pathname, 'd_inputs', subsys._dinputs.asarray(), 'd_outputs', subsys._doutputs.asarray(), 'd_residuals', subsys._dresiduals.asarray(), flush=True) else: b_vec.set_val(0.0) else: # subsys not local diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 06614fb9b1..49a2955c75 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -22,7 +22,6 @@ from openmdao.core.constants import INT_DTYPE from openmdao.utils.array_utils import shape_to_len - class PETScTransfer(DefaultTransfer): """ PETSc Transfer implementation for running in parallel. @@ -69,7 +68,8 @@ def _setup_transfers(group, desvars, responses): desvars : dict Dictionary of all design variable metadata. Keyed by absolute source name or alias. responses : dict - Dictionary of all response variable metadata. Keyed by absolute source name or alias. + Dictionary of all response variable metadata. Keyed by absolute source name or + alias. """ rev = group._mode != 'fwd' uses_approx = group._owns_approx_jac @@ -209,24 +209,28 @@ def get_xfer_ranks(name, io): iowninput = myrank == group._owning_rank[abs_in] sub_out = abs_out[mypathlen:].partition('.')[0] - if inp_is_dup and (abs_out not in abs2meta_out or (distrib_out and not iowninput)): + if inp_is_dup and (abs_out not in abs2meta_out or + (distrib_out and not iowninput)): rev_xfer_in[sub_out] rev_xfer_out[sub_out] elif out_is_dup and inp_is_dup and inp_missing > 0 and iowninput: - # if this proc owns the input and both the output and input have duplicates, - # then we send the owning input to each duplicated output that doesn't have - # a corresponding connected input on the same proc. + # if this proc owns the input and both the output and input have + # duplicates, then we send the owning input to each duplicated output + # that doesn't have a corresponding connected input on the same proc. oidxlist = [] iidxlist = [] oidxlist_nc = [] iidxlist_nc = [] oidxlist.append(output_inds) iidxlist.append(input_inds) - for rnk, osize, isize in zip(range(group.comm.size), sizes_out[:, idx_out], sizes_in[:, idx_in]): + for rnk, osize, isize in zip(range(group.comm.size), + sizes_out[:, idx_out], + sizes_in[:, idx_in]): if osize > 0 and isize == 0: offset = offsets_out[rnk, idx_out] if src_indices is None: - oarr = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) + oarr = np.arange(offset, offset + meta_in['size'], + dtype=INT_DTYPE) iarr = input_inds elif src_indices.size > 0: oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) @@ -240,19 +244,30 @@ def get_xfer_ranks(name, io): oidxlist_nc.append(oarr) iidxlist_nc.append(iarr) - input_inds = np.concatenate(iidxlist) if len(iidxlist) > 1 else iidxlist[0] - output_inds = np.concatenate(oidxlist) if len(oidxlist) > 1 else oidxlist[0] + if len(iidxlist) > 1: + input_inds = np.concatenate(iidxlist) + output_inds = np.concatenate(oidxlist) + else: + input_inds = iidxlist[0] + output_inds = oidxlist[0] + rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) if has_rev_par_coloring and iidxlist_nc: - # keep transfers separate that shouldn't happen when partial coloring is active - input_inds = np.concatenate(iidxlist_nc) if len(iidxlist_nc) > 1 else iidxlist_nc[0] - output_inds = np.concatenate(oidxlist_nc) if len(oidxlist_nc) > 1 else oidxlist_nc[0] + # keep transfers separate that shouldn't happen when partial + # coloring is active + if len(iidxlist_nc) > 1: + input_inds = np.concatenate(iidxlist_nc) + output_inds = np.concatenate(oidxlist_nc) + else: + input_inds = iidxlist_nc[0] + output_inds = oidxlist_nc[0] rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) - elif out_is_dup and (not inp_is_dup or inp_missing > 0) and (iowninput or distrib_in): + elif out_is_dup and (not inp_is_dup or inp_missing > 0) and (iowninput or + distrib_in): oidxlist = [] iidxlist = [] oidxlist_nc = [] @@ -260,10 +275,12 @@ def get_xfer_ranks(name, io): for rnk in get_xfer_ranks(abs_out, 'output'): offset = offsets_out[rnk, idx_out] if src_indices is None: - oarr = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) + oarr = np.arange(offset, offset + meta_in['size'], + dtype=INT_DTYPE) iarr = input_inds elif src_indices.size > 0: - if distrib_in and not distrib_out and len(on_iprocs) == 1 and on_iprocs[0] == rnk: + if (distrib_in and not distrib_out and len(on_iprocs) == 1 and + on_iprocs[0] == rnk): offset -= np.sum(sizes_out[:rnk, idx_out]) oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) iarr = input_inds @@ -288,8 +305,12 @@ def get_xfer_ranks(name, io): rev_xfer_out[sub_out].append(output_inds) if has_rev_par_coloring and iidxlist_nc: - input_inds = np.concatenate(iidxlist_nc) if len(iidxlist_nc) > 1 else iidxlist_nc[0] - output_inds = np.concatenate(oidxlist_nc) if len(oidxlist_nc) > 1 else oidxlist_nc[0] + if len(iidxlist_nc) > 1: + input_inds = np.concatenate(iidxlist_nc) + output_inds = np.concatenate(oidxlist_nc) + else: + input_inds = iidxlist_nc[0] + output_inds = oidxlist_nc[0] rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) @@ -430,7 +451,3 @@ def _transfer(self, in_vec, out_vec, mode='fwd'): if in_vec._alloc_complex: data = in_vec._get_data() data[:] = in_petsc.array - - # print(in_vec._system().comm.rank, 'TRANSFER from', in_vec._system().pathname, self._in_inds, self._out_inds, flush=True) - # print('IN', in_vec._data.real, flush=True) - # print('OUT', out_vec._data.real, flush=True) From 834ca2bf62594f19d8ce95b973c6916d3263732e Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Tue, 3 Oct 2023 14:40:50 -0400 Subject: [PATCH 16/70] fix for approx derivs when part is nonlocal --- openmdao/approximation_schemes/approximation_scheme.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openmdao/approximation_schemes/approximation_scheme.py b/openmdao/approximation_schemes/approximation_scheme.py index a5f9e35460..b15b39e664 100644 --- a/openmdao/approximation_schemes/approximation_scheme.py +++ b/openmdao/approximation_schemes/approximation_scheme.py @@ -445,6 +445,8 @@ def _vec_ind_iter(self, vec_ind_list): entry = [[None, None]] ent0 = entry[0] for vec, vec_idxs in vec_ind_list: + if vec_idxs is None: + continue for vinds in vec_idxs: ent0[0] = vec ent0[1] = vinds From 7e2bdadcf283a52c1ed689eba8e3b6bb34f6d322 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Wed, 4 Oct 2023 09:01:04 -0400 Subject: [PATCH 17/70] adding iterator of names and dist offsets --- openmdao/core/system.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/openmdao/core/system.py b/openmdao/core/system.py index 07b27fa523..c5fc4d1dec 100644 --- a/openmdao/core/system.py +++ b/openmdao/core/system.py @@ -6276,3 +6276,29 @@ def comm_info_iter(self): for s in self._subsystems_myproc: yield from s.comm_info_iter() + + def dist_offset_iter(self, io): + """ + Yield names and distributed offsets of all local and remote variables in this system. + + Parameters + ---------- + io : str + Either 'input' or 'output'. + + Yields + ------ + tuple + A tuple of the form (abs_name, rank, offset). + """ + system = self._system() + sizes = system._var_sizes + vmeta = system._var_allprocs_abs2meta + + total = 0 + for rank in range(system.comm.size): + for ivar, vname in enumerate(vmeta[io]): + sz = sizes[io][rank, ivar] + if sz > 0: + yield vname, rank, total + total += sz From 873896b4e432c78b9ca18cf896b709453e8ba8c2 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Thu, 5 Oct 2023 10:32:08 -0400 Subject: [PATCH 18/70] added dist_conns command tool to view distributed connections --- openmdao/core/system.py | 15 +- openmdao/devtools/debug.py | 239 ++++++++++++++++++++++++++- openmdao/utils/om.py | 7 +- openmdao/utils/range_collection.py | 251 +++++++++++++++-------------- 4 files changed, 379 insertions(+), 133 deletions(-) diff --git a/openmdao/core/system.py b/openmdao/core/system.py index c5fc4d1dec..68e70be4fd 100644 --- a/openmdao/core/system.py +++ b/openmdao/core/system.py @@ -6277,9 +6277,9 @@ def comm_info_iter(self): for s in self._subsystems_myproc: yield from s.comm_info_iter() - def dist_offset_iter(self, io): + def dist_range_iter(self, io): """ - Yield names and distributed offsets of all local and remote variables in this system. + Yield names and distributed ranges of all local and remote variables in this system. Parameters ---------- @@ -6289,16 +6289,15 @@ def dist_offset_iter(self, io): Yields ------ tuple - A tuple of the form (abs_name, rank, offset). + A tuple of the form ((abs_name, rank), start, end). """ - system = self._system() - sizes = system._var_sizes - vmeta = system._var_allprocs_abs2meta + sizes = self._var_sizes + vmeta = self._var_allprocs_abs2meta total = 0 - for rank in range(system.comm.size): + for rank in range(self.comm.size): for ivar, vname in enumerate(vmeta[io]): sz = sizes[io][rank, ivar] if sz > 0: - yield vname, rank, total + yield (vname, rank), total, total + sz total += sz diff --git a/openmdao/devtools/debug.py b/openmdao/devtools/debug.py index e79efbf963..33f2926011 100644 --- a/openmdao/devtools/debug.py +++ b/openmdao/devtools/debug.py @@ -9,11 +9,12 @@ from contextlib import contextmanager from collections import Counter -from openmdao.core.constants import _SetupStatus +from openmdao.core.constants import _SetupStatus, _DEFAULT_OUT_STREAM from openmdao.utils.mpi import MPI from openmdao.utils.om_warnings import issue_warning, MPIWarning from openmdao.utils.reports_system import register_report -from openmdao.utils.file_utils import text2html +from openmdao.utils.file_utils import text2html, _load_and_exec +from openmdao.utils.range_collection import DataRangeMapper from openmdao.visualization.tables.table_builder import generate_table @@ -636,3 +637,237 @@ def comm_info(system, outfile=None, verbose=False, table_format='box_grid'): else: with open(outfile, 'w') as f: print("No MPI process info available.", file=f) + + +def is_full_slice(range, inds): + size = range[1] - range[0] + inds = np.asarray(inds) + if len(inds) > 1 and inds[0] == 0 and inds[-1] == size - 1: + step = inds[1] - inds[0] + diffs = inds[1:] - inds[:-1] + return np.all(diffs == step) + + return len(inds) == 1 and inds[0] == 0 + + +def show_dist_var_conns(group, rev=False, out_stream=_DEFAULT_OUT_STREAM): + """ + Show all distributed variable connections in the given group and below. + + Parameters + ---------- + group : Group + The group to be searched. + rev : bool + If True show reverse transfers. + out_stream : file-like + Where the output will go. + + Returns + ------- + dict or None + Dictionary containing the data for the connections. None is returned on all ranks except + rank 0. + """ + from openmdao.core.group import Group + + if out_stream is _DEFAULT_OUT_STREAM: + out_stream = sys.stdout + + if out_stream is None: + printer = lambda *args, **kwargs: None + else: + printer = print + + direction = 'rev' if rev else 'fwd' + arrowdir = {'fwd': '->', 'rev': '<-'} + arrow = arrowdir[direction] + + gdict = {} + + for g in group.system_iter(typ=Group, include_self=True): + if g._transfers[direction]: + in_ranges = list(g.dist_range_iter('input')) + out_ranges = list(g.dist_range_iter('output')) + + inmapper = DataRangeMapper.create(in_ranges) + outmapper = DataRangeMapper.create(out_ranges) + + gprint = False + + gprefix = g.pathname + '.' if g.pathname else '' + skip = len(gprefix) + + for sub, transfer in g._transfers[direction].items(): + if sub is not None: + if not gprint: + gdict[g.pathname] = {} + gprint = True + + conns = {} + for iidx, oidx in zip(transfer._in_inds, transfer._out_inds): + idata, irind = inmapper.index2data_and_rel_ind(iidx) + ivar, irank = idata + odata, orind = outmapper.index2data_and_rel_ind(oidx) + ovar, orank = odata + + if odata not in conns: + conns[odata] = {} + if idata not in conns[odata]: + conns[odata][idata] = [] + conns[odata][idata].append((orind, irind)) + + strs = {} # use to avoid duplicate printouts for different ranks + + for odata, odict in conns.items(): + ovar, orank = odata + ovar = ovar[skip:] + + for idata, dlist in odict.items(): + ivar, irank = idata + ivar = ivar[skip:] + ranktup = (orank, irank) + + oinds = [d[0] for d in dlist] + iinds = [d[1] for d in dlist] + + orange = outmapper.data2range(odata) + irange = inmapper.data2range(idata) + + oslc = is_full_slice(orange, oinds) + if oslc: + islc = is_full_slice(irange, iinds) + if islc: + s = f"{ovar}[:] {arrow} {ivar}[:]" + if s not in strs: + strs[s] = set() + strs[s].add(ranktup) + continue + + for oidx, iidx in zip(oinds, iinds): + s = f"{ovar}[{oidx}] {arrow} {ivar}[{iidx}]" + if s not in strs: + strs[s] = set() + strs[s].add(ranktup) + + gdict[g.pathname][sub] = strs + + if group.comm.size > 1: + final = {} + gatherlist = group.comm.gather(gdict, root=0) + if group.comm.rank == 0: + for dct in gatherlist: + for gpath, subdct in dct.items(): + if gpath not in final: + final[gpath] = subdct + else: + fgpath = final[gpath] + for sub, strs in subdct.items(): + if sub not in fgpath: + fgpath[sub] = strs + else: + fgpathsub = fgpath[sub] + for s, ranks in strs.items(): + if s not in fgpathsub: + fgpathsub[s] = ranks + else: + fgpathsub[s] |= ranks + + gdict = final + + if group.comm.rank == 0: + fwd = direction == 'fwd' + for gpath, subdct in gdict.items(): + indent = 0 if gpath == '' else gpath.count('.') + 1 + pad = ' ' * indent + printer(f"{pad}In Group '{gpath}'", file=out_stream) + for sub, strs in subdct.items(): + if fwd: + printer(f"{pad} {arrow} {sub}", file=out_stream) + else: + printer(f"{pad} {sub} {arrow}", file=out_stream) + for s, ranks in strs.items(): + oranks = np.empty(len(ranks), dtype=int) + iranks = np.empty(len(ranks), dtype=int) + for i, (ornk, irnk) in enumerate(sorted(ranks)): + oranks[i] = ornk + iranks[i] = irnk + + if np.all(oranks == oranks[0]): + orstr = str(oranks[0]) + else: + orstr = str(sorted(oranks)) + + if np.all(iranks == iranks[0]): + irstr = str(iranks[0]) + else: + irstr = str(sorted(iranks)) + + if orstr == irstr and '[' not in orstr: + printer(f"{pad} {s} rank {orstr}", file=out_stream) + else: + printer(f"{pad} {s} ranks {orstr} {arrow} {irstr}", file=out_stream) + + return gdict + + +def _dist_conns_setup_parser(parser): + """ + Set up the openmdao subparser for the 'openmdao dist_conns' command. + + Parameters + ---------- + parser : argparse subparser + The parser we're adding options to. + """ + parser.add_argument('file', nargs=1, help='Python file containing the model.') + parser.add_argument('-o', default=None, action='store', dest='outfile', + help='Name of output file. By default, output goes to stdout.') + parser.add_argument('-r', '--rev', action='store_true', dest='rev', + help='If set, use "rev" transfer direction instead of "fwd".') + parser.add_argument('-p', '--problem', action='store', dest='problem', help='Problem name') + + +def _dist_conns_cmd(options, user_args): + """ + Run the `openmdao dist_conns` command. + + The command shows connections, with relative indexing information, between all + variables in the model across all MPI processes. + + Parameters + ---------- + options : argparse Namespace + Command line options. + user_args : list of str + Args to be passed to the user script. + """ + import openmdao.utils.hooks as hooks + + def _dist_conns(prob): + model = prob.model + if options.problem: + if model._problem_meta['name'] != options.problem and \ + model._problem_meta['pathname'] != options.problem: + return + elif '/' in model._problem_meta['pathname']: + # by default, only display comm info for a top level problem + return + + if options.outfile is None: + out_stream = sys.stdout + else: + out_stream = open(options.outfile, 'w') + + try: + show_dist_var_conns(model, rev=options.rev, out_stream=out_stream) + finally: + if out_stream is not sys.stdout: + out_stream.close() + + exit() + + # register the hook to be called right after final_setup on the problem + hooks._register_hook('final_setup', 'Problem', post=_dist_conns) + + _load_and_exec(options.file[0], user_args) diff --git a/openmdao/utils/om.py b/openmdao/utils/om.py index f5ec1d416e..c24914fd73 100644 --- a/openmdao/utils/om.py +++ b/openmdao/utils/om.py @@ -43,7 +43,8 @@ from openmdao.components.meta_model_structured_comp import MetaModelStructuredComp from openmdao.components.meta_model_unstructured_comp import MetaModelUnStructuredComp from openmdao.core.component import Component -from openmdao.devtools.debug import config_summary, tree, comm_info +from openmdao.devtools.debug import config_summary, tree, comm_info, _dist_conns_cmd, \ + _dist_conns_setup_parser from openmdao.devtools.itrace import _itrace_exec, _itrace_setup_parser from openmdao.devtools.iprofile_app.iprofile_app import _iprof_exec, _iprof_setup_parser from openmdao.devtools.iprofile import _iprof_totals_exec, _iprof_totals_setup_parser @@ -440,7 +441,7 @@ def _list_pre_post(prob): def _get_deps(dep_dict: dict, package_name: str) -> None: """ - Recursively determine all installed depenency versions and add newly found ones to dep_dict. + Recursively determine all installed dependency versions and add newly found ones to dep_dict. Parameters ---------- @@ -536,6 +537,8 @@ def _set_dyn_hook(prob): 'Print MPI communicator info for systems.'), 'compute_entry_points': (_compute_entry_points_setup_parser, _compute_entry_points_exec, 'Compute entry point declarations to add to the setup.py file.'), + 'dist_conns': (_dist_conns_setup_parser, _dist_conns_cmd, + 'Display connection information for variables across multiple MPI processes.'), 'find_plugins': (_find_plugins_setup_parser, _find_plugins_exec, 'Find openmdao plugins on github.'), 'iprof': (_iprof_setup_parser, _iprof_exec, diff --git a/openmdao/utils/range_collection.py b/openmdao/utils/range_collection.py index 739b73a888..bbaf7d9000 100644 --- a/openmdao/utils/range_collection.py +++ b/openmdao/utils/range_collection.py @@ -2,71 +2,76 @@ from openmdao.utils.array_utils import shape_to_len -# if the total array size is less than this, we'll just use a flat list mapping -# indices to names instead of a binary search tree +# default size of array for which we use a FlatRangeMapper instead of a RangeTree _MAX_FLAT_RANGE_SIZE = 10000 -class NameRangeMapper(object): +class DataRangeMapper(object): + """ + A mapper of indices to variable names and vice versa. + """ def __init__(self, ranges): - self._name2range = {} - self._range2name = {} + self._data2range = {} + self._range2data = {} self.size = ranges[-1][2] - ranges[0][1] @staticmethod - def create(ranges): + def create(ranges, max_flat_range_size=_MAX_FLAT_RANGE_SIZE): """ Return a mapper that maps indices to variable names and relative indices. Parameters ---------- - ranges : list of (name, start, stop) - Ordered list of (name, start, stop) tuples, where start and stop define the range of - indices for that name. Ranges must be contiguous. + ranges : list of (data, start, stop) + Ordered list of (data, start, stop) tuples, where start and stop define the range + of indices for the data. Ranges must be contiguous. + max_flat_range_size : int + If the total array size is less than this, a FlatRangeMapper will be returned instead + of a RangeTree. Returns ------- FlatRangeMapper or RangeTree - A mapper that maps indices to variable names and relative indices. + A mapper that maps indices to variable data and relative indices. """ size = ranges[-1][2] - ranges[0][1] - return FlatRangeMapper(ranges) if size < _MAX_FLAT_RANGE_SIZE else RangeTree(ranges) + return FlatRangeMapper(ranges) if size < max_flat_range_size else RangeTree(ranges) - def add_range(self, name, start, stop): + def add_range(self, data, start, stop): """ Add a range to the mapper. Parameters ---------- - name : str - Name of the variable. + data : object (must be hashable) + Data corresponding to an index range. start : int Starting index of the variable. stop : int Ending index of the variable. """ - self._name2range[name] = (start, stop) - self._range2name[(start, stop)] = name + self._data2range[data] = (start, stop) + self._range2data[start, stop] = data - def name2range(self, name): + def data2range(self, data): """ - Get the range corresponding to the given name. + Get the range corresponding to the given name and rank. Parameters ---------- - name : str - The name of the variable. + data : object (must be hashable) + Data corresponding to an index range. Returns ------- tuple of (int, int) - The range of indices corresponding to the given name. + The range of indices corresponding to the given data. """ - return self._name2range[name] + return self._data2range[data] - def index2name(self, idx): + def index2data(self, idx): """ - Find the name corresponding to the given index. + Find the data corresponding to the given index. Parameters ---------- @@ -75,14 +80,14 @@ def index2name(self, idx): Returns ------- - str or None - The name corresponding to the given index, or None if not found. + object or None + The data corresponding to the given index, or None if not found. """ - raise NotImplementedError("index2name method must be implemented by subclass.") + raise NotImplementedError("index2data method must be implemented by subclass.") - def index2names(self, idxs): + def indices2data(self, idxs): """ - Find the names corresponding to the given indices. + Find the data objects corresponding to the given indices. Parameters ---------- @@ -91,25 +96,66 @@ def index2names(self, idxs): Returns ------- - list of str - The names corresponding to the given indices. + list of (object, int) + The data corresponding to each of the given indices. """ - names = {self.index2name(idx) for idx in idxs} - if None in names: + data = [self.index2data(idx) for idx in idxs] + if None in data: missing = [] for idx in idxs: - if self.index2name(idx) is None: + d = self.index2data(idx) + if d is None: missing.append(idx) - raise RuntimeError("Indices %s are not in any range." % sorted(missing)) + raise RuntimeError(f"Indices {sorted(missing)} are not in any range.") + + return data - return names + +class RangeTreeNode(DataRangeMapper): + """ + A node in a binary search tree of ranges, mapping data to an index range. + + Parameters + ---------- + data : object + Data corresponding to an index range. + start : int + Starting index of the variable. + stop : int + Ending index of the variable. + + Attributes + ---------- + data : object + Data corresponding to an index range. + start : int + Starting index of the variable. + stop : int + Ending index of the variable. + left : RangeTreeNode or None + Left child node. + right : RangeTreeNode or None + Right child node. + """ + + __slots__ = ['data', 'start', 'stop', 'left', 'right'] + + def __init__(self, data, start, stop): + """ + Initialize a RangeTreeNode. + """ + self.data = data + self.start = start + self.stop = stop + self.left = None + self.right = None -class RangeTree(NameRangeMapper): +class RangeTree(DataRangeMapper): """ - A binary search tree of ranges, mapping a name to an index range. + A binary search tree of ranges, mapping data to an index range. - Allows for fast lookup of the name corresponding to a given index. The ranges must be + Allows for fast lookup of the data corresponding to a given index. The ranges must be contiguous, but they can be of different sizes. Search complexity is O(log2 n). Uses less memory than FlatRangeMapper when total array size is @@ -121,17 +167,17 @@ def __init__(self, ranges): Parameters ---------- - ranges : list of (name, start, stop) - List of (name, start, stop) tuples, where name is the variable name and start and stop - define the range of indices for that variable. + ranges : list of (data, start, stop) + List of (data, start, stop) tuples, where start and stop define the range of indices + for the data. Ranges must be contiguous and data must be hashable. """ super().__init__(ranges) self.size = ranges[-1][2] - ranges[0][1] self.root = self.build(ranges) - def index2name(self, idx): + def index2data(self, idx): """ - Find the name corresponding to the given index. + Find the data corresponding to the given index. Parameters ---------- @@ -140,8 +186,10 @@ def index2name(self, idx): Returns ------- - str or None - The name corresponding to the given index, or None if not found. + object or None + The data corresponding to the given index, or None if not found. + int or None + The rank corresponding to the given index, or None if not found. """ node = self.root while node is not None: @@ -150,11 +198,11 @@ def index2name(self, idx): elif idx >= node.stop: node = node.right else: - return node.name + return node.data - def index2name_and_rel_ind(self, idx): + def index2data_and_rel_ind(self, idx): """ - Find the name and relative index corresponding to the matched range. + Find the data and relative index corresponding to the matched range. Parameters ---------- @@ -163,8 +211,8 @@ def index2name_and_rel_ind(self, idx): Returns ------- - str or None - The name corresponding to the matched range, or None if not found. + obj or None + The data corresponding to the matched range, or None if not found. int or None The relative index into the matched range, or None if not found. """ @@ -175,19 +223,20 @@ def index2name_and_rel_ind(self, idx): elif idx >= node.stop: node = node.right else: - return node.name, idx - node.start + return node.data, idx - node.start return None, None def build(self, ranges): """ - Build a binary search tree to map indices to variable names. + Build a binary search tree to map indices to variable data. Parameters ---------- - ranges : list of (name, start, stop) - List of (name, start, stop) tuples, where name is the variable name and start and stop - define the range of indices for that variable. Ranges must be contiguous. + ranges : list of (data, start, stop) + List of (data, start, stop) tuples, where start and stop + define the range of indices for the data. Ranges must be contiguous. + data must be hashable. Returns ------- @@ -195,10 +244,10 @@ def build(self, ranges): Root node of the binary search tree. """ half = len(ranges) // 2 - name, start, stop = ranges[half] + data, start, stop = ranges[half] - node = RangeTreeNode(name, start, stop) - self.add_range(name, start, stop) + node = RangeTreeNode(data, start, stop) + self.add_range(data, start, stop) left_slices = ranges[:half] if left_slices: @@ -211,63 +260,23 @@ def build(self, ranges): return node -class RangeTreeNode(NameRangeMapper): - """ - A node in a binary search tree of ranges, mapping a name to an index range. - - Parameters - ---------- - name : str - Name of the variable. - start : int - Starting index of the variable. - stop : int - Ending index of the variable. - - Attributes - ---------- - name : str - Name of the variable. - start : int - Starting index of the variable. - stop : int - Ending index of the variable. - left : RangeTreeNode or None - Left child node. - right : RangeTreeNode or None - Right child node. - """ - - __slots__ = ['name', 'start', 'stop', 'left', 'right'] - - def __init__(self, name, start, stop): - """ - Initialize a RangeTreeNode. - """ - self.name = name - self.start = start - self.stop = stop - self.left = None - self.right = None - - -class FlatRangeMapper(NameRangeMapper): +class FlatRangeMapper(DataRangeMapper): """ - A flat list mapping indices to variable names and relative indices. + A flat list mapping indices to variable data and relative indices. Parameters ---------- - ranges : list of (name, start, stop) - Ordered list of (name, start, stop) tuples, where start and stop define the range of - indices for that name. Ranges must be contiguous. + ranges : list of (data, start, stop) + Ordered list of (data, start, stop) tuples, where start and stop define the range of + indices for that data. Ranges must be contiguous. data must be hashable. Attributes ---------- size : int Total size of all of the ranges combined. - ranges : list of (name, start, stop) - List of (name, start, stop) tuples, where start and stop define the range of - indices for that name. Ranges must be contiguous. + ranges : list of (data, start, stop) + List of (data, start, stop) tuples, where start and stop define the range of + indices for that data. Ranges must be contiguous. data must be hashable. """ def __init__(self, ranges): @@ -277,13 +286,13 @@ def __init__(self, ranges): super().__init__(ranges) self.ranges = [None] * self.size for rng in ranges: - name, start, stop = rng + data, start, stop = rng self.ranges[start:stop] = [rng] * (stop - start) - self.add_range(name, start, stop) + self.add_range(data, start, stop) - def index2name(self, idx): + def index2data(self, idx): """ - Find the name corresponding to the given index. + Find the data corresponding to the given index. Parameters ---------- @@ -292,17 +301,17 @@ def index2name(self, idx): Returns ------- - str or None - The name corresponding to the given index, or None if not found. + object or None + The data corresponding to the given index, or None if not found. """ try: return self.ranges[idx][0] except IndexError: return None - def index2name_and_rel_ind(self, idx): + def index2data_and_rel_ind(self, idx): """ - Find the name and relative index corresponding to the matched range. + Find the data and relative index corresponding to the matched range. Parameters ---------- @@ -311,17 +320,17 @@ def index2name_and_rel_ind(self, idx): Returns ------- - str or None - The name corresponding to the matched range, or None if not found. + object or None + The data corresponding to the matched range, or None if not found. int or None The relative index into the matched range, or None if not found. """ try: - name, start, _ = self.ranges[idx] + data, start, _ = self.ranges[idx] except IndexError: return (None, None) - return (name, idx - start) + return (data, idx - start) def metas2ranges(meta_iter, shape_name='shape'): @@ -385,7 +394,7 @@ def metas2shapes(meta_iter, shape_name='shape'): flat = FlatRangeMapper(ranges) for i in range(34): - rname, rind = rtree.index2name_and_rel_ind(i) - fname, find = flat.index2name_and_rel_ind(i) + rname, rind = rtree.index2data_and_rel_ind(i) + fname, find = flat.index2data_and_rel_ind(i) assert rname == fname and rind == find, f'i = {i}, rname = {rname}, rind = {rind}, fname = {fname}, find = {find}' print(i, rname, rind, fname, find) From 84640877693c54c12760f3c55dc6deee5f433c1f Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Thu, 5 Oct 2023 11:05:16 -0400 Subject: [PATCH 19/70] fixed output for no mpi and made all ranks be relative to top group --- openmdao/core/system.py | 11 ++++++-- openmdao/devtools/debug.py | 58 ++++++++++++++++++++++---------------- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/openmdao/core/system.py b/openmdao/core/system.py index 68e70be4fd..00e99f5654 100644 --- a/openmdao/core/system.py +++ b/openmdao/core/system.py @@ -6277,7 +6277,7 @@ def comm_info_iter(self): for s in self._subsystems_myproc: yield from s.comm_info_iter() - def dist_range_iter(self, io): + def dist_range_iter(self, io, top_comm): """ Yield names and distributed ranges of all local and remote variables in this system. @@ -6294,10 +6294,17 @@ def dist_range_iter(self, io): sizes = self._var_sizes vmeta = self._var_allprocs_abs2meta + topranks = np.arange(top_comm.size) + + myrank = self.comm.rank + toprank = top_comm.rank + + mytopranks = topranks[toprank - myrank: toprank - myrank + self.comm.size] + total = 0 for rank in range(self.comm.size): for ivar, vname in enumerate(vmeta[io]): sz = sizes[io][rank, ivar] if sz > 0: - yield (vname, rank), total, total + sz + yield (vname, mytopranks[rank]), total, total + sz total += sz diff --git a/openmdao/devtools/debug.py b/openmdao/devtools/debug.py index 33f2926011..5df9fbb45d 100644 --- a/openmdao/devtools/debug.py +++ b/openmdao/devtools/debug.py @@ -654,12 +654,14 @@ def show_dist_var_conns(group, rev=False, out_stream=_DEFAULT_OUT_STREAM): """ Show all distributed variable connections in the given group and below. + The ranks displayed will be relative to the communicator of the given top group. + Parameters ---------- group : Group - The group to be searched. + The top level group to be searched. Connections in all subgroups will also be displayed. rev : bool - If True show reverse transfers. + If True show reverse transfers instead of forward transfers. out_stream : file-like Where the output will go. @@ -687,8 +689,8 @@ def show_dist_var_conns(group, rev=False, out_stream=_DEFAULT_OUT_STREAM): for g in group.system_iter(typ=Group, include_self=True): if g._transfers[direction]: - in_ranges = list(g.dist_range_iter('input')) - out_ranges = list(g.dist_range_iter('output')) + in_ranges = list(g.dist_range_iter('input', group.comm)) + out_ranges = list(g.dist_range_iter('output', group.comm)) inmapper = DataRangeMapper.create(in_ranges) outmapper = DataRangeMapper.create(out_ranges) @@ -752,7 +754,10 @@ def show_dist_var_conns(group, rev=False, out_stream=_DEFAULT_OUT_STREAM): gdict[g.pathname][sub] = strs + do_ranks = False + if group.comm.size > 1: + do_ranks = True final = {} gatherlist = group.comm.gather(gdict, root=0) if group.comm.rank == 0: @@ -777,36 +782,39 @@ def show_dist_var_conns(group, rev=False, out_stream=_DEFAULT_OUT_STREAM): if group.comm.rank == 0: fwd = direction == 'fwd' - for gpath, subdct in gdict.items(): + for gpath, subdct in sorted(gdict.items(), key=lambda x: x[0]): indent = 0 if gpath == '' else gpath.count('.') + 1 pad = ' ' * indent printer(f"{pad}In Group '{gpath}'", file=out_stream) - for sub, strs in subdct.items(): + for sub, strs in sorted(subdct.items(), key=lambda x: x[0]): if fwd: printer(f"{pad} {arrow} {sub}", file=out_stream) else: printer(f"{pad} {sub} {arrow}", file=out_stream) for s, ranks in strs.items(): - oranks = np.empty(len(ranks), dtype=int) - iranks = np.empty(len(ranks), dtype=int) - for i, (ornk, irnk) in enumerate(sorted(ranks)): - oranks[i] = ornk - iranks[i] = irnk - - if np.all(oranks == oranks[0]): - orstr = str(oranks[0]) - else: - orstr = str(sorted(oranks)) - - if np.all(iranks == iranks[0]): - irstr = str(iranks[0]) - else: - irstr = str(sorted(iranks)) - - if orstr == irstr and '[' not in orstr: - printer(f"{pad} {s} rank {orstr}", file=out_stream) + if do_ranks: + oranks = np.empty(len(ranks), dtype=int) + iranks = np.empty(len(ranks), dtype=int) + for i, (ornk, irnk) in enumerate(sorted(ranks)): + oranks[i] = ornk + iranks[i] = irnk + + if np.all(oranks == oranks[0]): + orstr = str(oranks[0]) + else: + orstr = str(sorted(oranks)) + + if np.all(iranks == iranks[0]): + irstr = str(iranks[0]) + else: + irstr = str(sorted(iranks)) + + if orstr == irstr and '[' not in orstr: + printer(f"{pad} {s} rank {orstr}", file=out_stream) + else: + printer(f"{pad} {s} ranks {orstr} {arrow} {irstr}", file=out_stream) else: - printer(f"{pad} {s} ranks {orstr} {arrow} {irstr}", file=out_stream) + printer(f"{pad} {s}", file=out_stream) return gdict From c83dd2fe202bcaaf7725b9b70ae497ba1da2712b Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Thu, 5 Oct 2023 12:51:25 -0400 Subject: [PATCH 20/70] sorting of check_totals --- openmdao/core/problem.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index ed9ef629ac..af1ccc8e9f 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -1638,7 +1638,7 @@ def check_partials(self, out_stream=_DEFAULT_OUT_STREAM, includes=None, excludes def check_totals(self, of=None, wrt=None, out_stream=_DEFAULT_OUT_STREAM, compact_print=False, driver_scaling=False, abs_err_tol=1e-6, rel_err_tol=1e-6, method='fd', step=None, form=None, step_calc='abs', show_progress=False, - show_only_incorrect=False, directional=False): + show_only_incorrect=False, directional=False, sort=False): """ Check total derivatives for the model vs. finite difference. @@ -1688,6 +1688,8 @@ def check_totals(self, of=None, wrt=None, out_stream=_DEFAULT_OUT_STREAM, compac directional : bool If True, compute a single directional derivative for each 'of' in rev mode or each 'wrt' in fwd mode. + sort : bool + If True, sort the subjacobian keys alphabetically. Returns ------- @@ -1866,8 +1868,12 @@ def check_totals(self, of=None, wrt=None, out_stream=_DEFAULT_OUT_STREAM, compac resp = self.driver._responses do_steps = len(Jfds) > 1 + Jcalc_items = Jcalc.items() + if sort: + Jcalc_items = sorted(Jcalc_items, key=lambda x: x[0]) + for Jfd, step in Jfds: - for key, val in Jcalc.items(): + for key, val in Jcalc_items: if key not in data['']: data[''][key] = {} meta = data[''][key] @@ -1917,7 +1923,7 @@ def check_totals(self, of=None, wrt=None, out_stream=_DEFAULT_OUT_STREAM, compac _assemble_derivative_data(data, rel_err_tol, abs_err_tol, out_stream, compact_print, [model], {'': fd_args}, totals=total_info, lcons=lcons, - show_only_incorrect=show_only_incorrect) + show_only_incorrect=show_only_incorrect, sort=sort) if not do_steps: _fix_check_data(data) @@ -2931,7 +2937,7 @@ def _fix_check_data(data): def _assemble_derivative_data(derivative_data, rel_error_tol, abs_error_tol, out_stream, compact_print, system_list, global_options, totals=False, indep_key=None, print_reverse=False, - show_only_incorrect=False, lcons=None): + show_only_incorrect=False, lcons=None, sort=False): """ Compute the relative and absolute errors in the given derivatives and print to the out_stream. @@ -2962,6 +2968,8 @@ def _assemble_derivative_data(derivative_data, rel_error_tol, abs_error_tol, out Set to True if output should print only the subjacs found to be incorrect. lcons : list or None For total derivatives only, list of outputs that are actually linear constraints. + sort : bool + If True, sort subjacobian keys alphabetically. """ suppress_output = out_stream is None From af988b18f9575e69844e87401a317c5a00f94666 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 6 Oct 2023 11:23:21 -0400 Subject: [PATCH 21/70] fix for ken's mpi issue --- openmdao/core/system.py | 24 +++++++++++++++++++++ openmdao/core/total_jac.py | 30 +++++++++------------------ openmdao/devtools/debug.py | 4 ++-- openmdao/utils/range_collection.py | 31 +++++++++++++++++++--------- openmdao/vectors/default_transfer.py | 2 ++ openmdao/vectors/petsc_transfer.py | 8 +------ 6 files changed, 60 insertions(+), 39 deletions(-) diff --git a/openmdao/core/system.py b/openmdao/core/system.py index 00e99f5654..7cd13b387b 100644 --- a/openmdao/core/system.py +++ b/openmdao/core/system.py @@ -6285,6 +6285,8 @@ def dist_range_iter(self, io, top_comm): ---------- io : str Either 'input' or 'output'. + top_comm : MPI.Comm or None + The top-level MPI communicator. Yields ------ @@ -6308,3 +6310,25 @@ def dist_range_iter(self, io, top_comm): if sz > 0: yield (vname, mytopranks[rank]), total, total + sz total += sz + + def local_range_iter(self, io): + """ + Yield names and local ranges of all local variables in this system. + + Parameters + ---------- + io : str + Either 'input' or 'output'. + + Yields + ------ + tuple + A tuple of the form (abs_name, start, end). + """ + vmeta = self._var_allprocs_abs2meta + + offset = 0 + for vname, size in zip(vmeta[io], self._var_sizes[io][self.comm.rank]): + if size > 0: + yield vname, offset, offset + size + offset += size diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index 91940d8e20..b1e84c51e1 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -376,11 +376,16 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, # duplicated vars self.rev_allreduce_mask = np.ones(J.shape[1], dtype=bool) + voimeta = self.output_meta['rev'] start = end = 0 - for name in self.wrt: - meta = all_abs2meta_out[name] - end += meta['global_size'] - if not meta['distributed'] and model._owning_rank[name] != model.comm.rank: + for name, pname in zip(wrt, prom_wrt): + vmeta = all_abs2meta_out[name] + if pname in voimeta: + meta = voimeta[pname] + end += meta['size'] + else: + end += vmeta['global_size'] + if not vmeta['distributed'] and model._owning_rank[name] != model.comm.rank: self.rev_allreduce_mask[start:end] = False start = end @@ -1295,6 +1300,7 @@ def simple_single_jac_scatter(self, i, mode): """ deriv_idxs, jac_idxs, _ = self.sol2jac_map[mode] deriv_val = self.output_vec[mode].asarray() + if not self.get_remote: loc_idx = self.loc_jac_idxs[mode][i] if loc_idx >= 0: @@ -1334,22 +1340,6 @@ def _jac_setter_dist(self, i, mode): scratch[:] = 0.0 scratch[self.rev_allreduce_mask] = self.J[i][self.rev_allreduce_mask] self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) - # else: - # scatter = self.jac_scatters[mode] - # if scatter is not None: - # if self.dist_idx_map[mode][i]: # distrib var, skip scatter - # return - # loc = self.loc_jac_idxs[mode][i] - # if loc >= 0: - # self.tgt_petsc[mode].array[:] = self.J[loc, :][self.nondist_loc_map[mode]] - # self.src_petsc[mode].array[:] = self.J[loc, :][self.nondist_loc_map[mode]] - # else: - # self.src_petsc[mode].array[:] = 0.0 - - # scatter.scatter(self.src_petsc[mode], self.tgt_petsc[mode], - # addv=True, mode=False) - # if loc >= 0: - # self.J[loc, :][self.nondist_loc_map[mode]] = self.tgt_petsc[mode].array def single_jac_setter(self, i, mode, meta): """ diff --git a/openmdao/devtools/debug.py b/openmdao/devtools/debug.py index 5df9fbb45d..7fbcbd2412 100644 --- a/openmdao/devtools/debug.py +++ b/openmdao/devtools/debug.py @@ -708,9 +708,9 @@ def show_dist_var_conns(group, rev=False, out_stream=_DEFAULT_OUT_STREAM): conns = {} for iidx, oidx in zip(transfer._in_inds, transfer._out_inds): - idata, irind = inmapper.index2data_and_rel_ind(iidx) + idata, irind = inmapper.index2rel_data(iidx) ivar, irank = idata - odata, orind = outmapper.index2data_and_rel_ind(oidx) + odata, orind = outmapper.index2rel_data(oidx) ovar, orank = odata if odata not in conns: diff --git a/openmdao/utils/range_collection.py b/openmdao/utils/range_collection.py index bbaf7d9000..daace2808d 100644 --- a/openmdao/utils/range_collection.py +++ b/openmdao/utils/range_collection.py @@ -69,7 +69,7 @@ def data2range(self, data): """ return self._data2range[data] - def index2data(self, idx): + def _index2data(self, idx): """ Find the data corresponding to the given index. @@ -83,7 +83,18 @@ def index2data(self, idx): object or None The data corresponding to the given index, or None if not found. """ - raise NotImplementedError("index2data method must be implemented by subclass.") + raise NotImplementedError("_index2data method must be implemented by subclass.") + + def __getitem__(self, idx): + """ + Find the data corresponding to the given index. + + Parameters + ---------- + idx : int + The index into the full array. + """ + return self._index2data(idx) def indices2data(self, idxs): """ @@ -99,11 +110,11 @@ def indices2data(self, idxs): list of (object, int) The data corresponding to each of the given indices. """ - data = [self.index2data(idx) for idx in idxs] + data = [self._index2data(idx) for idx in idxs] if None in data: missing = [] for idx in idxs: - d = self.index2data(idx) + d = self._index2data(idx) if d is None: missing.append(idx) raise RuntimeError(f"Indices {sorted(missing)} are not in any range.") @@ -175,7 +186,7 @@ def __init__(self, ranges): self.size = ranges[-1][2] - ranges[0][1] self.root = self.build(ranges) - def index2data(self, idx): + def _index2data(self, idx): """ Find the data corresponding to the given index. @@ -200,7 +211,7 @@ def index2data(self, idx): else: return node.data - def index2data_and_rel_ind(self, idx): + def index2rel_data(self, idx): """ Find the data and relative index corresponding to the matched range. @@ -290,7 +301,7 @@ def __init__(self, ranges): self.ranges[start:stop] = [rng] * (stop - start) self.add_range(data, start, stop) - def index2data(self, idx): + def _index2data(self, idx): """ Find the data corresponding to the given index. @@ -309,7 +320,7 @@ def index2data(self, idx): except IndexError: return None - def index2data_and_rel_ind(self, idx): + def index2rel_data(self, idx): """ Find the data and relative index corresponding to the matched range. @@ -394,7 +405,7 @@ def metas2shapes(meta_iter, shape_name='shape'): flat = FlatRangeMapper(ranges) for i in range(34): - rname, rind = rtree.index2data_and_rel_ind(i) - fname, find = flat.index2data_and_rel_ind(i) + rname, rind = rtree.index2rel_data(i) + fname, find = flat.index2rel_data(i) assert rname == fname and rind == find, f'i = {i}, rname = {rname}, rind = {rind}, fname = {fname}, find = {find}' print(i, rname, rind, fname, find) diff --git a/openmdao/vectors/default_transfer.py b/openmdao/vectors/default_transfer.py index cf58264698..81016013f6 100644 --- a/openmdao/vectors/default_transfer.py +++ b/openmdao/vectors/default_transfer.py @@ -121,6 +121,8 @@ def _setup_transfers(group, desvars, responses): sub_out = abs_out[mypathlen:].split('.', 1)[0] rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) + else: + continue tot_size = 0 for sname, inds in fwd_xfer_in.items(): diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 49a2955c75..9ecba18dea 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -72,7 +72,6 @@ def _setup_transfers(group, desvars, responses): alias. """ rev = group._mode != 'fwd' - uses_approx = group._owns_approx_jac for subsys in group._subgroups_myproc: subsys._setup_transfers(desvars, responses) @@ -102,8 +101,6 @@ def _setup_transfers(group, desvars, responses): rev_xfer_in_nocolor = defaultdict(list) rev_xfer_out_nocolor = defaultdict(list) - # rev_conns = get_rev_conns(group._conn_abs_in2out) - allprocs_abs2idx = group._var_allprocs_abs2idx sizes_in = group._var_sizes['input'] sizes_out = group._var_sizes['output'] @@ -301,6 +298,7 @@ def get_xfer_ranks(name, io): output_inds = oidxlist[0] else: input_inds = output_inds = np.zeros(0, dtype=INT_DTYPE) + rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) @@ -392,10 +390,6 @@ def get_xfer_ranks(name, io): vectors['input']['nonlinear'], vectors['output']['nonlinear'], rev_xfer_in_nocolor[sname], inds, group.comm) - # from om_devtools.dist_idxs import dump_dist_idxs - # print(f"DIST IDXS for '{group.pathname}', rank {group.comm.rank}:", flush=True) - # dump_dist_idxs(group) - def _transfer(self, in_vec, out_vec, mode='fwd'): """ Perform transfer. From 75a57259e694f1166f6ace279a0674efc63b0340 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 6 Oct 2023 15:16:52 -0400 Subject: [PATCH 22/70] fixed rev transfer issue --- openmdao/vectors/petsc_transfer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 9ecba18dea..57ac9ce778 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -313,6 +313,10 @@ def get_xfer_ranks(name, io): rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) else: + if inp_is_dup and out_is_dup and src_indices is not None and src_indices.size > 0: + offset = offsets_out[myrank, idx_out] + output_inds = np.asarray(src_indices + offset, dtype=INT_DTYPE) + rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) else: From 0d3ad2219284c72b97c2b189fdbf60e940befc73 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 9 Oct 2023 12:56:13 -0400 Subject: [PATCH 23/70] saving some transfer memory --- openmdao/vectors/petsc_transfer.py | 70 ++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 57ac9ce778..88fbd734a0 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -333,23 +333,33 @@ def get_xfer_ranks(name, io): rev_xfer_in_nocolor[sub_out] rev_xfer_out_nocolor[sub_out] + total_fwd = 0 for sname, inds in fwd_xfer_in.items(): - fwd_xfer_in[sname] = _merge(inds) + inds = _merge(inds) + fwd_xfer_in[sname] = inds fwd_xfer_out[sname] = _merge(fwd_xfer_out[sname]) + total_fwd += len(inds) if rev: + total_rev = 0 for sname, inds in rev_xfer_out.items(): + inds = _merge(inds) rev_xfer_in[sname] = _merge(rev_xfer_in[sname]) - rev_xfer_out[sname] = _merge(inds) + rev_xfer_out[sname] = inds + total_rev += len(inds) + + total_rev_nocolor = 0 for sname, inds in rev_xfer_out_nocolor.items(): + inds = _merge(inds) rev_xfer_in_nocolor[sname] = _merge(rev_xfer_in_nocolor[sname]) - rev_xfer_out_nocolor[sname] = _merge(inds) + rev_xfer_out_nocolor[sname] = inds + total_rev_nocolor += len(inds) + + xfer_in = np.empty(total_fwd, dtype=INT_DTYPE) + xfer_out = np.empty(total_fwd, dtype=INT_DTYPE) if fwd_xfer_in: - xfer_in = np.concatenate(list(fwd_xfer_in.values())) - xfer_out = np.concatenate(list(fwd_xfer_out.values())) - else: - xfer_in = xfer_out = np.zeros(0, dtype=INT_DTYPE) + xfer_in, xfer_out = _merge_indices(total_fwd, fwd_xfer_in, fwd_xfer_out) out_vec = vectors['output']['nonlinear'] @@ -365,11 +375,7 @@ def get_xfer_ranks(name, io): inds, fwd_xfer_out[sname], group.comm) if rev: - if rev_xfer_in: - xfer_in = np.concatenate(list(rev_xfer_in.values())) - xfer_out = np.concatenate(list(rev_xfer_out.values())) - else: - xfer_in = xfer_out = np.zeros(0, dtype=INT_DTYPE) + xfer_in, xfer_out = _merge_indices(total_rev, rev_xfer_in, rev_xfer_out) xfer_all = PETScTransfer(vectors['input']['nonlinear'], out_vec, xfer_in, xfer_out, group.comm) @@ -383,8 +389,8 @@ def get_xfer_ranks(name, io): rev_xfer_in[sname], inds, group.comm) if has_rev_par_coloring and rev_xfer_in_nocolor: - xfer_in = np.concatenate(list(rev_xfer_in_nocolor.values())) - xfer_out = np.concatenate(list(rev_xfer_out_nocolor.values())) + xfer_in, xfer_out = _merge_indices(total_rev_nocolor, rev_xfer_in_nocolor, + rev_xfer_out_nocolor) xrev[(None, 'nocolor')] = PETScTransfer(vectors['input']['nonlinear'], out_vec, xfer_in, xfer_out, group.comm) @@ -449,3 +455,39 @@ def _transfer(self, in_vec, out_vec, mode='fwd'): if in_vec._alloc_complex: data = in_vec._get_data() data[:] = in_petsc.array + + +def _merge_indices(total_size, in_inds, out_inds): + """ + Merge indices into contiguous arrays and update index dicts to use subviews of the same array. + + Parameters + ---------- + total_size : int + Total size of the merged indices. + in_inds : dict + Input indices. + out_inds : dict + Output indices. + + Returns + ------- + int ndarray + Merged input indices. + int ndarray + Merged output indices. + """ + xfer_in = np.empty(total_size, dtype=INT_DTYPE) + xfer_out = np.empty(total_size, dtype=INT_DTYPE) + start = end = 0 + for idata, odata in zip(in_inds.items(), out_inds.items()): + sname, inp = idata + _, out = odata + end += len(inp) + xfer_in[start:end] = inp + xfer_out[start:end] = out + in_inds[sname] = xfer_in[start:end] + out_inds[sname] = xfer_out[start:end] + start = end + + return xfer_in, xfer_out From 466ae192b03f1007a81fda5712791a73809c7bc5 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Wed, 11 Oct 2023 09:39:03 -0400 Subject: [PATCH 24/70] fix for when dist desvar connected to dist input --- openmdao/core/tests/test_parallel_groups.py | 77 ++++++++++++++++++++- openmdao/core/total_jac.py | 24 ++++++- openmdao/vectors/petsc_transfer.py | 3 - 3 files changed, 98 insertions(+), 6 deletions(-) diff --git a/openmdao/core/tests/test_parallel_groups.py b/openmdao/core/tests/test_parallel_groups.py index 9907430963..951b96e457 100644 --- a/openmdao/core/tests/test_parallel_groups.py +++ b/openmdao/core/tests/test_parallel_groups.py @@ -24,8 +24,9 @@ from openmdao.test_suite.groups.parallel_groups import \ FanOutGrouped, FanInGrouped2, Diamond, ConvergeDiverge -from openmdao.utils.assert_utils import assert_near_equal +from openmdao.utils.assert_utils import assert_near_equal, assert_check_totals from openmdao.utils.logger_utils import TestLogger +from openmdao.utils.array_utils import evenly_distrib_idxs from openmdao.error_checking.check_config import _default_checks @@ -646,6 +647,80 @@ def setup(self): self.assertEqual(con_names, ['phases.climb.comp.y', 'phases.cruise.comp.y', 'phases.descent.comp.y']) + +class SimpleDistComp(om.ExplicitComponent): + + def setup(self): + comm = self.comm + rank = comm.rank + + sizes, _ = evenly_distrib_idxs(comm.size, 3) + io_size = sizes[rank] + + # src_indices will be computed automatically + self.add_input('x', val=np.ones(io_size), distributed=True) + self.add_input('a', val=-3.0 * np.ones(io_size), distributed=True) + + self.add_output('y', val=np.ones(io_size), distributed=True) + + self.declare_partials('y', ['x', 'a']) + + def compute(self, inputs, outputs): + x = inputs['x'] + a = inputs['a'] + + outputs['y'] = 2*x*a + x + a + + def compute_partials(self, inputs, partials): + x = inputs['x'] + a = inputs['a'] + + partials['y', 'x'] = np.diag(2.0 * a + 1.0) + partials['y', 'a'] = np.diag(2.0 * x + 1.0) + + +@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") +class TestFD(unittest.TestCase): + + N_PROCS = 2 + + def test_fd_rev_mode(self): + size = 3 + + p = om.Problem() + model = p.model + + model.add_subsystem('p', om.IndepVarComp('x', np.ones(size)), promotes=['*']) + sub = model.add_subsystem('sub', om.Group(), promotes=['*']) + + sub.add_subsystem('p2', om.IndepVarComp('a', -3.0 + 0.6 * np.arange(size)), promotes=['*']) + + sub.add_subsystem('C1', om.ExecComp(['xd = x'], + x=np.ones(size), xd=np.ones(size)), + promotes_inputs=['*']) + + sub.add_subsystem("D1", SimpleDistComp(), promotes_outputs=['*'], promotes_inputs=['a']) + + sub.connect('C1.xd', 'D1.x') + + model.add_design_var('x', lower=-50.0, upper=50.0) + model.add_constraint('y', lower=0.0) + + sub.approx_totals(method='fd') + + p.setup(mode='rev', force_alloc_complex=True) + + p.run_model() + + data = p.check_totals(method='fd', out_stream=None) + + print("sub jacobian:") + import pprint + pprint.pprint(sub._jacobian._subjacs_info) + + assert_check_totals(data, atol=1e-5) + + if __name__ == "__main__": from openmdao.utils.mpi import mpirun_tests mpirun_tests() diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index b1e84c51e1..9fdfc6399d 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -364,6 +364,16 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, if self.simul_coloring is not None: # when simul coloring, need two scratch arrays self.jac_scratch['fwd'].append(scratch[1][:J.shape[0]]) if 'rev' in modes: + from openmdao.core.group import Group + + # find all groups doing FD + fdgroups = [s.pathname for s in model.system_iter(recurse=True, typ=Group) + if s._owns_approx_jac] + allfdgroups = set() + for grps in model.comm.allgather(fdgroups): + allfdgroups.update(grps) + fdprefixes = [n + '.' for n in allfdgroups] + self.jac_scratch['rev'] = [scratch[0][:J.shape[1]]] if self.simul_coloring is not None: # when simul coloring, need two scratch arrays self.jac_scratch['rev'].append(scratch[1][:J.shape[1]]) @@ -378,6 +388,7 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, voimeta = self.output_meta['rev'] start = end = 0 + has_dist = False for name, pname in zip(wrt, prom_wrt): vmeta = all_abs2meta_out[name] if pname in voimeta: @@ -385,13 +396,22 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, end += meta['size'] else: end += vmeta['global_size'] - if not vmeta['distributed'] and model._owning_rank[name] != model.comm.rank: + + is_fdvar = False + for prefix in fdprefixes: + if name.startswith(prefix): + is_fdvar = True + break + + dist = vmeta['distributed'] + has_dist |= dist + if not is_fdvar and not dist and model._owning_rank[name] != model.comm.rank: self.rev_allreduce_mask[start:end] = False start = end # if rev_allreduce_mask isn't all True on all procs, then we need to do an Allreduce need_allreduce = not np.all(self.rev_allreduce_mask) - if not any(model.comm.allgather(need_allreduce)): + if not (has_dist or any(model.comm.allgather(need_allreduce))): self.rev_allreduce_mask = None if not approx: diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 88fbd734a0..781a84079a 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -355,9 +355,6 @@ def get_xfer_ranks(name, io): rev_xfer_out_nocolor[sname] = inds total_rev_nocolor += len(inds) - xfer_in = np.empty(total_fwd, dtype=INT_DTYPE) - xfer_out = np.empty(total_fwd, dtype=INT_DTYPE) - if fwd_xfer_in: xfer_in, xfer_out = _merge_indices(total_fwd, fwd_xfer_in, fwd_xfer_out) From aa5baf4833f28a34f8e46808a275378fcaff12cb Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Wed, 11 Oct 2023 13:14:38 -0400 Subject: [PATCH 25/70] updated default xfers to have 2 separate toplevel arrays for fwd/rev and all subsystem xfers are just views into those arrays. --- openmdao/core/tests/test_distrib_derivs.py | 30 ++++++ openmdao/utils/range_collection.py | 51 +++++++-- openmdao/vectors/default_transfer.py | 115 +++++++++++---------- openmdao/vectors/petsc_transfer.py | 36 +++---- openmdao/vectors/transfer.py | 7 +- 5 files changed, 152 insertions(+), 87 deletions(-) diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index 359e69caaf..f80033e46c 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -2299,6 +2299,36 @@ def test_constraint_aliases(self): totals = prob.check_totals(method='cs', out_stream=None) self._compare_totals(totals) + def test_dist_desvar_dist_input(self): + class SimpleSum(om.ExplicitComponent): + """Simple component to sum distributed vector""" + + def setup(self): + # Inputs + self.add_input('x', 1.0, shape=[2], distributed=True) + + # Outputs + self.add_output('sum', 0.0) + + def compute(self, inputs, outputs): + outputs['sum'] = self.comm.allreduce(np.sum(inputs["x"])) + + def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): + if mode == "fwd": + d_outputs['sum'] += self.comm.allreduce(np.sum(d_inputs["x"])) + if mode == "rev": + d_inputs["x"] += d_outputs['sum'] * np.ones(2) + + prob = om.Problem() + prob.model.add_subsystem("ivc", om.IndepVarComp("x", 1.0, shape=[2], distributed=True)) + prob.model.connect("ivc.x", "ParallelSum.x") + prob.model.add_subsystem("ParallelSum", SimpleSum()) + + prob.setup(mode='rev') + + prob.run_model() + # This check totals fails, some of the derivative terms from proc 2 are missing + assert_check_totals(prob.check_totals("ParallelSum.sum", "ivc.x")) if __name__ == "__main__": from openmdao.utils.mpi import mpirun_tests diff --git a/openmdao/utils/range_collection.py b/openmdao/utils/range_collection.py index daace2808d..ee6bdd4b98 100644 --- a/openmdao/utils/range_collection.py +++ b/openmdao/utils/range_collection.py @@ -1,3 +1,6 @@ +""" +A collection of classes for mapping indices to variable names and vice versa. +""" from openmdao.utils.array_utils import shape_to_len @@ -9,8 +12,27 @@ class DataRangeMapper(object): """ A mapper of indices to variable names and vice versa. + + Parameters + ---------- + ranges : list of (data, start, stop) + Ordered list of (data, start, stop) tuples, where start and stop define the range of + indices for the data. Ranges must be contiguous. data must be hashable. + + Attributes + ---------- + size : int + Total size of all of the ranges combined. + _data2range : dict + Dictionary mapping data to an index range. + _range2data : dict + Dictionary mapping an index range to data. """ + def __init__(self, ranges): + """ + Initialize a DataRangeMapper. + """ self._data2range = {} self._range2data = {} self.size = ranges[-1][2] - ranges[0][1] @@ -171,16 +193,24 @@ class RangeTree(DataRangeMapper): Search complexity is O(log2 n). Uses less memory than FlatRangeMapper when total array size is large. + + Parameters + ---------- + ranges : list of (data, start, stop) + Ordered list of (data, start, stop) tuples, where start and stop define the range of + indices for the data. Ranges must be contiguous. data must be hashable. + + Attributes + ---------- + size : int + Total size of all of the ranges combined. + root : RangeTreeNode + Root node of the binary search tree. """ + def __init__(self, ranges): """ Initialize a RangeTree. - - Parameters - ---------- - ranges : list of (data, start, stop) - List of (data, start, stop) tuples, where start and stop define the range of indices - for the data. Ranges must be contiguous and data must be hashable. """ super().__init__(ranges) self.size = ranges[-1][2] - ranges[0][1] @@ -283,8 +313,6 @@ class FlatRangeMapper(DataRangeMapper): Attributes ---------- - size : int - Total size of all of the ranges combined. ranges : list of (data, start, stop) List of (data, start, stop) tuples, where start and stop define the range of indices for that data. Ranges must be contiguous. data must be hashable. @@ -384,6 +412,12 @@ def metas2shapes(meta_iter, shape_name='shape'): Name of the metadata entry that contains the shape of the variable. Value can be either 'shape' or 'global_shape'. Default is 'shape'. The value of the metadata entry must be a tuple of integers. + + Yields + ------ + tuple + Tuple of the form (name, shape), where name is the variable name and shape is the shape + of the variable. """ for name, meta in meta_iter: yield (name, meta[shape_name]) @@ -407,5 +441,4 @@ def metas2shapes(meta_iter, shape_name='shape'): for i in range(34): rname, rind = rtree.index2rel_data(i) fname, find = flat.index2rel_data(i) - assert rname == fname and rind == find, f'i = {i}, rname = {rname}, rind = {rind}, fname = {fname}, find = {find}' print(i, rname, rind, fname, find) diff --git a/openmdao/vectors/default_transfer.py b/openmdao/vectors/default_transfer.py index 81016013f6..e227a58008 100644 --- a/openmdao/vectors/default_transfer.py +++ b/openmdao/vectors/default_transfer.py @@ -19,6 +19,58 @@ def _merge(indices_list): return _empty_idx_array +def _fill(arr, indices_list): + start = end = 0 + for inds in indices_list: + end += len(inds) + arr[start:end] = inds + start = end + + return end + + +def _setup_index_arrays(tot_size, in_xfers, out_xfers, vectors): + if tot_size > 0: + xfer_in = np.empty(tot_size, dtype=INT_DTYPE) + xfer_out = np.empty(tot_size, dtype=INT_DTYPE) + else: + xfer_in = xfer_out = np.zeros(0, dtype=INT_DTYPE) + + start = end = 0 + for sname, lst in in_xfers.items(): + # lst is a list of range objects + rstart = rend = start + for rng in lst: + rend += len(rng) + xfer_in[rstart:rend] = rng + rstart = rend + + end += rend - start + _fill(xfer_out[start:end], out_xfers[sname]) + in_xfers[sname] = xfer_in[start:end] + out_xfers[sname] = xfer_out[start:end] + start = end + + if tot_size > 0: + xfer_all = DefaultTransfer(vectors['input']['nonlinear'], + vectors['output']['nonlinear'], xfer_in, xfer_out) + else: + xfer_all = None + + xfer_dict = {} + xfer_dict[None] = xfer_all + + for sname, inds in in_xfers.items(): + if inds.size > 0: + xfer_dict[sname] = DefaultTransfer(vectors['input']['nonlinear'], + vectors['output']['nonlinear'], + inds, out_xfers[sname]) + else: + xfer_dict[sname] = None + + return xfer_dict + + class DefaultTransfer(Transfer): """ Default NumPy transfer. @@ -33,8 +85,6 @@ class DefaultTransfer(Transfer): Input indices for the transfer. out_inds : int ndarray Output indices for the transfer. - comm : MPI.Comm or - Communicator of the system that owns this transfer. """ @staticmethod @@ -65,8 +115,6 @@ def _setup_transfers(group, desvars, responses): mypathlen = len(group.pathname + '.' if group.pathname else '') # Initialize empty lists for the transfer indices - xfer_in = [] - xfer_out = [] fwd_xfer_in = defaultdict(list) fwd_xfer_out = defaultdict(list) if rev: @@ -83,12 +131,13 @@ def _setup_transfers(group, desvars, responses): if offsets_out.size > 0: offsets_out = offsets_out[iproc] + start = end = 0 + # Loop through all connections owned by this group for abs_in, abs_out in group._conn_abs_in2out.items(): # This weeds out discrete vars (all vars are local if using this Transfer) if abs_in in abs2meta['input']: - indices = None # Get meta meta_in = abs2meta['input'][abs_in] @@ -108,10 +157,9 @@ def _setup_transfers(group, desvars, responses): output_inds = src_indices + offset # 2. Compute the input indices - input_inds = np.arange(offsets_in[idx_in], - offsets_in[idx_in] + sizes_in[idx_in], dtype=INT_DTYPE) - if indices is not None: - input_inds = input_inds.reshape(indices.shape) + # all input indices can be simple ranges during this part in order to save memory + input_inds = range(offsets_in[idx_in], offsets_in[idx_in] + sizes_in[idx_in]) + end += sizes_in[idx_in] # Now the indices are ready - input_inds, output_inds sub_in = abs_in[mypathlen:].split('.', 1)[0] @@ -121,55 +169,14 @@ def _setup_transfers(group, desvars, responses): sub_out = abs_out[mypathlen:].split('.', 1)[0] rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) - else: - continue - tot_size = 0 - for sname, inds in fwd_xfer_in.items(): - fwd_xfer_in[sname] = arr = _merge(inds) - fwd_xfer_out[sname] = _merge(fwd_xfer_out[sname]) - tot_size += arr.size + start = end - if rev: - for sname, inds in rev_xfer_in.items(): - rev_xfer_in[sname] = _merge(inds) - rev_xfer_out[sname] = _merge(rev_xfer_out[sname]) - - if tot_size > 0: - try: - xfer_in = np.concatenate(list(fwd_xfer_in.values())) - xfer_out = np.concatenate(list(fwd_xfer_out.values())) - except ValueError: - xfer_in = xfer_out = np.zeros(0, dtype=INT_DTYPE) - - xfer_all = DefaultTransfer(vectors['input']['nonlinear'], - vectors['output']['nonlinear'], xfer_in, xfer_out, - group.comm) - else: - xfer_all = None + tot_size = end - transfers['fwd'] = xfwd = {} - xfwd[None] = xfer_all + transfers['fwd'] = _setup_index_arrays(tot_size, fwd_xfer_in, fwd_xfer_out, vectors) if rev: - transfers['rev'] = xrev = {} - xrev[None] = xfer_all - - for sname, inds in fwd_xfer_in.items(): - if inds.size > 0: - xfwd[sname] = DefaultTransfer(vectors['input']['nonlinear'], - vectors['output']['nonlinear'], - inds, fwd_xfer_out[sname], group.comm) - else: - xfwd[sname] = None - - if rev: - for sname, inds in rev_xfer_out.items(): - if inds.size > 0: - xrev[sname] = DefaultTransfer(vectors['input']['nonlinear'], - vectors['output']['nonlinear'], - rev_xfer_in[sname], inds, group.comm) - else: - xrev[sname] = None + transfers['rev'] = _setup_index_arrays(tot_size, rev_xfer_in, rev_xfer_out, vectors) @staticmethod def _setup_discrete_transfers(group): diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 781a84079a..819da9f8d0 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -49,9 +49,9 @@ def __init__(self, in_vec, out_vec, in_inds, out_inds, comm): """ Initialize all attributes. """ - super().__init__(in_vec, out_vec, in_inds, out_inds, comm) - in_indexset = PETSc.IS().createGeneral(self._in_inds, comm=self._comm) - out_indexset = PETSc.IS().createGeneral(self._out_inds, comm=self._comm) + super().__init__(in_vec, out_vec, in_inds, out_inds) + in_indexset = PETSc.IS().createGeneral(self._in_inds, comm=comm) + out_indexset = PETSc.IS().createGeneral(self._out_inds, comm=comm) self._scatter = PETSc.Scatter().create(out_vec._petsc, out_indexset, in_vec._petsc, in_indexset).scatter @@ -218,28 +218,28 @@ def get_xfer_ranks(name, io): iidxlist = [] oidxlist_nc = [] iidxlist_nc = [] - oidxlist.append(output_inds) - iidxlist.append(input_inds) for rnk, osize, isize in zip(range(group.comm.size), sizes_out[:, idx_out], sizes_in[:, idx_in]): - if osize > 0 and isize == 0: + if rnk == myrank: + oidxlist.append(output_inds) + iidxlist.append(input_inds) + elif osize > 0 and isize == 0: offset = offsets_out[rnk, idx_out] if src_indices is None: oarr = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) - iarr = input_inds elif src_indices.size > 0: oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) - iarr = input_inds else: continue - if rnk == myrank or not has_rev_par_coloring: - oidxlist.append(oarr) - iidxlist.append(iarr) - else: + + if has_rev_par_coloring: oidxlist_nc.append(oarr) - iidxlist_nc.append(iarr) + iidxlist_nc.append(input_inds) + else: + oidxlist.append(oarr) + iidxlist.append(input_inds) if len(iidxlist) > 1: input_inds = np.concatenate(iidxlist) @@ -263,6 +263,7 @@ def get_xfer_ranks(name, io): rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) + elif out_is_dup and (not inp_is_dup or inp_missing > 0) and (iowninput or distrib_in): oidxlist = [] @@ -274,21 +275,19 @@ def get_xfer_ranks(name, io): if src_indices is None: oarr = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) - iarr = input_inds elif src_indices.size > 0: if (distrib_in and not distrib_out and len(on_iprocs) == 1 and on_iprocs[0] == rnk): offset -= np.sum(sizes_out[:rnk, idx_out]) oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) - iarr = input_inds else: continue if rnk == myrank or not has_rev_par_coloring: oidxlist.append(oarr) - iidxlist.append(iarr) + iidxlist.append(input_inds) else: oidxlist_nc.append(oarr) - iidxlist_nc.append(iarr) + iidxlist_nc.append(input_inds) if len(iidxlist) > 1: input_inds = np.concatenate(iidxlist) @@ -313,7 +312,8 @@ def get_xfer_ranks(name, io): rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) else: - if inp_is_dup and out_is_dup and src_indices is not None and src_indices.size > 0: + if (inp_is_dup and out_is_dup and src_indices is not None and + src_indices.size > 0): offset = offsets_out[myrank, idx_out] output_inds = np.asarray(src_indices + offset, dtype=INT_DTYPE) diff --git a/openmdao/vectors/transfer.py b/openmdao/vectors/transfer.py index df959547ef..f44a26be8b 100644 --- a/openmdao/vectors/transfer.py +++ b/openmdao/vectors/transfer.py @@ -15,8 +15,6 @@ class Transfer(object): Input indices for the transfer. out_inds : int ndarray Output indices for the transfer. - comm : MPI.Comm or - Communicator of the system that owns this transfer. Attributes ---------- @@ -24,17 +22,14 @@ class Transfer(object): input indices for the transfer. _out_inds : int ndarray output indices for the transfer. - _comm : MPI.Comm or FakeComm - communicator of the system that owns this transfer. """ - def __init__(self, in_vec, out_vec, in_inds, out_inds, comm): + def __init__(self, in_vec, out_vec, in_inds, out_inds): """ Initialize all attributes. """ self._in_inds = in_inds self._out_inds = out_inds - self._comm = comm def __str__(self): """ From 3aea2c1eddbf1f6fe1f3c3284430f8ec4a68b677 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Wed, 11 Oct 2023 15:09:29 -0400 Subject: [PATCH 26/70] cleanup --- openmdao/vectors/default_transfer.py | 95 ++++++++++++------- openmdao/vectors/petsc_transfer.py | 137 ++++++++++++--------------- 2 files changed, 126 insertions(+), 106 deletions(-) diff --git a/openmdao/vectors/default_transfer.py b/openmdao/vectors/default_transfer.py index e227a58008..1f4db1b2a2 100644 --- a/openmdao/vectors/default_transfer.py +++ b/openmdao/vectors/default_transfer.py @@ -9,56 +9,91 @@ from openmdao.utils.array_utils import _global2local_offsets from openmdao.utils.mpi import MPI -_empty_idx_array = np.array([], dtype=INT_DTYPE) - - -def _merge(indices_list): - if len(indices_list) > 0: - return np.concatenate(indices_list) - else: - return _empty_idx_array - def _fill(arr, indices_list): + """ + Fill the given array with the given list of indices. + + Parameters + ---------- + arr : ndarray + Array to be filled. + indices_list : list of int ndarrays + List of ranges/indices to be placed into arr. + """ start = end = 0 for inds in indices_list: end += len(inds) arr[start:end] = inds start = end - return end +def _setup_index_views(tot_size, in_xfers, out_xfers): + """ + Create index views for all subsystems and allocate full transfer arrays. -def _setup_index_arrays(tot_size, in_xfers, out_xfers, vectors): - if tot_size > 0: - xfer_in = np.empty(tot_size, dtype=INT_DTYPE) - xfer_out = np.empty(tot_size, dtype=INT_DTYPE) - else: - xfer_in = xfer_out = np.zeros(0, dtype=INT_DTYPE) + Parameters + ---------- + tot_size : int + Total size of each full array. + in_xfers : dict + Mapping of subsystem name to input index arrays. + out_xfers : dict + Mapping of subsystem name to output index arrays. + """ + full_in = np.empty(tot_size, dtype=INT_DTYPE) + full_out = np.empty(tot_size, dtype=INT_DTYPE) start = end = 0 - for sname, lst in in_xfers.items(): - # lst is a list of range objects + for sname, ranges in in_xfers.items(): + # input inds are always ranges. output inds may be ranges or ndarrays. rstart = rend = start - for rng in lst: + for rng in ranges: rend += len(rng) - xfer_in[rstart:rend] = rng + full_in[rstart:rend] = rng rstart = rend end += rend - start - _fill(xfer_out[start:end], out_xfers[sname]) - in_xfers[sname] = xfer_in[start:end] - out_xfers[sname] = xfer_out[start:end] + _fill(full_out[start:end], out_xfers[sname]) + + # change subsystem transfer entries to be views of the full transfer arrays + in_xfers[sname] = full_in[start:end] + out_xfers[sname] = full_out[start:end] start = end + return full_in, full_out + + +def _setup_index_arrays(tot_size, in_xfers, out_xfers, vectors): + """ + Create index arrays for all subsystems. + + Parameters + ---------- + tot_size : int + Total size of each full array. + in_xfers : dict + Mapping of subsystem name to input index arrays. + out_xfers : dict + Mapping of subsystem name to output index arrays. + vectors : dict + Dictionary of input and output vectors. + + Returns + ------- + dict + Mapping of subsystem name to Transfer object. None key maps to the + 'full' transfer across all subsystems. + """ + xfer_in, xfer_out = _setup_index_views(tot_size, in_xfers, out_xfers) + if tot_size > 0: xfer_all = DefaultTransfer(vectors['input']['nonlinear'], vectors['output']['nonlinear'], xfer_in, xfer_out) else: xfer_all = None - xfer_dict = {} - xfer_dict[None] = xfer_all + xfer_dict = {None: xfer_all} for sname, inds in in_xfers.items(): if inds.size > 0: @@ -131,7 +166,7 @@ def _setup_transfers(group, desvars, responses): if offsets_out.size > 0: offsets_out = offsets_out[iproc] - start = end = 0 + tot_size = 0 # Loop through all connections owned by this group for abs_in, abs_out in group._conn_abs_in2out.items(): @@ -152,14 +187,14 @@ def _setup_transfers(group, desvars, responses): # 1. Compute the output indices offset = offsets_out[idx_out] if src_indices is None: - output_inds = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) + output_inds = range(offset, offset + meta_in['size']) else: output_inds = src_indices + offset # 2. Compute the input indices # all input indices can be simple ranges during this part in order to save memory input_inds = range(offsets_in[idx_in], offsets_in[idx_in] + sizes_in[idx_in]) - end += sizes_in[idx_in] + tot_size += sizes_in[idx_in] # Now the indices are ready - input_inds, output_inds sub_in = abs_in[mypathlen:].split('.', 1)[0] @@ -170,10 +205,6 @@ def _setup_transfers(group, desvars, responses): rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) - start = end - - tot_size = end - transfers['fwd'] = _setup_index_arrays(tot_size, fwd_xfer_in, fwd_xfer_out, vectors) if rev: transfers['rev'] = _setup_index_arrays(tot_size, rev_xfer_in, rev_xfer_out, vectors) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 819da9f8d0..eb32462016 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -1,7 +1,11 @@ """Define the PETSc Transfer class.""" +import numpy as np from openmdao.utils.mpi import check_mpi_env +from openmdao.core.constants import INT_DTYPE use_mpi = check_mpi_env() +_empty_idx_array = np.array([], dtype=INT_DTYPE) + if use_mpi is False: PETScTransfer = None @@ -14,12 +18,10 @@ if use_mpi is True: raise ImportError("Importing petsc4py failed and OPENMDAO_USE_MPI is true.") - import numpy as np from petsc4py import PETSc from collections import defaultdict - from openmdao.vectors.default_transfer import DefaultTransfer, _merge - from openmdao.core.constants import INT_DTYPE + from openmdao.vectors.default_transfer import DefaultTransfer, _fill, _setup_index_views from openmdao.utils.array_utils import shape_to_len class PETScTransfer(DefaultTransfer): @@ -119,6 +121,8 @@ def get_xfer_ranks(name, io): return [] return np.nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]])[0] + total_fwd = total_rev = total_rev_nocolor = 0 + # Loop through all connections owned by this system for abs_in, abs_out in group._conn_abs_in2out.items(): # Only continue if the input exists on this processor @@ -162,8 +166,7 @@ def get_xfer_ranks(name, io): else: rank = myrank if local_out else owner offset = offsets_out[rank, idx_out] - output_inds = np.arange(offset, offset + meta_in['size'], - dtype=INT_DTYPE) + output_inds = range(offset, offset + meta_in['size']) else: output_inds = np.zeros(src_indices.size, INT_DTYPE) start = end = 0 @@ -191,9 +194,10 @@ def get_xfer_ranks(name, io): start = end # 2. Compute the input indices - input_inds = np.arange(offsets_in[myrank, idx_in], - offsets_in[myrank, idx_in] + - sizes_in[myrank, idx_in], dtype=INT_DTYPE) + input_inds = range(offsets_in[myrank, idx_in], + offsets_in[myrank, idx_in] + sizes_in[myrank, idx_in]) + + total_fwd += len(input_inds) # Now the indices are ready - input_inds, output_inds sub_in = abs_in[mypathlen:].partition('.')[0] @@ -218,6 +222,7 @@ def get_xfer_ranks(name, io): iidxlist = [] oidxlist_nc = [] iidxlist_nc = [] + size = size_nc = 0 for rnk, osize, isize in zip(range(group.comm.size), sizes_out[:, idx_out], sizes_in[:, idx_in]): @@ -227,8 +232,7 @@ def get_xfer_ranks(name, io): elif osize > 0 and isize == 0: offset = offsets_out[rnk, idx_out] if src_indices is None: - oarr = np.arange(offset, offset + meta_in['size'], - dtype=INT_DTYPE) + oarr = range(offset, offset + meta_in['size']) elif src_indices.size > 0: oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) else: @@ -237,17 +241,21 @@ def get_xfer_ranks(name, io): if has_rev_par_coloring: oidxlist_nc.append(oarr) iidxlist_nc.append(input_inds) + size_nc += len(input_inds) else: oidxlist.append(oarr) iidxlist.append(input_inds) + size += len(input_inds) if len(iidxlist) > 1: - input_inds = np.concatenate(iidxlist) - output_inds = np.concatenate(oidxlist) + input_inds = _merge(iidxlist, size) + output_inds = _merge(oidxlist, size) else: input_inds = iidxlist[0] output_inds = oidxlist[0] + total_rev += len(input_inds) + rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) @@ -255,12 +263,14 @@ def get_xfer_ranks(name, io): # keep transfers separate that shouldn't happen when partial # coloring is active if len(iidxlist_nc) > 1: - input_inds = np.concatenate(iidxlist_nc) - output_inds = np.concatenate(oidxlist_nc) + input_inds = _merge(iidxlist_nc, size_nc) + output_inds = _merge(oidxlist_nc, size_nc) else: input_inds = iidxlist_nc[0] output_inds = oidxlist_nc[0] + total_rev_nocolor += len(input_inds) + rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) @@ -270,11 +280,11 @@ def get_xfer_ranks(name, io): iidxlist = [] oidxlist_nc = [] iidxlist_nc = [] + size = size_nc = 0 for rnk in get_xfer_ranks(abs_out, 'output'): offset = offsets_out[rnk, idx_out] if src_indices is None: - oarr = np.arange(offset, offset + meta_in['size'], - dtype=INT_DTYPE) + oarr = range(offset, offset + meta_in['size']) elif src_indices.size > 0: if (distrib_in and not distrib_out and len(on_iprocs) == 1 and on_iprocs[0] == rnk): @@ -285,30 +295,36 @@ def get_xfer_ranks(name, io): if rnk == myrank or not has_rev_par_coloring: oidxlist.append(oarr) iidxlist.append(input_inds) + size += len(input_inds) else: oidxlist_nc.append(oarr) iidxlist_nc.append(input_inds) + size_nc += len(input_inds) if len(iidxlist) > 1: - input_inds = np.concatenate(iidxlist) - output_inds = np.concatenate(oidxlist) + input_inds = _merge(iidxlist, size) + output_inds = _merge(oidxlist, size) elif len(iidxlist) == 1: input_inds = iidxlist[0] output_inds = oidxlist[0] else: - input_inds = output_inds = np.zeros(0, dtype=INT_DTYPE) + input_inds = output_inds = _empty_idx_array + + total_rev += len(input_inds) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) if has_rev_par_coloring and iidxlist_nc: if len(iidxlist_nc) > 1: - input_inds = np.concatenate(iidxlist_nc) - output_inds = np.concatenate(oidxlist_nc) + input_inds = _merge(iidxlist_nc, size_nc) + output_inds = _merge(oidxlist_nc, size_nc) else: input_inds = iidxlist_nc[0] output_inds = oidxlist_nc[0] + total_rev_nocolor += len(input_inds) + rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) else: @@ -317,6 +333,8 @@ def get_xfer_ranks(name, io): offset = offsets_out[myrank, idx_out] output_inds = np.asarray(src_indices + offset, dtype=INT_DTYPE) + total_rev += len(input_inds) + rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) else: @@ -333,30 +351,8 @@ def get_xfer_ranks(name, io): rev_xfer_in_nocolor[sub_out] rev_xfer_out_nocolor[sub_out] - total_fwd = 0 - for sname, inds in fwd_xfer_in.items(): - inds = _merge(inds) - fwd_xfer_in[sname] = inds - fwd_xfer_out[sname] = _merge(fwd_xfer_out[sname]) - total_fwd += len(inds) - - if rev: - total_rev = 0 - for sname, inds in rev_xfer_out.items(): - inds = _merge(inds) - rev_xfer_in[sname] = _merge(rev_xfer_in[sname]) - rev_xfer_out[sname] = inds - total_rev += len(inds) - - total_rev_nocolor = 0 - for sname, inds in rev_xfer_out_nocolor.items(): - inds = _merge(inds) - rev_xfer_in_nocolor[sname] = _merge(rev_xfer_in_nocolor[sname]) - rev_xfer_out_nocolor[sname] = inds - total_rev_nocolor += len(inds) - if fwd_xfer_in: - xfer_in, xfer_out = _merge_indices(total_fwd, fwd_xfer_in, fwd_xfer_out) + xfer_in, xfer_out = _setup_index_views(total_fwd, fwd_xfer_in, fwd_xfer_out) out_vec = vectors['output']['nonlinear'] @@ -372,7 +368,7 @@ def get_xfer_ranks(name, io): inds, fwd_xfer_out[sname], group.comm) if rev: - xfer_in, xfer_out = _merge_indices(total_rev, rev_xfer_in, rev_xfer_out) + xfer_in, xfer_out = _setup_index_views(total_rev, rev_xfer_in, rev_xfer_out) xfer_all = PETScTransfer(vectors['input']['nonlinear'], out_vec, xfer_in, xfer_out, group.comm) @@ -385,9 +381,9 @@ def get_xfer_ranks(name, io): vectors['input']['nonlinear'], vectors['output']['nonlinear'], rev_xfer_in[sname], inds, group.comm) - if has_rev_par_coloring and rev_xfer_in_nocolor: - xfer_in, xfer_out = _merge_indices(total_rev_nocolor, rev_xfer_in_nocolor, - rev_xfer_out_nocolor) + if rev_xfer_in_nocolor: + xfer_in, xfer_out = _setup_index_views(total_rev_nocolor, rev_xfer_in_nocolor, + rev_xfer_out_nocolor) xrev[(None, 'nocolor')] = PETScTransfer(vectors['input']['nonlinear'], out_vec, xfer_in, xfer_out, group.comm) @@ -454,37 +450,30 @@ def _transfer(self, in_vec, out_vec, mode='fwd'): data[:] = in_petsc.array -def _merge_indices(total_size, in_inds, out_inds): +def _merge(inds_list, tot_size): """ - Merge indices into contiguous arrays and update index dicts to use subviews of the same array. + Convert a list of indices and/or ranges into an array. Parameters ---------- - total_size : int - Total size of the merged indices. - in_inds : dict - Input indices. - out_inds : dict - Output indices. + inds_list : list of ranges or ndarrays + List of indices. + tot_size : int + Total size of the indices in the list. Returns ------- - int ndarray - Merged input indices. - int ndarray - Merged output indices. + ndarray + Array of indices. """ - xfer_in = np.empty(total_size, dtype=INT_DTYPE) - xfer_out = np.empty(total_size, dtype=INT_DTYPE) - start = end = 0 - for idata, odata in zip(in_inds.items(), out_inds.items()): - sname, inp = idata - _, out = odata - end += len(inp) - xfer_in[start:end] = inp - xfer_out[start:end] = out - in_inds[sname] = xfer_in[start:end] - out_inds[sname] = xfer_out[start:end] - start = end - - return xfer_in, xfer_out + if inds_list: + arr = np.empty(tot_size, dtype=INT_DTYPE) + start = end = 0 + for inds in inds_list: + end += len(inds) + arr[start:end] = inds + start = end + + return arr + + return _empty_idx_array From 9e15952fe448668828345550e0e6b52b117ba00b Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 13 Oct 2023 15:26:39 -0400 Subject: [PATCH 27/70] passing --- openmdao/core/group.py | 102 +++++++++++++++++++-- openmdao/core/problem.py | 3 + openmdao/core/tests/test_distrib_derivs.py | 18 ++-- openmdao/core/total_jac.py | 51 +++++------ openmdao/utils/graph_utils.py | 40 -------- openmdao/vectors/petsc_transfer.py | 48 ++++++++-- 6 files changed, 170 insertions(+), 92 deletions(-) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index cc65a6f9e9..67f6089f6f 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -30,7 +30,7 @@ _src_name_iter, meta2src_iter, get_rev_conns from openmdao.utils.units import is_compatible, unit_conversion, _has_val_mismatch, _find_unit, \ _is_unitless, simplify_unit -from openmdao.utils.graph_utils import get_sccs_topo, get_out_of_order_nodes, get_hybrid_graph +from openmdao.utils.graph_utils import get_sccs_topo, get_out_of_order_nodes from openmdao.utils.mpi import MPI, check_mpi_exceptions, multi_proc_exception_check import openmdao.utils.coloring as coloring_mod from openmdao.utils.indexer import indexer, Indexer @@ -192,6 +192,11 @@ class Group(System): Dict of absolute response metadata. _relevance_graph : nx.DiGraph Graph of relevance connections. Always None except in the top level Group. + _fd_subgroup_inputs : set + If one or more subgroups of this group is using finite difference to compute derivatives, + this is the set of inputs to those subgroups that are upstream of a distributed variable + within the same subgroup. These determine if an allreduce is necessary when transferring + data to a connected output in reverse mode. """ def __init__(self, **kwargs): @@ -222,6 +227,7 @@ def __init__(self, **kwargs): self._abs_desvars = None self._abs_responses = None self._relevance_graph = None + self._fd_subgroup_inputs = set() # TODO: we cannot set the solvers with property setters at the moment # because our lint check thinks that we are defining new attributes @@ -817,7 +823,8 @@ def get_relevance_graph(self, desvars, responses): return self._relevance_graph conns = self._conn_global_abs_in2out - graph = get_hybrid_graph(conns) + graph = self.get_hybrid_graph(conns) + outmeta = self._var_allprocs_abs2meta['output'] dvs = set(meta2src_iter(desvars.values())) resps = set(meta2src_iter(responses.values())) @@ -825,12 +832,14 @@ def get_relevance_graph(self, desvars, responses): # now add design vars and responses to the graph for dv in dvs: if dv not in graph: - graph.add_node(dv, type_='out') + graph.add_node(dv, type_='out', + dist=outmeta[dv]['distributed'] if dv in outmeta else None) graph.add_edge(dv.rpartition('.')[0], dv) for res in resps: if res not in graph: - graph.add_node(res, type_='out') + graph.add_node(res, type_='out', + dist=outmeta[res]['distributed'] if res in outmeta else None) graph.add_edge(res.rpartition('.')[0], res) # figure out if we can remove any edges based on zero partials we find @@ -866,6 +875,47 @@ def get_relevance_graph(self, desvars, responses): self._relevance_graph = graph return graph + def get_hybrid_graph(self, connections): + """ + Return a graph of all variables and components in the model. + + Each component is connected to each of its input and output variables, and + those variables are connected to other variables based on the connections + in the model. + + Parameters + ---------- + connections : dict + Dictionary of connections in the model, of the form {tgt: src}. + + Returns + ------- + networkx.DiGraph + Graph of all variables and components in the model. + """ + # Create a hybrid graph with components and all connected vars. If a var is connected, + # also connect it to its corresponding component. This results in a smaller graph + # (fewer edges) than would be the case for a pure variable graph where all inputs + # to a particular component would have to be connected to all outputs from that component. + graph = nx.DiGraph() + tgtmeta = self._var_allprocs_abs2meta['input'] + srcmeta = self._var_allprocs_abs2meta['output'] + + for tgt, src in connections.items(): + if src not in graph: + dist = srcmeta[src]['distributed'] if src in srcmeta else None + graph.add_node(src, type_='out', dist=dist) + + dist = tgtmeta[tgt]['distributed'] if tgt in tgtmeta else None + graph.add_node(tgt, type_='in', dist=dist) + + graph.add_edge(src.rpartition('.')[0], src) + graph.add_edge(tgt, tgt.rpartition('.')[0]) + + graph.add_edge(src, tgt) + + return graph + def get_relevant_vars(self, desvars, responses, mode): """ Find all relevant vars between desvars and responses. @@ -1265,6 +1315,8 @@ def _final_setup(self, comm, mode): # must call this before vector setup because it determines if we need to alloc commplex self._setup_partials() + self._fd_subgroup_inputs = set() + self._problem_meta['relevant'] = self._init_relevance(mode) self._setup_vectors(self._get_root_vectors()) @@ -1789,6 +1841,7 @@ def _setup_var_data(self): self._group_inputs[n] = lst.copy() # must copy the list manually self._has_distrib_vars = False + self._has_fd_group = self._owns_approx_jac abs_in2prom_info = self._problem_meta['abs_in2prom_info'] # sort the subsystems alphabetically in order to make the ordering @@ -1799,6 +1852,8 @@ def _setup_var_data(self): self._has_output_adder |= subsys._has_output_adder self._has_resid_scaling |= subsys._has_resid_scaling self._has_distrib_vars |= subsys._has_distrib_vars + if len(subsys._subsystems_allprocs) > 0: + self._has_fd_group |= subsys._has_fd_group var_maps = subsys._get_promotion_maps() @@ -1855,7 +1910,8 @@ def _setup_var_data(self): if self._gather_full_data(): raw = (allprocs_discrete, allprocs_prom2abs_list, allprocs_abs2meta, self._has_output_scaling, self._has_output_adder, - self._has_resid_scaling, self._group_inputs, self._has_distrib_vars) + self._has_resid_scaling, self._group_inputs, self._has_distrib_vars, + self._has_fd_group) else: raw = ( {'input': {}, 'output': {}}, @@ -1866,6 +1922,7 @@ def _setup_var_data(self): False, {}, False, + False, ) gathered = self.comm.allgather(raw) @@ -1879,11 +1936,13 @@ def _setup_var_data(self): myrank = self.comm.rank for rank, (proc_discrete, proc_prom2abs_list, proc_abs2meta, - oscale, oadd, rscale, ginputs, has_dist_vars) in enumerate(gathered): + oscale, oadd, rscale, ginputs, has_dist_vars, + has_fd_group) in enumerate(gathered): self._has_output_scaling |= oscale self._has_output_adder |= oadd self._has_resid_scaling |= rscale self._has_distrib_vars |= has_dist_vars + self._has_fd_group |= has_fd_group if rank != myrank: for p, mlist in ginputs.items(): @@ -3127,6 +3186,37 @@ def _transfer(self, vec_name, mode, sub=None): xfer = self._transfers['rev'][key] xfer._transfer(vec_inputs, self._vectors['output'][vec_name], mode) + if self._fd_subgroup_inputs and self.comm.size > 1: + seed_info = self._problem_meta['seed_var_info'] + if seed_info is not None: + seed_vars, has_distrib_seed = seed_info + if True: # has_distrib_seed: + if len(seed_vars) > 1: + raise RuntimeError("Multiple seed variables not supported " + "under MPI if they are distributed and in " + "a group doing finite difference.") + pre = '' if sub is None else sub + '.' + slices = self._doutputs.get_slice_dict() + outarr = self._doutputs.asarray() + data = {} + for inp in self._fd_subgroup_inputs: + src = self._conn_global_abs_in2out[inp] + if src.startswith(pre) and src in slices: + arr = outarr[slices[src]] + if np.any(arr): + data[src] = arr + else: + data[src] = None + + if data: + comm = self.comm + myrank = comm.rank + for rank, d in enumerate(comm.allgather(data)): + if rank != myrank: + for n, val in d.items(): + if val is not None and n in slices: + outarr[slices[n]] += val + if self._has_input_scaling: vec_inputs.scale_to_phys(mode='rev') diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index af1ccc8e9f..539d293c27 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -1005,6 +1005,9 @@ def setup(self, check=False, logger=None, mode='auto', force_alloc_complex=False 'singular_jac_behavior': 'warn', # How to handle singular jac conditions 'parallel_deriv_color': None, # None unless derivatives involving a parallel deriv # colored dv/response are currently being computed + 'seed_var_info': None, # list of tuples of the form (seed var names, any_are_distrib). + # The seed variables are those that are active in the current + # derivative solve. } if _prob_setup_stack: diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index f80033e46c..0bf6799d7e 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -688,12 +688,12 @@ def test_distrib_voi_group_fd(self): promotes_inputs=['*']) sub.add_subsystem("parab", DistParab(arr_size=size), promotes_outputs=['*'], promotes_inputs=['a']) - sub.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', + model.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, ))), promotes_outputs=['*']) - sub.promotes('sum', inputs=['f_xy'], src_indices=om.slicer[:]) + model.promotes('sum', inputs=['f_xy'], src_indices=om.slicer[:]) sub.connect('dummy.xd', 'parab.x') sub.connect('dummy.yd', 'parab.y') @@ -717,11 +717,7 @@ def test_distrib_voi_group_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - J = prob.check_totals(method='fd', show_only_incorrect=True) - assert_near_equal(J['sub.parab.f_xy', 'p.x']['abs error'].forward, 0.0, 1e-5) - assert_near_equal(J['sub.parab.f_xy', 'p.y']['abs error'].forward, 0.0, 1e-5) - assert_near_equal(J['sub.sum.f_sum', 'p.x']['abs error'].forward, 0.0, 1e-5) - assert_near_equal(J['sub.sum.f_sum', 'p.y']['abs error'].forward, 0.0, 1e-5) + #assert_check_totals(prob.check_totals(method='fd', out_stream=None)) # rev mode @@ -737,11 +733,9 @@ def test_distrib_voi_group_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - J = prob.check_totals(method='fd', show_only_incorrect=True) - assert_near_equal(J['sub.parab.f_xy', 'p.x']['abs error'].reverse, 0.0, 1e-5) - assert_near_equal(J['sub.parab.f_xy', 'p.y']['abs error'].reverse, 0.0, 1e-5) - assert_near_equal(J['sub.sum.f_sum', 'p.x']['abs error'].reverse, 0.0, 1e-5) - assert_near_equal(J['sub.sum.f_sum', 'p.y']['abs error'].reverse, 0.0, 1e-5) + # from openmdao.devtools.debug import trace_mpi + # trace_mpi() + assert_check_totals(prob.check_totals(method='fd', out_stream=None)) def test_simple_distrib_voi_group_fd(self): size = 3 diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index 9fdfc6399d..b9c967fd8e 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -91,7 +91,7 @@ class _TotalJacInfo(object): (local indices, local sizes). in_idx_map : dict Mapping of jacobian row/col index to a tuple of the form - (dist, relevant_systems, cache_linear_solutions_flag) + (relevant_systems, cache_linear_solutions_flag, voi name) total_relevant_systems : set The set of names of all systems relevant to the computation of the total derivatives. directional : bool @@ -372,7 +372,6 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, allfdgroups = set() for grps in model.comm.allgather(fdgroups): allfdgroups.update(grps) - fdprefixes = [n + '.' for n in allfdgroups] self.jac_scratch['rev'] = [scratch[0][:J.shape[1]]] if self.simul_coloring is not None: # when simul coloring, need two scratch arrays @@ -397,15 +396,9 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, else: end += vmeta['global_size'] - is_fdvar = False - for prefix in fdprefixes: - if name.startswith(prefix): - is_fdvar = True - break - dist = vmeta['distributed'] has_dist |= dist - if not is_fdvar and not dist and model._owning_rank[name] != model.comm.rank: + if not dist and model._owning_rank[name] != model.comm.rank: self.rev_allreduce_mask[start:end] = False start = end @@ -710,12 +703,13 @@ def _create_in_idx_map(self, mode): path = model._conn_global_abs_in2out[abs_in] in_var_meta = all_abs2meta_out[path] + dist = in_var_meta['distributed'] if name in vois: # if name is in vois, then it has been declared as either a design var or # a constraint or an objective. meta = vois[name] - if meta['distributed']: + if dist: end += meta['global_size'] else: end += meta['size'] @@ -753,7 +747,6 @@ def _create_in_idx_map(self, mode): offsets = var_offsets['output'] gstart = np.sum(sizes[:iproc, in_var_idx]) gend = gstart + sizes[iproc, in_var_idx] - has_rank_zeros = np.count_nonzero(sizes[:, in_var_idx]) < sizes.shape[0] # if we're doing parallel deriv coloring, we only want to set the seed on one proc # for each var in a given color @@ -765,7 +758,6 @@ def _create_in_idx_map(self, mode): else: relev = None - dist = in_var_meta['distributed'] if not dist: # if the var is not distributed, convert the indices to global. # We don't iterate over the full distributed size in this case. @@ -780,7 +772,7 @@ def _create_in_idx_map(self, mode): if gend > gstart and (relev is None or relev): loc = np.nonzero(np.logical_and(irange >= gstart, irange < gend))[0] if in_idxs is None: - if in_var_meta['distributed']: + if dist: loc_i[loc] = np.arange(0, gend - gstart, dtype=INT_DTYPE) else: loc_i[loc] = irange[loc] - gstart @@ -804,17 +796,22 @@ def _create_in_idx_map(self, mode): imeta = defaultdict(bool) imeta['par_deriv_color'] = parallel_deriv_color imeta['idx_list'] = [(start, end)] + imeta['seed_info'] = [[name], dist] idx_iter_dict[parallel_deriv_color] = (imeta, it) else: imeta = idx_iter_dict[parallel_deriv_color][0] imeta['idx_list'].append((start, end)) + imeta['seed_info'][0].append(name) + imeta['seed_info'][1] |= dist elif self.directional: imeta = defaultdict(bool) imeta['idx_list'] = range(start, end) + imeta['seed_info'] = [(name,), dist] idx_iter_dict[name] = (imeta, self.directional_iter) elif not simul_coloring: # plain old single index iteration imeta = defaultdict(bool) imeta['idx_list'] = range(start, end) + imeta['seed_info'] = [(name,), dist] idx_iter_dict[name] = (imeta, self.single_index_iter) if path in relevant and not non_rel_outs: @@ -843,14 +840,16 @@ def _create_in_idx_map(self, mode): imeta = defaultdict(bool) imeta['coloring'] = simul_coloring all_rel_systems = set() + all_vois = set() cache = False imeta['itermeta'] = itermeta = [] locs = None for ilist in simul_coloring.color_iter(mode): for i in ilist: - rel_systems, cache_lin_sol, _ = idx_map[i] + rel_systems, cache_lin_sol, voiname = idx_map[i] _update_rel_systems(all_rel_systems, rel_systems) cache |= cache_lin_sol + all_vois.add(voiname) iterdict = defaultdict(bool) @@ -862,6 +861,7 @@ def _create_in_idx_map(self, mode): iterdict['relevant'] = all_rel_systems iterdict['cache_lin_solve'] = cache + iterdict['seed_info'] = all_vois itermeta.append(iterdict) idx_iter_dict['@simul_coloring'] = (imeta, self.simul_coloring_iter) @@ -1067,7 +1067,7 @@ def single_index_iter(self, imeta, mode): Iteration metadata. """ for i in imeta['idx_list']: - yield i, self.single_input_setter, self.single_jac_setter, None + yield i, self.single_input_setter, self.single_jac_setter, imeta def simul_coloring_iter(self, imeta, mode): """ @@ -1096,11 +1096,7 @@ def simul_coloring_iter(self, imeta, mode): jac_setter = self.simul_coloring_jac_setter for color, ilist in enumerate(coloring.color_iter(mode)): - if len(ilist) == 1: - yield ilist, input_setter, jac_setter, None - else: - # yield all indices for a color at once - yield ilist, input_setter, jac_setter, imeta['itermeta'][color] + yield ilist, input_setter, jac_setter, imeta['itermeta'][color] def par_deriv_iter(self, imeta, mode): """ @@ -1216,7 +1212,7 @@ def simul_coloring_input_setter(self, inds, itermeta, mode): int or None key used for storage of cached linear solve (if active, else None). """ - if itermeta is None: + if len(inds) == 1: return self.single_input_setter(inds[0], None, mode) self._zero_vecs(mode) @@ -1255,8 +1251,6 @@ def par_deriv_input_setter(self, inds, imeta, mode): dist = self.comm.size > 1 - self.model._problem_meta['parallel_deriv_color'] = imeta['par_deriv_color'] - for i in inds: if not dist or self.in_loc_idxs[mode][i] >= 0: rel_systems, vnames, _ = self.single_input_setter(i, imeta, mode) @@ -1264,6 +1258,8 @@ def par_deriv_input_setter(self, inds, imeta, mode): if vnames is not None: vec_names.add(vnames[0]) + self.model._problem_meta['parallel_deriv_color'] = imeta['par_deriv_color'] + if vec_names: return all_rel_systems, sorted(vec_names), (inds[0], mode) else: @@ -1419,8 +1415,6 @@ def par_deriv_jac_setter(self, inds, mode, meta): for i in inds: self.simple_single_jac_scatter(i, mode) - self.model._problem_meta['parallel_deriv_color'] = None - def simul_coloring_jac_setter(self, inds, mode, meta): """ Set the appropriate part of the total jacobian for simul coloring input indices. @@ -1547,7 +1541,8 @@ def compute_totals(self): for key, idx_info in self.idx_iter_dict[mode].items(): imeta, idx_iter = idx_info for inds, input_setter, jac_setter, itermeta in idx_iter(imeta, mode): - rel_systems, vec_names, cache_key = input_setter(inds, itermeta, mode) + self.model._problem_meta['seed_var_info'] = itermeta['seed_info'] + rel_systems, _, cache_key = input_setter(inds, itermeta, mode) if debug_print: if par_print and key in par_print: @@ -1584,6 +1579,10 @@ def compute_totals(self): jac_setter(inds, mode, imeta) + # reset any Problem level data for the current iteration + self.model._problem_meta['parallel_deriv_color'] = None + self.model._problem_meta['seed_var_info'] = None + # Driver scaling. if self.has_scaling: self._do_driver_scaling(self.J_dict) diff --git a/openmdao/utils/graph_utils.py b/openmdao/utils/graph_utils.py index 5690644e94..1c04fa0538 100644 --- a/openmdao/utils/graph_utils.py +++ b/openmdao/utils/graph_utils.py @@ -54,43 +54,3 @@ def get_out_of_order_nodes(graph, orders): out_of_order.append((u, v)) return strongcomps, out_of_order - - -def get_hybrid_graph(connections): - """ - Return a graph of all variables and components in the model. - - Each component is connected each of its input and output variables, and - those variables are connected to other variables based on the connections - in the model. - - Parameters - ---------- - connections : dict - Dictionary of connections in the model, of the form {tgt: src}. - - Returns - ------- - networkx.DiGraph - Graph of all variables and components in the model. - """ - # Create a hybrid graph with components and all connected vars. If a var is connected, - # also connect it to its corresponding component. This results in a smaller graph - # (fewer edges) than would be the case for a pure variable graph where all inputs - # to a particular component would have to be connected to all outputs from that component. - graph = nx.DiGraph() - for tgt, src in connections.items(): - if src not in graph: - graph.add_node(src, type_='out') - - graph.add_node(tgt, type_='in') - - src_sys, _, _ = src.rpartition('.') - graph.add_edge(src_sys, src) - - tgt_sys, _, _ = tgt.rpartition('.') - graph.add_edge(tgt, tgt_sys) - - graph.add_edge(src, tgt) - - return graph diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index eb32462016..5781c1c421 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -1,6 +1,8 @@ """Define the PETSc Transfer class.""" import numpy as np +import networkx as nx from openmdao.utils.mpi import check_mpi_env +from openmdao.utils.general_utils import common_subpath from openmdao.core.constants import INT_DTYPE use_mpi = check_mpi_env() @@ -116,11 +118,39 @@ def is_dup(name, io): nz = np.count_nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]]) return nz > 1, group._var_sizes[io].shape[0] - nz, False - def get_xfer_ranks(name, io): - if group._var_allprocs_abs2meta[io][name]['distributed']: - return [] + def get_nonzero_ranks(name, io): return np.nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]])[0] + if rev and group._owns_approx_jac and group._has_distrib_vars and group.pathname != '': + # inp_boundary_set is the set of input variables that are connected to sources + # outside of this group. + inp_boundary_set = set(group._var_allprocs_abs2meta['input']) + inp_boundary_set = inp_boundary_set.difference(group._conn_global_abs_in2out) + model = group._problem_meta['model_ref']() + relgraph = model._relevance_graph + + # inp_dep_dist is the set of input variables that are upstream of distributed + # variables. + inp_dep_dist = set() + for inp in inp_boundary_set: + found = False + for _, succs in nx.bfs_successors(relgraph, inp): + for successor in succs: + ndata = relgraph.nodes[successor] + if 'dist' in ndata and ndata['dist']: + inp_dep_dist.add(inp) + found = True + break + if found: + break + + # look in model for the connections to the inp_dep_dist inputs + for inp in inp_dep_dist: + src = model._conn_global_abs_in2out[inp] + gname = common_subpath((src, inp)) + owning_group = model._get_subsystem(gname) + owning_group._fd_subgroup_inputs.add(inp) + total_fwd = total_rev = total_rev_nocolor = 0 # Loop through all connections owned by this system @@ -180,7 +210,7 @@ def get_xfer_ranks(name, io): if np.any(on_iproc): # This converts from iproc-then-ivar to ivar-then-iproc ordering - # Subtract off part of previous procs + # Subtract off part of this variable from previous procs # Then add all variables on previous procs # Then all previous variables on this proc # - np.sum(out_sizes[:iproc, idx_out]) @@ -203,7 +233,9 @@ def get_xfer_ranks(name, io): sub_in = abs_in[mypathlen:].partition('.')[0] fwd_xfer_in[sub_in].append(input_inds) fwd_xfer_out[sub_in].append(output_inds) - if rev: + if rev and group._owns_approx_jac: + pass # no rev transfers needed for FD group + elif rev: inp_is_dup, inp_missing, distrib_in = is_dup(abs_in, 'input') out_is_dup, _, distrib_out = is_dup(abs_out, 'output') @@ -281,7 +313,7 @@ def get_xfer_ranks(name, io): oidxlist_nc = [] iidxlist_nc = [] size = size_nc = 0 - for rnk in get_xfer_ranks(abs_out, 'output'): + for rnk in get_nonzero_ranks(abs_out, 'output'): offset = offsets_out[rnk, idx_out] if src_indices is None: oarr = range(offset, offset + meta_in['size']) @@ -343,7 +375,7 @@ def get_xfer_ranks(name, io): sub_in = abs_in[mypathlen:].partition('.')[0] fwd_xfer_in[sub_in] # defaultdict will create an empty list there fwd_xfer_out[sub_in] - if rev: + if rev and not group._owns_approx_jac: sub_out = abs_out[mypathlen:].partition('.')[0] rev_xfer_in[sub_out] rev_xfer_out[sub_out] @@ -367,7 +399,7 @@ def get_xfer_ranks(name, io): vectors['input']['nonlinear'], vectors['output']['nonlinear'], inds, fwd_xfer_out[sname], group.comm) - if rev: + if rev and not group._owns_approx_jac: xfer_in, xfer_out = _setup_index_views(total_rev, rev_xfer_in, rev_xfer_out) xfer_all = PETScTransfer(vectors['input']['nonlinear'], out_vec, From 3f062290286ea0d826f9e5c8cee74633c8e43cec Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 13 Oct 2023 15:33:16 -0400 Subject: [PATCH 28/70] cleaned up test --- openmdao/core/tests/test_distrib_derivs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index 0bf6799d7e..aa4e644806 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -717,7 +717,7 @@ def test_distrib_voi_group_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - #assert_check_totals(prob.check_totals(method='fd', out_stream=None)) + assert_check_totals(prob.check_totals(method='fd', out_stream=None)) # rev mode @@ -733,8 +733,6 @@ def test_distrib_voi_group_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - # from openmdao.devtools.debug import trace_mpi - # trace_mpi() assert_check_totals(prob.check_totals(method='fd', out_stream=None)) def test_simple_distrib_voi_group_fd(self): From b7cdb7589db5497671de0adf4f19c8a52df5c655 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 13 Oct 2023 16:12:06 -0400 Subject: [PATCH 29/70] new tests --- openmdao/core/group.py | 56 ++++++------ openmdao/core/problem.py | 2 +- openmdao/core/tests/test_distrib_derivs.py | 100 +++++++++++++++++++++ openmdao/core/total_jac.py | 15 ++-- openmdao/vectors/petsc_transfer.py | 12 +-- 5 files changed, 142 insertions(+), 43 deletions(-) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 67f6089f6f..65c39d791d 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -3187,35 +3187,33 @@ def _transfer(self, vec_name, mode, sub=None): xfer._transfer(vec_inputs, self._vectors['output'][vec_name], mode) if self._fd_subgroup_inputs and self.comm.size > 1: - seed_info = self._problem_meta['seed_var_info'] - if seed_info is not None: - seed_vars, has_distrib_seed = seed_info - if True: # has_distrib_seed: - if len(seed_vars) > 1: - raise RuntimeError("Multiple seed variables not supported " - "under MPI if they are distributed and in " - "a group doing finite difference.") - pre = '' if sub is None else sub + '.' - slices = self._doutputs.get_slice_dict() - outarr = self._doutputs.asarray() - data = {} - for inp in self._fd_subgroup_inputs: - src = self._conn_global_abs_in2out[inp] - if src.startswith(pre) and src in slices: - arr = outarr[slices[src]] - if np.any(arr): - data[src] = arr - else: - data[src] = None - - if data: - comm = self.comm - myrank = comm.rank - for rank, d in enumerate(comm.allgather(data)): - if rank != myrank: - for n, val in d.items(): - if val is not None and n in slices: - outarr[slices[n]] += val + seed_vars = self._problem_meta['seed_vars'] + if seed_vars is not None: + if len(seed_vars) > 1: + raise RuntimeError("Multiple seed variables not supported " + "under MPI if they are distributed and in " + "a group doing finite difference.") + pre = '' if sub is None else sub + '.' + slices = self._doutputs.get_slice_dict() + outarr = self._doutputs.asarray() + data = {} + for inp in self._fd_subgroup_inputs: + src = self._conn_global_abs_in2out[inp] + if src.startswith(pre) and src in slices: + arr = outarr[slices[src]] + if np.any(arr): + data[src] = arr + else: + data[src] = None + + if data: + comm = self.comm + myrank = comm.rank + for rank, d in enumerate(comm.allgather(data)): + if rank != myrank: + for n, val in d.items(): + if val is not None and n in slices: + outarr[slices[n]] += val if self._has_input_scaling: vec_inputs.scale_to_phys(mode='rev') diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index 539d293c27..042fa4d003 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -1005,7 +1005,7 @@ def setup(self, check=False, logger=None, mode='auto', force_alloc_complex=False 'singular_jac_behavior': 'warn', # How to handle singular jac conditions 'parallel_deriv_color': None, # None unless derivatives involving a parallel deriv # colored dv/response are currently being computed - 'seed_var_info': None, # list of tuples of the form (seed var names, any_are_distrib). + 'seed_vars': None, # list of tuples of the form (seed var names, any_are_distrib). # The seed variables are those that are active in the current # derivative solve. } diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index aa4e644806..76fa5875e3 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -735,6 +735,60 @@ def test_distrib_voi_group_fd(self): assert_check_totals(prob.check_totals(method='fd', out_stream=None)) + def test_distrib_voi_group_fd2(self): + size = 7 + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + ivc.add_output('y', np.ones((size, ))) + + model.add_subsystem('p', ivc, promotes=['*']) + sub = model.add_subsystem('sub', om.Group(), promotes=['*']) + + ivc2 = om.IndepVarComp() + ivc2.add_output('a', -3.0 + 0.6 * np.arange(size)) + + sub.add_subsystem('p2', ivc2, promotes=['*']) + sub.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], + x=np.ones(size), xd=np.ones(size), + y=np.ones(size), yd=np.ones(size)), + promotes_inputs=['*']) + + sub.add_subsystem("parab", DistParab(arr_size=size), promotes_outputs=['*'], promotes_inputs=['a']) + model.add_subsystem('sum', om.ExecComp('f_sum = sum(xd)', + f_sum=np.ones((size, )), + xd=np.ones((size, ))), + promotes_outputs=['*']) + + model.promotes('sum', inputs=['xd']) + + sub.connect('dummy.xd', 'parab.x') + sub.connect('dummy.yd', 'parab.y') + + model.add_design_var('x', lower=-50.0, upper=50.0) + model.add_design_var('y', lower=-50.0, upper=50.0) + model.add_constraint('f_xy', lower=0.0) + model.add_objective('f_sum', index=-1) + + sub.approx_totals(method='fd') + + prob.setup(mode='fwd', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None)) + + # rev mode + + prob.setup(mode='rev', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None)) + def test_simple_distrib_voi_group_fd(self): size = 3 @@ -775,6 +829,52 @@ def test_simple_distrib_voi_group_fd(self): assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=1e-5) + def test_nondistrib_voi_group_fd(self): + size = 7 + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + ivc.add_output('y', np.ones((size, ))) + + model.add_subsystem('p', ivc, promotes=['*']) + sub = model.add_subsystem('sub', om.Group(), promotes=['*']) + + ivc2 = om.IndepVarComp() + ivc2.add_output('a', -3.0 + 0.6 * np.arange(size)) + + sub.add_subsystem('p2', ivc2, promotes=['*']) + sub.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], + x=np.ones(size), xd=np.ones(size), + y=np.ones(size), yd=np.ones(size)), + promotes_inputs=['*']) + + sub.add_subsystem("parab", om.ExecComp('f_xy = x**2 + 3.*xy - y*y + a', shape=size), promotes_outputs=['*'], promotes_inputs=['a']) + model.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', + f_sum=np.ones((size, )), + f_xy=np.ones((size, ))), + promotes_outputs=['*']) + + model.promotes('sum', inputs=['f_xy'], src_indices=om.slicer[:]) + + sub.connect('dummy.xd', 'parab.x') + sub.connect('dummy.yd', 'parab.y') + + model.add_design_var('x', lower=-50.0, upper=50.0) + model.add_design_var('y', lower=-50.0, upper=50.0) + model.add_constraint('f_xy', lower=0.0) + model.add_objective('f_sum', index=-1) + + sub.approx_totals(method='fd') + + prob.setup(mode='rev', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None)) + def test_distrib_group_fd_unsupported_config(self): size = 7 diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index b9c967fd8e..47ce335493 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -796,22 +796,21 @@ def _create_in_idx_map(self, mode): imeta = defaultdict(bool) imeta['par_deriv_color'] = parallel_deriv_color imeta['idx_list'] = [(start, end)] - imeta['seed_info'] = [[name], dist] + imeta['seed_vars'] = [name] idx_iter_dict[parallel_deriv_color] = (imeta, it) else: imeta = idx_iter_dict[parallel_deriv_color][0] imeta['idx_list'].append((start, end)) - imeta['seed_info'][0].append(name) - imeta['seed_info'][1] |= dist + imeta['seed_vars'].append(name) elif self.directional: imeta = defaultdict(bool) imeta['idx_list'] = range(start, end) - imeta['seed_info'] = [(name,), dist] + imeta['seed_vars'] = (name,) idx_iter_dict[name] = (imeta, self.directional_iter) elif not simul_coloring: # plain old single index iteration imeta = defaultdict(bool) imeta['idx_list'] = range(start, end) - imeta['seed_info'] = [(name,), dist] + imeta['seed_vars'] = (name,) idx_iter_dict[name] = (imeta, self.single_index_iter) if path in relevant and not non_rel_outs: @@ -861,7 +860,7 @@ def _create_in_idx_map(self, mode): iterdict['relevant'] = all_rel_systems iterdict['cache_lin_solve'] = cache - iterdict['seed_info'] = all_vois + iterdict['seed_vars'] = all_vois itermeta.append(iterdict) idx_iter_dict['@simul_coloring'] = (imeta, self.simul_coloring_iter) @@ -1541,7 +1540,7 @@ def compute_totals(self): for key, idx_info in self.idx_iter_dict[mode].items(): imeta, idx_iter = idx_info for inds, input_setter, jac_setter, itermeta in idx_iter(imeta, mode): - self.model._problem_meta['seed_var_info'] = itermeta['seed_info'] + self.model._problem_meta['seed_vars'] = itermeta['seed_vars'] rel_systems, _, cache_key = input_setter(inds, itermeta, mode) if debug_print: @@ -1581,7 +1580,7 @@ def compute_totals(self): # reset any Problem level data for the current iteration self.model._problem_meta['parallel_deriv_color'] = None - self.model._problem_meta['seed_var_info'] = None + self.model._problem_meta['seed_vars'] = None # Driver scaling. if self.has_scaling: diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 5781c1c421..a199d5314a 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -128,6 +128,7 @@ def get_nonzero_ranks(name, io): inp_boundary_set = inp_boundary_set.difference(group._conn_global_abs_in2out) model = group._problem_meta['model_ref']() relgraph = model._relevance_graph + prefix = group.pathname + '.' # inp_dep_dist is the set of input variables that are upstream of distributed # variables. @@ -136,11 +137,12 @@ def get_nonzero_ranks(name, io): found = False for _, succs in nx.bfs_successors(relgraph, inp): for successor in succs: - ndata = relgraph.nodes[successor] - if 'dist' in ndata and ndata['dist']: - inp_dep_dist.add(inp) - found = True - break + if successor.startswith(prefix): + ndata = relgraph.nodes[successor] + if 'dist' in ndata and ndata['dist']: + inp_dep_dist.add(inp) + found = True + break if found: break From 39d7eea20686a66beebe5823076340e1422ac1a4 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 16 Oct 2023 16:58:39 -0400 Subject: [PATCH 30/70] FD group tests passing for rev transfers --- openmdao/core/group.py | 10 +- openmdao/core/tests/test_distrib_derivs.py | 208 +++++++++++++++++++++ openmdao/vectors/petsc_transfer.py | 76 +++++--- 3 files changed, 263 insertions(+), 31 deletions(-) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 65c39d791d..311265d1c3 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -835,12 +835,14 @@ def get_relevance_graph(self, desvars, responses): graph.add_node(dv, type_='out', dist=outmeta[dv]['distributed'] if dv in outmeta else None) graph.add_edge(dv.rpartition('.')[0], dv) + graph.nodes[dv]['isdv'] = True for res in resps: if res not in graph: graph.add_node(res, type_='out', dist=outmeta[res]['distributed'] if res in outmeta else None) - graph.add_edge(res.rpartition('.')[0], res) + graph.add_edge(res.rpartition('.')[0], res, isresponse=True) + graph.nodes[res]['isresponse'] = True # figure out if we can remove any edges based on zero partials we find # in components. By default all component connected outputs @@ -904,10 +906,12 @@ def get_hybrid_graph(self, connections): for tgt, src in connections.items(): if src not in graph: dist = srcmeta[src]['distributed'] if src in srcmeta else None - graph.add_node(src, type_='out', dist=dist) + graph.add_node(src, type_='out', dist=dist, + isdv=False, isresponse=False) dist = tgtmeta[tgt]['distributed'] if tgt in tgtmeta else None - graph.add_node(tgt, type_='in', dist=dist) + graph.add_node(tgt, type_='in', dist=dist, + isdv=False, isresponse=False) graph.add_edge(src.rpartition('.')[0], src) graph.add_edge(tgt, tgt.rpartition('.')[0]) diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index 76fa5875e3..4c4b34c2ba 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -789,6 +789,214 @@ def test_distrib_voi_group_fd2(self): assert_check_totals(prob.check_totals(method='fd', out_stream=None)) + def test_distrib_voi_group_fd4(self): + # distrib comp is inside of fd group but not a response, and 2 nondistrib + # constraints connect to it downstream, both inside and outside of the fd group. + size = 7 + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + ivc.add_output('y', np.ones((size, ))) + + model.add_subsystem('p', ivc) + sub = model.add_subsystem('sub', om.Group()) + + sub.add_subsystem('p2', om.IndepVarComp('a', -3.0 + 0.6 * np.arange(size))) + sub.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], + x=np.ones(size), xd=np.ones(size), + y=np.ones(size), yd=np.ones(size))) + + sub.add_subsystem("parab", DistParab(arr_size=size)) + sub.add_subsystem("cons", om.ExecComp("c = x*3. + 7.", x=np.ones(size), c=np.ones(size))) + # model.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, )))) + + model.connect('p.x', 'sub.dummy.x') + model.connect('p.y', 'sub.dummy.y') + model.connect('sub.p2.a', 'sub.parab.a') + model.connect('sub.dummy.xd', 'sub.parab.x') + model.connect('sub.dummy.yd', 'sub.parab.y') + # model.connect('sub.parab.f_xy', 'sum.f_xy', src_indices=om.slicer[:]) + model.connect('sub.parab.f_xy', 'sub.cons.x', src_indices=om.slicer[:]) + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p.y', lower=-50.0, upper=50.0) + model.add_constraint('sub.cons.c', lower=0.0) + # model.add_objective('sum.f_sum', index=-1) + + sub.approx_totals(method='fd') + + prob.setup(mode='fwd', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + # rev mode + + prob.setup(mode='rev', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_distrib_voi_group_fd5(self): + # distrib comp is inside of fd group but not a response, and 2 nondistrib + # constraints connect to it downstream, both inside of the fd group. + size = 7 + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + ivc.add_output('y', np.ones((size, ))) + + model.add_subsystem('p', ivc) + sub = model.add_subsystem('sub', om.Group()) + + sub.add_subsystem('p2', om.IndepVarComp('a', -3.0 + 0.6 * np.arange(size))) + sub.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], + x=np.ones(size), xd=np.ones(size), + y=np.ones(size), yd=np.ones(size))) + + sub.add_subsystem("parab", DistParab(arr_size=size)) + sub.add_subsystem("cons", om.ExecComp("c = x*3. + 7.", x=np.ones(size), c=np.ones(size))) + # sub.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, )))) + + model.connect('p.x', 'sub.dummy.x') + model.connect('p.y', 'sub.dummy.y') + model.connect('sub.p2.a', 'sub.parab.a') + model.connect('sub.dummy.xd', 'sub.parab.x') + model.connect('sub.dummy.yd', 'sub.parab.y') + # model.connect('sub.parab.f_xy', 'sub.sum.f_xy', src_indices=om.slicer[:]) + model.connect('sub.parab.f_xy', 'sub.cons.x', src_indices=om.slicer[:]) + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p.y', lower=-50.0, upper=50.0) + model.add_constraint('sub.cons.c', lower=0.0) + # model.add_objective('sub.sum.f_sum', index=-1) + + sub.approx_totals(method='fd') + + prob.setup(mode='fwd', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + # rev mode + + prob.setup(mode='rev', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_distrib_voi_group_nofd(self): + # distrib comp output feeds two nondist inputs + size = 7 + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + ivc.add_output('y', np.ones((size, ))) + + model.add_subsystem('p', ivc) + + model.add_subsystem('p2', om.IndepVarComp('a', -3.0 + 0.6 * np.arange(size))) + model.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], + x=np.ones(size), xd=np.ones(size), + y=np.ones(size), yd=np.ones(size))) + + model.add_subsystem("parab", DistParab(arr_size=size)) + model.add_subsystem("cons", om.ExecComp("c = x*3. + 7.", x=np.ones(size), c=np.ones(size))) + model.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, )))) + + model.connect('p.x', 'dummy.x') + model.connect('p.y', 'dummy.y') + model.connect('p2.a', 'parab.a') + model.connect('dummy.xd', 'parab.x') + model.connect('dummy.yd', 'parab.y') + model.connect('parab.f_xy', 'sum.f_xy', src_indices=om.slicer[:]) + model.connect('parab.f_xy', 'cons.x', src_indices=om.slicer[:]) + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p.y', lower=-50.0, upper=50.0) + model.add_constraint('cons.c', lower=0.0) + model.add_objective('sum.f_sum', index=-1) + + prob.setup(mode='fwd', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='cs', out_stream=None), atol=3e-6) + + # rev mode + + prob.setup(mode='rev', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='cs', out_stream=None), atol=3e-6) + + def test_nondistrib_voi_group_fd2(self): + # nondistrib comp is inside of fd group but not a response, and 2 nondistrib + # constraints connect to it downstream, both inside of the fd group. + size = 7 + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + ivc.add_output('y', np.ones((size, ))) + + model.add_subsystem('p', ivc) + sub = model.add_subsystem('sub', om.Group()) + + sub.add_subsystem('p2', om.IndepVarComp('a', -3.0 + 0.6 * np.arange(size))) + sub.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], + x=np.ones(size), xd=np.ones(size), + y=np.ones(size), yd=np.ones(size))) + + sub.add_subsystem("parab", om.ExecComp('f_xy = x**2 + 3.*xy - y*y + a', shape=size)) + sub.add_subsystem("cons", om.ExecComp("c = x*3. + 7.", x=np.ones(size), c=np.ones(size))) + sub.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, )))) + + model.connect('p.x', 'sub.dummy.x') + model.connect('p.y', 'sub.dummy.y') + model.connect('sub.p2.a', 'sub.parab.a') + model.connect('sub.dummy.xd', 'sub.parab.x') + model.connect('sub.dummy.yd', 'sub.parab.y') + model.connect('sub.parab.f_xy', 'sub.sum.f_xy', src_indices=om.slicer[:]) + model.connect('sub.parab.f_xy', 'sub.cons.x', src_indices=om.slicer[:]) + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p.y', lower=-50.0, upper=50.0) + model.add_constraint('sub.cons.c', lower=0.0) + model.add_objective('sub.sum.f_sum', index=-1) + + sub.approx_totals(method='fd') + + prob.setup(mode='fwd', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + # rev mode + + prob.setup(mode='rev', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + def test_simple_distrib_voi_group_fd(self): size = 3 diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index a199d5314a..3429ab391f 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -121,37 +121,57 @@ def is_dup(name, io): def get_nonzero_ranks(name, io): return np.nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]])[0] + # for an FD group, we use the relevance graph to determine which inputs on the + # boundary of the group are upstream of distributed variables within the group so + # that we can perform any necessary allreduce operations on the outputs that + # are connected to those inputs. if rev and group._owns_approx_jac and group._has_distrib_vars and group.pathname != '': - # inp_boundary_set is the set of input variables that are connected to sources - # outside of this group. - inp_boundary_set = set(group._var_allprocs_abs2meta['input']) - inp_boundary_set = inp_boundary_set.difference(group._conn_global_abs_in2out) model = group._problem_meta['model_ref']() relgraph = model._relevance_graph - prefix = group.pathname + '.' - - # inp_dep_dist is the set of input variables that are upstream of distributed - # variables. - inp_dep_dist = set() - for inp in inp_boundary_set: - found = False - for _, succs in nx.bfs_successors(relgraph, inp): - for successor in succs: - if successor.startswith(prefix): - ndata = relgraph.nodes[successor] - if 'dist' in ndata and ndata['dist']: - inp_dep_dist.add(inp) - found = True - break - if found: - break - - # look in model for the connections to the inp_dep_dist inputs - for inp in inp_dep_dist: - src = model._conn_global_abs_in2out[inp] - gname = common_subpath((src, inp)) - owning_group = model._get_subsystem(gname) - owning_group._fd_subgroup_inputs.add(inp) + group_path = group.pathname + '.' + + inner_resps = [] + outer_dvs = [] + inner_dists = set() + for n, data in relgraph.nodes(data=True): + inside = n.startswith(group_path) + if inside: + if 'isresponse' in data and data['isresponse']: + inner_resps.append(n) + if 'dist' in data and data['dist']: + inner_dists.add(n) + else: # not inside + if 'isdv' in data and data['isdv']: + outer_dvs.append(n) + + if inner_resps and outer_dvs and inner_dists: + # inp_boundary_set is the set of input variables that are connected to sources + # outside of this group. (all group inputs minus group inputs that are connected + # internal to the group) + inp_boundary_set = set(group._var_allprocs_abs2meta['input']) + inp_boundary_set = inp_boundary_set.difference(group._conn_global_abs_in2out) + + # inp_dep_dist is the set of group boundary inputs that are upstream of + # distributed variables and between an external design var and an internal + # response. + inp_dep_dist = set() + + relevant = group._relevant + for resp in inner_resps: + for dv in outer_dvs: + rel = relevant[resp][dv][0] + if inner_dists.intersection(rel['input']) or \ + inner_dists.intersection(rel['output']): + # dist var is in between the dv and response + if resp in inner_dists: + inp_dep_dist.update(inp_boundary_set.intersection(rel['input'])) + + # look in model for the connections to the group boundary inputs from outside + for inp in inp_dep_dist: + src = model._conn_global_abs_in2out[inp] + gname = common_subpath((src, inp)) + owning_group = model._get_subsystem(gname) + owning_group._fd_subgroup_inputs.add(inp) total_fwd = total_rev = total_rev_nocolor = 0 From 120e42b208fd1f45381e87b65d3ef3896bb98d6f Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Tue, 17 Oct 2023 12:38:51 -0400 Subject: [PATCH 31/70] passing --- openmdao/core/tests/test_driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmdao/core/tests/test_driver.py b/openmdao/core/tests/test_driver.py index 56f9e81e23..3508d2f688 100644 --- a/openmdao/core/tests/test_driver.py +++ b/openmdao/core/tests/test_driver.py @@ -11,7 +11,7 @@ import openmdao.api as om from openmdao.core.driver import Driver from openmdao.utils.units import convert_units -from openmdao.utils.assert_utils import assert_near_equal, assert_warning, assert_check_totals +from openmdao.utils.assert_utils import assert_near_equal, assert_warnings, assert_check_totals from openmdao.utils.general_utils import printoptions from openmdao.utils.testing_utils import use_tempdirs from openmdao.test_suite.components.paraboloid import Paraboloid From 0744c8dc51362dabe0cb629cbaf5215c32f49f1b Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Wed, 18 Oct 2023 08:49:17 -0400 Subject: [PATCH 32/70] added new failing tests --- openmdao/core/group.py | 5 + openmdao/core/tests/test_distrib_derivs.py | 141 ++++++++++++++++++++- 2 files changed, 143 insertions(+), 3 deletions(-) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 311265d1c3..ba9472e08c 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -4140,6 +4140,9 @@ def _jac_of_iter(self): else: path = of + if not total and path not in self._var_abs2meta['output']: + continue + meta = abs2meta[path] if meta['distributed']: dist_sizes = sizes[:, abs2idx[path]] @@ -4214,6 +4217,8 @@ def _jac_wrt_iter(self, wrt_matches=None): elif wrt in local_outs: vec = self._outputs else: + if not total: + continue vec = None if wrt in approx_wrt_idx: sub_wrt_idx = approx_wrt_idx[wrt] diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index 4c4b34c2ba..61e5285f4e 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -864,20 +864,155 @@ def test_distrib_voi_group_fd5(self): sub.add_subsystem("parab", DistParab(arr_size=size)) sub.add_subsystem("cons", om.ExecComp("c = x*3. + 7.", x=np.ones(size), c=np.ones(size))) - # sub.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, )))) + sub.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, )))) model.connect('p.x', 'sub.dummy.x') model.connect('p.y', 'sub.dummy.y') model.connect('sub.p2.a', 'sub.parab.a') model.connect('sub.dummy.xd', 'sub.parab.x') model.connect('sub.dummy.yd', 'sub.parab.y') - # model.connect('sub.parab.f_xy', 'sub.sum.f_xy', src_indices=om.slicer[:]) + model.connect('sub.parab.f_xy', 'sub.sum.f_xy', src_indices=om.slicer[:]) model.connect('sub.parab.f_xy', 'sub.cons.x', src_indices=om.slicer[:]) model.add_design_var('p.x', lower=-50.0, upper=50.0) model.add_design_var('p.y', lower=-50.0, upper=50.0) model.add_constraint('sub.cons.c', lower=0.0) - # model.add_objective('sub.sum.f_sum', index=-1) + model.add_objective('sub.sum.f_sum', index=-1) + + sub.approx_totals(method='fd') + + prob.setup(mode='fwd', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + # rev mode + + prob.setup(mode='rev', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_group_fd_inner_par(self): + size = 7 + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + + model.add_subsystem('p', ivc) + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1*.5 - x2*3.', shape=size)) + + model.connect('p.x', 'sub.par.C1.x') + model.connect('p.x', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x1') + model.connect('sub.par.C2.y', 'sub.C3.x2') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='fd') + + prob.setup(mode='fwd', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + # rev mode + + prob.setup(mode='rev', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_group_fd_inner_par2(self): + size = 7 + + prob = om.Problem() + model = prob.model + + model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) + model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x*.5', shape=size)) + sub.add_subsystem('C4', om.ExecComp('y = x*3.', shape=size)) + + model.connect('p.x', 'sub.par.C1.x') + model.connect('p2.x', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x') + model.connect('sub.par.C2.y', 'sub.C4.x') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p2.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_constraint('sub.C4.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='fd') + + prob.setup(mode='fwd', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + # rev mode + + prob.setup(mode='rev', force_alloc_complex=True) + + prob.run_model() + + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_distrib_voi_group_fd_loop(self): + # distrib comp is inside of fd group and part of a loop. + size = 7 + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + + model.add_subsystem('p', ivc) + sub = model.add_subsystem('sub', om.Group()) + + sub.add_subsystem('p2', om.IndepVarComp('a', -3.0 + 0.6 * np.arange(size))) + sub.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], + x=np.ones(size), xd=np.ones(size), + y=np.ones(size), yd=np.ones(size))) + + sub.add_subsystem("parab", DistParab(arr_size=size)) + sub.add_subsystem("cons", om.ExecComp("c = x*3. + 7.", x=np.ones(size), c=np.ones(size))) + sub.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, )))) + + model.connect('p.x', 'sub.dummy.x') + model.connect('sub.p2.a', 'sub.parab.a') + model.connect('sub.dummy.xd', 'sub.parab.x') + model.connect('sub.dummy.yd', 'sub.parab.y') + model.connect('sub.parab.f_xy', 'sub.sum.f_xy', src_indices=om.slicer[:]) + model.connect('sub.parab.f_xy', 'sub.cons.x', src_indices=om.slicer[:]) + model.connect('sub.cons.c', 'sub.dummy.y') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.cons.c', lower=0.0) + model.add_objective('sub.sum.f_sum', index=-1) sub.approx_totals(method='fd') From b0406a5b563d71e560cc22de46d329c0d589dea8 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Wed, 18 Oct 2023 11:15:01 -0400 Subject: [PATCH 33/70] cleanup --- .../approximation_schemes/approximation_scheme.py | 9 +++++---- openmdao/core/component.py | 2 +- openmdao/core/group.py | 11 +++++++---- openmdao/vectors/petsc_transfer.py | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/openmdao/approximation_schemes/approximation_scheme.py b/openmdao/approximation_schemes/approximation_scheme.py index 7ce5843aa0..dacd3a7590 100644 --- a/openmdao/approximation_schemes/approximation_scheme.py +++ b/openmdao/approximation_schemes/approximation_scheme.py @@ -155,9 +155,9 @@ def _init_colored_approximations(self, system): if is_total: ccol2vcol = np.empty(coloring._shape[1], dtype=INT_DTYPE) - ordered_wrt_iter = list(system._jac_wrt_iter()) + # ordered_wrt_iter = list(system._jac_wrt_iter()) colored_start = colored_end = 0 - for abs_wrt, cstart, cend, _, cinds, _ in ordered_wrt_iter: + for abs_wrt, cstart, cend, _, cinds, _ in system._jac_wrt_iter(): if wrt_matches is None or abs_wrt in wrt_matches: colored_end += cend - cstart ccol2jcol[colored_start:colored_end] = np.arange(cstart, cend, dtype=INT_DTYPE) @@ -479,9 +479,10 @@ def _uncolored_column_iter(self, system, approx_groups): solution array corresponding to the jacobian column at the given column index """ total = system.pathname == '' - ordered_of_iter = list(system._jac_of_iter()) if total: - tot_result = np.zeros(ordered_of_iter[-1][2]) + for _, _, end, _, _ in system._jac_of_iter(): + pass + tot_result = np.zeros(end) total_or_semi = total or _is_group(system) diff --git a/openmdao/core/component.py b/openmdao/core/component.py index fd01358fea..22d0351156 100644 --- a/openmdao/core/component.py +++ b/openmdao/core/component.py @@ -1758,7 +1758,7 @@ def _get_dist_nz_dresids(self): """ nzresids = [] dresids = self._dresiduals.asarray() - for of, start, end, _full_slice, dist_sizes in self._jac_of_iter(): + for of, start, end, _, dist_sizes in self._jac_of_iter(): if dist_sizes is not None: if np.any(dresids[start:end]): nzresids.append(of) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index ba9472e08c..bf77a0a58c 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -1215,11 +1215,13 @@ def _get_var_offsets(self): def _get_jac_col_scatter(self): """ - Return source and target indices for a scatter from the output vector to a jacobian column. + Return source and target indices for a scatter from output vector to total jacobian column. If the transfer involves remote or distributed variables, the indices will be global. Otherwise they will be converted to local. + This is only called on the top level system. + Returns ------- ndarray @@ -3194,9 +3196,10 @@ def _transfer(self, vec_name, mode, sub=None): seed_vars = self._problem_meta['seed_vars'] if seed_vars is not None: if len(seed_vars) > 1: - raise RuntimeError("Multiple seed variables not supported " - "under MPI if they are distributed and in " - "a group doing finite difference.") + raise RuntimeError(f"Multiple seed variables {sorted(seed_vars)} are " + "not supported under MPI in reverse mode if they " + "depend on an group doing finite difference and " + "containing distributed variables.") pre = '' if sub is None else sub + '.' slices = self._doutputs.get_slice_dict() outarr = self._doutputs.asarray() diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 3429ab391f..f3dc2a6768 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -220,7 +220,7 @@ def get_nonzero_ranks(name, io): offset = offsets_out[rank, idx_out] output_inds = range(offset, offset + meta_in['size']) else: - output_inds = np.zeros(src_indices.size, INT_DTYPE) + output_inds = np.empty(src_indices.size, INT_DTYPE) start = end = 0 for iproc in range(group.comm.size): end += sizes_out[iproc, idx_out] From 2b87b63b701e3d071133cde3ceec55cbbee6409a Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Wed, 18 Oct 2023 13:27:45 -0400 Subject: [PATCH 34/70] cleanup --- openmdao/core/system.py | 41 ++++++++++++++++++++++++++++++ openmdao/vectors/petsc_transfer.py | 18 ++++--------- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/openmdao/core/system.py b/openmdao/core/system.py index 0406c63bf8..479166df22 100644 --- a/openmdao/core/system.py +++ b/openmdao/core/system.py @@ -6355,3 +6355,44 @@ def local_range_iter(self, io): if size > 0: yield vname, offset, offset + size offset += size + + def get_var_dup_info(self, name, io): + """ + Return information about how the given variable is duplicated across MPI processes. + + Parameters + ---------- + name : str + Name of the variable. + io : str + Either 'input' or 'output'. + + Returns + ------- + tuple + A tuple of the form (is_duplicated, num_zeros, is_distributed). + """ + nz = np.count_nonzero(self._var_sizes[io][:, self._var_allprocs_abs2idx[name]]) + + if self._var_allprocs_abs2meta[io][name]['distributed']: + return False, self._var_sizes[io].shape[0] - nz, True # distributed vars are never dups + + return nz > 1, self._var_sizes[io].shape[0] - nz, False + + def get_var_sizes(self, name, io): + """ + Return the sizes of the given variable on all procs. + + Parameters + ---------- + name : str + Name of the variable. + io : str + Either 'input' or 'output'. + + Returns + ------- + ndarray + Array of sizes of the variable on all procs. + """ + return self._var_sizes[io][:, self._var_allprocs_abs2idx[name]] diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index f3dc2a6768..365b02ade0 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -111,16 +111,6 @@ def _setup_transfers(group, desvars, responses): offsets_in = offsets['input'] offsets_out = offsets['output'] - def is_dup(name, io): - # return if given var is duplicated and number of procs where var doesn't exist - if group._var_allprocs_abs2meta[io][name]['distributed']: - return False, 0, True # distributed vars are never dups - nz = np.count_nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]]) - return nz > 1, group._var_sizes[io].shape[0] - nz, False - - def get_nonzero_ranks(name, io): - return np.nonzero(group._var_sizes[io][:, allprocs_abs2idx[name]])[0] - # for an FD group, we use the relevance graph to determine which inputs on the # boundary of the group are upstream of distributed variables within the group so # that we can perform any necessary allreduce operations on the outputs that @@ -258,8 +248,8 @@ def get_nonzero_ranks(name, io): if rev and group._owns_approx_jac: pass # no rev transfers needed for FD group elif rev: - inp_is_dup, inp_missing, distrib_in = is_dup(abs_in, 'input') - out_is_dup, _, distrib_out = is_dup(abs_out, 'output') + inp_is_dup, inp_missing, distrib_in = group.get_var_dup_info(abs_in, 'input') + out_is_dup, _, distrib_out = group.get_var_dup_info(abs_out, 'output') iowninput = myrank == group._owning_rank[abs_in] sub_out = abs_out[mypathlen:].partition('.')[0] @@ -335,7 +325,9 @@ def get_nonzero_ranks(name, io): oidxlist_nc = [] iidxlist_nc = [] size = size_nc = 0 - for rnk in get_nonzero_ranks(abs_out, 'output'): + for rnk, sz in enumerate(group.get_var_sizes(abs_out, 'output')): + if sz == 0: + continue offset = offsets_out[rnk, idx_out] if src_indices is None: oarr = range(offset, offset + meta_in['size']) From a9b71d0609993a77fa0883952b4d7f924f467902 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Wed, 18 Oct 2023 15:19:09 -0400 Subject: [PATCH 35/70] split fwd and rev transfer routines for petsc --- .../approximation_scheme.py | 3 +- openmdao/core/group.py | 1 + openmdao/core/tests/test_distrib_derivs.py | 33 ++-- openmdao/vectors/petsc_transfer.py | 187 ++++++++++++++---- 4 files changed, 170 insertions(+), 54 deletions(-) diff --git a/openmdao/approximation_schemes/approximation_scheme.py b/openmdao/approximation_schemes/approximation_scheme.py index dacd3a7590..586a4c26b7 100644 --- a/openmdao/approximation_schemes/approximation_scheme.py +++ b/openmdao/approximation_schemes/approximation_scheme.py @@ -454,7 +454,7 @@ def _vec_ind_iter(self, vec_ind_list): def _uncolored_column_iter(self, system, approx_groups): """ - Perform approximations and yields (column_index, column) for each jac column. + Perform approximations and yield (column_index, column) for each jac column. Parameters ---------- @@ -592,6 +592,7 @@ def compute_approximations(self, system, jac=None): for ic, col in self.compute_approx_col_iter(system, under_cs=system._outputs._under_complex_step): if system._tot_jac is None: + print(ic, col) jac.set_col(system, ic, col) else: system._tot_jac.set_col(ic, col) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index bf77a0a58c..a8fd30f59f 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -3877,6 +3877,7 @@ def _solve_linear(self, mode, rel_systems, scope_out=_UNDEFINED, scope_in=_UNDEF # ExplicitComponent jacobian defined with -1 on diagonal. d_residuals *= -1.0 + print("after solve_linear:", d_outputs.asarray()) else: self._linear_solver._set_matvec_scope(scope_out, scope_in) self._linear_solver.solve(mode, rel_systems) diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index 61e5285f4e..50dd758000 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -895,8 +895,8 @@ def test_distrib_voi_group_fd5(self): assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def test_group_fd_inner_par(self): - size = 7 + def _setup_inner_par(self, size=7): + # one IVC feeds two parallel comps, which feed a third comp prob = om.Problem() model = prob.model @@ -923,23 +923,22 @@ def test_group_fd_inner_par(self): sub.approx_totals(method='fd') - prob.setup(mode='fwd', force_alloc_complex=True) + return prob + def test_group_fd_inner_par_fwd(self): + prob = self._setup_inner_par(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - # rev mode - + def test_group_fd_inner_par_rev(self): + prob = self._setup_inner_par(size=7) prob.setup(mode='rev', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def test_group_fd_inner_par2(self): - size = 7 - + def _setup_inner_par2(self, size=7): + # 2 IVCs feed two parallel comps, which feed two duplicated comps prob = om.Problem() model = prob.model @@ -966,18 +965,18 @@ def test_group_fd_inner_par2(self): sub.approx_totals(method='fd') - prob.setup(mode='fwd', force_alloc_complex=True) + return prob + def test_group_fd_inner_par2_fwd(self): + prob = self._setup_inner_par2(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - # rev mode - + def test_group_fd_inner_par2_rev(self): + prob = self._setup_inner_par2(size=7) prob.setup(mode='rev', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) def test_distrib_voi_group_fd_loop(self): diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 365b02ade0..ff61637e94 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -80,12 +80,20 @@ def _setup_transfers(group, desvars, responses): for subsys in group._subgroups_myproc: subsys._setup_transfers(desvars, responses) + group._transfers = {} + + PETScTransfer._setup_transfers_fwd(group, desvars, responses) + if rev: + PETScTransfer._setup_transfers_rev(group, desvars, responses) + + @staticmethod + def _setup_transfers_fwd(group, desvars, responses): abs2meta_in = group._var_abs2meta['input'] abs2meta_out = group._var_abs2meta['output'] allprocs_abs2meta_out = group._var_allprocs_abs2meta['output'] myrank = group.comm.rank - transfers = group._transfers = {} + transfers = group._transfers vectors = group._vectors offsets = group._get_var_offsets() mypathlen = len(group.pathname) + 1 if group.pathname else 0 @@ -95,15 +103,142 @@ def _setup_transfers(group, desvars, responses): xfer_out = [] fwd_xfer_in = defaultdict(list) fwd_xfer_out = defaultdict(list) - if rev: - has_rev_par_coloring = any([m['parallel_deriv_color'] is not None - for m in responses.values()]) - rev_xfer_in = defaultdict(list) - rev_xfer_out = defaultdict(list) - # xfers that are only active when parallel coloring is not - rev_xfer_in_nocolor = defaultdict(list) - rev_xfer_out_nocolor = defaultdict(list) + allprocs_abs2idx = group._var_allprocs_abs2idx + sizes_in = group._var_sizes['input'] + sizes_out = group._var_sizes['output'] + offsets_in = offsets['input'] + offsets_out = offsets['output'] + + total_fwd = total_rev = total_rev_nocolor = 0 + + # Loop through all connections owned by this system + for abs_in, abs_out in group._conn_abs_in2out.items(): + # Only continue if the input exists on this processor + if abs_in in abs2meta_in: + # Get meta + meta_in = abs2meta_in[abs_in] + meta_out = allprocs_abs2meta_out[abs_out] + + idx_in = allprocs_abs2idx[abs_in] + idx_out = allprocs_abs2idx[abs_out] + + local_out = abs_out in abs2meta_out + + # Read in and process src_indices + src_indices = meta_in['src_indices'] + if src_indices is None: + owner = group._owning_rank[abs_out] + # if the input is larger than the output on a single proc, we have + # to just loop over the procs in the same way we do when src_indices + # is defined. + if meta_in['size'] > sizes_out[owner, idx_out]: + src_indices = np.arange(meta_in['size'], dtype=INT_DTYPE) + else: + src_indices = src_indices.shaped_array() + + on_iprocs = [] + + # 1. Compute the output indices + # NOTE: src_indices are relative to a single, possibly distributed variable, + # while the output_inds that we compute are relative to the full distributed + # array that contains all local variables from each rank stacked in rank order. + if src_indices is None: + if meta_out['distributed']: + # input in this case is non-distributed (else src_indices would be + # defined by now). dist output to non-distributed input conns w/o + # src_indices are not allowed. + raise RuntimeError(f"{group.msginfo}: Can't connect distributed output " + f"'{abs_out}' to non-distributed input '{abs_in}' " + "without declaring src_indices.", + ident=(abs_out, abs_in)) + else: + rank = myrank if local_out else owner + offset = offsets_out[rank, idx_out] + output_inds = range(offset, offset + meta_in['size']) + else: + output_inds = np.empty(src_indices.size, INT_DTYPE) + start = end = 0 + for iproc in range(group.comm.size): + end += sizes_out[iproc, idx_out] + if start == end: + continue + + # The part of src on iproc + on_iproc = np.logical_and(start <= src_indices, src_indices < end) + + if np.any(on_iproc): + # This converts from iproc-then-ivar to ivar-then-iproc ordering + # Subtract off part of this variable from previous procs + # Then add all variables on previous procs + # Then all previous variables on this proc + # - np.sum(out_sizes[:iproc, idx_out]) + # + np.sum(out_sizes[:iproc, :]) + # + np.sum(out_sizes[iproc, :idx_out]) + # + inds + offset = offsets_out[iproc, idx_out] - start + output_inds[on_iproc] = src_indices[on_iproc] + offset + on_iprocs.append(iproc) + + start = end + + # 2. Compute the input indices + input_inds = range(offsets_in[myrank, idx_in], + offsets_in[myrank, idx_in] + sizes_in[myrank, idx_in]) + + total_fwd += len(input_inds) + + # Now the indices are ready - input_inds, output_inds + sub_in = abs_in[mypathlen:].partition('.')[0] + fwd_xfer_in[sub_in].append(input_inds) + fwd_xfer_out[sub_in].append(output_inds) + else: + # not a local input but still need entries in the transfer dicts to + # avoid hangs + sub_in = abs_in[mypathlen:].partition('.')[0] + fwd_xfer_in[sub_in] # defaultdict will create an empty list there + fwd_xfer_out[sub_in] + + if fwd_xfer_in: + xfer_in, xfer_out = _setup_index_views(total_fwd, fwd_xfer_in, fwd_xfer_out) + + out_vec = vectors['output']['nonlinear'] + + xfer_all = PETScTransfer(vectors['input']['nonlinear'], out_vec, + xfer_in, xfer_out, group.comm) + + transfers['fwd'] = xfwd = {} + xfwd[None] = xfer_all + + for sname, inds in fwd_xfer_in.items(): + transfers['fwd'][sname] = PETScTransfer( + vectors['input']['nonlinear'], vectors['output']['nonlinear'], + inds, fwd_xfer_out[sname], group.comm) + + @staticmethod + def _setup_transfers_rev(group, desvars, responses): + abs2meta_in = group._var_abs2meta['input'] + abs2meta_out = group._var_abs2meta['output'] + allprocs_abs2meta_out = group._var_allprocs_abs2meta['output'] + myrank = group.comm.rank + + transfers = group._transfers + vectors = group._vectors + offsets = group._get_var_offsets() + mypathlen = len(group.pathname) + 1 if group.pathname else 0 + + # Initialize empty lists for the transfer indices + xfer_in = [] + xfer_out = [] + + has_rev_par_coloring = any([m['parallel_deriv_color'] is not None + for m in responses.values()]) + rev_xfer_in = defaultdict(list) + rev_xfer_out = defaultdict(list) + + # xfers that are only active when parallel coloring is not + rev_xfer_in_nocolor = defaultdict(list) + rev_xfer_out_nocolor = defaultdict(list) allprocs_abs2idx = group._var_allprocs_abs2idx sizes_in = group._var_sizes['input'] @@ -115,7 +250,7 @@ def _setup_transfers(group, desvars, responses): # boundary of the group are upstream of distributed variables within the group so # that we can perform any necessary allreduce operations on the outputs that # are connected to those inputs. - if rev and group._owns_approx_jac and group._has_distrib_vars and group.pathname != '': + if group._owns_approx_jac and group._has_distrib_vars and group.pathname != '': model = group._problem_meta['model_ref']() relgraph = model._relevance_graph group_path = group.pathname + '.' @@ -242,12 +377,9 @@ def _setup_transfers(group, desvars, responses): total_fwd += len(input_inds) # Now the indices are ready - input_inds, output_inds - sub_in = abs_in[mypathlen:].partition('.')[0] - fwd_xfer_in[sub_in].append(input_inds) - fwd_xfer_out[sub_in].append(output_inds) - if rev and group._owns_approx_jac: + if group._owns_approx_jac: pass # no rev transfers needed for FD group - elif rev: + else: inp_is_dup, inp_missing, distrib_in = group.get_var_dup_info(abs_in, 'input') out_is_dup, _, distrib_out = group.get_var_dup_info(abs_out, 'output') @@ -386,10 +518,7 @@ def _setup_transfers(group, desvars, responses): else: # not a local input but still need entries in the transfer dicts to # avoid hangs - sub_in = abs_in[mypathlen:].partition('.')[0] - fwd_xfer_in[sub_in] # defaultdict will create an empty list there - fwd_xfer_out[sub_in] - if rev and not group._owns_approx_jac: + if not group._owns_approx_jac: sub_out = abs_out[mypathlen:].partition('.')[0] rev_xfer_in[sub_out] rev_xfer_out[sub_out] @@ -397,25 +526,11 @@ def _setup_transfers(group, desvars, responses): rev_xfer_in_nocolor[sub_out] rev_xfer_out_nocolor[sub_out] - if fwd_xfer_in: - xfer_in, xfer_out = _setup_index_views(total_fwd, fwd_xfer_in, fwd_xfer_out) - - out_vec = vectors['output']['nonlinear'] - - xfer_all = PETScTransfer(vectors['input']['nonlinear'], out_vec, - xfer_in, xfer_out, group.comm) - - transfers['fwd'] = xfwd = {} - xfwd[None] = xfer_all - - for sname, inds in fwd_xfer_in.items(): - transfers['fwd'][sname] = PETScTransfer( - vectors['input']['nonlinear'], vectors['output']['nonlinear'], - inds, fwd_xfer_out[sname], group.comm) - - if rev and not group._owns_approx_jac: + if not group._owns_approx_jac: xfer_in, xfer_out = _setup_index_views(total_rev, rev_xfer_in, rev_xfer_out) + out_vec = vectors['output']['nonlinear'] + xfer_all = PETScTransfer(vectors['input']['nonlinear'], out_vec, xfer_in, xfer_out, group.comm) From 7cb812adf4fa0d062ff6f784288b4c8bb665facd Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Thu, 19 Oct 2023 09:06:24 -0400 Subject: [PATCH 36/70] cleanup --- openmdao/vectors/petsc_transfer.py | 113 ++++++++++++++--------------- 1 file changed, 54 insertions(+), 59 deletions(-) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index ff61637e94..0c44e3036e 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -90,7 +90,6 @@ def _setup_transfers(group, desvars, responses): def _setup_transfers_fwd(group, desvars, responses): abs2meta_in = group._var_abs2meta['input'] abs2meta_out = group._var_abs2meta['output'] - allprocs_abs2meta_out = group._var_allprocs_abs2meta['output'] myrank = group.comm.rank transfers = group._transfers @@ -110,7 +109,7 @@ def _setup_transfers_fwd(group, desvars, responses): offsets_in = offsets['input'] offsets_out = offsets['output'] - total_fwd = total_rev = total_rev_nocolor = 0 + total_len = 0 # Loop through all connections owned by this system for abs_in, abs_out in group._conn_abs_in2out.items(): @@ -118,17 +117,14 @@ def _setup_transfers_fwd(group, desvars, responses): if abs_in in abs2meta_in: # Get meta meta_in = abs2meta_in[abs_in] - meta_out = allprocs_abs2meta_out[abs_out] idx_in = allprocs_abs2idx[abs_in] idx_out = allprocs_abs2idx[abs_out] - - local_out = abs_out in abs2meta_out + owner = group._owning_rank[abs_out] # Read in and process src_indices src_indices = meta_in['src_indices'] if src_indices is None: - owner = group._owning_rank[abs_out] # if the input is larger than the output on a single proc, we have # to just loop over the procs in the same way we do when src_indices # is defined. @@ -137,56 +133,15 @@ def _setup_transfers_fwd(group, desvars, responses): else: src_indices = src_indices.shaped_array() - on_iprocs = [] - - # 1. Compute the output indices - # NOTE: src_indices are relative to a single, possibly distributed variable, - # while the output_inds that we compute are relative to the full distributed - # array that contains all local variables from each rank stacked in rank order. - if src_indices is None: - if meta_out['distributed']: - # input in this case is non-distributed (else src_indices would be - # defined by now). dist output to non-distributed input conns w/o - # src_indices are not allowed. - raise RuntimeError(f"{group.msginfo}: Can't connect distributed output " - f"'{abs_out}' to non-distributed input '{abs_in}' " - "without declaring src_indices.", - ident=(abs_out, abs_in)) - else: - rank = myrank if local_out else owner - offset = offsets_out[rank, idx_out] - output_inds = range(offset, offset + meta_in['size']) - else: - output_inds = np.empty(src_indices.size, INT_DTYPE) - start = end = 0 - for iproc in range(group.comm.size): - end += sizes_out[iproc, idx_out] - if start == end: - continue - - # The part of src on iproc - on_iproc = np.logical_and(start <= src_indices, src_indices < end) - - if np.any(on_iproc): - # This converts from iproc-then-ivar to ivar-then-iproc ordering - # Subtract off part of this variable from previous procs - # Then add all variables on previous procs - # Then all previous variables on this proc - # - np.sum(out_sizes[:iproc, idx_out]) - # + np.sum(out_sizes[:iproc, :]) - # + np.sum(out_sizes[iproc, :idx_out]) - # + inds - offset = offsets_out[iproc, idx_out] - start - output_inds[on_iproc] = src_indices[on_iproc] + offset - on_iprocs.append(iproc) - - start = end + rank = myrank if abs_out in abs2meta_out else owner + output_inds = _get_output_inds(group, abs_out, abs_in, src_indices, rank, + sizes_out[:, idx_out], offsets_out[:, idx_out]) # 2. Compute the input indices input_inds = range(offsets_in[myrank, idx_in], offsets_in[myrank, idx_in] + sizes_in[myrank, idx_in]) - total_fwd += len(input_inds) + total_len += len(input_inds) # Now the indices are ready - input_inds, output_inds sub_in = abs_in[mypathlen:].partition('.')[0] @@ -200,7 +155,7 @@ def _setup_transfers_fwd(group, desvars, responses): fwd_xfer_out[sub_in] if fwd_xfer_in: - xfer_in, xfer_out = _setup_index_views(total_fwd, fwd_xfer_in, fwd_xfer_out) + xfer_in, xfer_out = _setup_index_views(total_len, fwd_xfer_in, fwd_xfer_out) out_vec = vectors['output']['nonlinear'] @@ -227,10 +182,6 @@ def _setup_transfers_rev(group, desvars, responses): offsets = group._get_var_offsets() mypathlen = len(group.pathname) + 1 if group.pathname else 0 - # Initialize empty lists for the transfer indices - xfer_in = [] - xfer_out = [] - has_rev_par_coloring = any([m['parallel_deriv_color'] is not None for m in responses.values()]) rev_xfer_in = defaultdict(list) @@ -298,7 +249,7 @@ def _setup_transfers_rev(group, desvars, responses): owning_group = model._get_subsystem(gname) owning_group._fd_subgroup_inputs.add(inp) - total_fwd = total_rev = total_rev_nocolor = 0 + total_rev = total_rev_nocolor = 0 # Loop through all connections owned by this system for abs_in, abs_out in group._conn_abs_in2out.items(): @@ -374,8 +325,6 @@ def _setup_transfers_rev(group, desvars, responses): input_inds = range(offsets_in[myrank, idx_in], offsets_in[myrank, idx_in] + sizes_in[myrank, idx_in]) - total_fwd += len(input_inds) - # Now the indices are ready - input_inds, output_inds if group._owns_approx_jac: pass # no rev transfers needed for FD group @@ -638,3 +587,49 @@ def _merge(inds_list, tot_size): return arr return _empty_idx_array + + +def _get_output_inds(group, abs_out, abs_in, src_indices, rank, sizes, offsets): + meta_out = group._var_allprocs_abs2meta['output'][abs_out] + + # NOTE: src_indices are relative to a single, possibly distributed variable, + # while the output_inds that we compute are relative to the full distributed + # array that contains all local variables from each rank stacked in rank order. + if src_indices is None: + if meta_out['distributed']: + # input in this case is non-distributed (else src_indices would be + # defined by now). dist output to non-distributed input conns w/o + # src_indices are not allowed. + raise RuntimeError(f"{group.msginfo}: Can't connect distributed output " + f"'{abs_out}' to non-distributed input '{abs_in}' " + "without declaring src_indices.", + ident=(abs_out, abs_in)) + else: + offset = offsets[rank] + output_inds = range(offset, offset + sizes[rank]) + else: + output_inds = np.empty(src_indices.size, INT_DTYPE) + start = end = 0 + for iproc in range(group.comm.size): + end += sizes[iproc] + if start == end: + continue + + # The part of src on iproc + on_iproc = np.logical_and(start <= src_indices, src_indices < end) + + if np.any(on_iproc): + # This converts from iproc-then-ivar to ivar-then-iproc ordering + # Subtract off part of this variable from previous procs + # Then add all variables on previous procs + # Then all previous variables on this proc + # - np.sum(out_sizes[:iproc, idx_out]) + # + np.sum(out_sizes[:iproc, :]) + # + np.sum(out_sizes[iproc, :idx_out]) + # + inds + offset = offsets[iproc] - start + output_inds[on_iproc] = src_indices[on_iproc] + offset + + start = end + + return output_inds From 196d08cbdf056114dd5ede071bd24c96a44b495b Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Thu, 19 Oct 2023 09:44:08 -0400 Subject: [PATCH 37/70] cleanup --- openmdao/vectors/petsc_transfer.py | 287 ++++++++++++++--------------- 1 file changed, 143 insertions(+), 144 deletions(-) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 0c44e3036e..68af0dc5e9 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -249,6 +249,10 @@ def _setup_transfers_rev(group, desvars, responses): owning_group = model._get_subsystem(gname) owning_group._fd_subgroup_inputs.add(inp) + if group._owns_approx_jac: + # FD groups don't need to do anything here + return + total_rev = total_rev_nocolor = 0 # Loop through all connections owned by this system @@ -326,182 +330,177 @@ def _setup_transfers_rev(group, desvars, responses): offsets_in[myrank, idx_in] + sizes_in[myrank, idx_in]) # Now the indices are ready - input_inds, output_inds - if group._owns_approx_jac: - pass # no rev transfers needed for FD group - else: - inp_is_dup, inp_missing, distrib_in = group.get_var_dup_info(abs_in, 'input') - out_is_dup, _, distrib_out = group.get_var_dup_info(abs_out, 'output') - - iowninput = myrank == group._owning_rank[abs_in] - sub_out = abs_out[mypathlen:].partition('.')[0] - - if inp_is_dup and (abs_out not in abs2meta_out or - (distrib_out and not iowninput)): - rev_xfer_in[sub_out] - rev_xfer_out[sub_out] - elif out_is_dup and inp_is_dup and inp_missing > 0 and iowninput: - # if this proc owns the input and both the output and input have - # duplicates, then we send the owning input to each duplicated output - # that doesn't have a corresponding connected input on the same proc. - oidxlist = [] - iidxlist = [] - oidxlist_nc = [] - iidxlist_nc = [] - size = size_nc = 0 - for rnk, osize, isize in zip(range(group.comm.size), - sizes_out[:, idx_out], - sizes_in[:, idx_in]): - if rnk == myrank: - oidxlist.append(output_inds) - iidxlist.append(input_inds) - elif osize > 0 and isize == 0: - offset = offsets_out[rnk, idx_out] - if src_indices is None: - oarr = range(offset, offset + meta_in['size']) - elif src_indices.size > 0: - oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) - else: - continue - - if has_rev_par_coloring: - oidxlist_nc.append(oarr) - iidxlist_nc.append(input_inds) - size_nc += len(input_inds) - else: - oidxlist.append(oarr) - iidxlist.append(input_inds) - size += len(input_inds) - - if len(iidxlist) > 1: - input_inds = _merge(iidxlist, size) - output_inds = _merge(oidxlist, size) - else: - input_inds = iidxlist[0] - output_inds = oidxlist[0] - - total_rev += len(input_inds) + inp_is_dup, inp_missing, distrib_in = group.get_var_dup_info(abs_in, 'input') + out_is_dup, _, distrib_out = group.get_var_dup_info(abs_out, 'output') - rev_xfer_in[sub_out].append(input_inds) - rev_xfer_out[sub_out].append(output_inds) + iowninput = myrank == group._owning_rank[abs_in] + sub_out = abs_out[mypathlen:].partition('.')[0] - if has_rev_par_coloring and iidxlist_nc: - # keep transfers separate that shouldn't happen when partial - # coloring is active - if len(iidxlist_nc) > 1: - input_inds = _merge(iidxlist_nc, size_nc) - output_inds = _merge(oidxlist_nc, size_nc) - else: - input_inds = iidxlist_nc[0] - output_inds = oidxlist_nc[0] - - total_rev_nocolor += len(input_inds) - - rev_xfer_in_nocolor[sub_out].append(input_inds) - rev_xfer_out_nocolor[sub_out].append(output_inds) - - elif out_is_dup and (not inp_is_dup or inp_missing > 0) and (iowninput or - distrib_in): - oidxlist = [] - iidxlist = [] - oidxlist_nc = [] - iidxlist_nc = [] - size = size_nc = 0 - for rnk, sz in enumerate(group.get_var_sizes(abs_out, 'output')): - if sz == 0: - continue + if inp_is_dup and (abs_out not in abs2meta_out or + (distrib_out and not iowninput)): + rev_xfer_in[sub_out] + rev_xfer_out[sub_out] + elif out_is_dup and inp_is_dup and inp_missing > 0 and iowninput: + # if this proc owns the input and both the output and input have + # duplicates, then we send the owning input to each duplicated output + # that doesn't have a corresponding connected input on the same proc. + oidxlist = [] + iidxlist = [] + oidxlist_nc = [] + iidxlist_nc = [] + size = size_nc = 0 + for rnk, osize, isize in zip(range(group.comm.size), + sizes_out[:, idx_out], + sizes_in[:, idx_in]): + if rnk == myrank: + oidxlist.append(output_inds) + iidxlist.append(input_inds) + elif osize > 0 and isize == 0: offset = offsets_out[rnk, idx_out] if src_indices is None: oarr = range(offset, offset + meta_in['size']) elif src_indices.size > 0: - if (distrib_in and not distrib_out and len(on_iprocs) == 1 and - on_iprocs[0] == rnk): - offset -= np.sum(sizes_out[:rnk, idx_out]) oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) else: continue - if rnk == myrank or not has_rev_par_coloring: - oidxlist.append(oarr) - iidxlist.append(input_inds) - size += len(input_inds) - else: + + if has_rev_par_coloring: oidxlist_nc.append(oarr) iidxlist_nc.append(input_inds) size_nc += len(input_inds) + else: + oidxlist.append(oarr) + iidxlist.append(input_inds) + size += len(input_inds) + + if len(iidxlist) > 1: + input_inds = _merge(iidxlist, size) + output_inds = _merge(oidxlist, size) + else: + input_inds = iidxlist[0] + output_inds = oidxlist[0] + + total_rev += len(input_inds) + + rev_xfer_in[sub_out].append(input_inds) + rev_xfer_out[sub_out].append(output_inds) - if len(iidxlist) > 1: - input_inds = _merge(iidxlist, size) - output_inds = _merge(oidxlist, size) - elif len(iidxlist) == 1: - input_inds = iidxlist[0] - output_inds = oidxlist[0] + if has_rev_par_coloring and iidxlist_nc: + # keep transfers separate that shouldn't happen when partial + # coloring is active + if len(iidxlist_nc) > 1: + input_inds = _merge(iidxlist_nc, size_nc) + output_inds = _merge(oidxlist_nc, size_nc) + else: + input_inds = iidxlist_nc[0] + output_inds = oidxlist_nc[0] + + total_rev_nocolor += len(input_inds) + + rev_xfer_in_nocolor[sub_out].append(input_inds) + rev_xfer_out_nocolor[sub_out].append(output_inds) + + elif out_is_dup and (not inp_is_dup or inp_missing > 0) and (iowninput or + distrib_in): + oidxlist = [] + iidxlist = [] + oidxlist_nc = [] + iidxlist_nc = [] + size = size_nc = 0 + for rnk, sz in enumerate(group.get_var_sizes(abs_out, 'output')): + if sz == 0: + continue + offset = offsets_out[rnk, idx_out] + if src_indices is None: + oarr = range(offset, offset + meta_in['size']) + elif src_indices.size > 0: + if (distrib_in and not distrib_out and len(on_iprocs) == 1 and + on_iprocs[0] == rnk): + offset -= np.sum(sizes_out[:rnk, idx_out]) + oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) else: - input_inds = output_inds = _empty_idx_array + continue + if rnk == myrank or not has_rev_par_coloring: + oidxlist.append(oarr) + iidxlist.append(input_inds) + size += len(input_inds) + else: + oidxlist_nc.append(oarr) + iidxlist_nc.append(input_inds) + size_nc += len(input_inds) + + if len(iidxlist) > 1: + input_inds = _merge(iidxlist, size) + output_inds = _merge(oidxlist, size) + elif len(iidxlist) == 1: + input_inds = iidxlist[0] + output_inds = oidxlist[0] + else: + input_inds = output_inds = _empty_idx_array - total_rev += len(input_inds) + total_rev += len(input_inds) - rev_xfer_in[sub_out].append(input_inds) - rev_xfer_out[sub_out].append(output_inds) + rev_xfer_in[sub_out].append(input_inds) + rev_xfer_out[sub_out].append(output_inds) - if has_rev_par_coloring and iidxlist_nc: - if len(iidxlist_nc) > 1: - input_inds = _merge(iidxlist_nc, size_nc) - output_inds = _merge(oidxlist_nc, size_nc) - else: - input_inds = iidxlist_nc[0] - output_inds = oidxlist_nc[0] + if has_rev_par_coloring and iidxlist_nc: + if len(iidxlist_nc) > 1: + input_inds = _merge(iidxlist_nc, size_nc) + output_inds = _merge(oidxlist_nc, size_nc) + else: + input_inds = iidxlist_nc[0] + output_inds = oidxlist_nc[0] - total_rev_nocolor += len(input_inds) + total_rev_nocolor += len(input_inds) - rev_xfer_in_nocolor[sub_out].append(input_inds) - rev_xfer_out_nocolor[sub_out].append(output_inds) - else: - if (inp_is_dup and out_is_dup and src_indices is not None and - src_indices.size > 0): - offset = offsets_out[myrank, idx_out] - output_inds = np.asarray(src_indices + offset, dtype=INT_DTYPE) + rev_xfer_in_nocolor[sub_out].append(input_inds) + rev_xfer_out_nocolor[sub_out].append(output_inds) + else: + if (inp_is_dup and out_is_dup and src_indices is not None and + src_indices.size > 0): + offset = offsets_out[myrank, idx_out] + output_inds = np.asarray(src_indices + offset, dtype=INT_DTYPE) - total_rev += len(input_inds) + total_rev += len(input_inds) - rev_xfer_in[sub_out].append(input_inds) - rev_xfer_out[sub_out].append(output_inds) + rev_xfer_in[sub_out].append(input_inds) + rev_xfer_out[sub_out].append(output_inds) else: # not a local input but still need entries in the transfer dicts to # avoid hangs - if not group._owns_approx_jac: - sub_out = abs_out[mypathlen:].partition('.')[0] - rev_xfer_in[sub_out] - rev_xfer_out[sub_out] - if has_rev_par_coloring: - rev_xfer_in_nocolor[sub_out] - rev_xfer_out_nocolor[sub_out] + sub_out = abs_out[mypathlen:].partition('.')[0] + rev_xfer_in[sub_out] + rev_xfer_out[sub_out] + if has_rev_par_coloring: + rev_xfer_in_nocolor[sub_out] + rev_xfer_out_nocolor[sub_out] - if not group._owns_approx_jac: - xfer_in, xfer_out = _setup_index_views(total_rev, rev_xfer_in, rev_xfer_out) + xfer_in, xfer_out = _setup_index_views(total_rev, rev_xfer_in, rev_xfer_out) - out_vec = vectors['output']['nonlinear'] + out_vec = vectors['output']['nonlinear'] - xfer_all = PETScTransfer(vectors['input']['nonlinear'], out_vec, - xfer_in, xfer_out, group.comm) + xfer_all = PETScTransfer(vectors['input']['nonlinear'], out_vec, + xfer_in, xfer_out, group.comm) - transfers['rev'] = xrev = {} - xrev[None] = xfer_all + transfers['rev'] = xrev = {} + xrev[None] = xfer_all - for sname, inds in rev_xfer_out.items(): - transfers['rev'][sname] = PETScTransfer( - vectors['input']['nonlinear'], vectors['output']['nonlinear'], - rev_xfer_in[sname], inds, group.comm) + for sname, inds in rev_xfer_out.items(): + transfers['rev'][sname] = PETScTransfer( + vectors['input']['nonlinear'], vectors['output']['nonlinear'], + rev_xfer_in[sname], inds, group.comm) - if rev_xfer_in_nocolor: - xfer_in, xfer_out = _setup_index_views(total_rev_nocolor, rev_xfer_in_nocolor, - rev_xfer_out_nocolor) + if rev_xfer_in_nocolor: + xfer_in, xfer_out = _setup_index_views(total_rev_nocolor, rev_xfer_in_nocolor, + rev_xfer_out_nocolor) - xrev[(None, 'nocolor')] = PETScTransfer(vectors['input']['nonlinear'], out_vec, - xfer_in, xfer_out, group.comm) + xrev[(None, 'nocolor')] = PETScTransfer(vectors['input']['nonlinear'], out_vec, + xfer_in, xfer_out, group.comm) - for sname, inds in rev_xfer_out_nocolor.items(): - transfers['rev'][(sname, 'nocolor')] = PETScTransfer( - vectors['input']['nonlinear'], vectors['output']['nonlinear'], - rev_xfer_in_nocolor[sname], inds, group.comm) + for sname, inds in rev_xfer_out_nocolor.items(): + transfers['rev'][(sname, 'nocolor')] = PETScTransfer( + vectors['input']['nonlinear'], vectors['output']['nonlinear'], + rev_xfer_in_nocolor[sname], inds, group.comm) def _transfer(self, in_vec, out_vec, mode='fwd'): """ From d01f9b157505a47f38731d1bbe15453a1717679a Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Thu, 19 Oct 2023 10:34:22 -0400 Subject: [PATCH 38/70] cleanup --- openmdao/vectors/petsc_transfer.py | 140 +++++++++++------------------ 1 file changed, 53 insertions(+), 87 deletions(-) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 68af0dc5e9..f5c37d6525 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -80,11 +80,13 @@ def _setup_transfers(group, desvars, responses): for subsys in group._subgroups_myproc: subsys._setup_transfers(desvars, responses) - group._transfers = {} + group._transfers = { + 'fwd': PETScTransfer._setup_transfers_fwd(group, desvars, responses) + } - PETScTransfer._setup_transfers_fwd(group, desvars, responses) if rev: - PETScTransfer._setup_transfers_rev(group, desvars, responses) + group._transfers['rev'] = PETScTransfer._setup_transfers_rev(group, desvars, + responses) @staticmethod def _setup_transfers_fwd(group, desvars, responses): @@ -92,7 +94,6 @@ def _setup_transfers_fwd(group, desvars, responses): abs2meta_out = group._var_abs2meta['output'] myrank = group.comm.rank - transfers = group._transfers vectors = group._vectors offsets = group._get_var_offsets() mypathlen = len(group.pathname) + 1 if group.pathname else 0 @@ -134,8 +135,9 @@ def _setup_transfers_fwd(group, desvars, responses): src_indices = src_indices.shaped_array() rank = myrank if abs_out in abs2meta_out else owner - output_inds = _get_output_inds(group, abs_out, abs_in, src_indices, rank, - sizes_out[:, idx_out], offsets_out[:, idx_out]) + output_inds, _ = _get_output_inds(group, abs_out, abs_in, src_indices, rank, + sizes_out[:, idx_out], + offsets_out[:, idx_out]) # 2. Compute the input indices input_inds = range(offsets_in[myrank, idx_in], @@ -157,18 +159,20 @@ def _setup_transfers_fwd(group, desvars, responses): if fwd_xfer_in: xfer_in, xfer_out = _setup_index_views(total_len, fwd_xfer_in, fwd_xfer_out) - out_vec = vectors['output']['nonlinear'] - - xfer_all = PETScTransfer(vectors['input']['nonlinear'], out_vec, - xfer_in, xfer_out, group.comm) - - transfers['fwd'] = xfwd = {} - xfwd[None] = xfer_all + transfers = { + # full transfer (transfer to all subsystems at once) + None: PETScTransfer(vectors['input']['nonlinear'], + vectors['output']['nonlinear'], + xfer_in, xfer_out, group.comm) + } + # transfers to individual subsystems for sname, inds in fwd_xfer_in.items(): - transfers['fwd'][sname] = PETScTransfer( - vectors['input']['nonlinear'], vectors['output']['nonlinear'], - inds, fwd_xfer_out[sname], group.comm) + transfers[sname] = PETScTransfer(vectors['input']['nonlinear'], + vectors['output']['nonlinear'], + inds, fwd_xfer_out[sname], group.comm) + + return transfers @staticmethod def _setup_transfers_rev(group, desvars, responses): @@ -251,27 +255,25 @@ def _setup_transfers_rev(group, desvars, responses): if group._owns_approx_jac: # FD groups don't need to do anything here - return + return {} total_rev = total_rev_nocolor = 0 # Loop through all connections owned by this system for abs_in, abs_out in group._conn_abs_in2out.items(): + sub_out = abs_out[mypathlen:].partition('.')[0] + # Only continue if the input exists on this processor if abs_in in abs2meta_in: # Get meta meta_in = abs2meta_in[abs_in] - meta_out = allprocs_abs2meta_out[abs_out] - idx_in = allprocs_abs2idx[abs_in] idx_out = allprocs_abs2idx[abs_out] - - local_out = abs_out in abs2meta_out + owner = group._owning_rank[abs_out] # Read in and process src_indices src_indices = meta_in['src_indices'] if src_indices is None: - owner = group._owning_rank[abs_out] # if the input is larger than the output on a single proc, we have # to just loop over the procs in the same way we do when src_indices # is defined. @@ -280,50 +282,10 @@ def _setup_transfers_rev(group, desvars, responses): else: src_indices = src_indices.shaped_array() - on_iprocs = [] - - # 1. Compute the output indices - # NOTE: src_indices are relative to a single, possibly distributed variable, - # while the output_inds that we compute are relative to the full distributed - # array that contains all local variables from each rank stacked in rank order. - if src_indices is None: - if meta_out['distributed']: - # input in this case is non-distributed (else src_indices would be - # defined by now). dist output to non-distributed input conns w/o - # src_indices are not allowed. - raise RuntimeError(f"{group.msginfo}: Can't connect distributed output " - f"'{abs_out}' to non-distributed input '{abs_in}' " - "without declaring src_indices.", - ident=(abs_out, abs_in)) - else: - rank = myrank if local_out else owner - offset = offsets_out[rank, idx_out] - output_inds = range(offset, offset + meta_in['size']) - else: - output_inds = np.empty(src_indices.size, INT_DTYPE) - start = end = 0 - for iproc in range(group.comm.size): - end += sizes_out[iproc, idx_out] - if start == end: - continue - - # The part of src on iproc - on_iproc = np.logical_and(start <= src_indices, src_indices < end) - - if np.any(on_iproc): - # This converts from iproc-then-ivar to ivar-then-iproc ordering - # Subtract off part of this variable from previous procs - # Then add all variables on previous procs - # Then all previous variables on this proc - # - np.sum(out_sizes[:iproc, idx_out]) - # + np.sum(out_sizes[:iproc, :]) - # + np.sum(out_sizes[iproc, :idx_out]) - # + inds - offset = offsets_out[iproc, idx_out] - start - output_inds[on_iproc] = src_indices[on_iproc] + offset - on_iprocs.append(iproc) - - start = end + rank = myrank if abs_out in abs2meta_out else owner + output_inds, on_iprocs = _get_output_inds(group, abs_out, abs_in, src_indices, + rank, sizes_out[:, idx_out], + offsets_out[:, idx_out]) # 2. Compute the input indices input_inds = range(offsets_in[myrank, idx_in], @@ -334,7 +296,6 @@ def _setup_transfers_rev(group, desvars, responses): out_is_dup, _, distrib_out = group.get_var_dup_info(abs_out, 'output') iowninput = myrank == group._owning_rank[abs_in] - sub_out = abs_out[mypathlen:].partition('.')[0] if inp_is_dup and (abs_out not in abs2meta_out or (distrib_out and not iowninput)): @@ -468,39 +429,42 @@ def _setup_transfers_rev(group, desvars, responses): else: # not a local input but still need entries in the transfer dicts to # avoid hangs - sub_out = abs_out[mypathlen:].partition('.')[0] rev_xfer_in[sub_out] rev_xfer_out[sub_out] if has_rev_par_coloring: rev_xfer_in_nocolor[sub_out] rev_xfer_out_nocolor[sub_out] - xfer_in, xfer_out = _setup_index_views(total_rev, rev_xfer_in, rev_xfer_out) - - out_vec = vectors['output']['nonlinear'] + full_xfer_in, full_xfer_out = _setup_index_views(total_rev, rev_xfer_in, rev_xfer_out) - xfer_all = PETScTransfer(vectors['input']['nonlinear'], out_vec, - xfer_in, xfer_out, group.comm) - - transfers['rev'] = xrev = {} - xrev[None] = xfer_all + transfers = { + None: PETScTransfer(vectors['input']['nonlinear'], + vectors['output']['nonlinear'], + full_xfer_in, full_xfer_out, group.comm) + } for sname, inds in rev_xfer_out.items(): - transfers['rev'][sname] = PETScTransfer( - vectors['input']['nonlinear'], vectors['output']['nonlinear'], - rev_xfer_in[sname], inds, group.comm) + transfers[sname] = PETScTransfer(vectors['input']['nonlinear'], + vectors['output']['nonlinear'], + rev_xfer_in[sname], inds, group.comm) if rev_xfer_in_nocolor: - xfer_in, xfer_out = _setup_index_views(total_rev_nocolor, rev_xfer_in_nocolor, - rev_xfer_out_nocolor) + full_xfer_in, full_xfer_out = _setup_index_views(total_rev_nocolor, + rev_xfer_in_nocolor, + rev_xfer_out_nocolor) - xrev[(None, 'nocolor')] = PETScTransfer(vectors['input']['nonlinear'], out_vec, - xfer_in, xfer_out, group.comm) + transfers[(None, 'nocolor')] = PETScTransfer(vectors['input']['nonlinear'], + vectors['output']['nonlinear'], + full_xfer_in, full_xfer_out, + group.comm) for sname, inds in rev_xfer_out_nocolor.items(): - transfers['rev'][(sname, 'nocolor')] = PETScTransfer( - vectors['input']['nonlinear'], vectors['output']['nonlinear'], - rev_xfer_in_nocolor[sname], inds, group.comm) + transfers[(sname, 'nocolor')] = PETScTransfer(vectors['input']['nonlinear'], + vectors['output']['nonlinear'], + rev_xfer_in_nocolor[sname], inds, + group.comm) + + return transfers def _transfer(self, in_vec, out_vec, mode='fwd'): """ @@ -590,6 +554,7 @@ def _merge(inds_list, tot_size): def _get_output_inds(group, abs_out, abs_in, src_indices, rank, sizes, offsets): meta_out = group._var_allprocs_abs2meta['output'][abs_out] + on_iprocs = [] # NOTE: src_indices are relative to a single, possibly distributed variable, # while the output_inds that we compute are relative to the full distributed @@ -628,7 +593,8 @@ def _get_output_inds(group, abs_out, abs_in, src_indices, rank, sizes, offsets): # + inds offset = offsets[iproc] - start output_inds[on_iproc] = src_indices[on_iproc] + offset + on_iprocs.append(iproc) start = end - return output_inds + return output_inds, on_iprocs From c7a174fe8487be3def05b0d2b368705531b93d4e Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Thu, 19 Oct 2023 11:04:33 -0400 Subject: [PATCH 39/70] cleanup --- openmdao/vectors/petsc_transfer.py | 51 ++++++++++++++---------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index f5c37d6525..b156442b38 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -98,9 +98,6 @@ def _setup_transfers_fwd(group, desvars, responses): offsets = group._get_var_offsets() mypathlen = len(group.pathname) + 1 if group.pathname else 0 - # Initialize empty lists for the transfer indices - xfer_in = [] - xfer_out = [] fwd_xfer_in = defaultdict(list) fwd_xfer_out = defaultdict(list) @@ -156,21 +153,19 @@ def _setup_transfers_fwd(group, desvars, responses): fwd_xfer_in[sub_in] # defaultdict will create an empty list there fwd_xfer_out[sub_in] + transfers = {} if fwd_xfer_in: xfer_in, xfer_out = _setup_index_views(total_len, fwd_xfer_in, fwd_xfer_out) - - transfers = { # full transfer (transfer to all subsystems at once) - None: PETScTransfer(vectors['input']['nonlinear'], - vectors['output']['nonlinear'], - xfer_in, xfer_out, group.comm) - } + transfers[None] = PETScTransfer(vectors['input']['nonlinear'], + vectors['output']['nonlinear'], + xfer_in, xfer_out, group.comm) - # transfers to individual subsystems - for sname, inds in fwd_xfer_in.items(): - transfers[sname] = PETScTransfer(vectors['input']['nonlinear'], - vectors['output']['nonlinear'], - inds, fwd_xfer_out[sname], group.comm) + # transfers to individual subsystems + for sname, inds in fwd_xfer_in.items(): + transfers[sname] = PETScTransfer(vectors['input']['nonlinear'], + vectors['output']['nonlinear'], + inds, fwd_xfer_out[sname], group.comm) return transfers @@ -178,7 +173,6 @@ def _setup_transfers_fwd(group, desvars, responses): def _setup_transfers_rev(group, desvars, responses): abs2meta_in = group._var_abs2meta['input'] abs2meta_out = group._var_abs2meta['output'] - allprocs_abs2meta_out = group._var_allprocs_abs2meta['output'] myrank = group.comm.rank transfers = group._transfers @@ -254,18 +248,17 @@ def _setup_transfers_rev(group, desvars, responses): owning_group._fd_subgroup_inputs.add(inp) if group._owns_approx_jac: - # FD groups don't need to do anything here + # FD groups don't need reverse transfers return {} - total_rev = total_rev_nocolor = 0 + total_size = total_size_nocolor = 0 # Loop through all connections owned by this system for abs_in, abs_out in group._conn_abs_in2out.items(): sub_out = abs_out[mypathlen:].partition('.')[0] - + # Only continue if the input exists on this processor if abs_in in abs2meta_in: - # Get meta meta_in = abs2meta_in[abs_in] idx_in = allprocs_abs2idx[abs_in] idx_out = allprocs_abs2idx[abs_out] @@ -311,12 +304,14 @@ def _setup_transfers_rev(group, desvars, responses): iidxlist_nc = [] size = size_nc = 0 for rnk, osize, isize in zip(range(group.comm.size), - sizes_out[:, idx_out], - sizes_in[:, idx_in]): + group.get_var_sizes(abs_out, 'output'), + group.get_var_sizes(abs_in, 'input')): if rnk == myrank: oidxlist.append(output_inds) iidxlist.append(input_inds) elif osize > 0 and isize == 0: + # dup output exists on this rank but there is no corresponding + # input, so we send the owning input to the dup output offset = offsets_out[rnk, idx_out] if src_indices is None: oarr = range(offset, offset + meta_in['size']) @@ -341,7 +336,7 @@ def _setup_transfers_rev(group, desvars, responses): input_inds = iidxlist[0] output_inds = oidxlist[0] - total_rev += len(input_inds) + total_size += len(input_inds) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) @@ -356,7 +351,7 @@ def _setup_transfers_rev(group, desvars, responses): input_inds = iidxlist_nc[0] output_inds = oidxlist_nc[0] - total_rev_nocolor += len(input_inds) + total_size_nocolor += len(input_inds) rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) @@ -399,7 +394,7 @@ def _setup_transfers_rev(group, desvars, responses): else: input_inds = output_inds = _empty_idx_array - total_rev += len(input_inds) + total_size += len(input_inds) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) @@ -412,7 +407,7 @@ def _setup_transfers_rev(group, desvars, responses): input_inds = iidxlist_nc[0] output_inds = oidxlist_nc[0] - total_rev_nocolor += len(input_inds) + total_size_nocolor += len(input_inds) rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) @@ -422,7 +417,7 @@ def _setup_transfers_rev(group, desvars, responses): offset = offsets_out[myrank, idx_out] output_inds = np.asarray(src_indices + offset, dtype=INT_DTYPE) - total_rev += len(input_inds) + total_size += len(input_inds) rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) @@ -435,7 +430,7 @@ def _setup_transfers_rev(group, desvars, responses): rev_xfer_in_nocolor[sub_out] rev_xfer_out_nocolor[sub_out] - full_xfer_in, full_xfer_out = _setup_index_views(total_rev, rev_xfer_in, rev_xfer_out) + full_xfer_in, full_xfer_out = _setup_index_views(total_size, rev_xfer_in, rev_xfer_out) transfers = { None: PETScTransfer(vectors['input']['nonlinear'], @@ -449,7 +444,7 @@ def _setup_transfers_rev(group, desvars, responses): rev_xfer_in[sname], inds, group.comm) if rev_xfer_in_nocolor: - full_xfer_in, full_xfer_out = _setup_index_views(total_rev_nocolor, + full_xfer_in, full_xfer_out = _setup_index_views(total_size_nocolor, rev_xfer_in_nocolor, rev_xfer_out_nocolor) From 2a3b72b5829988e81e7677cf239e5a33cd1fbb15 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Thu, 19 Oct 2023 13:50:25 -0400 Subject: [PATCH 40/70] cleanup of rev xfers --- .../approximation_scheme.py | 1 - openmdao/core/group.py | 2 - openmdao/vectors/petsc_transfer.py | 77 +++---------------- 3 files changed, 11 insertions(+), 69 deletions(-) diff --git a/openmdao/approximation_schemes/approximation_scheme.py b/openmdao/approximation_schemes/approximation_scheme.py index 586a4c26b7..b0771f8366 100644 --- a/openmdao/approximation_schemes/approximation_scheme.py +++ b/openmdao/approximation_schemes/approximation_scheme.py @@ -592,7 +592,6 @@ def compute_approximations(self, system, jac=None): for ic, col in self.compute_approx_col_iter(system, under_cs=system._outputs._under_complex_step): if system._tot_jac is None: - print(ic, col) jac.set_col(system, ic, col) else: system._tot_jac.set_col(ic, col) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index a8fd30f59f..7422b0a3c3 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -3876,8 +3876,6 @@ def _solve_linear(self, mode, rel_systems, scope_out=_UNDEFINED, scope_in=_UNDEF # ExplicitComponent jacobian defined with -1 on diagonal. d_residuals *= -1.0 - - print("after solve_linear:", d_outputs.asarray()) else: self._linear_solver._set_matvec_scope(scope_out, scope_in) self._linear_solver.solve(mode, rel_systems) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index b156442b38..b15c7acab1 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -164,8 +164,8 @@ def _setup_transfers_fwd(group, desvars, responses): # transfers to individual subsystems for sname, inds in fwd_xfer_in.items(): transfers[sname] = PETScTransfer(vectors['input']['nonlinear'], - vectors['output']['nonlinear'], - inds, fwd_xfer_out[sname], group.comm) + vectors['output']['nonlinear'], + inds, fwd_xfer_out[sname], group.comm) return transfers @@ -291,13 +291,14 @@ def _setup_transfers_rev(group, desvars, responses): iowninput = myrank == group._owning_rank[abs_in] if inp_is_dup and (abs_out not in abs2meta_out or - (distrib_out and not iowninput)): + (distrib_out and not iowninput)): rev_xfer_in[sub_out] rev_xfer_out[sub_out] - elif out_is_dup and inp_is_dup and inp_missing > 0 and iowninput: - # if this proc owns the input and both the output and input have - # duplicates, then we send the owning input to each duplicated output - # that doesn't have a corresponding connected input on the same proc. + elif out_is_dup and inp_missing > 0 and (iowninput or distrib_in): + # if this proc owns the input or the input is distributyed, + # and the output is duplicated, then we send the owning/distrib input + # to each duplicated output that doesn't have a corresponding connected + # input on the same proc. oidxlist = [] iidxlist = [] oidxlist_nc = [] @@ -309,6 +310,7 @@ def _setup_transfers_rev(group, desvars, responses): if rnk == myrank: oidxlist.append(output_inds) iidxlist.append(input_inds) + size += len(input_inds) elif osize > 0 and isize == 0: # dup output exists on this rank but there is no corresponding # input, so we send the owning input to the dup output @@ -353,62 +355,6 @@ def _setup_transfers_rev(group, desvars, responses): total_size_nocolor += len(input_inds) - rev_xfer_in_nocolor[sub_out].append(input_inds) - rev_xfer_out_nocolor[sub_out].append(output_inds) - - elif out_is_dup and (not inp_is_dup or inp_missing > 0) and (iowninput or - distrib_in): - oidxlist = [] - iidxlist = [] - oidxlist_nc = [] - iidxlist_nc = [] - size = size_nc = 0 - for rnk, sz in enumerate(group.get_var_sizes(abs_out, 'output')): - if sz == 0: - continue - offset = offsets_out[rnk, idx_out] - if src_indices is None: - oarr = range(offset, offset + meta_in['size']) - elif src_indices.size > 0: - if (distrib_in and not distrib_out and len(on_iprocs) == 1 and - on_iprocs[0] == rnk): - offset -= np.sum(sizes_out[:rnk, idx_out]) - oarr = np.asarray(src_indices + offset, dtype=INT_DTYPE) - else: - continue - if rnk == myrank or not has_rev_par_coloring: - oidxlist.append(oarr) - iidxlist.append(input_inds) - size += len(input_inds) - else: - oidxlist_nc.append(oarr) - iidxlist_nc.append(input_inds) - size_nc += len(input_inds) - - if len(iidxlist) > 1: - input_inds = _merge(iidxlist, size) - output_inds = _merge(oidxlist, size) - elif len(iidxlist) == 1: - input_inds = iidxlist[0] - output_inds = oidxlist[0] - else: - input_inds = output_inds = _empty_idx_array - - total_size += len(input_inds) - - rev_xfer_in[sub_out].append(input_inds) - rev_xfer_out[sub_out].append(output_inds) - - if has_rev_par_coloring and iidxlist_nc: - if len(iidxlist_nc) > 1: - input_inds = _merge(iidxlist_nc, size_nc) - output_inds = _merge(oidxlist_nc, size_nc) - else: - input_inds = iidxlist_nc[0] - output_inds = oidxlist_nc[0] - - total_size_nocolor += len(input_inds) - rev_xfer_in_nocolor[sub_out].append(input_inds) rev_xfer_out_nocolor[sub_out].append(output_inds) else: @@ -560,9 +506,8 @@ def _get_output_inds(group, abs_out, abs_in, src_indices, rank, sizes, offsets): # defined by now). dist output to non-distributed input conns w/o # src_indices are not allowed. raise RuntimeError(f"{group.msginfo}: Can't connect distributed output " - f"'{abs_out}' to non-distributed input '{abs_in}' " - "without declaring src_indices.", - ident=(abs_out, abs_in)) + f"'{abs_out}' to non-distributed input '{abs_in}' " + "without declaring src_indices.", ident=(abs_out, abs_in)) else: offset = offsets[rank] output_inds = range(offset, offset + sizes[rank]) From 9e4b5d93d575f60827cbe21cc7a61ee05732b717 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Thu, 19 Oct 2023 14:30:11 -0400 Subject: [PATCH 41/70] some renaming --- openmdao/vectors/petsc_transfer.py | 91 ++++++++++++++---------------- 1 file changed, 42 insertions(+), 49 deletions(-) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index b15c7acab1..76cb80e974 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -98,8 +98,8 @@ def _setup_transfers_fwd(group, desvars, responses): offsets = group._get_var_offsets() mypathlen = len(group.pathname) + 1 if group.pathname else 0 - fwd_xfer_in = defaultdict(list) - fwd_xfer_out = defaultdict(list) + xfer_in = defaultdict(list) + xfer_out = defaultdict(list) allprocs_abs2idx = group._var_allprocs_abs2idx sizes_in = group._var_sizes['input'] @@ -144,28 +144,28 @@ def _setup_transfers_fwd(group, desvars, responses): # Now the indices are ready - input_inds, output_inds sub_in = abs_in[mypathlen:].partition('.')[0] - fwd_xfer_in[sub_in].append(input_inds) - fwd_xfer_out[sub_in].append(output_inds) + xfer_in[sub_in].append(input_inds) + xfer_out[sub_in].append(output_inds) else: # not a local input but still need entries in the transfer dicts to # avoid hangs sub_in = abs_in[mypathlen:].partition('.')[0] - fwd_xfer_in[sub_in] # defaultdict will create an empty list there - fwd_xfer_out[sub_in] + xfer_in[sub_in] # defaultdict will create an empty list there + xfer_out[sub_in] transfers = {} - if fwd_xfer_in: - xfer_in, xfer_out = _setup_index_views(total_len, fwd_xfer_in, fwd_xfer_out) + if xfer_in: + full_xfer_in, full_xfer_out = _setup_index_views(total_len, xfer_in, xfer_out) # full transfer (transfer to all subsystems at once) transfers[None] = PETScTransfer(vectors['input']['nonlinear'], vectors['output']['nonlinear'], - xfer_in, xfer_out, group.comm) + full_xfer_in, full_xfer_out, group.comm) # transfers to individual subsystems - for sname, inds in fwd_xfer_in.items(): + for sname, inds in xfer_in.items(): transfers[sname] = PETScTransfer(vectors['input']['nonlinear'], vectors['output']['nonlinear'], - inds, fwd_xfer_out[sname], group.comm) + inds, xfer_out[sname], group.comm) return transfers @@ -182,12 +182,12 @@ def _setup_transfers_rev(group, desvars, responses): has_rev_par_coloring = any([m['parallel_deriv_color'] is not None for m in responses.values()]) - rev_xfer_in = defaultdict(list) - rev_xfer_out = defaultdict(list) + xfer_in = defaultdict(list) + xfer_out = defaultdict(list) # xfers that are only active when parallel coloring is not - rev_xfer_in_nocolor = defaultdict(list) - rev_xfer_out_nocolor = defaultdict(list) + xfer_in_nocolor = defaultdict(list) + xfer_out_nocolor = defaultdict(list) allprocs_abs2idx = group._var_allprocs_abs2idx sizes_in = group._var_sizes['input'] @@ -276,9 +276,9 @@ def _setup_transfers_rev(group, desvars, responses): src_indices = src_indices.shaped_array() rank = myrank if abs_out in abs2meta_out else owner - output_inds, on_iprocs = _get_output_inds(group, abs_out, abs_in, src_indices, - rank, sizes_out[:, idx_out], - offsets_out[:, idx_out]) + output_inds, _ = _get_output_inds(group, abs_out, abs_in, src_indices, + rank, sizes_out[:, idx_out], + offsets_out[:, idx_out]) # 2. Compute the input indices input_inds = range(offsets_in[myrank, idx_in], @@ -292,8 +292,8 @@ def _setup_transfers_rev(group, desvars, responses): if inp_is_dup and (abs_out not in abs2meta_out or (distrib_out and not iowninput)): - rev_xfer_in[sub_out] - rev_xfer_out[sub_out] + xfer_in[sub_out] + xfer_out[sub_out] elif out_is_dup and inp_missing > 0 and (iowninput or distrib_in): # if this proc owns the input or the input is distributyed, # and the output is duplicated, then we send the owning/distrib input @@ -313,7 +313,7 @@ def _setup_transfers_rev(group, desvars, responses): size += len(input_inds) elif osize > 0 and isize == 0: # dup output exists on this rank but there is no corresponding - # input, so we send the owning input to the dup output + # input, so we send the owning/distrib input to the dup output offset = offsets_out[rnk, idx_out] if src_indices is None: oarr = range(offset, offset + meta_in['size']) @@ -340,8 +340,8 @@ def _setup_transfers_rev(group, desvars, responses): total_size += len(input_inds) - rev_xfer_in[sub_out].append(input_inds) - rev_xfer_out[sub_out].append(output_inds) + xfer_in[sub_out].append(input_inds) + xfer_out[sub_out].append(output_inds) if has_rev_par_coloring and iidxlist_nc: # keep transfers separate that shouldn't happen when partial @@ -355,8 +355,8 @@ def _setup_transfers_rev(group, desvars, responses): total_size_nocolor += len(input_inds) - rev_xfer_in_nocolor[sub_out].append(input_inds) - rev_xfer_out_nocolor[sub_out].append(output_inds) + xfer_in_nocolor[sub_out].append(input_inds) + xfer_out_nocolor[sub_out].append(output_inds) else: if (inp_is_dup and out_is_dup and src_indices is not None and src_indices.size > 0): @@ -365,18 +365,18 @@ def _setup_transfers_rev(group, desvars, responses): total_size += len(input_inds) - rev_xfer_in[sub_out].append(input_inds) - rev_xfer_out[sub_out].append(output_inds) + xfer_in[sub_out].append(input_inds) + xfer_out[sub_out].append(output_inds) else: # not a local input but still need entries in the transfer dicts to # avoid hangs - rev_xfer_in[sub_out] - rev_xfer_out[sub_out] + xfer_in[sub_out] + xfer_out[sub_out] if has_rev_par_coloring: - rev_xfer_in_nocolor[sub_out] - rev_xfer_out_nocolor[sub_out] + xfer_in_nocolor[sub_out] + xfer_out_nocolor[sub_out] - full_xfer_in, full_xfer_out = _setup_index_views(total_size, rev_xfer_in, rev_xfer_out) + full_xfer_in, full_xfer_out = _setup_index_views(total_size, xfer_in, xfer_out) transfers = { None: PETScTransfer(vectors['input']['nonlinear'], @@ -384,25 +384,25 @@ def _setup_transfers_rev(group, desvars, responses): full_xfer_in, full_xfer_out, group.comm) } - for sname, inds in rev_xfer_out.items(): + for sname, inds in xfer_out.items(): transfers[sname] = PETScTransfer(vectors['input']['nonlinear'], vectors['output']['nonlinear'], - rev_xfer_in[sname], inds, group.comm) + xfer_in[sname], inds, group.comm) - if rev_xfer_in_nocolor: + if xfer_in_nocolor: full_xfer_in, full_xfer_out = _setup_index_views(total_size_nocolor, - rev_xfer_in_nocolor, - rev_xfer_out_nocolor) + xfer_in_nocolor, + xfer_out_nocolor) transfers[(None, 'nocolor')] = PETScTransfer(vectors['input']['nonlinear'], vectors['output']['nonlinear'], full_xfer_in, full_xfer_out, group.comm) - for sname, inds in rev_xfer_out_nocolor.items(): + for sname, inds in xfer_out_nocolor.items(): transfers[(sname, 'nocolor')] = PETScTransfer(vectors['input']['nonlinear'], vectors['output']['nonlinear'], - rev_xfer_in_nocolor[sname], inds, + xfer_in_nocolor[sname], inds, group.comm) return transfers @@ -523,18 +523,11 @@ def _get_output_inds(group, abs_out, abs_in, src_indices, rank, sizes, offsets): on_iproc = np.logical_and(start <= src_indices, src_indices < end) if np.any(on_iproc): - # This converts from iproc-then-ivar to ivar-then-iproc ordering - # Subtract off part of this variable from previous procs - # Then add all variables on previous procs - # Then all previous variables on this proc - # - np.sum(out_sizes[:iproc, idx_out]) - # + np.sum(out_sizes[:iproc, :]) - # + np.sum(out_sizes[iproc, :idx_out]) - # + inds - offset = offsets[iproc] - start - output_inds[on_iproc] = src_indices[on_iproc] + offset on_iprocs.append(iproc) + # This converts from global to variable specific ordering + output_inds[on_iproc] = src_indices[on_iproc] + (offsets[iproc] - start) + start = end return output_inds, on_iprocs From eeaa60d62bf6ded36c482fdfd17d38460ed655fa Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Thu, 19 Oct 2023 16:50:34 -0400 Subject: [PATCH 42/70] cleanup --- openmdao/vectors/petsc_transfer.py | 91 ++++++++++-------------------- 1 file changed, 29 insertions(+), 62 deletions(-) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 76cb80e974..9ba882ae2e 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -90,81 +90,58 @@ def _setup_transfers(group, desvars, responses): @staticmethod def _setup_transfers_fwd(group, desvars, responses): + transfers = {} + + if not group._conn_abs_in2out: + return transfers + abs2meta_in = group._var_abs2meta['input'] - abs2meta_out = group._var_abs2meta['output'] myrank = group.comm.rank - vectors = group._vectors - offsets = group._get_var_offsets() + offsets_in = group._get_var_offsets()['input'][myrank, :] mypathlen = len(group.pathname) + 1 if group.pathname else 0 xfer_in = defaultdict(list) xfer_out = defaultdict(list) allprocs_abs2idx = group._var_allprocs_abs2idx - sizes_in = group._var_sizes['input'] - sizes_out = group._var_sizes['output'] - offsets_in = offsets['input'] - offsets_out = offsets['output'] + sizes_in = group._var_sizes['input'][myrank, :] total_len = 0 # Loop through all connections owned by this system for abs_in, abs_out in group._conn_abs_in2out.items(): + sub_in = abs_in[mypathlen:].partition('.')[0] + # Only continue if the input exists on this processor if abs_in in abs2meta_in: - # Get meta - meta_in = abs2meta_in[abs_in] - idx_in = allprocs_abs2idx[abs_in] - idx_out = allprocs_abs2idx[abs_out] - owner = group._owning_rank[abs_out] - - # Read in and process src_indices - src_indices = meta_in['src_indices'] - if src_indices is None: - # if the input is larger than the output on a single proc, we have - # to just loop over the procs in the same way we do when src_indices - # is defined. - if meta_in['size'] > sizes_out[owner, idx_out]: - src_indices = np.arange(meta_in['size'], dtype=INT_DTYPE) - else: - src_indices = src_indices.shaped_array() - - rank = myrank if abs_out in abs2meta_out else owner - output_inds, _ = _get_output_inds(group, abs_out, abs_in, src_indices, rank, - sizes_out[:, idx_out], - offsets_out[:, idx_out]) + output_inds, _ = _get_output_inds(group, abs_out, abs_in) - # 2. Compute the input indices - input_inds = range(offsets_in[myrank, idx_in], - offsets_in[myrank, idx_in] + sizes_in[myrank, idx_in]) + idx_in = allprocs_abs2idx[abs_in] + input_inds = range(offsets_in[idx_in], offsets_in[idx_in] + sizes_in[idx_in]) total_len += len(input_inds) - # Now the indices are ready - input_inds, output_inds - sub_in = abs_in[mypathlen:].partition('.')[0] xfer_in[sub_in].append(input_inds) xfer_out[sub_in].append(output_inds) else: # not a local input but still need entries in the transfer dicts to # avoid hangs - sub_in = abs_in[mypathlen:].partition('.')[0] xfer_in[sub_in] # defaultdict will create an empty list there xfer_out[sub_in] - transfers = {} if xfer_in: full_xfer_in, full_xfer_out = _setup_index_views(total_len, xfer_in, xfer_out) # full transfer (transfer to all subsystems at once) - transfers[None] = PETScTransfer(vectors['input']['nonlinear'], - vectors['output']['nonlinear'], + transfers[None] = PETScTransfer(group._vectors['input']['nonlinear'], + group._vectors['output']['nonlinear'], full_xfer_in, full_xfer_out, group.comm) # transfers to individual subsystems for sname, inds in xfer_in.items(): - transfers[sname] = PETScTransfer(vectors['input']['nonlinear'], - vectors['output']['nonlinear'], + transfers[sname] = PETScTransfer(group._vectors['input']['nonlinear'], + group._vectors['output']['nonlinear'], inds, xfer_out[sname], group.comm) return transfers @@ -262,23 +239,8 @@ def _setup_transfers_rev(group, desvars, responses): meta_in = abs2meta_in[abs_in] idx_in = allprocs_abs2idx[abs_in] idx_out = allprocs_abs2idx[abs_out] - owner = group._owning_rank[abs_out] - - # Read in and process src_indices - src_indices = meta_in['src_indices'] - if src_indices is None: - # if the input is larger than the output on a single proc, we have - # to just loop over the procs in the same way we do when src_indices - # is defined. - if meta_in['size'] > sizes_out[owner, idx_out]: - src_indices = np.arange(meta_in['size'], dtype=INT_DTYPE) - else: - src_indices = src_indices.shaped_array() - rank = myrank if abs_out in abs2meta_out else owner - output_inds, _ = _get_output_inds(group, abs_out, abs_in, src_indices, - rank, sizes_out[:, idx_out], - offsets_out[:, idx_out]) + output_inds, src_indices = _get_output_inds(group, abs_out, abs_in) # 2. Compute the input indices input_inds = range(offsets_in[myrank, idx_in], @@ -493,15 +455,22 @@ def _merge(inds_list, tot_size): return _empty_idx_array -def _get_output_inds(group, abs_out, abs_in, src_indices, rank, sizes, offsets): - meta_out = group._var_allprocs_abs2meta['output'][abs_out] - on_iprocs = [] +def _get_output_inds(group, abs_out, abs_in): + owner = group._owning_rank[abs_out] + src_indices = group._var_abs2meta['input'][abs_in]['src_indices'] + if src_indices is not None: + src_indices = src_indices.shaped_array() + + rank = group.comm.rank if abs_out in group._var_abs2meta['output'] else owner + out_idx = group._var_allprocs_abs2idx[abs_out] + offsets = group._get_var_offsets()['output'][:, out_idx] + sizes = group._var_sizes['output'][:, out_idx] # NOTE: src_indices are relative to a single, possibly distributed variable, # while the output_inds that we compute are relative to the full distributed # array that contains all local variables from each rank stacked in rank order. if src_indices is None: - if meta_out['distributed']: + if group._var_allprocs_abs2meta['output'][abs_out]['distributed']: # input in this case is non-distributed (else src_indices would be # defined by now). dist output to non-distributed input conns w/o # src_indices are not allowed. @@ -523,11 +492,9 @@ def _get_output_inds(group, abs_out, abs_in, src_indices, rank, sizes, offsets): on_iproc = np.logical_and(start <= src_indices, src_indices < end) if np.any(on_iproc): - on_iprocs.append(iproc) - # This converts from global to variable specific ordering output_inds[on_iproc] = src_indices[on_iproc] + (offsets[iproc] - start) start = end - return output_inds, on_iprocs + return output_inds, src_indices From 0e44bf8557d308f726ac3e75004b71e2546b1c30 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 20 Oct 2023 08:49:02 -0400 Subject: [PATCH 43/70] updated comment --- openmdao/vectors/petsc_transfer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 9ba882ae2e..e060b9f2ac 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -168,7 +168,6 @@ def _setup_transfers_rev(group, desvars, responses): allprocs_abs2idx = group._var_allprocs_abs2idx sizes_in = group._var_sizes['input'] - sizes_out = group._var_sizes['output'] offsets_in = offsets['input'] offsets_out = offsets['output'] @@ -218,6 +217,8 @@ def _setup_transfers_rev(group, desvars, responses): inp_dep_dist.update(inp_boundary_set.intersection(rel['input'])) # look in model for the connections to the group boundary inputs from outside + # and update the _fd_subgroup_inputs in the group that owns connections to + # those inputs. for inp in inp_dep_dist: src = model._conn_global_abs_in2out[inp] gname = common_subpath((src, inp)) From fb63db8555a858d44b4918b16814bb08797ad58b Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 27 Oct 2023 16:20:02 -0400 Subject: [PATCH 44/70] passing --- .../approximation_scheme.py | 43 +- openmdao/core/group.py | 119 ++-- openmdao/core/implicitcomponent.py | 4 +- openmdao/core/problem.py | 11 +- openmdao/core/system.py | 15 +- openmdao/core/tests/test_deriv_transfers.py | 3 - openmdao/core/tests/test_distrib_derivs.py | 572 ++++++++++++------ .../core/tests/test_parallel_derivatives.py | 2 +- openmdao/core/tests/test_parallel_groups.py | 4 - openmdao/core/total_jac.py | 41 +- openmdao/jacobians/dictionary_jacobian.py | 114 +++- openmdao/jacobians/jacobian.py | 69 ++- openmdao/solvers/nonlinear/newton.py | 48 +- openmdao/utils/general_utils.py | 65 +- openmdao/vectors/petsc_transfer.py | 136 +++-- openmdao/vectors/vector.py | 29 +- 16 files changed, 839 insertions(+), 436 deletions(-) diff --git a/openmdao/approximation_schemes/approximation_scheme.py b/openmdao/approximation_schemes/approximation_scheme.py index b0771f8366..e4832e8f7a 100644 --- a/openmdao/approximation_schemes/approximation_scheme.py +++ b/openmdao/approximation_schemes/approximation_scheme.py @@ -1,9 +1,11 @@ """Base class used to define the interface for derivative approximation schemes.""" import time from collections import defaultdict +from itertools import repeat import numpy as np from openmdao.core.constants import INT_DTYPE +from openmdao.vectors.vector import _full_slice from openmdao.utils.array_utils import get_input_idx_split import openmdao.utils.coloring as coloring_mod from openmdao.utils.general_utils import _convert_auto_ivc_to_conn_name, LocalRangeIterable @@ -134,8 +136,8 @@ def add_approximation(self, abs_key, system, kwargs): raise NotImplementedError("add_approximation has not been implemented") def _init_colored_approximations(self, system): - is_group = _is_group(system) - is_total = is_group and system.pathname == '' + is_total = system.pathname == '' + is_semi = _is_group(system) and not is_total self._colored_approx_groups = [] # don't do anything if the coloring doesn't exist yet @@ -155,17 +157,17 @@ def _init_colored_approximations(self, system): if is_total: ccol2vcol = np.empty(coloring._shape[1], dtype=INT_DTYPE) - # ordered_wrt_iter = list(system._jac_wrt_iter()) colored_start = colored_end = 0 - for abs_wrt, cstart, cend, _, cinds, _ in system._jac_wrt_iter(): + for abs_wrt, cstart, cend, vec, cinds, _ in system._jac_wrt_iter(): if wrt_matches is None or abs_wrt in wrt_matches: colored_end += cend - cstart - ccol2jcol[colored_start:colored_end] = np.arange(cstart, cend, dtype=INT_DTYPE) + ccol2jcol[colored_start:colored_end] = range(cstart, cend) if is_total and abs_wrt in out_slices: slc = out_slices[abs_wrt] - rng = np.arange(slc.start, slc.stop) - if cinds is not None: - rng = rng[cinds] + if cinds is None or cinds is _full_slice: + rng = range(slc.start, slc.stop) + else: + rng = np.arange(slc.start, slc.stop)[cinds] ccol2vcol[colored_start:colored_end] = rng colored_start = colored_end @@ -198,7 +200,6 @@ def _init_colored_approximations(self, system): inputs = system._inputs from openmdao.core.implicitcomponent import ImplicitComponent - is_semi = is_group and not is_total use_full_cols = is_semi or isinstance(system, ImplicitComponent) for cols, nzrows in coloring.color_nonzero_iter('fwd'): @@ -259,7 +260,7 @@ def _init_approximations(self, system): if wrt in approx_wrt_idx: if vec is None: - vec_idx = None + vec_idx = repeat(None, approx_wrt_idx[wrt].shaped_array().size) else: # local index into var vec_idx = approx_wrt_idx[wrt].shaped_array(copy=True) @@ -271,15 +272,18 @@ def _init_approximations(self, system): in_idx = [list(in_idx)] vec_idx = [vec_idx] else: - if vec is None: # remote wrt - if wrt in abs2meta['input']: - vec_idx = range(abs2meta['input'][wrt]['global_size']) - else: - vec_idx = range(abs2meta['output'][wrt]['global_size']) - else: - vec_idx = LocalRangeIterable(system, wrt) - if directional: - vec_idx = [v for v in vec_idx if v is not None] + # if vec is None: # remote wrt + # if wrt in abs2meta['input']: + # vec_idx = range(abs2meta['input'][wrt]['global_size']) + # else: + # vec_idx = range(abs2meta['output'][wrt]['global_size']) + # else: + # vec_idx = LocalRangeIterable(system, wrt) + # if directional: + # vec_idx = [v for v in vec_idx if v is not None] + vec_idx = LocalRangeIterable(system, wrt) + if directional and vec is not None: + vec_idx = [v for v in vec_idx if v is not None] # Directional derivatives for quick deriv checking. # Place the indices in a list so that they are all stepped at the same time. @@ -569,7 +573,6 @@ def _uncolored_column_iter(self, system, approx_groups): yield jinds[0], res else: yield jinds, res - # print(f"APPROX: {wrt} {jinds} {res}") def compute_approximations(self, system, jac=None): """ diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 7422b0a3c3..64db9cb8c1 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -3,7 +3,7 @@ from collections import Counter, defaultdict from collections.abc import Iterable -from itertools import product, chain +from itertools import product, chain, repeat from numbers import Number import inspect from fnmatch import fnmatchcase @@ -26,7 +26,7 @@ from openmdao.utils.array_utils import array_connection_compatible, _flatten_src_indices, \ shape_to_len from openmdao.utils.general_utils import common_subpath, all_ancestors, \ - convert_src_inds, ContainsAll, shape2tuple, get_connection_owner, ensure_compatible, \ + convert_src_inds, _contains_all, shape2tuple, get_connection_owner, ensure_compatible, \ _src_name_iter, meta2src_iter, get_rev_conns from openmdao.utils.units import is_compatible, unit_conversion, _has_val_mismatch, _find_unit, \ _is_unitless, simplify_unit @@ -192,11 +192,15 @@ class Group(System): Dict of absolute response metadata. _relevance_graph : nx.DiGraph Graph of relevance connections. Always None except in the top level Group. - _fd_subgroup_inputs : set + _fd_rev_xfer_correction_dist : set If one or more subgroups of this group is using finite difference to compute derivatives, - this is the set of inputs to those subgroups that are upstream of a distributed variable + this is the set of inputs to those subgroups that are upstream of a distributed response within the same subgroup. These determine if an allreduce is necessary when transferring data to a connected output in reverse mode. + _fd_rev_xfer_correction_dup : dict + If one or more subgroups of this group is using finite difference to compute derivatives, + this is the set of inputs to those subgroups that are upstream of a duplicated response + within the same subgroup. """ def __init__(self, **kwargs): @@ -227,7 +231,8 @@ def __init__(self, **kwargs): self._abs_desvars = None self._abs_responses = None self._relevance_graph = None - self._fd_subgroup_inputs = set() + self._fd_rev_xfer_correction_dist = set() + self._fd_rev_xfer_correction_dup = {} # TODO: we cannot set the solvers with property setters at the moment # because our lint check thinks that we are defining new attributes @@ -801,7 +806,7 @@ def _init_relevance(self, mode): return self.get_relevant_vars(self._abs_desvars, self._check_alias_overlaps(self._abs_responses), mode) - return {'@all': ({'input': ContainsAll(), 'output': ContainsAll()}, ContainsAll())} + return {'@all': ({'input': _contains_all, 'output': _contains_all}, _contains_all)} def get_relevance_graph(self, desvars, responses): """ @@ -956,20 +961,19 @@ def get_relevant_vars(self, desvars, responses, mode): for dvmeta in desvars.values(): desvar = dvmeta['source'] dvset = set(self.all_connected_nodes(graph, desvar)) - parallel_deriv_color = dvmeta['parallel_deriv_color'] - if parallel_deriv_color: + if dvmeta['parallel_deriv_color']: pd_dv_locs[desvar] = set(self.all_connected_nodes(graph, desvar, local=True)) - pd_err_chk[parallel_deriv_color][desvar] = pd_dv_locs[desvar] + pd_err_chk[dvmeta['parallel_deriv_color']][desvar] = pd_dv_locs[desvar] for resmeta in responses.values(): response = resmeta['source'] if response not in rescache: rescache[response] = set(self.all_connected_nodes(grev, response)) - parallel_deriv_color = resmeta['parallel_deriv_color'] - if parallel_deriv_color: + if resmeta['parallel_deriv_color']: pd_res_locs[response] = set(self.all_connected_nodes(grev, response, local=True)) - pd_err_chk[parallel_deriv_color][response] = pd_res_locs[response] + pd_err_chk[resmeta['parallel_deriv_color']][response] = \ + pd_res_locs[response] common = dvset.intersection(rescache[response]) @@ -1321,7 +1325,8 @@ def _final_setup(self, comm, mode): # must call this before vector setup because it determines if we need to alloc commplex self._setup_partials() - self._fd_subgroup_inputs = set() + self._fd_rev_xfer_correction_dist = set() + self._fd_rev_xfer_correction_dup = {} self._problem_meta['relevant'] = self._init_relevance(mode) @@ -3192,36 +3197,6 @@ def _transfer(self, vec_name, mode, sub=None): xfer = self._transfers['rev'][key] xfer._transfer(vec_inputs, self._vectors['output'][vec_name], mode) - if self._fd_subgroup_inputs and self.comm.size > 1: - seed_vars = self._problem_meta['seed_vars'] - if seed_vars is not None: - if len(seed_vars) > 1: - raise RuntimeError(f"Multiple seed variables {sorted(seed_vars)} are " - "not supported under MPI in reverse mode if they " - "depend on an group doing finite difference and " - "containing distributed variables.") - pre = '' if sub is None else sub + '.' - slices = self._doutputs.get_slice_dict() - outarr = self._doutputs.asarray() - data = {} - for inp in self._fd_subgroup_inputs: - src = self._conn_global_abs_in2out[inp] - if src.startswith(pre) and src in slices: - arr = outarr[slices[src]] - if np.any(arr): - data[src] = arr - else: - data[src] = None - - if data: - comm = self.comm - myrank = comm.rank - for rank, d in enumerate(comm.allgather(data)): - if rank != myrank: - for n, val in d.items(): - if val is not None and n in slices: - outarr[slices[n]] += val - if self._has_input_scaling: vec_inputs.scale_to_phys(mode='rev') @@ -3813,12 +3788,55 @@ def _apply_linear(self, jac, rel_systems, mode, scope_out=None, scope_in=None): if jac is not None: with self._matvec_context(scope_out, scope_in, mode) as vecs: d_inputs, d_outputs, d_residuals = vecs + jac._apply(self, d_inputs, d_outputs, d_residuals, mode) + + # _fd_rev_xfer_correction_dist is used to correct for the fact that we don't + # do reverse transfers internal to an FD group. Reverse transfers + # are constructed such that derivative values are correct when transferred into + # system output variables, taking into account distributed inputs. + # Since the transfers are not correcting for those issues, we need to do it here. + + # If we have a distributed constraint/obj within the FD group, + # we perform essentially an allreduce on the d_inputs vars that connect to + # outside systems so they'll include the contribution from all procs. + if self._fd_rev_xfer_correction_dist: + seed_vars = self._problem_meta['seed_vars'] + if seed_vars is not None: + if len(seed_vars) > 1: + prefix = self.pathname + '.' + seed_vars = [n for n in seed_vars if n.startswith(prefix)] + if len(seed_vars) > 1: + raise RuntimeError("Multiple simultaneous seed variables " + f"{sorted(seed_vars)} within an FD group are not" + " supported under MPI in reverse mode if they " + "depend on an group doing finite difference and " + "containing distributed constraints/objectives.") + slices = self._dinputs.get_slice_dict() + inarr = self._dinputs.asarray() + data = {} + for inp in self._fd_rev_xfer_correction_dist: + if inp in slices: + arr = inarr[slices[inp]] + if np.any(arr): + data[inp] = arr + else: + data[inp] = None + + if data: + comm = self.comm + myrank = comm.rank + for rank, d in enumerate(comm.allgather(data)): + if rank != myrank: + for n, val in d.items(): + if val is not None and n in slices: + inarr[slices[n]] += val + # Apply recursion else: if mode == 'fwd': self._transfer('linear', mode) - if rel_systems is not None: + if rel_systems is not None and rel_systems is not _contains_all: for s in self._solver_subsystem_iter(local_only=True): if s.pathname not in rel_systems: # zero out dvecs of irrelevant subsystems @@ -3830,7 +3848,7 @@ def _apply_linear(self, jac, rel_systems, mode, scope_out=None, scope_in=None): if mode == 'rev': self._transfer('linear', mode) - if rel_systems is not None: + if rel_systems is not None and rel_systems is not _contains_all: for s in self._solver_subsystem_iter(local_only=True): if s.pathname not in rel_systems: # zero out dvecs of irrelevant subsystems @@ -3880,7 +3898,7 @@ def _solve_linear(self, mode, rel_systems, scope_out=_UNDEFINED, scope_in=_UNDEF self._linear_solver._set_matvec_scope(scope_out, scope_in) self._linear_solver.solve(mode, rel_systems) - def _linearize(self, jac, sub_do_ln=True, rel_systems=ContainsAll()): + def _linearize(self, jac, sub_do_ln=True, rel_systems=_contains_all): """ Compute jacobian / factorization. The model is assumed to be in a scaled state. @@ -4219,9 +4237,10 @@ def _jac_wrt_iter(self, wrt_matches=None): elif wrt in local_outs: vec = self._outputs else: - if not total: - continue - vec = None + # ??? + # if not total: + # continue + vec = None # remote wrt if wrt in approx_wrt_idx: sub_wrt_idx = approx_wrt_idx[wrt] size = sub_wrt_idx.indexed_src_size @@ -4229,6 +4248,8 @@ def _jac_wrt_iter(self, wrt_matches=None): else: sub_wrt_idx = _full_slice size = abs2meta[io][wrt][szname] + if vec is None: + sub_wrt_idx = repeat(None, size) end += size dist_sizes = sizes[io][:, toidx[wrt]] if meta['distributed'] else None yield wrt, start, end, vec, sub_wrt_idx, dist_sizes diff --git a/openmdao/core/implicitcomponent.py b/openmdao/core/implicitcomponent.py index c50ec8019c..a6bb217873 100644 --- a/openmdao/core/implicitcomponent.py +++ b/openmdao/core/implicitcomponent.py @@ -5,6 +5,7 @@ from openmdao.core.component import Component, _allowed_types from openmdao.core.constants import _UNDEFINED, _SetupStatus +from openmdao.vectors.vector import _full_slice from openmdao.recorders.recording_iteration_stack import Recording from openmdao.utils.class_util import overrides_method from openmdao.utils.array_utils import shape_to_len @@ -12,9 +13,6 @@ from openmdao.utils.units import simplify_unit -_full_slice = slice(None) - - def _get_slice_shape_dict(name_shape_iter): """ Return a dict of (slice, shape) tuples using provided names and shapes. diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index 6c179cf9be..b6655eae3d 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -24,7 +24,7 @@ from openmdao.core.explicitcomponent import ExplicitComponent from openmdao.core.system import System, _OptStatus from openmdao.core.group import Group -from openmdao.core.total_jac import _TotalJacInfo +from openmdao.core.total_jac import _TotalJacInfo, _contains_all from openmdao.core.constants import _DEFAULT_OUT_STREAM, _UNDEFINED from openmdao.jacobians.dictionary_jacobian import _CheckingJacobian from openmdao.approximation_schemes.complex_step import ComplexStep @@ -48,7 +48,7 @@ from openmdao.utils.class_util import overrides_method from openmdao.utils.reports_system import get_reports_to_activate, activate_reports, \ clear_reports, get_reports_dir, _load_report_plugins -from openmdao.utils.general_utils import ContainsAll, pad_name, LocalRangeIterable, \ +from openmdao.utils.general_utils import _contains_all, pad_name, LocalRangeIterable, \ _find_dict_meta, env_truthy, add_border, match_includes_excludes, inconsistent_across_procs from openmdao.utils.om_warnings import issue_warning, DerivativesWarning, warn_deprecation, \ OMInvalidCheckDerivativesOptionsWarning @@ -62,7 +62,6 @@ from openmdao.utils.name_maps import rel_key2abs_key, rel_name2abs_name -_contains_all = ContainsAll() CITATION = """@article{openmdao_2019, Author={Justin S. Gray and John T. Hwang and Joaquim R. R. A. @@ -1006,9 +1005,9 @@ def setup(self, check=False, logger=None, mode='auto', force_alloc_complex=False 'singular_jac_behavior': 'warn', # How to handle singular jac conditions 'parallel_deriv_color': None, # None unless derivatives involving a parallel deriv # colored dv/response are currently being computed - 'seed_vars': None, # list of tuples of the form (seed var names, any_are_distrib). - # The seed variables are those that are active in the current - # derivative solve. + 'seed_vars': None, # set of names of seed variables. Seed variables are those that + # have their derivative value set to 1.0 at the beginning of the + # current derivative solve. } if _prob_setup_stack: diff --git a/openmdao/core/system.py b/openmdao/core/system.py index 479166df22..eff5820e48 100644 --- a/openmdao/core/system.py +++ b/openmdao/core/system.py @@ -35,7 +35,7 @@ from openmdao.utils.om_warnings import issue_warning, \ DerivativesWarning, PromotionWarning, UnusedOptionWarning, UnitsWarning from openmdao.utils.general_utils import determine_adder_scaler, \ - format_as_float_or_array, ContainsAll, all_ancestors, make_set, match_prom_or_abs, \ + format_as_float_or_array, _contains_all, all_ancestors, make_set, match_prom_or_abs, \ ensure_compatible, env_truthy, make_traceback, _is_slicer_op from openmdao.approximation_schemes.complex_step import ComplexStep from openmdao.approximation_schemes.finite_difference import FiniteDifference @@ -672,8 +672,8 @@ def _jac_wrt_iter(self, wrt_matches=None): Starting index. int Ending index. - Vector - Either the _outputs or _inputs vector. + Vector or None + Either the _outputs or _inputs vector if var is local else None. slice A full slice. ndarray or None @@ -2592,9 +2592,8 @@ def _matvec_context(self, scope_out, scope_in, mode, clear=True): """ Context manager for vectors. - For the given vec_name, return vectors that use a set of - internal variables that are relevant to the current matrix-vector - product. This is called only from _apply_linear. + Return vectors that use a set of internal variables that are relevant to the current + matrix-vector product. This is called only from _apply_linear. Parameters ---------- @@ -4576,7 +4575,7 @@ def run_apply_linear(self, mode, scope_out=None, scope_in=None): If None, all are in the scope. """ with self._scaled_context_all(): - self._apply_linear(None, ContainsAll(), mode, scope_out, scope_in) + self._apply_linear(None, _contains_all, mode, scope_out, scope_in) def run_solve_linear(self, mode): """ @@ -4590,7 +4589,7 @@ def run_solve_linear(self, mode): 'fwd' or 'rev'. """ with self._scaled_context_all(): - self._solve_linear(mode, ContainsAll()) + self._solve_linear(mode, _contains_all) def run_linearize(self, sub_do_ln=True): """ diff --git a/openmdao/core/tests/test_deriv_transfers.py b/openmdao/core/tests/test_deriv_transfers.py index b9b64c5a34..22696cefc0 100644 --- a/openmdao/core/tests/test_deriv_transfers.py +++ b/openmdao/core/tests/test_deriv_transfers.py @@ -242,9 +242,6 @@ def test_par_dup(self, mode): J = prob.compute_totals(of=of, wrt=wrt) - import pprint - pprint.pprint(J) - assert_near_equal(J['C1.y', 'par.indep1.x'][0][0], 2.5, 1e-6) assert_near_equal(J['C1.y', 'par.indep2.x'][0][0], 3.5, 1e-6) assert_near_equal(prob['C1.y'], 6., 1e-6) diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index 50dd758000..a78e9969e3 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -211,6 +211,319 @@ def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): d_inputs['in_nd'] += dg_dIs * d_outputs['out_nd'] +def _setup2ivc2par2dup(size=7): + # 2 IVCs feed two parallel comps, which feed two duplicated comps + prob = om.Problem() + model = prob.model + + model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) + model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x*.5', shape=size)) + sub.add_subsystem('C4', om.ExecComp('y = x*3.', shape=size)) + + model.connect('p.x', 'sub.par.C1.x') + model.connect('p2.x', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x') + model.connect('sub.par.C2.y', 'sub.C4.x') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p2.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_constraint('sub.C4.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='fd') + + return prob + + +def _setup_ivc_subivc_dist_parab_sum(): + size = 7 + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + ivc.add_output('y', np.ones((size, ))) + + model.add_subsystem('p', ivc, promotes=['*']) + sub = model.add_subsystem('sub', om.Group(), promotes=['*']) + + ivc2 = om.IndepVarComp() + ivc2.add_output('a', -3.0 + 0.6 * np.arange(size)) + + sub.add_subsystem('p2', ivc2, promotes=['*']) + sub.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], + x=np.ones(size), xd=np.ones(size), + y=np.ones(size), yd=np.ones(size)), + promotes_inputs=['*']) + + sub.add_subsystem("parab", DistParab(arr_size=size), promotes_outputs=['*'], promotes_inputs=['a']) + model.add_subsystem('sum', om.ExecComp('f_sum = sum(xd)', + f_sum=np.ones((size, )), + xd=np.ones((size, ))), + promotes_outputs=['*']) + + model.promotes('sum', inputs=['xd']) + + sub.connect('dummy.xd', 'parab.x') + sub.connect('dummy.yd', 'parab.y') + + model.add_design_var('x', lower=-50.0, upper=50.0) + model.add_design_var('y', lower=-50.0, upper=50.0) + model.add_constraint('f_xy', lower=0.0) + model.add_objective('f_sum', index=-1) + + sub.approx_totals(method='fd') + + return prob + + +def _setup_ivc_sub_ivcdistparabcons_nosum(): + # distrib comp is inside of fd group but not a response, and 2 nondistrib + # constraints connect to it downstream, both inside and outside of the fd group. + size = 7 + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + ivc.add_output('y', np.ones((size, ))) + + model.add_subsystem('p', ivc) + sub = model.add_subsystem('sub', om.Group()) + + sub.add_subsystem('p2', om.IndepVarComp('a', -3.0 + 0.6 * np.arange(size))) + sub.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], + x=np.ones(size), xd=np.ones(size), + y=np.ones(size), yd=np.ones(size))) + + sub.add_subsystem("parab", DistParab(arr_size=size)) + sub.add_subsystem("cons", om.ExecComp("c = x*3. + 7.", x=np.ones(size), c=np.ones(size))) + # model.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, )))) + + model.connect('p.x', 'sub.dummy.x') + model.connect('p.y', 'sub.dummy.y') + model.connect('sub.p2.a', 'sub.parab.a') + model.connect('sub.dummy.xd', 'sub.parab.x') + model.connect('sub.dummy.yd', 'sub.parab.y') + # model.connect('sub.parab.f_xy', 'sum.f_xy', src_indices=om.slicer[:]) + model.connect('sub.parab.f_xy', 'sub.cons.x', src_indices=om.slicer[:]) + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p.y', lower=-50.0, upper=50.0) + model.add_constraint('sub.cons.c', lower=0.0) + # model.add_objective('sum.f_sum', index=-1) + + sub.approx_totals(method='fd') + + return prob + + +def _setup_ivc_subivcdistparabconssum_in_sub(): + # distrib comp is inside of fd group but not a response, and 2 nondistrib + # constraints connect to it downstream, both inside of the fd group. + size = 7 + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + ivc.add_output('y', np.ones((size, ))) + + model.add_subsystem('p', ivc) + sub = model.add_subsystem('sub', om.Group()) + + sub.add_subsystem('p2', om.IndepVarComp('a', -3.0 + 0.6 * np.arange(size))) + sub.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], + x=np.ones(size), xd=np.ones(size), + y=np.ones(size), yd=np.ones(size))) + + sub.add_subsystem("parab", DistParab(arr_size=size)) + sub.add_subsystem("cons", om.ExecComp("c = x*3. + 7.", x=np.ones(size), c=np.ones(size))) + sub.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, )))) + + model.connect('p.x', 'sub.dummy.x') + model.connect('p.y', 'sub.dummy.y') + model.connect('sub.p2.a', 'sub.parab.a') + model.connect('sub.dummy.xd', 'sub.parab.x') + model.connect('sub.dummy.yd', 'sub.parab.y') + model.connect('sub.parab.f_xy', 'sub.sum.f_xy', src_indices=om.slicer[:]) + model.connect('sub.parab.f_xy', 'sub.cons.x', src_indices=om.slicer[:]) + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p.y', lower=-50.0, upper=50.0) + model.add_constraint('sub.cons.c', lower=0.0) + model.add_objective('sub.sum.f_sum', index=-1) + + sub.approx_totals(method='fd') + + return prob + + +def _setup_inner_par_ivc_direct_conn(size=7): + # one IVC feeds two parallel comps, which feed a third comp. All but the IVC are + # in an FD subgroup. + + prob = om.Problem() + model = prob.model + + model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) + + model.connect('p.x', 'sub.par.C1.x') + model.connect('p.x', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x1') + model.connect('sub.par.C2.y', 'sub.C3.x2') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='fd') + + return prob + +def _setup_inner_par_2ivcs(size=7): + prob = om.Problem() + model = prob.model + + model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) + model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) + + model.connect('p.x', 'sub.par.C1.x') + model.connect('p2.x', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x1') + model.connect('sub.par.C2.y', 'sub.C3.x2') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p2.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='cs') + + return prob + +def _setup_inner_par_ivc_indirect_conn(size=7): + # one IVC feeds an intermediate dup comp, which feeds two parallel comps, which feed a third comp + # inside the FD group. + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + + model.add_subsystem('p', ivc) + model.add_subsystem('dum', om.ExecComp('y = x', shape=size)) + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) + + model.connect('p.x', 'dum.x') + model.connect('dum.y', 'sub.par.C1.x') + model.connect('dum.y', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x1') + model.connect('sub.par.C2.y', 'sub.C3.x2') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='fd') + + return prob + +def _setup_inner_par_ivc_indirect2_conn(size=7): + # one IVC feeds an intermediate dup comp, which feeds two parallel comps, which feed a third comp + # inside the FD group. + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + + model.add_subsystem('p', ivc) + model.add_subsystem('dum', om.ExecComp('y = x', shape=size)) + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) + model.add_subsystem('C4', om.ExecComp('y = x', shape=size)) + + model.connect('p.x', 'dum.x') + model.connect('dum.y', 'sub.par.C1.x') + model.connect('dum.y', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x1') + model.connect('sub.par.C2.y', 'sub.C3.x2') + model.connect('sub.C3.y', 'C4.x') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_objective('C4.y', index=-1) + + sub.approx_totals(method='fd') + + return prob + +def _setup_inner_par_2ivc_conn(size=7): + # one IVC feeds an intermediate dup comp, which feeds two parallel comps, which feed a third comp + # inside the FD group, which feeds another comp outside the FD group. + + prob = om.Problem() + model = prob.model + + model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) + model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) + + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) + + model.connect('p.x', 'sub.par.C1.x') + model.connect('p2.x', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x1') + model.connect('sub.par.C2.y', 'sub.C3.x2') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p2.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='fd') + + return prob + + def _test_func_name(func, num, param): args = [] for p in param.args: @@ -736,248 +1049,115 @@ def test_distrib_voi_group_fd(self): assert_check_totals(prob.check_totals(method='fd', out_stream=None)) def test_distrib_voi_group_fd2(self): - size = 7 - - prob = om.Problem() - model = prob.model - - ivc = om.IndepVarComp() - ivc.add_output('x', np.ones((size, ))) - ivc.add_output('y', np.ones((size, ))) - - model.add_subsystem('p', ivc, promotes=['*']) - sub = model.add_subsystem('sub', om.Group(), promotes=['*']) - - ivc2 = om.IndepVarComp() - ivc2.add_output('a', -3.0 + 0.6 * np.arange(size)) - - sub.add_subsystem('p2', ivc2, promotes=['*']) - sub.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], - x=np.ones(size), xd=np.ones(size), - y=np.ones(size), yd=np.ones(size)), - promotes_inputs=['*']) - - sub.add_subsystem("parab", DistParab(arr_size=size), promotes_outputs=['*'], promotes_inputs=['a']) - model.add_subsystem('sum', om.ExecComp('f_sum = sum(xd)', - f_sum=np.ones((size, )), - xd=np.ones((size, ))), - promotes_outputs=['*']) - - model.promotes('sum', inputs=['xd']) - - sub.connect('dummy.xd', 'parab.x') - sub.connect('dummy.yd', 'parab.y') - - model.add_design_var('x', lower=-50.0, upper=50.0) - model.add_design_var('y', lower=-50.0, upper=50.0) - model.add_constraint('f_xy', lower=0.0) - model.add_objective('f_sum', index=-1) - - sub.approx_totals(method='fd') - + prob = _setup_ivc_subivc_dist_parab_sum() prob.setup(mode='fwd', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None)) - # rev mode - + def test_distrib_voi_group_fd2(self): + prob = _setup_ivc_subivc_dist_parab_sum() prob.setup(mode='rev', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None)) - def test_distrib_voi_group_fd4(self): - # distrib comp is inside of fd group but not a response, and 2 nondistrib - # constraints connect to it downstream, both inside and outside of the fd group. - size = 7 - - prob = om.Problem() - model = prob.model - - ivc = om.IndepVarComp() - ivc.add_output('x', np.ones((size, ))) - ivc.add_output('y', np.ones((size, ))) - - model.add_subsystem('p', ivc) - sub = model.add_subsystem('sub', om.Group()) - - sub.add_subsystem('p2', om.IndepVarComp('a', -3.0 + 0.6 * np.arange(size))) - sub.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], - x=np.ones(size), xd=np.ones(size), - y=np.ones(size), yd=np.ones(size))) - - sub.add_subsystem("parab", DistParab(arr_size=size)) - sub.add_subsystem("cons", om.ExecComp("c = x*3. + 7.", x=np.ones(size), c=np.ones(size))) - # model.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, )))) - - model.connect('p.x', 'sub.dummy.x') - model.connect('p.y', 'sub.dummy.y') - model.connect('sub.p2.a', 'sub.parab.a') - model.connect('sub.dummy.xd', 'sub.parab.x') - model.connect('sub.dummy.yd', 'sub.parab.y') - # model.connect('sub.parab.f_xy', 'sum.f_xy', src_indices=om.slicer[:]) - model.connect('sub.parab.f_xy', 'sub.cons.x', src_indices=om.slicer[:]) - - model.add_design_var('p.x', lower=-50.0, upper=50.0) - model.add_design_var('p.y', lower=-50.0, upper=50.0) - model.add_constraint('sub.cons.c', lower=0.0) - # model.add_objective('sum.f_sum', index=-1) - - sub.approx_totals(method='fd') - + def test_distrib_voi_group_fd4_fwd(self): + prob = _setup_ivc_sub_ivcdistparabcons_nosum() prob.setup(mode='fwd', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - # rev mode - + def test_distrib_voi_group_fd4_rev(self): + prob = _setup_ivc_sub_ivcdistparabcons_nosum() prob.setup(mode='rev', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def test_distrib_voi_group_fd5(self): - # distrib comp is inside of fd group but not a response, and 2 nondistrib - # constraints connect to it downstream, both inside of the fd group. - size = 7 - - prob = om.Problem() - model = prob.model - - ivc = om.IndepVarComp() - ivc.add_output('x', np.ones((size, ))) - ivc.add_output('y', np.ones((size, ))) - - model.add_subsystem('p', ivc) - sub = model.add_subsystem('sub', om.Group()) - - sub.add_subsystem('p2', om.IndepVarComp('a', -3.0 + 0.6 * np.arange(size))) - sub.add_subsystem('dummy', om.ExecComp(['xd = x', "yd = y"], - x=np.ones(size), xd=np.ones(size), - y=np.ones(size), yd=np.ones(size))) - - sub.add_subsystem("parab", DistParab(arr_size=size)) - sub.add_subsystem("cons", om.ExecComp("c = x*3. + 7.", x=np.ones(size), c=np.ones(size))) - sub.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, )))) - - model.connect('p.x', 'sub.dummy.x') - model.connect('p.y', 'sub.dummy.y') - model.connect('sub.p2.a', 'sub.parab.a') - model.connect('sub.dummy.xd', 'sub.parab.x') - model.connect('sub.dummy.yd', 'sub.parab.y') - model.connect('sub.parab.f_xy', 'sub.sum.f_xy', src_indices=om.slicer[:]) - model.connect('sub.parab.f_xy', 'sub.cons.x', src_indices=om.slicer[:]) - - model.add_design_var('p.x', lower=-50.0, upper=50.0) - model.add_design_var('p.y', lower=-50.0, upper=50.0) - model.add_constraint('sub.cons.c', lower=0.0) - model.add_objective('sub.sum.f_sum', index=-1) - - sub.approx_totals(method='fd') - + def test_distrib_voi_group_fd5_fwd(self): + prob = _setup_ivc_subivcdistparabconssum_in_sub() prob.setup(mode='fwd', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - # rev mode - + def test_distrib_voi_group_fd5_rev(self): + prob = _setup_ivc_subivcdistparabconssum_in_sub() prob.setup(mode='rev', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def _setup_inner_par(self, size=7): - # one IVC feeds two parallel comps, which feed a third comp - - prob = om.Problem() - model = prob.model - - ivc = om.IndepVarComp() - ivc.add_output('x', np.ones((size, ))) - - model.add_subsystem('p', ivc) - sub = model.add_subsystem('sub', om.Group()) - par = sub.add_subsystem('par', om.ParallelGroup()) - par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) - par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) - sub.add_subsystem('C3', om.ExecComp('y = x1*.5 - x2*3.', shape=size)) - - model.connect('p.x', 'sub.par.C1.x') - model.connect('p.x', 'sub.par.C2.x') - model.connect('sub.par.C1.y', 'sub.C3.x1') - model.connect('sub.par.C2.y', 'sub.C3.x2') - - model.add_design_var('p.x', lower=-50.0, upper=50.0) - model.add_constraint('sub.par.C1.y', lower=0.0) - model.add_constraint('sub.par.C2.y', lower=0.0) - model.add_objective('sub.C3.y', index=-1) - - sub.approx_totals(method='fd') + def test_group_fd_inner_par_fwd(self): + prob = _setup_inner_par_ivc_direct_conn(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd'), atol=3e-6) - return prob + def test_group_fd_inner_par_rev(self): + prob = _setup_inner_par_ivc_direct_conn(size=7) + prob.setup(mode='rev', force_alloc_complex=True) + prob.run_model() + # assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) - def test_group_fd_inner_par_fwd(self): - prob = self._setup_inner_par(size=7) + def test_group_fd_inner_par2_fwd(self): + prob = _setup2ivc2par2dup(size=7) prob.setup(mode='fwd', force_alloc_complex=True) prob.run_model() assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def test_group_fd_inner_par_rev(self): - prob = self._setup_inner_par(size=7) + def test_group_fd_inner_par2_rev(self): + prob = _setup2ivc2par2dup(size=7) prob.setup(mode='rev', force_alloc_complex=True) prob.run_model() assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def _setup_inner_par2(self, size=7): - # 2 IVCs feed two parallel comps, which feed two duplicated comps - prob = om.Problem() - model = prob.model + def test_group_fd_inner_par2ivcs_fwd(self): + prob = _setup_inner_par_2ivcs(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) - model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) - sub = model.add_subsystem('sub', om.Group()) - par = sub.add_subsystem('par', om.ParallelGroup()) - par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) - par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) - sub.add_subsystem('C3', om.ExecComp('y = x*.5', shape=size)) - sub.add_subsystem('C4', om.ExecComp('y = x*3.', shape=size)) + def test_group_fd_inner_par2ivcs_rev(self): + prob = _setup_inner_par_2ivcs(size=7) + prob.setup(mode='rev', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd'), atol=3e-6) - model.connect('p.x', 'sub.par.C1.x') - model.connect('p2.x', 'sub.par.C2.x') - model.connect('sub.par.C1.y', 'sub.C3.x') - model.connect('sub.par.C2.y', 'sub.C4.x') + def test_group_fd_inner_par_indirect_fwd(self): + prob = _setup_inner_par_ivc_indirect_conn(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - model.add_design_var('p.x', lower=-50.0, upper=50.0) - model.add_design_var('p2.x', lower=-50.0, upper=50.0) - model.add_constraint('sub.par.C1.y', lower=0.0) - model.add_constraint('sub.par.C2.y', lower=0.0) - model.add_constraint('sub.C4.y', lower=0.0) - model.add_objective('sub.C3.y', index=-1) + def test_group_fd_inner_par_indirect_rev(self): + prob = _setup_inner_par_ivc_indirect_conn(size=7) + prob.setup(mode='rev', force_alloc_complex=True) + prob.run_model() + # assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) - sub.approx_totals(method='fd') + def test_group_inner_par_2ivc_conn_fwd(self): + prob = _setup_inner_par_2ivc_conn(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - return prob + def test_group_inner_par_2ivc_conn_rev(self): + prob = _setup_inner_par_2ivc_conn(size=7) + prob.setup(mode='rev', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def test_group_fd_inner_par2_fwd(self): - prob = self._setup_inner_par2(size=7) + def test_group_fd_inner_par_indirect2_fwd(self): + prob = _setup_inner_par_ivc_indirect2_conn(size=7) prob.setup(mode='fwd', force_alloc_complex=True) prob.run_model() assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def test_group_fd_inner_par2_rev(self): - prob = self._setup_inner_par2(size=7) + def test_group_fd_inner_par_indirect2_rev(self): + prob = _setup_inner_par_ivc_indirect2_conn(size=7) prob.setup(mode='rev', force_alloc_complex=True) prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + # assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) def test_distrib_voi_group_fd_loop(self): # distrib comp is inside of fd group and part of a loop. diff --git a/openmdao/core/tests/test_parallel_derivatives.py b/openmdao/core/tests/test_parallel_derivatives.py index 0d56cc5a5b..2dd1604e1c 100644 --- a/openmdao/core/tests/test_parallel_derivatives.py +++ b/openmdao/core/tests/test_parallel_derivatives.py @@ -903,7 +903,7 @@ def compute_partials(self, inputs, partials): # from om_devtools.dist_idxs import dump_dist_idxs # dump_dist_idxs(prob, full=True) - assert_check_totals(prob.check_totals(method='cs', show_only_incorrect=True, out_stream=None)) + assert_check_totals(prob.check_totals(method='cs', out_stream=None)) if __name__ == "__main__": diff --git a/openmdao/core/tests/test_parallel_groups.py b/openmdao/core/tests/test_parallel_groups.py index 951b96e457..90c1867b29 100644 --- a/openmdao/core/tests/test_parallel_groups.py +++ b/openmdao/core/tests/test_parallel_groups.py @@ -713,10 +713,6 @@ def test_fd_rev_mode(self): p.run_model() data = p.check_totals(method='fd', out_stream=None) - - print("sub jacobian:") - import pprint - pprint.pprint(sub._jacobian._subjacs_info) assert_check_totals(data, atol=1e-5) diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index 47ce335493..a0b2210100 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -11,7 +11,7 @@ import numpy as np from openmdao.core.constants import INT_DTYPE -from openmdao.utils.general_utils import ContainsAll, _src_or_alias_dict, _src_or_alias_name +from openmdao.utils.general_utils import _contains_all, _src_or_alias_dict, _src_or_alias_name from openmdao.utils.mpi import MPI, check_mpi_env from openmdao.utils.coloring import _initialize_model_approx, Coloring @@ -29,7 +29,6 @@ elif use_mpi is False: PETSc = None -_contains_all = ContainsAll() _directional_rng = np.random.default_rng(99) @@ -366,12 +365,12 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, if 'rev' in modes: from openmdao.core.group import Group - # find all groups doing FD - fdgroups = [s.pathname for s in model.system_iter(recurse=True, typ=Group) - if s._owns_approx_jac] - allfdgroups = set() - for grps in model.comm.allgather(fdgroups): - allfdgroups.update(grps) + # # find all groups doing FD + # fdgroups = [s.pathname for s in model.system_iter(recurse=True, typ=Group) + # if s._owns_approx_jac] + # allfdgroups = set() + # for grps in model.comm.allgather(fdgroups): + # allfdgroups.update(grps) self.jac_scratch['rev'] = [scratch[0][:J.shape[1]]] if self.simul_coloring is not None: # when simul coloring, need two scratch arrays @@ -652,7 +651,6 @@ def _create_in_idx_map(self, mode): """ iproc = self.comm.rank model = self.model - owning_rank = model._owning_rank relevant = model._relevant has_par_deriv_color = False all_abs2meta_out = model._var_allprocs_abs2meta['output'] @@ -796,21 +794,21 @@ def _create_in_idx_map(self, mode): imeta = defaultdict(bool) imeta['par_deriv_color'] = parallel_deriv_color imeta['idx_list'] = [(start, end)] - imeta['seed_vars'] = [name] + imeta['seed_vars'] = {name} idx_iter_dict[parallel_deriv_color] = (imeta, it) else: imeta = idx_iter_dict[parallel_deriv_color][0] imeta['idx_list'].append((start, end)) - imeta['seed_vars'].append(name) + imeta['seed_vars'].add(name) elif self.directional: imeta = defaultdict(bool) imeta['idx_list'] = range(start, end) - imeta['seed_vars'] = (name,) + imeta['seed_vars'] = {name} idx_iter_dict[name] = (imeta, self.directional_iter) elif not simul_coloring: # plain old single index iteration imeta = defaultdict(bool) imeta['idx_list'] = range(start, end) - imeta['seed_vars'] = (name,) + imeta['seed_vars'] = {name} idx_iter_dict[name] = (imeta, self.single_index_iter) if path in relevant and not non_rel_outs: @@ -937,6 +935,7 @@ def _get_sol2jac_map(self, names, vois, allprocs_abs2meta_out, mode): if (path in abs2idx and path in slices and path not in self.remote_vois): var_idx = abs2idx[path] slc = slices[path] + slcsize = slc.stop - slc.start if MPI and meta['distributed'] and self.get_remote: if indices is not None: @@ -950,16 +949,20 @@ def _get_sol2jac_map(self, names, vois, allprocs_abs2meta_out, mode): name2jinds.append((path, jac_inds[-1])) else: dist_offset = np.sum(sizes[:myproc, var_idx]) - inds.append(np.arange(slc.start, slc.stop, dtype=INT_DTYPE)) + inds.append(range(slc.start, slc.stop) if slcsize > 0 + else np.zeros(0, dtype=INT_DTYPE)) jac_inds.append(np.arange(jstart + dist_offset, jstart + dist_offset + sizes[myproc, var_idx], dtype=INT_DTYPE)) name2jinds.append((path, jac_inds[-1])) else: - idx_array = np.arange(slc.start, slc.stop, dtype=INT_DTYPE) - if indices is not None: - idx_array = idx_array[indices.flat()] - inds.append(idx_array) + if indices is None: + sol_inds = range(slc.start, slc.stop) if slcsize > 0 \ + else np.zeros(0, dtype=INT_DTYPE) + else: + sol_inds = np.arange(slc.start, slc.stop, dtype=INT_DTYPE) + sol_inds = sol_inds[indices.flat()] + inds.append(sol_inds) jac_inds.append(np.arange(jstart, jstart + sz, dtype=INT_DTYPE)) if fwd or not self.get_remote: name2jinds.append((path, jac_inds[-1])) @@ -1344,9 +1347,11 @@ def _jac_setter_dist(self, i, mode): if self.jac_scatters[mode] is not None: self.src_petsc[mode].array = self.J[:, i] self.tgt_petsc[mode].array[:] = self.J[:, i] + # print(f"J[:, {i}] before:", self.J[:, i]) self.jac_scatters[mode].scatter(self.src_petsc[mode], self.tgt_petsc[mode], addv=False, mode=False) self.J[:, i] = self.tgt_petsc[mode].array + # print(f"J[:, {i}] after:", self.J[:, i]) else: # rev if self.get_remote: diff --git a/openmdao/jacobians/dictionary_jacobian.py b/openmdao/jacobians/dictionary_jacobian.py index 9a6677bf87..49d62934a5 100644 --- a/openmdao/jacobians/dictionary_jacobian.py +++ b/openmdao/jacobians/dictionary_jacobian.py @@ -2,7 +2,7 @@ import numpy as np import scipy.sparse as sp -from openmdao.jacobians.jacobian import Jacobian +from openmdao.jacobians.jacobian import Jacobian, _get_remote_vars from openmdao.core.constants import INT_DTYPE @@ -21,6 +21,8 @@ class DictionaryJacobian(Jacobian): ---------- _iter_keys : list of (vname, vname) tuples List of tuples of variable names that match subjacs in the this Jacobian. + _key_owner : dict + Dict mapping subjac keys to the rank where that subjac is local. """ def __init__(self, system, **kwargs): @@ -29,6 +31,7 @@ def __init__(self, system, **kwargs): """ super().__init__(system, **kwargs) self._iter_keys = None + self._key_owner = None def _iter_abs_keys(self, system): """ @@ -47,17 +50,60 @@ def _iter_abs_keys(self, system): List of keys matching this jacobian for the current system. """ if self._iter_keys is None: + include_remotes = system.pathname and \ + system.comm.size > 1 and system._owns_approx_jac and system._subsystems_allprocs subjacs = self._subjacs_info keys = [] - for res_name in system._var_abs2meta['output']: + if include_remotes: + ofnames = system._var_allprocs_abs2meta['output'] + wrtnames = system._var_allprocs_abs2meta + else: + ofnames = system._var_abs2meta['output'] + wrtnames = system._var_abs2meta + + for res_name in ofnames: for type_ in ('output', 'input'): - for name in system._var_abs2meta[type_]: + for name in wrtnames[type_]: key = (res_name, name) if key in subjacs: keys.append(key) self._iter_keys = keys + if include_remotes: + local_out = system._var_abs2meta['output'] + local_in = system._var_abs2meta['input'] + remote_keys = [] + for key in keys: + of, wrt = key + if of not in local_out or (wrt not in local_in and wrt not in local_out): + remote_keys.append(key) + + abs2idx = system._var_allprocs_abs2idx + sizes_out = system._var_sizes['output'] + sizes_in = system._var_sizes['input'] + owner_dict = {} + for keys in system.comm.allgather(remote_keys): + for key in keys: + if key not in owner_dict: + of, wrt = key + ofsizes = sizes_out[:, abs2idx[of]] + if wrt in ofnames: + wrtsizes = sizes_out[:, abs2idx[wrt]] + else: + wrtsizes = sizes_in[:, abs2idx[wrt]] + for rank, (ofsz, wrtsz) in enumerate(zip(ofsizes, wrtsizes)): + # find first rank where both of and wrt are local + if ofsz and wrtsz: + owner_dict[key] = rank + break + else: # no rank was found where both were local... + owner_dict[key] = None + + self._key_owner = owner_dict + else: + self._key_owner = {} + return self._iter_keys def _apply(self, system, d_inputs, d_outputs, d_residuals, mode): @@ -78,8 +124,8 @@ def _apply(self, system, d_inputs, d_outputs, d_residuals, mode): 'fwd' or 'rev'. """ fwd = mode == 'fwd' - d_res_names = d_residuals._names d_out_names = d_outputs._names + d_res_names = d_residuals._names d_inp_names = d_inputs._names if not d_out_names and not d_inp_names: @@ -94,33 +140,27 @@ def _apply(self, system, d_inputs, d_outputs, d_residuals, mode): with system._unscaled_context(outputs=[d_outputs], residuals=[d_residuals]): for abs_key in self._iter_abs_keys(system): res_name, other_name = abs_key - if res_name in d_res_names: - if other_name in d_out_names: + ofvec = rflat(res_name) if res_name in d_res_names else None + + if other_name in d_out_names: + wrtvec = oflat(other_name) + elif other_name in d_inp_names: + wrtvec = iflat(other_name) + else: + wrtvec = None + + if fwd: + if is_explicit and res_name is other_name and wrtvec is not None: # skip the matvec mult completely for identity subjacs - if is_explicit and res_name is other_name: - if fwd: - val = rflat(res_name) - val -= oflat(other_name) - else: - val = oflat(other_name) - val -= rflat(res_name) - continue - if fwd: - left_vec = rflat(res_name) - right_vec = oflat(other_name) - else: - left_vec = oflat(other_name) - right_vec = rflat(res_name) - elif other_name in d_inp_names: - if fwd: - left_vec = rflat(res_name) - right_vec = iflat(other_name) - else: - left_vec = iflat(other_name) - right_vec = rflat(res_name) - else: + ofvec -= wrtvec continue + left_vec = ofvec + right_vec = wrtvec + else: # rev + left_vec = wrtvec + right_vec = ofvec + if left_vec is not None and right_vec is not None: subjac_info = subjacs_info[abs_key] if self._randgen: subjac = self._randomize_subjac(subjac_info['val'], abs_key) @@ -147,7 +187,25 @@ def _apply(self, system, d_inputs, d_outputs, d_residuals, mode): left_vec += subjac.dot(right_vec) else: # rev subjac = subjac.transpose() + # print("subjac (T): ", abs_key, self[abs_key].T) + # print("TIMES") + # print("dresids: ", right_vec) + # print("dinputs BEFORE:", left_vec) left_vec += subjac.dot(right_vec) + # print("dinputs AFTER:", left_vec) + + hasremote = fwd and abs_key in self._key_owner + if hasremote: + if fwd: + owner = self._key_owner[abs_key] + if owner == system.comm.rank: + # print("SENDING", left_vec, "from", owner, abs_key) + system.comm.bcast(left_vec, root=owner) + elif owner is not None: + left_vec = system.comm.bcast(None, root=owner) + if res_name in d_res_names: + d_residuals._abs_set_val(res_name, left_vec) + # print("RECEIVED", left_vec, "from", owner, abs_key) class _CheckingJacobian(DictionaryJacobian): diff --git a/openmdao/jacobians/jacobian.py b/openmdao/jacobians/jacobian.py index 9fc2570479..8c7217e2af 100644 --- a/openmdao/jacobians/jacobian.py +++ b/openmdao/jacobians/jacobian.py @@ -354,7 +354,7 @@ def _setup_index_maps(self, system): if ridxs is not _full_slice or cidxs is not _full_slice: # replace our local subjac with a smaller one but don't # change the subjac belonging to the system (which has values - # shared with subsystems) + # shared with other systems) if self._subjacs_info is system._subjacs_info: self._subjacs_info = system._subjacs_info.copy() meta = self._subjacs_info[key] = meta.copy() @@ -415,7 +415,10 @@ def set_col(self, system, icol, column): if key in self._subjacs_info: subjac = self._subjacs_info[key] if subjac['cols'] is None: # dense - subjac['val'][:, loc_idx] = column[start:end] + try: + subjac['val'][:, loc_idx] = column[start:end] + except Exception as ex: + print(ex) else: # our COO format match_inds = np.nonzero(subjac['cols'] == loc_idx)[0] if match_inds.size > 0: @@ -456,3 +459,65 @@ def _restore_approx_sparsity(self): """ self._subjacs_info = self._system()._subjacs_info self._col_varnames = None # force recompute of internal index maps on next set_col + + def get_wrt_names(self): + """ + Get the list of all wrt names. + + Returns + ------- + list + List of all wrt names. + """ + return self._col_varnames + + def get_of_names(self): + """ + Get the list of all of names. + + Returns + ------- + list + List of all of names. + """ + seen = set() + ofs = [] + for of, _ in self._subjacs_info.keys(): + if of not in seen: + seen.add(of) + ofs.append(of) + + return ofs + + +def _get_remote_vars(system, varnames): + # vnames must be absolute names (no aliases) + + nprocs = system.comm.size + remote_vars = set() + + if nprocs > 1: + # If we have remote vars, pick an owning rank for each and use that + # to bcast to others later + all_abs2meta_out = system._var_allprocs_abs2meta['output'] + all_abs2meta_in = system._var_allprocs_abs2meta['input'] + abs2meta_out = system._var_abs2meta['output'] + abs2meta_in = system._var_abs2meta['input'] + + remote_lst = [n for n in varnames if n not in abs2meta_in and n not in abs2meta_out] + + seen = set() + + for vnames in system.comm.allgather(remote_lst): + for vname in vnames: + if vname not in seen: + seen.add(vname) + if vname in all_abs2meta_out: + dist = all_abs2meta_out[vname]['distributed'] + else: + dist = all_abs2meta_in[vname]['distributed'] + + if not dist: + remote_vars.add(vname) + + return remote_vars diff --git a/openmdao/solvers/nonlinear/newton.py b/openmdao/solvers/nonlinear/newton.py index 161e68058f..1162cf5069 100644 --- a/openmdao/solvers/nonlinear/newton.py +++ b/openmdao/solvers/nonlinear/newton.py @@ -76,7 +76,6 @@ def _setup_solvers(self, system, depth): depth of the current system (already incremented). """ super()._setup_solvers(system, depth) - rank = MPI.COMM_WORLD.rank if MPI is not None else 0 self._disallow_discrete_outputs() @@ -220,35 +219,36 @@ def _single_iteration(self): approx_status = system._owns_approx_jac system._owns_approx_jac = False - system._dresiduals.set_vec(system._residuals) - system._dresiduals *= -1.0 - my_asm_jac = self.linear_solver._assembled_jac + try: + system._dresiduals.set_vec(system._residuals) + system._dresiduals *= -1.0 + my_asm_jac = self.linear_solver._assembled_jac - system._linearize(my_asm_jac, sub_do_ln=do_sub_ln) - if (my_asm_jac is not None and system.linear_solver._assembled_jac is not my_asm_jac): - my_asm_jac._update(system) + system._linearize(my_asm_jac, sub_do_ln=do_sub_ln) + if (my_asm_jac is not None and system.linear_solver._assembled_jac is not my_asm_jac): + my_asm_jac._update(system) - self._linearize() + self._linearize() - self.linear_solver.solve('fwd') + self.linear_solver.solve('fwd') - if self.linesearch and not system.under_complex_step: - self.linesearch._do_subsolve = do_subsolve - self.linesearch.solve() - else: - system._outputs += system._doutputs + if self.linesearch and not system.under_complex_step: + self.linesearch._do_subsolve = do_subsolve + self.linesearch.solve() + else: + system._outputs += system._doutputs - self._solver_info.pop() + self._solver_info.pop() - # Hybrid newton support. - if do_subsolve: - with Recording('Newton_subsolve', 0, self): - self._solver_info.append_solver() - self._gs_iter() - self._solver_info.pop() - - # Enable local fd - system._owns_approx_jac = approx_status + # Hybrid newton support. + if do_subsolve: + with Recording('Newton_subsolve', 0, self): + self._solver_info.append_solver() + self._gs_iter() + self._solver_info.pop() + finally: + # Enable local fd + system._owns_approx_jac = approx_status def _set_complex_step_mode(self, active): """ diff --git a/openmdao/utils/general_utils.py b/openmdao/utils/general_utils.py index 1288ed6199..7817e7d9e0 100644 --- a/openmdao/utils/general_utils.py +++ b/openmdao/utils/general_utils.py @@ -320,6 +320,9 @@ def __contains__(self, name): return True +_contains_all = ContainsAll() + + def all_ancestors(pathname, delim='.'): """ Return a generator of pathnames of the starting object and all of its parents. @@ -1173,10 +1176,12 @@ def wing_dbg(): class LocalRangeIterable(object): """ - Iterable object yielding local indices while iterating over local or distributed vars. + Iterable object yielding local indices while iterating over local, distributed, or remote vars. The number of iterations for a distributed variable will be the full distributed size of the - variable but None will be returned for any indices that are not local to the given rank. + variable. + + None will be returned for any indices that are not local to the given rank. Parameters ---------- @@ -1185,15 +1190,17 @@ class LocalRangeIterable(object): vname : str Name of the variable. use_vec_offset : bool - If True, return indices for the given variable within its vector, else just return + If True, return indices for the given variable within its parent vector, else just return indices within the variable itself, i.e. range(var_size). Attributes ---------- + _vname : str + Name of the variable. _inds : ndarray Variable indices (unused for distributed variables). - _dist_size : int - Full size of distributed variable. + _var_size : int + Full size of distributed or remote variable. _start : int Starting index of distributed variable on this rank. _end : int @@ -1208,18 +1215,21 @@ def __init__(self, system, vname, use_vec_offset=True): """ Initialize the iterator. """ - self._dist_size = 0 + self._vname = vname + self._var_size = 0 - abs2meta = system._var_allprocs_abs2meta['output'] - if vname in abs2meta: + all_abs2meta = system._var_allprocs_abs2meta['output'] + if vname in all_abs2meta: sizes = system._var_sizes['output'] slices = system._outputs.get_slice_dict() + abs2meta = system._var_abs2meta['output'] else: - abs2meta = system._var_allprocs_abs2meta['input'] + all_abs2meta = system._var_allprocs_abs2meta['input'] sizes = system._var_sizes['input'] slices = system._inputs.get_slice_dict() + abs2meta = system._var_abs2meta['input'] - if abs2meta[vname]['distributed']: + if all_abs2meta[vname]['distributed']: var_idx = system._var_allprocs_abs2idx[vname] rank = system.comm.rank self._offset = np.sum(sizes[rank, :var_idx]) if use_vec_offset else 0 @@ -1227,13 +1237,32 @@ def __init__(self, system, vname, use_vec_offset=True): self._iter = self._dist_iter self._start = np.sum(sizes[:rank, var_idx]) self._end = self._start + sizes[rank, var_idx] - self._dist_size = np.sum(sizes[:, var_idx]) + self._var_size = np.sum(sizes[:, var_idx]) + elif vname not in abs2meta: # variable is remote + self._iter = self._remote_iter + self._var_size = all_abs2meta[vname]['global_size'] else: self._iter = self._serial_iter if use_vec_offset: self._inds = range(slices[vname].start, slices[vname].stop) else: self._inds = range(slices[vname].stop - slices[vname].start) + self._var_size = all_abs2meta[vname]['global_size'] + + def __str__(self): + """ + Return a string representation of the iterator. + + Returns + ------- + str + String representation of the iterator. + """ + if self._iter is self._dist_iter: + return f"LocalRangeIterable({self._vname}, dist: {self._start} to {self._end})" + elif self._iter is self._remote_iter: + return f"LocalRangeIterable({self._vname}, remote: size={self._var_size})" + return f"LocalRangeIterable({self._vname}, serial: size={self._var_size})" def _serial_iter(self): """ @@ -1258,12 +1287,24 @@ def _dist_iter(self): start = self._start end = self._end - for i in range(self._dist_size): + for i in range(self._var_size): if i >= start and i < end: yield i - start + self._offset else: yield None + def _remote_iter(self): + """ + Iterate over a remote variable. + + Yields + ------ + None + Always yields None. + """ + for _ in range(self._var_size): + yield None + def __iter__(self): """ Return an iterator. diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index e060b9f2ac..6703614236 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -150,7 +150,72 @@ def _setup_transfers_fwd(group, desvars, responses): def _setup_transfers_rev(group, desvars, responses): abs2meta_in = group._var_abs2meta['input'] abs2meta_out = group._var_abs2meta['output'] + allprocs_abs2idx = group._var_allprocs_abs2idx + allprocs_abs2prom = group._var_allprocs_abs2prom myrank = group.comm.rank + commsize = group.comm.size + + # for an FD group, we use the relevance graph to determine which inputs on the + # boundary of the group are upstream of responses within the group so + # that we can perform any necessary corrections to the derivative inputs. + if commsize > 1 and group._owns_approx_jac and group.pathname != '': + all_abs2meta_out = group._var_allprocs_abs2meta['output'] + all_abs2meta_in = group._var_allprocs_abs2meta['input'] + model = group._problem_meta['model_ref']() + all_conns = model._conn_global_abs_in2out + + # connections internal to this group + conns = group._conn_global_abs_in2out + relevant = group._relevant + + inner_srcs = {src for _, src in conns.items() if src in all_abs2meta_out} + out_boundary_set = {n for n, m in all_abs2meta_out.items() if not m['distributed']} + out_boundary_set = out_boundary_set.difference(inner_srcs) + inp_boundary_set = set(all_abs2meta_in).difference(conns) + + boundary_relevance = {} + for resp, dvdct in relevant.items(): + for dv, tup in dvdct.items(): + rel = tup[0] + rel_boundary_ins = inp_boundary_set.intersection(rel['input']) + for out in out_boundary_set.intersection(rel['output']): + if out not in boundary_relevance: + boundary_relevance[out] = set() + boundary_relevance[out].update(rel_boundary_ins) + + external_srcs = {all_conns[inp] for inp in inp_boundary_set} + + dup_dep_inputs = defaultdict(dict) + + for resp, dvdct in relevant.items(): + if resp in all_abs2meta_out: # resp is continuous and inside this group + is_dist_resp = all_abs2meta_out[resp]['distributed'] + is_dup_resp = False + if not is_dist_resp and resp in allprocs_abs2idx: + ndups = _get_output_dups(group, resp) + is_dup_resp = ndups > 1 + + for dv, tup in dvdct.items(): + # use only dvs outside of this group. + if dv not in allprocs_abs2prom: + rel = tup[0] + if is_dist_resp: + for inp in inp_boundary_set.intersection(rel['input']): + if inp in abs2meta_in: + group._fd_rev_xfer_correction_dist.add(inp) + # elif is_dup_resp: + # rel_boundary_ins = inp_boundary_set.intersection(rel['input']) + # for resinp, nnz in dup_ins.items(): + # if resinp in rel['input']: + # for inp in rel_boundary_ins: + # if inp in abs2meta_in: + # dup_dep_inputs[inp][resp] = nnz + + group._fd_rev_xfer_correction_dup = dup_dep_inputs + + if group._owns_approx_jac: + # FD groups don't need reverse transfers + return {} transfers = group._transfers vectors = group._vectors @@ -166,69 +231,10 @@ def _setup_transfers_rev(group, desvars, responses): xfer_in_nocolor = defaultdict(list) xfer_out_nocolor = defaultdict(list) - allprocs_abs2idx = group._var_allprocs_abs2idx sizes_in = group._var_sizes['input'] offsets_in = offsets['input'] offsets_out = offsets['output'] - # for an FD group, we use the relevance graph to determine which inputs on the - # boundary of the group are upstream of distributed variables within the group so - # that we can perform any necessary allreduce operations on the outputs that - # are connected to those inputs. - if group._owns_approx_jac and group._has_distrib_vars and group.pathname != '': - model = group._problem_meta['model_ref']() - relgraph = model._relevance_graph - group_path = group.pathname + '.' - - inner_resps = [] - outer_dvs = [] - inner_dists = set() - for n, data in relgraph.nodes(data=True): - inside = n.startswith(group_path) - if inside: - if 'isresponse' in data and data['isresponse']: - inner_resps.append(n) - if 'dist' in data and data['dist']: - inner_dists.add(n) - else: # not inside - if 'isdv' in data and data['isdv']: - outer_dvs.append(n) - - if inner_resps and outer_dvs and inner_dists: - # inp_boundary_set is the set of input variables that are connected to sources - # outside of this group. (all group inputs minus group inputs that are connected - # internal to the group) - inp_boundary_set = set(group._var_allprocs_abs2meta['input']) - inp_boundary_set = inp_boundary_set.difference(group._conn_global_abs_in2out) - - # inp_dep_dist is the set of group boundary inputs that are upstream of - # distributed variables and between an external design var and an internal - # response. - inp_dep_dist = set() - - relevant = group._relevant - for resp in inner_resps: - for dv in outer_dvs: - rel = relevant[resp][dv][0] - if inner_dists.intersection(rel['input']) or \ - inner_dists.intersection(rel['output']): - # dist var is in between the dv and response - if resp in inner_dists: - inp_dep_dist.update(inp_boundary_set.intersection(rel['input'])) - - # look in model for the connections to the group boundary inputs from outside - # and update the _fd_subgroup_inputs in the group that owns connections to - # those inputs. - for inp in inp_dep_dist: - src = model._conn_global_abs_in2out[inp] - gname = common_subpath((src, inp)) - owning_group = model._get_subsystem(gname) - owning_group._fd_subgroup_inputs.add(inp) - - if group._owns_approx_jac: - # FD groups don't need reverse transfers - return {} - total_size = total_size_nocolor = 0 # Loop through all connections owned by this system @@ -499,3 +505,15 @@ def _get_output_inds(group, abs_out, abs_in): start = end return output_inds, src_indices + + +def _get_output_dups(group, output): + return np.count_nonzero(group._var_sizes['output'][:, group._var_allprocs_abs2idx[output]]) + + +def _get_comp_inputs(graph, output): + compname = output.rpartition('.')[0] + if compname in graph: + return list(graph.predecessors(compname)) + else: # response will be connected directly to resp component inputs + return list(graph.predecessors(output)) diff --git a/openmdao/vectors/vector.py b/openmdao/vectors/vector.py index 9e456e9807..d8df6dd2b2 100644 --- a/openmdao/vectors/vector.py +++ b/openmdao/vectors/vector.py @@ -226,18 +226,23 @@ def items(self): Yields ------ str - Name of each variable. + Relative name of each variable. ndarray or float Value of each variable. """ + if self._system().pathname: + plen = len(self._system().pathname) + 1 + else: + plen = 0 + if self._under_complex_step: for n, v in self._views.items(): if n in self._names: - yield n, v + yield n[plen:], v else: for n, v in self._views.items(): if n in self._names: - yield n, v.real + yield n[plen:], v.real def _name2abs_name(self, name): """ @@ -397,6 +402,24 @@ def _abs_get_val(self, name, flat=True): else: return self._views[name].real + def _abs_set_val(self, name, val): + """ + Get the variable value using the absolute name. + + No error checking is performed on the name. + + Parameters + ---------- + name : str + Absolute name in the owning system's namespace. + val : float or ndarray + Value to set. + """ + if self._under_complex_step: + self._views[name][:] = val + else: + self._views[name].real[:] = val + def __setitem__(self, name, value): """ Set the variable value. From 89b36e92db4259a624fa486cef816c7889571eb0 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 30 Oct 2023 09:27:04 -0400 Subject: [PATCH 45/70] moved some tests --- openmdao/core/tests/test_approx_derivs.py | 273 ++++++++++++++++++++- openmdao/core/tests/test_distrib_derivs.py | 261 -------------------- 2 files changed, 271 insertions(+), 263 deletions(-) diff --git a/openmdao/core/tests/test_approx_derivs.py b/openmdao/core/tests/test_approx_derivs.py index a2c56bafb2..7fe8f9fe42 100644 --- a/openmdao/core/tests/test_approx_derivs.py +++ b/openmdao/core/tests/test_approx_derivs.py @@ -14,10 +14,10 @@ from openmdao.test_suite.components.unit_conv import SrcComp, TgtCompC, TgtCompF, TgtCompK from openmdao.test_suite.groups.parallel_groups import FanInSubbedIDVC from openmdao.test_suite.parametric_suite import parametric_suite -from openmdao.utils.assert_utils import assert_near_equal, assert_warnings, assert_check_partials, assert_warning +from openmdao.utils.assert_utils import assert_near_equal, assert_check_partials, \ + assert_check_totals, assert_warnings from openmdao.utils.general_utils import set_pyoptsparse_opt from openmdao.utils.mpi import MPI -from openmdao.utils.om_warnings import OMDeprecationWarning from openmdao.utils.testing_utils import use_tempdirs try: @@ -2609,5 +2609,274 @@ def compute_partials(self, inputs, partials): check = prob.check_totals(compact_print=True) +def _setup_inner_par_ivc_direct_conn(size=7): + # one IVC feeds two parallel comps, which feed a third comp. All but the IVC are + # in an FD subgroup. + + prob = om.Problem() + model = prob.model + + model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) + + model.connect('p.x', 'sub.par.C1.x') + model.connect('p.x', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x1') + model.connect('sub.par.C2.y', 'sub.C3.x2') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='fd') + + return prob + +def _setup_inner_par_2ivcs(size=7): + prob = om.Problem() + model = prob.model + + model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) + model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) + + model.connect('p.x', 'sub.par.C1.x') + model.connect('p2.x', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x1') + model.connect('sub.par.C2.y', 'sub.C3.x2') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p2.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='cs') + + return prob + +def _setup_inner_par_ivc_indirect_conn(size=7): + # one IVC feeds an intermediate dup comp, which feeds two parallel comps, which feed a third comp + # inside the FD group. + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + + model.add_subsystem('p', ivc) + model.add_subsystem('dum', om.ExecComp('y = x', shape=size)) + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) + + model.connect('p.x', 'dum.x') + model.connect('dum.y', 'sub.par.C1.x') + model.connect('dum.y', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x1') + model.connect('sub.par.C2.y', 'sub.C3.x2') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='fd') + + return prob + +def _setup_inner_par_ivc_indirect2_conn(size=7): + # one IVC feeds an intermediate dup comp, which feeds two parallel comps, which feed a third comp + # inside the FD group. + + prob = om.Problem() + model = prob.model + + ivc = om.IndepVarComp() + ivc.add_output('x', np.ones((size, ))) + + model.add_subsystem('p', ivc) + model.add_subsystem('dum', om.ExecComp('y = x', shape=size)) + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) + model.add_subsystem('C4', om.ExecComp('y = x', shape=size)) + + model.connect('p.x', 'dum.x') + model.connect('dum.y', 'sub.par.C1.x') + model.connect('dum.y', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x1') + model.connect('sub.par.C2.y', 'sub.C3.x2') + model.connect('sub.C3.y', 'C4.x') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_objective('C4.y', index=-1) + + sub.approx_totals(method='fd') + + return prob + +def _setup_inner_par_2ivc_conn(size=7): + # one IVC feeds an intermediate dup comp, which feeds two parallel comps, which feed a third comp + # inside the FD group, which feeds another comp outside the FD group. + + prob = om.Problem() + model = prob.model + + model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) + model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) + + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) + + model.connect('p.x', 'sub.par.C1.x') + model.connect('p2.x', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x1') + model.connect('sub.par.C2.y', 'sub.C3.x2') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p2.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='fd') + + return prob + + +def _setup2ivc2par2dup(size=7): + # 2 IVCs feed two parallel comps, which feed two duplicated comps + prob = om.Problem() + model = prob.model + + model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) + model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x*.5', shape=size)) + sub.add_subsystem('C4', om.ExecComp('y = x*3.', shape=size)) + + model.connect('p.x', 'sub.par.C1.x') + model.connect('p2.x', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x') + model.connect('sub.par.C2.y', 'sub.C4.x') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p2.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_constraint('sub.C4.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='fd') + + return prob + + +@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") +class TestFDWithParallelSubGroups(unittest.TestCase): + + N_PROCS = 2 + + + def test_group_fd_inner_par_fwd(self): + prob = _setup_inner_par_ivc_direct_conn(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd'), atol=3e-6) + + def test_group_fd_inner_par_rev(self): + prob = _setup_inner_par_ivc_direct_conn(size=7) + prob.setup(mode='rev', force_alloc_complex=True) + prob.run_model() + # assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) + + def test_group_fd_inner_par2_fwd(self): + prob = _setup2ivc2par2dup(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_group_fd_inner_par2_rev(self): + prob = _setup2ivc2par2dup(size=7) + prob.setup(mode='rev', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_group_fd_inner_par2ivcs_fwd(self): + prob = _setup_inner_par_2ivcs(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_group_fd_inner_par2ivcs_rev(self): + prob = _setup_inner_par_2ivcs(size=7) + prob.setup(mode='rev', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd'), atol=3e-6) + + def test_group_fd_inner_par_indirect_fwd(self): + prob = _setup_inner_par_ivc_indirect_conn(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_group_fd_inner_par_indirect_rev(self): + prob = _setup_inner_par_ivc_indirect_conn(size=7) + prob.setup(mode='rev', force_alloc_complex=True) + prob.run_model() + # assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) + + def test_group_inner_par_2ivc_conn_fwd(self): + prob = _setup_inner_par_2ivc_conn(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_group_inner_par_2ivc_conn_rev(self): + prob = _setup_inner_par_2ivc_conn(size=7) + prob.setup(mode='rev', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_group_fd_inner_par_indirect2_fwd(self): + prob = _setup_inner_par_ivc_indirect2_conn(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_group_fd_inner_par_indirect2_rev(self): + prob = _setup_inner_par_ivc_indirect2_conn(size=7) + prob.setup(mode='rev', force_alloc_complex=True) + prob.run_model() + # assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) + + + if __name__ == "__main__": unittest.main() diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index a78e9969e3..56f141f722 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -211,37 +211,6 @@ def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): d_inputs['in_nd'] += dg_dIs * d_outputs['out_nd'] -def _setup2ivc2par2dup(size=7): - # 2 IVCs feed two parallel comps, which feed two duplicated comps - prob = om.Problem() - model = prob.model - - model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) - model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) - sub = model.add_subsystem('sub', om.Group()) - par = sub.add_subsystem('par', om.ParallelGroup()) - par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) - par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) - sub.add_subsystem('C3', om.ExecComp('y = x*.5', shape=size)) - sub.add_subsystem('C4', om.ExecComp('y = x*3.', shape=size)) - - model.connect('p.x', 'sub.par.C1.x') - model.connect('p2.x', 'sub.par.C2.x') - model.connect('sub.par.C1.y', 'sub.C3.x') - model.connect('sub.par.C2.y', 'sub.C4.x') - - model.add_design_var('p.x', lower=-50.0, upper=50.0) - model.add_design_var('p2.x', lower=-50.0, upper=50.0) - model.add_constraint('sub.par.C1.y', lower=0.0) - model.add_constraint('sub.par.C2.y', lower=0.0) - model.add_constraint('sub.C4.y', lower=0.0) - model.add_objective('sub.C3.y', index=-1) - - sub.approx_totals(method='fd') - - return prob - - def _setup_ivc_subivc_dist_parab_sum(): size = 7 @@ -369,161 +338,6 @@ def _setup_ivc_subivcdistparabconssum_in_sub(): return prob -def _setup_inner_par_ivc_direct_conn(size=7): - # one IVC feeds two parallel comps, which feed a third comp. All but the IVC are - # in an FD subgroup. - - prob = om.Problem() - model = prob.model - - model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) - sub = model.add_subsystem('sub', om.Group()) - par = sub.add_subsystem('par', om.ParallelGroup()) - par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) - par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) - sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) - - model.connect('p.x', 'sub.par.C1.x') - model.connect('p.x', 'sub.par.C2.x') - model.connect('sub.par.C1.y', 'sub.C3.x1') - model.connect('sub.par.C2.y', 'sub.C3.x2') - - model.add_design_var('p.x', lower=-50.0, upper=50.0) - model.add_constraint('sub.par.C1.y', lower=0.0) - model.add_constraint('sub.par.C2.y', lower=0.0) - model.add_objective('sub.C3.y', index=-1) - - sub.approx_totals(method='fd') - - return prob - -def _setup_inner_par_2ivcs(size=7): - prob = om.Problem() - model = prob.model - - model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) - model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) - sub = model.add_subsystem('sub', om.Group()) - par = sub.add_subsystem('par', om.ParallelGroup()) - par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) - par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) - sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) - - model.connect('p.x', 'sub.par.C1.x') - model.connect('p2.x', 'sub.par.C2.x') - model.connect('sub.par.C1.y', 'sub.C3.x1') - model.connect('sub.par.C2.y', 'sub.C3.x2') - - model.add_design_var('p.x', lower=-50.0, upper=50.0) - model.add_design_var('p2.x', lower=-50.0, upper=50.0) - model.add_constraint('sub.par.C1.y', lower=0.0) - model.add_constraint('sub.par.C2.y', lower=0.0) - model.add_objective('sub.C3.y', index=-1) - - sub.approx_totals(method='cs') - - return prob - -def _setup_inner_par_ivc_indirect_conn(size=7): - # one IVC feeds an intermediate dup comp, which feeds two parallel comps, which feed a third comp - # inside the FD group. - - prob = om.Problem() - model = prob.model - - ivc = om.IndepVarComp() - ivc.add_output('x', np.ones((size, ))) - - model.add_subsystem('p', ivc) - model.add_subsystem('dum', om.ExecComp('y = x', shape=size)) - sub = model.add_subsystem('sub', om.Group()) - par = sub.add_subsystem('par', om.ParallelGroup()) - par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) - par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) - sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) - - model.connect('p.x', 'dum.x') - model.connect('dum.y', 'sub.par.C1.x') - model.connect('dum.y', 'sub.par.C2.x') - model.connect('sub.par.C1.y', 'sub.C3.x1') - model.connect('sub.par.C2.y', 'sub.C3.x2') - - model.add_design_var('p.x', lower=-50.0, upper=50.0) - model.add_constraint('sub.par.C1.y', lower=0.0) - model.add_constraint('sub.par.C2.y', lower=0.0) - model.add_objective('sub.C3.y', index=-1) - - sub.approx_totals(method='fd') - - return prob - -def _setup_inner_par_ivc_indirect2_conn(size=7): - # one IVC feeds an intermediate dup comp, which feeds two parallel comps, which feed a third comp - # inside the FD group. - - prob = om.Problem() - model = prob.model - - ivc = om.IndepVarComp() - ivc.add_output('x', np.ones((size, ))) - - model.add_subsystem('p', ivc) - model.add_subsystem('dum', om.ExecComp('y = x', shape=size)) - sub = model.add_subsystem('sub', om.Group()) - par = sub.add_subsystem('par', om.ParallelGroup()) - par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) - par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) - sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) - model.add_subsystem('C4', om.ExecComp('y = x', shape=size)) - - model.connect('p.x', 'dum.x') - model.connect('dum.y', 'sub.par.C1.x') - model.connect('dum.y', 'sub.par.C2.x') - model.connect('sub.par.C1.y', 'sub.C3.x1') - model.connect('sub.par.C2.y', 'sub.C3.x2') - model.connect('sub.C3.y', 'C4.x') - - model.add_design_var('p.x', lower=-50.0, upper=50.0) - model.add_constraint('sub.par.C1.y', lower=0.0) - model.add_constraint('sub.par.C2.y', lower=0.0) - model.add_objective('C4.y', index=-1) - - sub.approx_totals(method='fd') - - return prob - -def _setup_inner_par_2ivc_conn(size=7): - # one IVC feeds an intermediate dup comp, which feeds two parallel comps, which feed a third comp - # inside the FD group, which feeds another comp outside the FD group. - - prob = om.Problem() - model = prob.model - - model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) - model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) - - sub = model.add_subsystem('sub', om.Group()) - par = sub.add_subsystem('par', om.ParallelGroup()) - par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) - par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) - sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) - - model.connect('p.x', 'sub.par.C1.x') - model.connect('p2.x', 'sub.par.C2.x') - model.connect('sub.par.C1.y', 'sub.C3.x1') - model.connect('sub.par.C2.y', 'sub.C3.x2') - - model.add_design_var('p.x', lower=-50.0, upper=50.0) - model.add_design_var('p2.x', lower=-50.0, upper=50.0) - model.add_constraint('sub.par.C1.y', lower=0.0) - model.add_constraint('sub.par.C2.y', lower=0.0) - model.add_objective('sub.C3.y', index=-1) - - sub.approx_totals(method='fd') - - return prob - - def _test_func_name(func, num, param): args = [] for p in param.args: @@ -1084,81 +898,6 @@ def test_distrib_voi_group_fd5_rev(self): prob.run_model() assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def test_group_fd_inner_par_fwd(self): - prob = _setup_inner_par_ivc_direct_conn(size=7) - prob.setup(mode='fwd', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd'), atol=3e-6) - - def test_group_fd_inner_par_rev(self): - prob = _setup_inner_par_ivc_direct_conn(size=7) - prob.setup(mode='rev', force_alloc_complex=True) - prob.run_model() - # assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) - - def test_group_fd_inner_par2_fwd(self): - prob = _setup2ivc2par2dup(size=7) - prob.setup(mode='fwd', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - - def test_group_fd_inner_par2_rev(self): - prob = _setup2ivc2par2dup(size=7) - prob.setup(mode='rev', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - - def test_group_fd_inner_par2ivcs_fwd(self): - prob = _setup_inner_par_2ivcs(size=7) - prob.setup(mode='fwd', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - - def test_group_fd_inner_par2ivcs_rev(self): - prob = _setup_inner_par_2ivcs(size=7) - prob.setup(mode='rev', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd'), atol=3e-6) - - def test_group_fd_inner_par_indirect_fwd(self): - prob = _setup_inner_par_ivc_indirect_conn(size=7) - prob.setup(mode='fwd', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - - def test_group_fd_inner_par_indirect_rev(self): - prob = _setup_inner_par_ivc_indirect_conn(size=7) - prob.setup(mode='rev', force_alloc_complex=True) - prob.run_model() - # assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) - - def test_group_inner_par_2ivc_conn_fwd(self): - prob = _setup_inner_par_2ivc_conn(size=7) - prob.setup(mode='fwd', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - - def test_group_inner_par_2ivc_conn_rev(self): - prob = _setup_inner_par_2ivc_conn(size=7) - prob.setup(mode='rev', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - - def test_group_fd_inner_par_indirect2_fwd(self): - prob = _setup_inner_par_ivc_indirect2_conn(size=7) - prob.setup(mode='fwd', force_alloc_complex=True) - prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - - def test_group_fd_inner_par_indirect2_rev(self): - prob = _setup_inner_par_ivc_indirect2_conn(size=7) - prob.setup(mode='rev', force_alloc_complex=True) - prob.run_model() - # assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) - def test_distrib_voi_group_fd_loop(self): # distrib comp is inside of fd group and part of a loop. size = 7 From 96bebb2767dd6e5f23b21a287184828f26fd6d2c Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Tue, 31 Oct 2023 13:49:14 -0400 Subject: [PATCH 46/70] updated some tests --- openmdao/core/tests/test_approx_derivs.py | 259 ++++++++++++++++++---- 1 file changed, 210 insertions(+), 49 deletions(-) diff --git a/openmdao/core/tests/test_approx_derivs.py b/openmdao/core/tests/test_approx_derivs.py index 7fe8f9fe42..45a0377b53 100644 --- a/openmdao/core/tests/test_approx_derivs.py +++ b/openmdao/core/tests/test_approx_derivs.py @@ -2571,13 +2571,12 @@ def compute_partials(self, inputs, J): prob = om.Problem(model=model) prob.setup(force_alloc_complex=True) prob.run_model() - data = prob.check_totals(method='cs', out_stream=None) - assert_near_equal(data[('pg.dc1.y', 'iv.x')]['abs error'][0], 0.0, 1e-6) - assert_near_equal(data[('pg.dc3.y', 'iv.x')]['abs error'][0], 0.0, 1e-6) + assert_check_totals(prob.check_totals(method='cs', out_stream=None)) class CheckTotalsIndices(unittest.TestCase): def test_w_indices(self): + # just checks for indexing error. Doesn't raise exception if derivs are wrong. class TopComp(om.ExplicitComponent): def setup(self): @@ -2597,7 +2596,7 @@ def compute_partials(self, inputs, partials): prob = om.Problem() model = prob.model - geom = model.add_subsystem('tcomp', TopComp()) + model.add_subsystem('tcomp', TopComp()) # setting indices here caused an indexing error later on model.add_design_var('tcomp.theta_c2_C', lower=-20., upper=20., indices=range(2, 9)) @@ -2609,18 +2608,18 @@ def compute_partials(self, inputs, partials): check = prob.check_totals(compact_print=True) -def _setup_inner_par_ivc_direct_conn(size=7): - # one IVC feeds two parallel comps, which feed a third comp. All but the IVC are - # in an FD subgroup. +def _setup_1ivc_fdgroupwithpar_1sink(size=7): prob = om.Problem() model = prob.model model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) + sub = model.add_subsystem('sub', om.Group()) par = sub.add_subsystem('par', om.ParallelGroup()) par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) model.connect('p.x', 'sub.par.C1.x') @@ -2637,12 +2636,13 @@ def _setup_inner_par_ivc_direct_conn(size=7): return prob -def _setup_inner_par_2ivcs(size=7): +def _setup_2ivcs_fdgroupwithpar_1sink(size=7): prob = om.Problem() model = prob.model model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) + sub = model.add_subsystem('sub', om.Group()) par = sub.add_subsystem('par', om.ParallelGroup()) par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) @@ -2660,26 +2660,24 @@ def _setup_inner_par_2ivcs(size=7): model.add_constraint('sub.par.C2.y', lower=0.0) model.add_objective('sub.C3.y', index=-1) - sub.approx_totals(method='cs') + sub.approx_totals(method='fd') return prob -def _setup_inner_par_ivc_indirect_conn(size=7): - # one IVC feeds an intermediate dup comp, which feeds two parallel comps, which feed a third comp - # inside the FD group. +def _setup_1ivc_dum_fdgroupwithpar_1sink(size=7): prob = om.Problem() model = prob.model - ivc = om.IndepVarComp() - ivc.add_output('x', np.ones((size, ))) + model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) - model.add_subsystem('p', ivc) model.add_subsystem('dum', om.ExecComp('y = x', shape=size)) + sub = model.add_subsystem('sub', om.Group()) par = sub.add_subsystem('par', om.ParallelGroup()) par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) model.connect('p.x', 'dum.x') @@ -2697,9 +2695,7 @@ def _setup_inner_par_ivc_indirect_conn(size=7): return prob -def _setup_inner_par_ivc_indirect2_conn(size=7): - # one IVC feeds an intermediate dup comp, which feeds two parallel comps, which feed a third comp - # inside the FD group. +def _setup_1ivc_dum_fdgroupwithpar_1sink_c4(size=7): prob = om.Problem() model = prob.model @@ -2732,10 +2728,8 @@ def _setup_inner_par_ivc_indirect2_conn(size=7): return prob -def _setup_inner_par_2ivc_conn(size=7): - # one IVC feeds an intermediate dup comp, which feeds two parallel comps, which feed a third comp - # inside the FD group, which feeds another comp outside the FD group. +def _setup_2ivcs_fdgroupwithpar_2sinks(size=7): prob = om.Problem() model = prob.model @@ -2746,17 +2740,98 @@ def _setup_inner_par_2ivc_conn(size=7): par = sub.add_subsystem('par', om.ParallelGroup()) par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + + sub.add_subsystem('C3', om.ExecComp('y = x*.5', shape=size)) + sub.add_subsystem('C4', om.ExecComp('y = x*3.', shape=size)) + + model.connect('p.x', 'sub.par.C1.x') + model.connect('p2.x', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.C3.x') + model.connect('sub.par.C2.y', 'sub.C4.x') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p2.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_constraint('sub.C4.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='fd') + + return prob + + +def _setup_2ivcs_fdgroupwith2pars_2sinks(size=7): + prob = om.Problem() + model = prob.model + + model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) + model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) + + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + + par2 = sub.add_subsystem('par2', om.ParallelGroup()) + par2.add_subsystem('C1a', om.ExecComp('y = 2.*x', shape=size)) + par2.add_subsystem('C2a', om.ExecComp('y = 3.*x', shape=size)) + + sub.add_subsystem('C3', om.ExecComp('y = x*.5', shape=size)) + sub.add_subsystem('C4', om.ExecComp('y = x*3.', shape=size)) + + model.connect('p.x', 'sub.par.C1.x') + model.connect('p2.x', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.par2.C1a.x') + model.connect('sub.par.C2.y', 'sub.par2.C2a.x') + model.connect('sub.par2.C1a.y', 'sub.C3.x') + model.connect('sub.par2.C2a.y', 'sub.C4.x') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p2.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_constraint('sub.par2.C1a.y', lower=0.0) + model.add_constraint('sub.par2.C2a.y', lower=0.0) + model.add_constraint('sub.C4.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='fd') + + return prob + + +def _setup_2ivcs_fdgroupwith2pars_1sink(size=7): + prob = om.Problem() + model = prob.model + + model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) + model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) + + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + + par2 = sub.add_subsystem('par2', om.ParallelGroup()) + par2.add_subsystem('C1a', om.ExecComp('y = 2.*x', shape=size)) + par2.add_subsystem('C2a', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) model.connect('p.x', 'sub.par.C1.x') model.connect('p2.x', 'sub.par.C2.x') - model.connect('sub.par.C1.y', 'sub.C3.x1') - model.connect('sub.par.C2.y', 'sub.C3.x2') + model.connect('sub.par.C1.y', 'sub.par2.C1a.x') + model.connect('sub.par.C2.y', 'sub.par2.C2a.x') + model.connect('sub.par2.C1a.y', 'sub.C3.x1') + model.connect('sub.par2.C2a.y', 'sub.C3.x2') model.add_design_var('p.x', lower=-50.0, upper=50.0) model.add_design_var('p2.x', lower=-50.0, upper=50.0) model.add_constraint('sub.par.C1.y', lower=0.0) model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_constraint('sub.par2.C1a.y', lower=0.0) + model.add_constraint('sub.par2.C2a.y', lower=0.0) model.add_objective('sub.C3.y', index=-1) sub.approx_totals(method='fd') @@ -2764,29 +2839,38 @@ def _setup_inner_par_2ivc_conn(size=7): return prob -def _setup2ivc2par2dup(size=7): - # 2 IVCs feed two parallel comps, which feed two duplicated comps +def _setup_2ivcs_fdgroupwithcrisscrosspars_2sinks(size=7): prob = om.Problem() model = prob.model model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) + sub = model.add_subsystem('sub', om.Group()) par = sub.add_subsystem('par', om.ParallelGroup()) par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + + par2 = sub.add_subsystem('par2', om.ParallelGroup()) + par2.add_subsystem('C1a', om.ExecComp('y = 2.*x', shape=size)) + par2.add_subsystem('C2a', om.ExecComp('y = 3.*x', shape=size)) + sub.add_subsystem('C3', om.ExecComp('y = x*.5', shape=size)) sub.add_subsystem('C4', om.ExecComp('y = x*3.', shape=size)) model.connect('p.x', 'sub.par.C1.x') model.connect('p2.x', 'sub.par.C2.x') - model.connect('sub.par.C1.y', 'sub.C3.x') - model.connect('sub.par.C2.y', 'sub.C4.x') + model.connect('sub.par.C1.y', 'sub.par2.C2a.x') + model.connect('sub.par.C2.y', 'sub.par2.C1a.x') + model.connect('sub.par2.C1a.y', 'sub.C3.x') + model.connect('sub.par2.C2a.y', 'sub.C4.x') model.add_design_var('p.x', lower=-50.0, upper=50.0) model.add_design_var('p2.x', lower=-50.0, upper=50.0) model.add_constraint('sub.par.C1.y', lower=0.0) model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_constraint('sub.par2.C1a.y', lower=0.0) + model.add_constraint('sub.par2.C2a.y', lower=0.0) model.add_constraint('sub.C4.y', lower=0.0) model.add_objective('sub.C3.y', index=-1) @@ -2794,89 +2878,166 @@ def _setup2ivc2par2dup(size=7): return prob +def _setup_2ivcs_fdgroupwithcrisscrosspars_1sink(size=7): + prob = om.Problem() + model = prob.model + + model.add_subsystem('p', om.IndepVarComp('x', np.ones((size, )))) + model.add_subsystem('p2', om.IndepVarComp('x', np.ones((size, )))) + + sub = model.add_subsystem('sub', om.Group()) + par = sub.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('C1', om.ExecComp('y = 2.*x', shape=size)) + par.add_subsystem('C2', om.ExecComp('y = 3.*x', shape=size)) + + par2 = sub.add_subsystem('par2', om.ParallelGroup()) + par2.add_subsystem('C1a', om.ExecComp('y = 2.*x', shape=size)) + par2.add_subsystem('C2a', om.ExecComp('y = 3.*x', shape=size)) + + sub.add_subsystem('C3', om.ExecComp('y = x1 + x2', shape=size)) + + model.connect('p.x', 'sub.par.C1.x') + model.connect('p2.x', 'sub.par.C2.x') + model.connect('sub.par.C1.y', 'sub.par2.C2a.x') + model.connect('sub.par.C2.y', 'sub.par2.C1a.x') + model.connect('sub.par2.C1a.y', 'sub.C3.x1') + model.connect('sub.par2.C2a.y', 'sub.C3.x2') + + model.add_design_var('p.x', lower=-50.0, upper=50.0) + model.add_design_var('p2.x', lower=-50.0, upper=50.0) + model.add_constraint('sub.par.C1.y', lower=0.0) + model.add_constraint('sub.par.C2.y', lower=0.0) + model.add_constraint('sub.par2.C1a.y', lower=0.0) + model.add_constraint('sub.par2.C2a.y', lower=0.0) + model.add_objective('sub.C3.y', index=-1) + + sub.approx_totals(method='fd') + + return prob + @unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") class TestFDWithParallelSubGroups(unittest.TestCase): N_PROCS = 2 - def test_group_fd_inner_par_fwd(self): - prob = _setup_inner_par_ivc_direct_conn(size=7) + prob = _setup_1ivc_fdgroupwithpar_1sink(size=7) prob.setup(mode='fwd', force_alloc_complex=True) prob.run_model() assert_check_totals(prob.check_totals(method='fd'), atol=3e-6) def test_group_fd_inner_par_rev(self): - prob = _setup_inner_par_ivc_direct_conn(size=7) + prob = _setup_1ivc_fdgroupwithpar_1sink(size=7) prob.setup(mode='rev', force_alloc_complex=True) prob.run_model() # assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) def test_group_fd_inner_par2_fwd(self): - prob = _setup2ivc2par2dup(size=7) + prob = _setup_2ivcs_fdgroupwithpar_2sinks(size=7) prob.setup(mode='fwd', force_alloc_complex=True) prob.run_model() assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) def test_group_fd_inner_par2_rev(self): - prob = _setup2ivc2par2dup(size=7) + prob = _setup_2ivcs_fdgroupwithpar_2sinks(size=7) prob.setup(mode='rev', force_alloc_complex=True) prob.run_model() assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def test_group_fd_inner_par2ivcs_fwd(self): - prob = _setup_inner_par_2ivcs(size=7) + def test_2ivcs_fdgroupwithcrisscrosspars_2sinks_fwd(self): + prob = _setup_2ivcs_fdgroupwithcrisscrosspars_2sinks(size=7) prob.setup(mode='fwd', force_alloc_complex=True) prob.run_model() assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def test_group_fd_inner_par2ivcs_rev(self): - prob = _setup_inner_par_2ivcs(size=7) + def test_2ivcs_fdgroupwithcrisscrosspars_2sinks_rev(self): + prob = _setup_2ivcs_fdgroupwithcrisscrosspars_2sinks(size=7) prob.setup(mode='rev', force_alloc_complex=True) prob.run_model() - assert_check_totals(prob.check_totals(method='fd'), atol=3e-6) + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def test_group_fd_inner_par_indirect_fwd(self): - prob = _setup_inner_par_ivc_indirect_conn(size=7) + def test_2ivcs_fdgroupwith2pars_2sinks_fwd(self): + prob = _setup_2ivcs_fdgroupwith2pars_2sinks(size=7) prob.setup(mode='fwd', force_alloc_complex=True) prob.run_model() assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def test_group_fd_inner_par_indirect_rev(self): - prob = _setup_inner_par_ivc_indirect_conn(size=7) + def test_2ivcs_fdgroupwith2pars_2sinks_rev(self): + prob = _setup_2ivcs_fdgroupwith2pars_2sinks(size=7) prob.setup(mode='rev', force_alloc_complex=True) prob.run_model() - # assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def test_group_inner_par_2ivc_conn_fwd(self): - prob = _setup_inner_par_2ivc_conn(size=7) + def test_2ivcs_fdgroupwithcrisscrosspars_1sink_fwd(self): + prob = _setup_2ivcs_fdgroupwithcrisscrosspars_1sink(size=7) prob.setup(mode='fwd', force_alloc_complex=True) prob.run_model() assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) - def test_group_inner_par_2ivc_conn_rev(self): - prob = _setup_inner_par_2ivc_conn(size=7) + def test_2ivcs_fdgroupwithcrisscrosspars_1sink_rev(self): + prob = _setup_2ivcs_fdgroupwithcrisscrosspars_1sink(size=7) prob.setup(mode='rev', force_alloc_complex=True) prob.run_model() assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + def test_2ivcs_fdgroupwith2pars_1sink_fwd(self): + prob = _setup_2ivcs_fdgroupwith2pars_1sink(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_2ivcs_fdgroupwith2pars_1sink_rev(self): + prob = _setup_2ivcs_fdgroupwith2pars_1sink(size=7) + prob.setup(mode='rev', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_group_fd_inner_par2ivcs_fwd(self): + prob = _setup_2ivcs_fdgroupwithpar_1sink(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_group_fd_inner_par2ivcs_rev(self): + prob = _setup_2ivcs_fdgroupwithpar_1sink(size=7) + prob.setup(mode='rev', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd'), atol=3e-6) + + def test_group_fd_inner_par_indirect_fwd(self): + prob = _setup_1ivc_dum_fdgroupwithpar_1sink(size=7) + prob.setup(mode='fwd', force_alloc_complex=True) + prob.run_model() + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + + def test_group_fd_inner_par_indirect_rev(self): + prob = _setup_1ivc_dum_fdgroupwithpar_1sink(size=7) + prob.setup(mode='rev', force_alloc_complex=True) + prob.run_model() + # assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) + assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) + def test_group_fd_inner_par_indirect2_fwd(self): - prob = _setup_inner_par_ivc_indirect2_conn(size=7) + prob = _setup_1ivc_dum_fdgroupwithpar_1sink_c4(size=7) prob.setup(mode='fwd', force_alloc_complex=True) prob.run_model() assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) def test_group_fd_inner_par_indirect2_rev(self): - prob = _setup_inner_par_ivc_indirect2_conn(size=7) + prob = _setup_1ivc_dum_fdgroupwithpar_1sink_c4(size=7) prob.setup(mode='rev', force_alloc_complex=True) prob.run_model() # assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) +@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") +class TestFDWithParallelSubGroups4(TestFDWithParallelSubGroups): + + N_PROCS = 4 + if __name__ == "__main__": unittest.main() From dbeb33fde0ea10872cfed699dcc3897fdc7f4fe4 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Wed, 1 Nov 2023 13:43:02 -0400 Subject: [PATCH 47/70] passing --- openmdao/core/group.py | 43 ++++++++++++++++------- openmdao/core/tests/test_approx_derivs.py | 9 +++-- openmdao/jacobians/dictionary_jacobian.py | 25 +++++++++---- openmdao/vectors/petsc_transfer.py | 32 ----------------- 4 files changed, 55 insertions(+), 54 deletions(-) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 64db9cb8c1..a320c02f00 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -197,10 +197,6 @@ class Group(System): this is the set of inputs to those subgroups that are upstream of a distributed response within the same subgroup. These determine if an allreduce is necessary when transferring data to a connected output in reverse mode. - _fd_rev_xfer_correction_dup : dict - If one or more subgroups of this group is using finite difference to compute derivatives, - this is the set of inputs to those subgroups that are upstream of a duplicated response - within the same subgroup. """ def __init__(self, **kwargs): @@ -232,7 +228,6 @@ def __init__(self, **kwargs): self._abs_responses = None self._relevance_graph = None self._fd_rev_xfer_correction_dist = set() - self._fd_rev_xfer_correction_dup = {} # TODO: we cannot set the solvers with property setters at the moment # because our lint check thinks that we are defining new attributes @@ -1326,7 +1321,6 @@ def _final_setup(self, comm, mode): self._setup_partials() self._fd_rev_xfer_correction_dist = set() - self._fd_rev_xfer_correction_dup = {} self._problem_meta['relevant'] = self._init_relevance(mode) @@ -3794,7 +3788,7 @@ def _apply_linear(self, jac, rel_systems, mode, scope_out=None, scope_in=None): # _fd_rev_xfer_correction_dist is used to correct for the fact that we don't # do reverse transfers internal to an FD group. Reverse transfers # are constructed such that derivative values are correct when transferred into - # system output variables, taking into account distributed inputs. + # system doutput variables, taking into account distributed inputs. # Since the transfers are not correcting for those issues, we need to do it here. # If we have a distributed constraint/obj within the FD group, @@ -4104,12 +4098,13 @@ def _approx_subjac_keys_iter(self): of -= ivc for key in product(of, wrt.union(of)): + _of, _wrt = key + # Create approximations for the ones we need. if self._tot_jac is not None: yield key # get all combos if we're doing total derivs continue - _of, _wrt = key # Skip explicit res wrt outputs if _wrt in of and _wrt not in ivc: @@ -4306,6 +4301,9 @@ def _setup_approx_partials(self): abs2meta = self._var_allprocs_abs2meta info = self._coloring_info + total = self.pathname == '' + nprocs = self.comm.size + responses = self.get_responses(recurse=True, get_sizes=False, use_prom_ivc=False) if info['coloring'] is not None and (self._owns_approx_of is None or @@ -4321,13 +4319,32 @@ def _setup_approx_partials(self): approx._wrt_meta = {} approx._reset() + sizes_out = self._var_sizes['output'] + sizes_in = self._var_sizes['input'] + abs2idx = self._var_allprocs_abs2idx + + self._cross_keys = set() approx_keys = self._get_approx_subjac_keys() for key in approx_keys: left, right = key - if left in responses and responses[left]['alias'] is not None: - left = responses[left]['source'] - if right in responses and responses[right]['alias'] is not None: - right = responses[right]['source'] + if total: + if left in responses and responses[left]['alias'] is not None: + left = responses[left]['source'] + if right in responses and responses[right]['alias'] is not None: + right = responses[right]['source'] + elif nprocs > 1: + sout = sizes_out[:, abs2idx[left]] + sin = sizes_in[:, abs2idx[right]] + if np.count_nonzero(sout[sin == 0]) > 0 and np.count_nonzero(sin[sout == 0]) > 0: + # we have of and wrt that exist on different procs. Now see if they're relevant + # to each other + for rel in self._relevant.values(): + relins = rel['@all'][0]['input'] + relouts = rel['@all'][0]['output'] + if left in relouts: + if right in relins or right in relouts: + self._cross_keys.add(key) + break if key in self._subjacs_info: meta = self._subjacs_info[key] @@ -4363,7 +4380,7 @@ def _setup_approx_partials(self): approx.add_approximation(key, self, meta) - if self.pathname: + if not total: abs_outs = self._var_allprocs_abs2meta['output'] abs_ins = self._var_allprocs_abs2meta['input'] # we're taking semi-total derivs for this group. Update _owns_approx_of diff --git a/openmdao/core/tests/test_approx_derivs.py b/openmdao/core/tests/test_approx_derivs.py index 45a0377b53..3810df416c 100644 --- a/openmdao/core/tests/test_approx_derivs.py +++ b/openmdao/core/tests/test_approx_derivs.py @@ -2974,6 +2974,9 @@ def test_2ivcs_fdgroupwithcrisscrosspars_1sink_fwd(self): prob = _setup_2ivcs_fdgroupwithcrisscrosspars_1sink(size=7) prob.setup(mode='fwd', force_alloc_complex=True) prob.run_model() + prob.compute_totals() + import pprint + pprint.pprint({n: m['val'] for n,m in prob.model.sub._jacobian._subjacs_info.items()}) assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=3e-6) def test_2ivcs_fdgroupwithcrisscrosspars_1sink_rev(self): @@ -3033,10 +3036,10 @@ def test_group_fd_inner_par_indirect2_rev(self): assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) -@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") -class TestFDWithParallelSubGroups4(TestFDWithParallelSubGroups): +# @unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") +# class TestFDWithParallelSubGroups4(TestFDWithParallelSubGroups): - N_PROCS = 4 +# N_PROCS = 4 if __name__ == "__main__": diff --git a/openmdao/jacobians/dictionary_jacobian.py b/openmdao/jacobians/dictionary_jacobian.py index 110eb23558..1237b844ed 100644 --- a/openmdao/jacobians/dictionary_jacobian.py +++ b/openmdao/jacobians/dictionary_jacobian.py @@ -97,8 +97,8 @@ def _iter_abs_keys(self, system): if ofsz and wrtsz: owner_dict[key] = rank break - else: # no rank was found where both were local... - owner_dict[key] = None + else: # no rank was found where both were local. Use 'of' local rank + owner_dict[key] = np.min(np.nonzero(ofsizes)[0]) self._key_owner = owner_dict else: @@ -161,6 +161,13 @@ def _apply(self, system, d_inputs, d_outputs, d_residuals, mode): left_vec = wrtvec right_vec = ofvec + if abs_key in self._key_owner and abs_key in system._cross_keys: + wrtowner = system._owning_rank[other_name] + if system.comm.rank == wrtowner: + system.comm.bcast(right_vec, root=wrtowner) + else: + right_vec = system.comm.bcast(None, root=wrtowner) + if left_vec is not None and right_vec is not None: subjac_info = subjacs_info[abs_key] if randgen: @@ -195,17 +202,23 @@ def _apply(self, system, d_inputs, d_outputs, d_residuals, mode): left_vec += subjac.dot(right_vec) # print("dinputs AFTER:", left_vec) - hasremote = fwd and abs_key in self._key_owner + hasremote = abs_key in self._key_owner if hasremote: - if fwd: + if True: # fwd: owner = self._key_owner[abs_key] if owner == system.comm.rank: # print("SENDING", left_vec, "from", owner, abs_key) system.comm.bcast(left_vec, root=owner) elif owner is not None: left_vec = system.comm.bcast(None, root=owner) - if res_name in d_res_names: - d_residuals._abs_set_val(res_name, left_vec) + if fwd: + if res_name in d_res_names: + d_residuals._abs_set_val(res_name, left_vec) + else: # rev + if other_name in d_out_names: + d_outputs._abs_set_val(other_name, left_vec) + elif other_name in d_inp_names: + d_inputs._abs_set_val(other_name, left_vec) # print("RECEIVED", left_vec, "from", owner, abs_key) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 6703614236..28f4f06282 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -161,39 +161,16 @@ def _setup_transfers_rev(group, desvars, responses): if commsize > 1 and group._owns_approx_jac and group.pathname != '': all_abs2meta_out = group._var_allprocs_abs2meta['output'] all_abs2meta_in = group._var_allprocs_abs2meta['input'] - model = group._problem_meta['model_ref']() - all_conns = model._conn_global_abs_in2out # connections internal to this group conns = group._conn_global_abs_in2out relevant = group._relevant - inner_srcs = {src for _, src in conns.items() if src in all_abs2meta_out} - out_boundary_set = {n for n, m in all_abs2meta_out.items() if not m['distributed']} - out_boundary_set = out_boundary_set.difference(inner_srcs) inp_boundary_set = set(all_abs2meta_in).difference(conns) - boundary_relevance = {} - for resp, dvdct in relevant.items(): - for dv, tup in dvdct.items(): - rel = tup[0] - rel_boundary_ins = inp_boundary_set.intersection(rel['input']) - for out in out_boundary_set.intersection(rel['output']): - if out not in boundary_relevance: - boundary_relevance[out] = set() - boundary_relevance[out].update(rel_boundary_ins) - - external_srcs = {all_conns[inp] for inp in inp_boundary_set} - - dup_dep_inputs = defaultdict(dict) - for resp, dvdct in relevant.items(): if resp in all_abs2meta_out: # resp is continuous and inside this group is_dist_resp = all_abs2meta_out[resp]['distributed'] - is_dup_resp = False - if not is_dist_resp and resp in allprocs_abs2idx: - ndups = _get_output_dups(group, resp) - is_dup_resp = ndups > 1 for dv, tup in dvdct.items(): # use only dvs outside of this group. @@ -203,15 +180,6 @@ def _setup_transfers_rev(group, desvars, responses): for inp in inp_boundary_set.intersection(rel['input']): if inp in abs2meta_in: group._fd_rev_xfer_correction_dist.add(inp) - # elif is_dup_resp: - # rel_boundary_ins = inp_boundary_set.intersection(rel['input']) - # for resinp, nnz in dup_ins.items(): - # if resinp in rel['input']: - # for inp in rel_boundary_ins: - # if inp in abs2meta_in: - # dup_dep_inputs[inp][resp] = nnz - - group._fd_rev_xfer_correction_dup = dup_dep_inputs if group._owns_approx_jac: # FD groups don't need reverse transfers From 67724ded1d549a6f6d9e9a0dbee31f1d95195787 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Wed, 1 Nov 2023 13:58:46 -0400 Subject: [PATCH 48/70] cleanup --- openmdao/vectors/petsc_transfer.py | 43 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 28f4f06282..fb9291b187 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -150,41 +150,40 @@ def _setup_transfers_fwd(group, desvars, responses): def _setup_transfers_rev(group, desvars, responses): abs2meta_in = group._var_abs2meta['input'] abs2meta_out = group._var_abs2meta['output'] - allprocs_abs2idx = group._var_allprocs_abs2idx allprocs_abs2prom = group._var_allprocs_abs2prom - myrank = group.comm.rank - commsize = group.comm.size # for an FD group, we use the relevance graph to determine which inputs on the # boundary of the group are upstream of responses within the group so # that we can perform any necessary corrections to the derivative inputs. - if commsize > 1 and group._owns_approx_jac and group.pathname != '': - all_abs2meta_out = group._var_allprocs_abs2meta['output'] - all_abs2meta_in = group._var_allprocs_abs2meta['input'] + if group._owns_approx_jac: + if group.comm.size > 1 and group.pathname != '': + all_abs2meta_out = group._var_allprocs_abs2meta['output'] + all_abs2meta_in = group._var_allprocs_abs2meta['input'] - # connections internal to this group - conns = group._conn_global_abs_in2out - relevant = group._relevant + # connections internal to this group + conns = group._conn_global_abs_in2out + relevant = group._relevant - inp_boundary_set = set(all_abs2meta_in).difference(conns) + inp_boundary_set = set(all_abs2meta_in).difference(conns) - for resp, dvdct in relevant.items(): - if resp in all_abs2meta_out: # resp is continuous and inside this group - is_dist_resp = all_abs2meta_out[resp]['distributed'] + for resp, dvdct in relevant.items(): + if resp in all_abs2meta_out: # resp is continuous and inside this group + is_dist_resp = all_abs2meta_out[resp]['distributed'] - for dv, tup in dvdct.items(): - # use only dvs outside of this group. - if dv not in allprocs_abs2prom: - rel = tup[0] - if is_dist_resp: - for inp in inp_boundary_set.intersection(rel['input']): - if inp in abs2meta_in: - group._fd_rev_xfer_correction_dist.add(inp) + for dv, tup in dvdct.items(): + # use only dvs outside of this group. + if dv not in allprocs_abs2prom: + rel = tup[0] + if is_dist_resp: + for inp in inp_boundary_set.intersection(rel['input']): + if inp in abs2meta_in: + group._fd_rev_xfer_correction_dist.add(inp) - if group._owns_approx_jac: # FD groups don't need reverse transfers return {} + myrank = group.comm.rank + allprocs_abs2idx = group._var_allprocs_abs2idx transfers = group._transfers vectors = group._vectors offsets = group._get_var_offsets() From 5f20fe72990998299f6adc8d726db9a9488b6731 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Thu, 2 Nov 2023 14:09:19 -0400 Subject: [PATCH 49/70] updating transfer docs --- .../approximation_scheme.py | 9 ---- openmdao/core/explicitcomponent.py | 1 - openmdao/core/group.py | 26 +++++------- openmdao/core/problem.py | 3 -- openmdao/core/system.py | 3 +- openmdao/core/tests/test_approx_derivs.py | 6 +-- openmdao/core/tests/test_check_totals.py | 6 +-- openmdao/core/tests/test_distrib_derivs.py | 31 ++------------ openmdao/core/tests/test_driver.py | 15 +++---- openmdao/core/tests/test_mpi_coloring_bug.py | 26 ++++++++++-- .../core/tests/test_parallel_derivatives.py | 3 -- .../distributed_components.ipynb | 2 +- .../theory_manual/images/Par1.png | Bin 20202 -> 12590 bytes .../theory_manual/images/Par2.png | Bin 90636 -> 0 bytes .../theory_manual/images/Par3.png | Bin 33097 -> 0 bytes .../theory_manual/images/Par4.png | Bin 99969 -> 0 bytes .../theory_manual/images/nonpar_fwd.png | Bin 0 -> 19030 bytes .../theory_manual/images/nonpar_rev.png | Bin 0 -> 19194 bytes .../theory_manual/images/par_fwd.png | Bin 0 -> 34071 bytes .../theory_manual/images/par_rev.png | Bin 0 -> 34600 bytes .../openmdao_book/theory_manual/mpi.ipynb | 28 ++++++++----- openmdao/jacobians/dictionary_jacobian.py | 39 +++++++----------- 22 files changed, 82 insertions(+), 116 deletions(-) delete mode 100644 openmdao/docs/openmdao_book/theory_manual/images/Par2.png delete mode 100644 openmdao/docs/openmdao_book/theory_manual/images/Par3.png delete mode 100644 openmdao/docs/openmdao_book/theory_manual/images/Par4.png create mode 100644 openmdao/docs/openmdao_book/theory_manual/images/nonpar_fwd.png create mode 100644 openmdao/docs/openmdao_book/theory_manual/images/nonpar_rev.png create mode 100644 openmdao/docs/openmdao_book/theory_manual/images/par_fwd.png create mode 100644 openmdao/docs/openmdao_book/theory_manual/images/par_rev.png diff --git a/openmdao/approximation_schemes/approximation_scheme.py b/openmdao/approximation_schemes/approximation_scheme.py index e4832e8f7a..65ea042d49 100644 --- a/openmdao/approximation_schemes/approximation_scheme.py +++ b/openmdao/approximation_schemes/approximation_scheme.py @@ -272,15 +272,6 @@ def _init_approximations(self, system): in_idx = [list(in_idx)] vec_idx = [vec_idx] else: - # if vec is None: # remote wrt - # if wrt in abs2meta['input']: - # vec_idx = range(abs2meta['input'][wrt]['global_size']) - # else: - # vec_idx = range(abs2meta['output'][wrt]['global_size']) - # else: - # vec_idx = LocalRangeIterable(system, wrt) - # if directional: - # vec_idx = [v for v in vec_idx if v is not None] vec_idx = LocalRangeIterable(system, wrt) if directional and vec is not None: vec_idx = [v for v in vec_idx if v is not None] diff --git a/openmdao/core/explicitcomponent.py b/openmdao/core/explicitcomponent.py index f5cfc687c1..f6b6a1b3ee 100644 --- a/openmdao/core/explicitcomponent.py +++ b/openmdao/core/explicitcomponent.py @@ -8,7 +8,6 @@ from openmdao.utils.class_util import overrides_method from openmdao.recorders.recording_iteration_stack import Recording from openmdao.core.constants import INT_DTYPE, _UNDEFINED -from openmdao.utils.mpi import MPI class ExplicitComponent(Component): diff --git a/openmdao/core/group.py b/openmdao/core/group.py index a320c02f00..d5375510e3 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -885,6 +885,10 @@ def get_hybrid_graph(self, connections): those variables are connected to other variables based on the connections in the model. + This results in a smaller graph (fewer edges) than would be the case for a pure variable + graph where all inputs to a particular component would have to be connected to all outputs + from that component. + Parameters ---------- connections : dict @@ -895,10 +899,6 @@ def get_hybrid_graph(self, connections): networkx.DiGraph Graph of all variables and components in the model. """ - # Create a hybrid graph with components and all connected vars. If a var is connected, - # also connect it to its corresponding component. This results in a smaller graph - # (fewer edges) than would be the case for a pure variable graph where all inputs - # to a particular component would have to be connected to all outputs from that component. graph = nx.DiGraph() tgtmeta = self._var_allprocs_abs2meta['input'] srcmeta = self._var_allprocs_abs2meta['output'] @@ -913,9 +913,11 @@ def get_hybrid_graph(self, connections): graph.add_node(tgt, type_='in', dist=dist, isdv=False, isresponse=False) + # create edges to/from the component graph.add_edge(src.rpartition('.')[0], src) graph.add_edge(tgt, tgt.rpartition('.')[0]) + # connect the variables src and tgt graph.add_edge(src, tgt) return graph @@ -1584,12 +1586,12 @@ def _set_auto_order(self, strongcomps, orders): def _check_nondist_sizes(self): # verify that nondistributed variables have same size across all procs + abs2idx = self._var_allprocs_abs2idx for io in ('input', 'output'): sizes = self._var_sizes[io] - idxs = self._var_allprocs_abs2idx for abs_name, meta in self._var_allprocs_abs2meta[io].items(): if not meta['distributed']: - vsizes = sizes[:, idxs[abs_name]] + vsizes = sizes[:, abs2idx[abs_name]] unique = set(vsizes) unique.discard(0) if len(unique) > 1: @@ -3037,8 +3039,6 @@ def _setup_connections(self): # either owned by (implicit) or declared by (explicit) this Group. # This way, we don't repeat the error checking in multiple groups. - self._dist_in_sources = defaultdict(list) - for abs_in, abs_out in abs_in2out.items(): all_meta_out = allprocs_abs2meta_out[abs_out] all_meta_in = allprocs_abs2meta_in[abs_in] @@ -3076,8 +3076,6 @@ def _setup_connections(self): # get input shape and src_indices from the local meta dict # (input is always local) if meta_in['distributed']: - self._dist_in_sources[abs_out].append(abs_in) - # if output is non-distributed and input is distributed, make output shape the # full distributed shape, i.e., treat it in this regard as a distributed output out_shape = self._get_full_dist_shape(abs_out, all_meta_out['shape']) @@ -4098,13 +4096,12 @@ def _approx_subjac_keys_iter(self): of -= ivc for key in product(of, wrt.union(of)): - _of, _wrt = key - # Create approximations for the ones we need. if self._tot_jac is not None: yield key # get all combos if we're doing total derivs continue + _of, _wrt = key # Skip explicit res wrt outputs if _wrt in of and _wrt not in ivc: @@ -4232,9 +4229,6 @@ def _jac_wrt_iter(self, wrt_matches=None): elif wrt in local_outs: vec = self._outputs else: - # ??? - # if not total: - # continue vec = None # remote wrt if wrt in approx_wrt_idx: sub_wrt_idx = approx_wrt_idx[wrt] @@ -4332,7 +4326,7 @@ def _setup_approx_partials(self): left = responses[left]['source'] if right in responses and responses[right]['alias'] is not None: right = responses[right]['source'] - elif nprocs > 1: + elif nprocs > 1 and self._has_fd_group: sout = sizes_out[:, abs2idx[left]] sin = sizes_in[:, abs2idx[right]] if np.count_nonzero(sout[sin == 0]) > 0 and np.count_nonzero(sin[sout == 0]) > 0: diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index 0ce86664fd..f8bf109736 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -1124,9 +1124,6 @@ def final_setup(self): logger = TestLogger() self.check_config(logger, checks=checks) - # from om_devtools.dist_idxs import dump_dist_idxs - # dump_dist_idxs(self, full=True) - def check_partials(self, out_stream=_DEFAULT_OUT_STREAM, includes=None, excludes=None, compact_print=False, abs_err_tol=1e-6, rel_err_tol=1e-6, method='fd', step=None, form='forward', step_calc='abs', diff --git a/openmdao/core/system.py b/openmdao/core/system.py index eff5820e48..1d0254ff21 100644 --- a/openmdao/core/system.py +++ b/openmdao/core/system.py @@ -5597,8 +5597,7 @@ def _get_input_from_src(self, name, abs_ins, conns, units=None, indices=None, "`get_val(, get_remote=True)`.") else: if src_indices._flat_src: - sinds = src_indices.flat() - val = val.ravel()[sinds] + val = val.ravel()[src_indices.flat()] # if at component level, just keep shape of the target and don't flatten if not flat and not is_prom: shp = vmeta['shape'] diff --git a/openmdao/core/tests/test_approx_derivs.py b/openmdao/core/tests/test_approx_derivs.py index 3810df416c..2b94522798 100644 --- a/openmdao/core/tests/test_approx_derivs.py +++ b/openmdao/core/tests/test_approx_derivs.py @@ -3036,10 +3036,10 @@ def test_group_fd_inner_par_indirect2_rev(self): assert_check_totals(prob.check_totals(method='fd', show_only_incorrect=True), atol=3e-6) -# @unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") -# class TestFDWithParallelSubGroups4(TestFDWithParallelSubGroups): +@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") +class TestFDWithParallelSubGroups4(TestFDWithParallelSubGroups): -# N_PROCS = 4 + N_PROCS = 4 if __name__ == "__main__": diff --git a/openmdao/core/tests/test_check_totals.py b/openmdao/core/tests/test_check_totals.py index cbcb09836b..d91d9ef20a 100644 --- a/openmdao/core/tests/test_check_totals.py +++ b/openmdao/core/tests/test_check_totals.py @@ -222,9 +222,9 @@ def _remotevar_compute_totals(self, mode): assert_near_equal(prob['c4.y1'], 46.0, 1e-6) assert_near_equal(prob['c4.y2'], -93.0, 1e-6) - #J = prob.compute_totals(of=unknown_list, wrt=indep_list) - #for key, val in full_expected.items(): - #assert_near_equal(J[key], val, 1e-6) + J = prob.compute_totals(of=unknown_list, wrt=indep_list) + for key, val in full_expected.items(): + assert_near_equal(J[key], val, 1e-6) reduced_expected = {key: v for key, v in full_expected.items() if key[0] in prob.model._var_abs2meta['output']} diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index 56f141f722..21484c5cbc 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -815,12 +815,12 @@ def test_distrib_voi_group_fd(self): promotes_inputs=['*']) sub.add_subsystem("parab", DistParab(arr_size=size), promotes_outputs=['*'], promotes_inputs=['a']) - model.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', + sub.add_subsystem('sum', om.ExecComp('f_sum = sum(f_xy)', f_sum=np.ones((size, )), f_xy=np.ones((size, ))), promotes_outputs=['*']) - model.promotes('sum', inputs=['f_xy'], src_indices=om.slicer[:]) + sub.promotes('sum', inputs=['f_xy'], src_indices=om.slicer[:]) sub.connect('dummy.xd', 'parab.x') sub.connect('dummy.yd', 'parab.y') @@ -1240,35 +1240,13 @@ def compute(self, inputs, outputs): model.add_constraint('ndp2.g', lower=0.0) model.add_objective('f_sum', index=-1) - mode_idx = {'fwd': 0, 'rev': 1} for mode in ['fwd', 'rev']: prob.setup(mode=mode, force_alloc_complex=True) prob.run_model() - J = prob.check_totals(method='cs', show_only_incorrect=True) - assert_near_equal(J['parab.f_xy', 'p.x']['abs error'][mode_idx[mode]], 0.0, 1e-5) - assert_near_equal(J['parab.f_xy', 'p.y']['abs error'][mode_idx[mode]], 0.0, 1e-5) - assert_near_equal(J['ndp.g', 'p.x']['abs error'][mode_idx[mode]], 0.0, 2e-5) - assert_near_equal(J['ndp.g', 'p.y']['abs error'][mode_idx[mode]], 0.0, 2e-5) - assert_near_equal(J['parab2.f_xy', 'p.x2']['abs error'][mode_idx[mode]], 0.0, 1e-5) - assert_near_equal(J['parab2.f_xy', 'p.y2']['abs error'][mode_idx[mode]], 0.0, 1e-5) - assert_near_equal(J['ndp2.g', 'p.x2']['abs error'][mode_idx[mode]], 0.0, 2e-5) - assert_near_equal(J['ndp2.g', 'p.y2']['abs error'][mode_idx[mode]], 0.0, 2e-5) - assert_near_equal(J['sum.f_sum', 'p.x']['abs error'][mode_idx[mode]], 0.0, 1e-5) - assert_near_equal(J['sum.f_sum', 'p.y']['abs error'][mode_idx[mode]], 0.0, 1e-5) - - J = prob.check_totals(method='cs', show_only_incorrect=True) - assert_near_equal(J['parab.f_xy', 'p.x']['abs error'][mode_idx[mode]], 0.0, 1e-14) - assert_near_equal(J['parab.f_xy', 'p.y']['abs error'][mode_idx[mode]], 0.0, 1e-14) - assert_near_equal(J['ndp.g', 'p.x']['abs error'][mode_idx[mode]], 0.0, 1e-13) - assert_near_equal(J['ndp.g', 'p.y']['abs error'][mode_idx[mode]], 0.0, 1e-13) - assert_near_equal(J['parab2.f_xy', 'p.x2']['abs error'][mode_idx[mode]], 0.0, 1e-14) - assert_near_equal(J['parab2.f_xy', 'p.y2']['abs error'][mode_idx[mode]], 0.0, 1e-14) - assert_near_equal(J['ndp2.g', 'p.x2']['abs error'][mode_idx[mode]], 0.0, 1e-13) - assert_near_equal(J['ndp2.g', 'p.y2']['abs error'][mode_idx[mode]], 0.0, 1e-13) - assert_near_equal(J['sum.f_sum', 'p.x']['abs error'][mode_idx[mode]], 0.0, 1e-14) - assert_near_equal(J['sum.f_sum', 'p.y']['abs error'][mode_idx[mode]], 0.0, 1e-14) + assert_check_totals(prob.check_totals(method='fd', out_stream=None)) + assert_check_totals(prob.check_totals(method='cs', out_stream=None), rtol=1e-14) def run_mixed_distrib2_prob(self, mode, klass=MixedDistrib2): size = 5 @@ -2680,7 +2658,6 @@ def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): prob.setup(mode='rev') prob.run_model() - # This check totals fails, some of the derivative terms from proc 2 are missing assert_check_totals(prob.check_totals("ParallelSum.sum", "ivc.x")) if __name__ == "__main__": diff --git a/openmdao/core/tests/test_driver.py b/openmdao/core/tests/test_driver.py index 3508d2f688..e88d9c5778 100644 --- a/openmdao/core/tests/test_driver.py +++ b/openmdao/core/tests/test_driver.py @@ -993,11 +993,6 @@ def compute(self, inputs, outputs): outputs['y'] = np.sum(y_g) + (inputs['w']-10)**2 outputs['z'] = x**2 - print('-------------') - print(self.comm.rank, 'x', x) - print(self.comm.rank, 'w', inputs['w']) - print(self.comm.rank, 'y', outputs['y']) - print(self.comm.rank, 'z', outputs['z']) def compute_partials(self, inputs, J): x = inputs['x'] @@ -1014,10 +1009,12 @@ def compute_partials(self, inputs, J): promotes=['*']) d_ivc.add_output('x', 2*np.ones(size)) - # distributed indep var, 'w' + # non-distributed indep var, 'w' ivc = model.add_subsystem('ivc', om.IndepVarComp(distributed=False), promotes=['*']) - ivc.add_output('w') + # previous version of this test set w to 2 different values depending on rank, which + # is not valid for a non-distributed variable. Changed to be the same on all procs. + ivc.add_output('w', 3.0) # distributed component, 'dc' model.add_subsystem('dc', DistribComp(), promotes=['*']) @@ -1030,7 +1027,6 @@ def compute_partials(self, inputs, J): driver.supports._read_only = False driver.supports['distributed_design_vars'] = True - # run model p.setup() p.run_model() @@ -1058,10 +1054,9 @@ def compute_partials(self, inputs, J): assert_near_equal(driver.get_design_var_values(get_remote=True)['d_ivc.x'], [5, 5, 5, 9, 9]) - # run driver p.run_driver() - assert_near_equal(p.get_val('dc.y', get_remote=True), [113, 113]) + assert_near_equal(p.get_val('dc.y', get_remote=True), [81, 81]) assert_near_equal(p.get_val('dc.z', get_remote=True), [25, 25, 25, 81, 81]) def test_distrib_desvar_bug(self): diff --git a/openmdao/core/tests/test_mpi_coloring_bug.py b/openmdao/core/tests/test_mpi_coloring_bug.py index 366360ffc8..e819267126 100644 --- a/openmdao/core/tests/test_mpi_coloring_bug.py +++ b/openmdao/core/tests/test_mpi_coloring_bug.py @@ -302,8 +302,20 @@ def add_design_parameter(self, name, val, targets): self.design_parameter_options[name]['val'] = val self.design_parameter_options[name]['targets'] = targets + + def _setup_design_parameters(self): + if self.design_parameter_options: + indep = self.add_subsystem('design_params', subsys=om.IndepVarComp(), + promotes_outputs=['*']) + + for name, options in self.design_parameter_options.items(): + indep.add_output(name='design_parameters:{0}'.format(name), + val=options['val'], + shape=(1, np.prod(options['shape']))) + def setup(self): super().setup() + self._setup_design_parameters() phases_group = self.add_subsystem('phases', subsys=om.ParallelGroup(), promotes_inputs=['*'], promotes_outputs=['*']) @@ -437,13 +449,19 @@ def compute(self, inputs, outputs): def make_traj(): + t = GaussLobatto() + traj = Trajectory() - burn1 = Phase(ode_class=FiniteBurnODE, transcription=GaussLobatto()) + traj.add_design_parameter('c', val=1.5, targets={'burn1': ['c'], 'burn2': ['c']}) + + # First Phase (burn) + burn1 = Phase(ode_class=FiniteBurnODE, transcription=t) burn1 = traj.add_phase('burn1', burn1) burn1.add_state('deltav', rate_source='deltav_dot') - burn2 = Phase(ode_class=FiniteBurnODE, transcription=GaussLobatto()) + # Third Phase (burn) + burn2 = Phase(ode_class=FiniteBurnODE, transcription=t) traj.add_phase('burn2', burn2) burn2.add_state('deltav', rate_source='deltav_dot') @@ -479,13 +497,15 @@ def run(self): p.setup(mode='rev') + # Set Initial Guesses + p.set_val('design_parameters:c', val=1.5) + of = ['phases.burn2.indep_states.states:deltav', 'phases.burn1.collocation_constraint.defects:deltav', 'phases.burn2.collocation_constraint.defects:deltav', ] wrt = ['phases.burn1.indep_states.states:deltav', 'phases.burn2.indep_states.states:deltav'] p.run_model() p.run_driver() - # assert_check_totals(p.check_totals(of=of, wrt=wrt)) J = p.driver._compute_totals(of=of, wrt=wrt, return_format='dict') dd = J['phases.burn1.collocation_constraint.defects:deltav']['phases.burn1.indep_states.states:deltav'] diff --git a/openmdao/core/tests/test_parallel_derivatives.py b/openmdao/core/tests/test_parallel_derivatives.py index 2dd1604e1c..f8e0488d82 100644 --- a/openmdao/core/tests/test_parallel_derivatives.py +++ b/openmdao/core/tests/test_parallel_derivatives.py @@ -900,9 +900,6 @@ def compute_partials(self, inputs, partials): prob.run_model() - # from om_devtools.dist_idxs import dump_dist_idxs - # dump_dist_idxs(prob, full=True) - assert_check_totals(prob.check_totals(method='cs', out_stream=None)) diff --git a/openmdao/docs/openmdao_book/features/core_features/working_with_components/distributed_components.ipynb b/openmdao/docs/openmdao_book/features/core_features/working_with_components/distributed_components.ipynb index ae9cbe6fe1..f97f71faa1 100644 --- a/openmdao/docs/openmdao_book/features/core_features/working_with_components/distributed_components.ipynb +++ b/openmdao/docs/openmdao_book/features/core_features/working_with_components/distributed_components.ipynb @@ -33,7 +33,7 @@ "\n", "At times when you need to perform a computation using large input arrays, you may want to perform that computation in multiple processes, where each process operates on some subset of the input values. This may be done purely for performance reasons, or it may be necessary because the entire input will not fit in the memory of a single machine. In any case, this can be accomplished in OpenMDAO by declaring those inputs and outputs as distributed. By definition, a `distributed variable` is an input or output where each process contains only a part of the whole variable. Distributed variables are declared by setting the optional \"distributed\" argument to True when adding the variable to a component. A component that has at least one distributed variable can also be called a distributed component.\n", "\n", - "Any variable that is not distributed is called a `non-distributed variable`. When the model is run under MPI, every process contains a copy of the entire variable.\n", + "Any variable that is not distributed is called a `non-distributed variable`. When the model is run under MPI, every process contains a copy of the entire non-distributed variable. We also call these duplicated variables.\n", "\n", "We’ve already seen that by using [src_indices](connect-with-src-indices), we can connect an input to only a subset of an output variable. By giving different values for src_indices in each MPI process, we can distribute computations on a distributed output across the processes. All of the scenarios that involve connecting distributed and non-distributed variables are detailed in [Connections involving distributed variables](../working_with_groups/dist_serial.ipynb)." ] diff --git a/openmdao/docs/openmdao_book/theory_manual/images/Par1.png b/openmdao/docs/openmdao_book/theory_manual/images/Par1.png index 1ea865d5a69f9ad2987ed659cdec4127a3883596..8b4b7df5c4171ac74031045087b07e394038ab97 100644 GIT binary patch literal 12590 zcmd^lXIRr`)ORQfE~G751QpaOXi1;~ii`*qS7a)RY*YwCRt;f~pj2_7C}<)h0hzKf zVHj3G6vW6%fEZR(2oUy2KnR5Q4iws_m)HCKeO|tW|2@t*_nE(Q?wJ~&k=P`=2?Bvg zAkLn=0D*`aLLee)8^pkWe((*y4}tv7L!A8MqQBi}_vY8zkrj25(w}VBZJWH4xwq#- znMFVRt=siv*M8n}7rC^Q%tNxNcLG~KZh1Eq7whUOH+-eensDL$$8xsqo6X5$8(vxr zppAV>Kh7;Vwck22UZyc3L7EA5ZkV0cp@_GalV)x&eVnLtbLzRV72N;7>Ca`OhldBH zMEBQMKNa+%P>@uQAir6Pcf{ z;EI+Kk+6ITu;%4{+rU{foX^t%i#vVs+S|~zh=udr3nz~5U=wG87KJ94lqF_ihcak{ z218k0FOt-HvgX?IL!a58TTa#gwNWa%K1NYdg|dJuFt$tvilM&KveIJ?PwlAdiw9JM z4aI#W8WTNd7>i0O)?)qbF80kbUPa}zJwf%KUOCYJsQp|W9`mIpLJ=qdm9Hl*2u#IY5IilV4uh`66}^VwQ}_anJn;yV1KId%JGn0{SySMG9*SZF<8~#WTa)hcSTA00w$7v*UK)b;@oHvF()YYR-=D5 zGGe#y*>)?p+&;WtPd$t>KnSAUtV>12+?stWx7hWM?rZ#dMlQfn)#>AN$Cl(%WOk{I zCo5#us8MJ7<7YU}CV>5n6LQAj!4jdi3r#nwKfx(flW(C#l_Dju|4;p#JtxSg%f7)mH`p>JSBU4lr6GltZIEA8U z#;*`+gt{e={b@o#e{AaId^-*^u9kgf*Nv0JsSiqu)BO#bd*~m#LT0-Wl+eZK(c(^J zjZdXqngPA~{o8v|bdlCD5z%P8Z^xm4*>1n^{Fa!Orok1V-Y_S+e=c>cN*zR%w3%Cb zOupUhJHw83z`|>UV!6oW>FMtBRbo!Ghb4W3G5?OG_{g!E|imI1HKku?&yfT|L2<=%Uw%y$xbKD?hgkIxj#uiJlnZ5#fR(N z9W-9SsW6ZCD7Fj?u=86;u)P9j>_C00opc+y`unpzKc|r#6%Kef}?GW3tMD(Fj zHZxGgZqX9`N}&qhWp!Y8+O}u5MLM4{+P-_$fB&K%xlnigLIBTn@ zeiXYNyBV6j@-$yLmHWFBQ~6KPl;|<*A>}`u6W+=LDLHC%zDMENaOyKGZ&^R8?$ZtZ z;wFvNSgv9^4SzAZ*CL->vd4@nwG0~%jqhK6Zg$F{cpmfs+zf%!wI5wfy0LDh(AuC9u`& ziB`)>kbABvnfy+*PPLt;&5+Gl<%9Sax}7J6gHsW>?7Vm{pl5p7LU)90N^VIEwEgGS zdm&XQGId5C9yOxBto%=(HST%n!<0FIx1`CW6F08rfYxCOCp8&JHH6^()RSN}K;Q?* zLh9IgTE#hJ=vn8rCq7}0Vz%gnZKH>Aczk8)%)Hw}B@n}&K2w&aJ>&!u zq6yilCK1+utljxW{VeSYB3wa&pnmtaVN${v+H%yhWT9s2snYoP4-m-X3(lu;;+)F+ zZcw+rc!RY}HfMt^A%n_AlL<2hww0F^=0BfmdU0V<(0RCeJp^))d>9*x_si(` zPuSb|4AO18^#uAjxK!GOj*gDd@eZcg*)9$}qlv%OK&=^7N>vUiVJOF7U;n)LM zCf3Zxrta1&PZ`nlo{At9)Tm85;_P;_PWdp}n)w+Rs&%{8YBw0B$xNtA$wAzm=UxF5 z;v%V8bNzC!^4U(97mCN6q%O3%U?%*2Op3hqFn)y@1V`pv>B8xzi!{{mAM07atrAk8K5UtXW|9)}NbXO)9-aR4 zqe5y{i{SJ49zB`$J2jT8rT5VmHK>PFt5;&ZVkC=a5JklY*uzTIXlqio zXD_8JRX~#@BXOd`nqE>+{iz0{pLUTpW?nDU{ieOa z^mKKt(Q=hZ6z6QAS>W*JX|V_JSAPh5?mk8oZ>)46x2i%JRyp6L3%=@~N}V@Ct{v@C zeMfwcDnf<_d+)vFaCGmbhcn_Bagh%PKfPXYLB47Yz8O>-bw1vcC>EyLqV`45hK$ns zb`zN$JiY0=!ett97xu%?An;q@ml5>2AM`A9-8Jc|O{fwiE_fRC{XLE$uneny>%1Gx z3=*XIH}03Fd)UjizV{{tbN$NS-*Pf=L*GhfaTMZmAOsp=NqD8>rkPxvqx4-?j#e+{ z6s04v3J}u+e5-V8J5nTx+f?KW{jOHsY%lH_Bo1H)e}0x1TDl36>!?UIvGFz3=_H=h zk9X>x{_ZnEs~}WO{lqOuuFIysHCT2=BB^lI4Es2*BvfSq-J%LIx8lE=9 z{y4XEZ7*RZ3!m*$n|r4c`csbeok72e&E#!?ccKYtsm{M~nCBp2{r`Dp&!TevSXsSq zrw6XEs3yGU5 zV})>H@nr7-F%OV{nsoLz#7h@$e=^;utzzl$@JO_@eehwI&Od7;VvIDp)ff37SB)3! zGJ*3hdg57vnG()a*?6V4KgQ&g5G(->3&qsmu4sKncwP6NV@@##0t9mdF~X8<5yNZL z9yb)$J2_I`p);h!G~Ke$h|6{X2V4~Ami_S_r<#TJN2+VPBZG&nS?B!XhVv=MUHG5s zj$_@4^$OKfft3qv6^7sIt<&N13w!)o#Hs#xFNMWcGj0eszKPiP@`(J;{`HY7Apcd0bbj2DUsm+ z^&Vopr-HszgT5UhSf~;|{#a+CO0;_Z)u|9Mo-|i2+ayW1E#h@ww81T(`shK{p%OpH z6S_Vx@6Rt)hd#ZIf1>H#zdu||;+&$Di;Vyj?+O)xymOUP&+^xeX9Cn=45487yr5NQ z@~u=gAN#BcBbeqq$}zb=L{)zM`4&1>%r!8iQF91#ja%M+H*kQ6CUBA8zmpp!~B)C1t($)U!`(LBB^ym`@Ske4B0k6u=qSSdL-rycEE^EavB#&WT2+AId%>JiNZ}ySb>+B0scW zEfLWZFo6H;R)XS=j{vvbD(ELhV25(v+IiM59s*XFGZ19m+_E4jrvE zm;=D}68C!C{J2G0YjKo$Qm2%u$U}_@jZH_#oYF&09*@~-fd`Da1OTH7`WDYv92F0Z8J^v*QQkl7Iop71DC8D}J&rxr~ija0l&e%IXECwmZv zV$RMRg1W5v=I7_T4Tj_|8po+0DzTCO`FfRE;>62}L0SO*D(9eJPmWs4(s*gB(dA%u zWk!hcyrxh4&}GW}rNG#7Wt@G-Qn$f_#&t&s67(yE_V&)@LsTy5+qj#GmYD{s?>$%Z zku1~}AdX~Ti8ty2Pdj{S5hakppv?TQnB3n^Qi43k532C)c@~WfX(Y+sdRP|Q!uHgv z8oW{gfX)LBb&n+44{2$Uc=t?#_bt9Xm$9%oQ#<%`SleZL_}oN|l#o{>Y4B~q2C@J~uX<(L6t)B1@#<5rY8{B}O%R_gc>0NZitM|13;kJ{c-uJ_W}na*k8`S=tWgfO%7 zD`m^QEL+60Wj{&m_)8Ld-c5d;M2G<5%)gEZSG|!+j>=(XDdynh#TFvNes+M*eK67zoRI?E& znFnWj4c_{qT-M}qhg#XB3@Z7m3+w7eo+pdf4V{zhc1vLZB5eaV#!?80$zxY(Zw=-J zot|YLp4Y^6b>~~6XiECrqNd~gk))7Q7v91Rc{cuEZgUt@<4ZJ|gD}RpEIoX@OuJ|_ zW&7Z4bJg`cWP0xwGxoZjexwu=6gNbO(d|nysaP<1Bb#DUTG#^;gQt}QD++Rq%?m3qiL7fpPrJzlQnlC5i> zb)zb^ez&^mJkURU4n;7}xndS`^4oas39eTR3x3WZiv2|{ebZ*4q!X1npG)>Je!UDA zcCJj~+52$t3|HNJ4@>liU`47p=z&zJuCE*UgsPFyNL`8&0@@s;mkog z={paiDHj%5rnAR5PY#u87=)*0nq{Pi)AS|b>fcH)-Mv0GDvx}}Oua9p@|0TrU@H%F zN~Cp3VNiP^V7dF-YqkV+8#OFg3X-((mV>W*%g0xxC8u`>;72DrA3PG%iqk_Z9SyXJ zkaHX#h|pcQS~>=d8=rTX;%M5kUCt`~Ot^ZeZMSt!zF*s=%j;RB!PfC0+QqMN1t)M@ z+MQV*<=0Lx?u5~~gFIy1b_?oI)cKb1Kac#`swRI)9PW@saxVMu%VLj#uS8Nga4(iE z9G?)~1|7!kM5iJUHwL9qhwfN-mgsmfKe&lPwkaKRs@vg1 zEs2uj7u(>C0?5}e(XD40*}8t`rIZK%D6neZEvkSwlGP0jR&{AkdZ#;GaJ7c!S~EP{ zD((xs1A44t-V(F`x*Y-aMos3Kwxa3lH~*6Aoa)XZ-97`nq7q0->V7fu&Q&`Q_gA%- zlEIML$S6pNyS1+Trd66_I%|dx8YBNP%bABK?J(q1?QS}YPBR(tXUb=OZ5z6vtddBpfX@f3B#(gXeVe# z#hgz$yt37FrN(1lNDOa3 zuv!@_XIuh320LOT7w~5*y^DmHzECR?L--?2EhO$m!KEf>+X^Uj96o(p%_x+ zhurfCtHl}l)}sPwLBdrI8?^JZR6yF+pB5>qV3bn1IMb?{S-&Lb(+3f_iAgFv@?YI-RMgyZNQAf1O)UUn0>-cTr z3)v;^2BzEm?(!quM<-K{5-HY4xgGb{aGK4@8}~N8D)*RsDs(`_|3VLvb6jJ`i9xu+ z7e*v*s9Gf=Rxzex$>VYH>q|Y?X9IQx53@!NN$b$UGIL>Epr!b4nKv2 zQ^#$KMci%eGxgSRkb7Hwn1B6fk804SZ-nITTdU?D-*UP2Tf@QJ<0uSOL78vocQkL^ zwzUU58Y?zNzuv+US&rGOr0dz=U@~;ytG<|-Jxo3lc597R&m@w87B!V*q&2(_vf-cF z|3VVa_NZouB-DhQgP~!qMGu}`g>*(vmnI@+k8lJE-2>|&7XK!UbN=9x+gt>DQ@+GO z!?pEJahfJ-gRTm2rxX#>BRefY)!=Z$L6EAuD8y!rJSZ;dDc#YFUe^C(*WN($7y>iZ zS%V{E3^CoFVb$YfpyjsYz%;_lMhV}fC?qBJs4sKYzr_pk&w~dwYbk3V5EM}J6ve{K zH9jJ7OCN7ACoOR6TnqLu*YoMc81-3)oA?<2k6&So=^4ZjwY4)S&H?=-@LYlcbttZEbk+V*XMo( z+b!or4^-q$%SX%nU3UBP&pbh6?^TMGo?4YnRUI_FoJ2HUMBE$Yq%+Tes+9`rQG=(aZeRAId#b>T4ACY)XU;H z!Rv=GPkJPvu-SG0-zX>~tdZG-lngF^PXjXxdL(ska4A}L^-@UEbbDc!M?e?x6|m7$ zLy!KmIZvC3lsd6%Km)kSp~nAwWRu9Cg^9<$0Db~}$z;pZ3nBcO#c;QQ9>OyGT`(^(EYd;F6@XBpO&`LS&x$FuR3?6WH8M~na@X2WH0@VdjE!%8s{4hYkPlylov3po zy;u#X8B`QNZ2P3>dYPW7>?UCcK)1;O`xa%|3pj%ykG@5fu>+{)c00oe(!F|_Ei%eG z_j7&{o1ZQfHhA>hj)BJ4&x!pSkinl`Nj@D4w3I_LuW)>*=Y*Z2u=ZCs+2^T?*rJMJ z+8`sJg(2rmpFa0=S-fX0#m@OCmANOSY6;%0^o#V|I1ZXDfQ{-^W~%=)YV`j8p-#Qb zu`=asyz=N`1oBQVR0;?+3~w`&IoXyMj89zaY3=))X9r>qYtiNmnNoRl$btJzd z-e;Jnu5FB8*%)PAk1AsOP>t=Z&#!fbJ>Li|4AP=m1vIWwDfGnMwQcpwNA9`qW<6G- zbDde`X8qIJJH)JTk(uCR>7rTdWnsMbrFw;s!Zi;#oV9l1o@J|V{#nc--J+);Sclbm zV0B;LrfW%{WgdntOhi6n?=y%8#n0L|)ZgDq(%GKC#vfJ`Bi9;UWp|gK;MAG=BT*BcbbQ1KrJc*`I6C25YA?i!Dvjhl<~2>nq#kfh1k%3@BWV)+YcECYX)l7 zvOHdLoagQU^nJ`O9&|xVta`v4NMDtxq@lP>ygLy=6lP;WX2$e}@ z)n%tdEwjjgpK0uIEavryTt-h3`w!=G873GYit#9BDpcs?-I#lGdvlC%^r<5MJ;nqs zrC@mWk=D@5v|3D5;@tFrI8|5_2rC8K1uOeAI8HY};vV2!9~S`_I0HBJ4n0R0GFNez zST>j|nyy)sOE;KT5W7bymbHuyFe9dh0%&yxegPC|{a6-&B{@^pp5z1mCU8Kxm)2uT zt6yMN+sgA(Tt2ILcD?5E?f$G@{{Tk;@O}N`z53BShP`gAwx4IIVfc|;KPD2l-abRo zEIVac)gYS+qLGnQr>3CJDT;ELI(QY0Uxl+vDj4ml_rH2mCA9{VxF|BHZ`O^wp5`uc zwUxBI`w|1@XG$peVmz9LsK*EfFzLF}iU4-|Jj_L3r-&}|l7vtXv#RrwLv(d7CTi8dIx;~>iUrD4JOwVKxK_+w{*rk z0+AVEjF-UK7d7gZ^~j|ZXk3mEl5{<@RJ+s5(4OI(AAdX~u)x#d{_BX$is0Uzth1CQ zfFow8w!Jg^ zR?1|*5LZ{O{Py(#qD(HbUZ5+_rp+Sst^w&f4#31y^M~_Y}H+ucsI;3jK&%z9J?{8 zir)y{G)5oIFbu`t3vCRRG1D${pYteU1h!+HPTE{?&CVAZYtppv4l_z?uD0UEX11RY z^;LEMRN(lz>Lw@Td>lGC6hi~e6vAv2HTgVuy%ZdD61Wv$0;Mxdj#>D#+0yhOtva-1Z!Ot;7xr^2``%p&N|9 za{eBFC4lkQrZz#j;>z4=WcmD1cKu|Xd}6R*I?)7lTgXXxGz`E#@5K^tSeA*k$WMT} z4ia{5$YCyxs`VjmyGL%Ycf?va5#uqZ`{(+%u8rn6IZzK^ z8Mf`AXO{;C{s+GQ62 literal 20202 zcmafbWmuG3)b=pc(47*}l8&Hsi6RWBgh)5iDIg&=qk{@4=+IJvpo9nrBF%^b3MgO! zBAqG{qJ-4Dp7Xxn@9&3~=giErpS{;!d&Pa<+XU0|22Aw)^e7aH31@iP9EGAJqEHmJ z7#jHG%v|a@_=_&c&?W?h;vGZ&P$*kn^MDWeLiMdfF9dpohPwp2qr$_(6}$s{L)=_~ z+!X?YJ#&^d`BA9DDBS6j77@8iW8qmAb7MExaukY=o4t1CS>mLAc#T^22(KUOHKXp| zU)edt*xf{wk0*A~aFZQP&5qwHdQeS8buta(W6DZJU4pwzaG%i0=d;dm7*^Eq^w`+9 zAF@}9dY>H^fdS6m(rxW;#^L1r3`nALoM55V{GfZ*sk!I zi(6LOb`t40iWE+fEa++Ua5C`^Q?&2%I7W#4P%Jj=%T-I|IIjx-EsiTOJ@x}Fjw6ZV z2C=M0@Rzxo`{6YBpg6ck_XzU&TpGa?{v!2AQx+mWOuqmF!Ur;s8Hvc}7U%ge67WG; zuHye^fU+Jlo@&ZwiUrDEf>q-7H!XZJxu0A?_tY2}=YyjtdMcJYQ`5bGu3%7|H=Jx6 zp1Dc57oYpn=7DyQ3YOao!$`B=TTcjzdw|BW3|o*JnQv)rUNF~TU0!^?^rlxgZ+M0W z|F}M6={xyvWbFF`m7h$Z3X_pKlp(G||BCQ?E9{Q1K9iS3FUrGwrOgvCL?6F}SM=jfp##I0gdmd#zDsKdTvJUU_;THIlbNocauPi#KV#ZoU6hE}Paqy={-6-!-`;0v*b_@j|~e^tIk>tSId--`zdQ=UjmaVPP<#GtEwF~fiArl=%l2l(vjk-a->0`=7Da* zQJ222I_rKKQuXrVV^<=U7PcxMRMQ2CtWp$S@l(L2VZ*Ah&FDRXbj?B^?KoQDa2;tb zJb3A!6?bsknCs5X*^w_I@TkAifv=jE#$T>k!AaTr4?pBRKDzd zbamwvdaxAwuI)q8*>aUNXoB2Hb z{6)f^K>KI82W0?@%l;=y>*kYxYDNIdoE3H^f=GnoTta+xRYon9<*|k@ETK zS#Y&swfgo~VUE3T^&{j*Ywr#Gvr59g@^O^Mi{axajT0lsTc$Y%EV|r{5#!Wll(PJZ z>}H#1iCQbrLAObU>S#9N-DFS`-N_^Qaco`BD^vDbSnRr*K1&ckwlItI<&NYJ{CFWO zax2CNQxw-59<8q@JA9J-dU)*|oqWn;lq_DsPwRpwPK&cbO=DxdZQHT;4|{G%#FBR0 zi^1iZ9Ty2zPsi@gWggc3<^N8Bg{WP`v|I1__u=~-m>Cq$>-rd~rE}KYqE8GoXE*bmh+o;I4)tk1{03j{?jtrbW&*m->Np!kT9zrFUQ--pQB zoHn3m&CZ3XC2u6`?b=HFAm=*`&CZAE?p|I`7W31y7G8O!+0Aua%-e>h<1eRxqh$B; zA@+=_TN<)q2QnhMbDQ)o^`Dx^HFV|225+!6)j4wRYzyeM_ScrUUokk38c!P27Oh*- zHeQtZ=j5zA-Glf)XZ&fJjxE)Wa=-7E5y4CQ4>h~zgbl1&R5dpuqR~GSb~3Q_3hmGR z_*gQ-MRTvaZ90J;3Dk%BYSrC!E80L89`e~7?qAz+=~V3&^5g4`Kv!$Hocw2pIT;aI zR)MU6vkcyF?Tc8SM&a3Sa(WDQyICA`V{AO=17l~c%RlRDkG{bBH@{wv8-Bme-Mt(x zZ>H}a$mb&X@;cp^WB=jX*}HYT3vZg4T^#g!C#T4jesmxH+XL$&&xF2aA(!agNwl7< zgS~>wvy2#wQb>k-_#nxeqj|zjqx)0J*a6AYPSc*dp7YiD66tcP8=cDqhl!0-@xrzb zT7I%BJf2*YL*mRG33b-wlrdqB=3iZ3Dz9ITY`&9moV82p7VeMlmQ2`0!`(eD!kL=x z$maAeJJIl?vype`N0{B$ZM#@)6@QvGJ*P#YfP3lbn`JFCn7~{9JEi!U>CyL;$+xu> zj;IEhv>1ElxoQU*j3((WbU!`$F@?aM9BC+uHRIx6L@J4bjV0?#EsB+Q%?I z*}lB1U9ol!T3Xv9CU{RjqSQf0p~IiKPyH*`)tTHJZS=5d&B&yCxjRUeXM&t=8eHgM zqC7=4y0CZJidvwSE$em0T3DyQAd&TrGTMrRyRKj4c8IUdwUD{z2?O_43b1^=dWRXd zatA~kG*;C`{>(imR9nX=tV&Hj9uBc&^O$dWgsGYGso8V+D<`h*I+DF(g zX^*_W{Axr|x_3QZp!Qnz^29atMk~{CvAWaBmwJa=-D`FV-1_rgtJA=70c_Q)ppMBr?(!eMPVcND!G|>VyBJbd_wD^-Tl_K+Y(uA zIosJ|RjPma9%$AF8nN|rZPJTO9@zISAd^WnbWw6OB8ACqKB;{_-O1y0yNOwvUCHzP zERo|pet={Ci&w=x@$}O+p?q*WdVW?BN1ig=5A`?c$l6?fD`!aJ3+D>EcrI-FLV zHAF`UJQ5U?|BI5KK=viy$NV5EcB&9dvaX^o;{OrnG91wv2G*J4>AsNs>6KCKQlb7) z+0|#fqb7aydb>gf(?aWX4@B5B2yt@N5_7bR`evGqX?#ay^-G1`b+Y-1mR z_$3@?PK34ARW{Y^FzJS+#T)W1(&-H?T_Muth~wCv^_GhJ|DxCXIq9ZrHAy{5rKf6f z-=;ZC-mUAl%eRKheK{&A62=+TKc6-S{~0t08Pw;w{F=?+w1aD;g-BQwn^fw(;x|ym zcqfSwq6nAC>Ewuxn_o2W?We6Wj9lmYvU4wY@jbD7;h!=gec=%{V%0gIW$n!VZ=QoX zT-b@J_3rCtJfb*c7M;KJvK;>K==C}?yQuFLlCKcIuce={ta3aT7N+b!T0hsgkUvvV z)#kc3-FY?(OGp;fAf=|h!c$Q4ss&&ARFc6n#HKfSg={$a(X02KI6nn?s~sYL5*9t;-xpN>S0T~ zX6bWACJ)$M=rVtbKGi32y1`as^S|@!ZP{f9PVi59buxXJl@^!Jk~96 zs=~68Uen{_qjQ?OtFQdNHQZYsYrgRKe4eu$% zE5_qeC2PREA1bVl%2#Hz*CF$Eu|$mG5ER4x%b9~D)>Lk{7{mIRD%F4guwb9qIh&@hSq z@wvikHl*HnD4^MGmp){7t(#{@Z9C@gXRANUV~6nVd%pq&!iHU_g~I33t$q(02p{w4 z<9*unL*~m0gZ)Jy5Ig53aNfmUF;U{U?Tb{w$yyuM<1w3K zrM}W3nfAeSVg|{xZ9B`O0*``2LcGUY)Fz_WnF8DQl%;Zm^j=?F9`1-|P+RX)vZ;2a z0He&^GY}dHP#NNqwHzqFD9bHpjnW0vzwPNU%1#I=QVyB#+nU&8=B`SrGO|ta3_Ks| z{q106GBuAq`4o#*-K%3M&wYm;*}VwJiEEq7uw%SV9mU7X7)=$L(7=r|KQ`=cK}vk8 zwtDozm&mtFP^@_wzI-%T~2K zJFOI@;MkJIEU3OZ7Cv$Lz1jly(wn>QYHtVh4>`rX+uNLI-(F~5Y0IhY*B*U)iGBlx zO<-jnrO0&UF~=E2La_Z>Gl8PSC($wZE}&Ofwf|%=aLHIa@a)h37W9eQMAPkGDRl z_!l&J=2JeXxI1IPBfs+-*&6t_)Qg%#2cP>|1vLLsc8r|k3LEuT0XHSb^Eg^Rd3$M^ zhFRbZnLye5pr>;0&qotD4Ft82`Jf)@$E#qe3zfFdMYUcywZ}$pjCj_04PM*d+X)mh zQFVQdIj{Bilk2;3+xNSt7cFB>V5u9>5z-$L#c}b$jD_O<6^a8q)|g%DQu@~yPAjis z=c+$Hgw{7VZh!lUP3?`7*J&sUo4v|y>;3o^azmJ%qSvUU3ZfN{wybt2Ck^n-{~dB# zcq>&TSEfdlh4;z2!3p;{VuBYIkgPl;5+2 z!s9Na*ZMGiEG;k9Cz{_trWK07Q!=aKmHIVPH-3!|3O5$u{$QTvz2lYu}<_*+vD z(y#tg&M|tevzY79^Ufrm5Xh{iW#$r)HZKHZjG4at{p3^xLy#|9{s)=}=7{(K!Kl3r z7APf7*kL5nJx}0M@t14t?DZdL1X<+!j}UQ+J%4G3yM>}|GaXVM0JnPez{pjgkl}j> z`E-X|@p=(dNc-yz|whwx1x4 z$l1doI#``+O~t=2i4`(<2X+vZ!rAuyjYvIN1XJ@!OS*^~tTg?uKd~4W^X$=7hct`i zJEDEdQKG9t1BgFKuR|tIw_3wCsEpP#c8$?LIqP9Y+Dc}{=tzc&Xvrf#sYQNbu!bGJ z%SvO#6a4KauENmV@ettOkg?dqlQ33sNt-6(>PmI@Z7|s z0prv^4D6D{-Gz`XE2dv%uhzJ6Qre+UDpX|eD3-~K%U_L@W&<|tNs8coa?{1sNM(e4 zqGt5aaJ&VHy-Vc+Zti4D&SC&BO-bkme~@O70+zk>5c1GIJf!GwQIGctL+%}TF5HQt zow`k;Pw%RI-JGHMRV7xVo=X?$#!mlq?R}7jiQFv4u&R_PZPDCvLS}J3WoS#qseI5i9J&0{E_ewGxqi|g~4esdkHs~J@jHa z*84OU<8@MfAlCq0gK(B%);$Xn5oU3<{0!!LIw9m3qmj&g5u!q|3VYMtWdRtKQww?W z!kvM0pI}t+N6;_11jz(hBc`oGUWjXPuV%k-5-0J3z2WYFt187-4Dwwl@|`|RhU)z* zcrY}WK#LC}=fTjQ#lYo7ox$qN<&=SJ5};r!yem%LVdkvb=gw{ETBo@2cZcEY>CR5U$4y`O3#r zkGHQ2dqg@?o9aG#hr))uf6r13=iaA#M4D@k>H2~jZ(5qYF`=F)5uh#7!h$nD0~yWe z#2R)>7S=NOmU>yTg=H^k;7*+pggNJI{$Z^d)@x{0?<8g)M4@B1#{&Ej=0bCT-ef)e|=DLpOR-+*uJzsVWBN zv1}H`EHb|?@3P9mOCwQ|d56rTD?AxFsqnxYnK2!7k@_NmymjuT=IT)#s)+-xTSC>I zb#rOM1YJ*Ol*qSbZm=||Bp6U+neb_sAn8K)AU9;|%3^4985yaxCC+ND$EHzpeQYLw z->%m>x%J_^`Hk9mQ*%W=8VUDuze)Z{#Yz2++r{61tNF68s&std7lQ`FoHU|$MA`;s z+;jaNIgXG?uq5jfuP~;13qLrvdIGC6(Y8l7c4`5|;5HiYw)UXonZkKm=#FWQ{zT1Seja&h9DI&so(4@(kF6%-z9Fww zs+mg=6o^l(YeW%b`-x+#`VV&Vkt*;uVUDW?ny7E|vyxL^dol8(Xm9bp-LUJI6 zV^UQcK<)HKe)3JS@uhhl)5h_ZNHLxUKL7rgi2kXN9t}+EJ=TrpbM4Wlq z>sONuu(xH0Z=wn)t0{x0UiV^G#tlTOHbR1OFP)>WpVwvPgSrfr>8RbvZg?mVud3k$Lm?% z0>pQmc@3nLb>y85_u0NEL}v^HTiY@b3NT~l6+F*k7NGg0p6?2%zz|lOHAlmYbdim4 zXcAAil<)8>azeGzO&A)J5|=Mb9+|xC%s)W4^@MXlgDD~%Tj4!Uy~u-ZmwIA6%pajr zzjw%aDeQgbg+yENl7>rXGh^k0LfM~+l{7_?&zbSubAFByKC++WVoG8!Jc?y)5aMOO zEFWJ+#hBEDk=Acvyq(x|%VLve@MzZ0cXl%09;Mp&aClTjC04hki%TC%M(@*VGJU$$ zka5jI6_4N&j_0w` z7aR^1X}!A|n`8dChcRdq%&;m>0MZZLO7R`Jc z0@HU0fo0e$7&jVEXXMLujv#Iqv!ya(iL`jxFvJvk_CUT|3i}&idUG>{AeA5??5%;k zIv)rsMj9}uZSas#2-(G7-vOfsRn9fd7^}LK3Kbw-TmZ$$&ekUd?0e0?>MaK-#>LMS_qjK$7!eTA*#fq2<&B!}0@EU$Sj=l1<>$(*jnS2}tTV`ey`RdORF!x6ulUs_6(l>4)1a_C0=U z885@}|6xFB{N=2+`^;3#XnslqR5L{s1=nwZa8;MR4jrlXv$(h5m5NrHjLBUG_l!yP zs^~?AQ0lO;mo*w}O=-!L;|eoP;`(HThvpuNj7B7SCCY|e?1;6LGkZ1N*7LhF6{w>W zI}#67ac`lvQtZzsVz&a3iFNEjkyIy2Y#qf5yN#;MX@p5lGClBN|CyRbMWZKO3)OSn zf6ewkYt%O!@2O$JfYxzZYo1XjI#tdF4bl(sbf}cm)F2_sBjb@qsBTLL8u@_7lA3s$uf*DDa{#Pw|9a{67>+a zo7B@K{(!+W$@*y8xt!lDN@;1yVD+oMhFSExCnl@dhD32a(G>&(l#Jycx8gLq-AuWX zs#b6|h~u2dFB5ORwEExw1$ff@9~?_IG86I`RF~klACNF6j-vzI?L)UV7=sZG07+Z| zPcSuNiF`F#`M*D9G*bf(_Lr$9BQtG45?~I1_OseyNA z`qJV+Ap&#(K9Tc(7~dI(fXKR|h3>1;lfwVE0FT3RH=&8t0W|9vf-;p`RyIXOWhyu{ z%#w_~K0abvU+70V?b)xsRd=dC1iuBBCKVT7HqU0aZ3q&9X4q}4=}Np%?d6#>^Y3!H z0ls+i?%g@pE2BdI*6gp|)bZn@nPJvg!Xl%iB+uQi``iZD*s;KI&Dj0z1$2K-PEPao z-={XUl56pv+OHk}yFu#V?qF})+Zc^pZd_O#suG4?Om;O(Wjx}#kk-|KmdFroXb#1# ztb?J^q;iID?feU{F6fwle+nI2uU>L@AAE4~dgX==AUp`z0)UV9*<96Q-hVxA#n8FY z6rci95R+S2{2ZzRAXH8B4>#ri(*pDr=oDP6u!IpP0e-NVe-&W)#QJ3-mu#s;89UNi zW;@E7Sw6LY3qZxz_s0d+HLlcue|&xjKyHpsl;l?yon1155M85YC` z$#j{E00><>o3F`-f5oB`UGXbVBYYlI0n^!@`Gw&t<1O3sIXO<7xA~N|b|wza!|!?3 zLyrIwao$jV?sY6N<0N8K@Be!0-*3cP*57M9v3>4S5d(nYV}lf-Z_(@TEUFx}M22>@ z7YZW5(Xs(eQu64V91U!z21C+BZ_j1YHJ-7tX$qD0EQnDC_>(IKAP*m>czQ0xNdR8d z*;#3i(|a0zu(zocJZq>8sEq&B?}EW^M81L1awDb_=o&30v8)-UN!;>YU+bh1dId0> z5&#eFeidkhD+tH@VQXG)k_EAY%f0@nsdyezdiuqstB$eT?q9)~mKNqzVs|6DQiM(B zdeXmtsT@__S!vyLgpYl z`}fC}%HFw5r}t(Cg8DG0EZEHUgS{2wnDGw#tKUod^0jip$0Kj%g^xE{ERU9>O}K;Gif7$L$`A+wOJBQuenk zt4gOM4H7n(CUbYseb09O_4WDjfNxnRe7~%M_XeKWSROdV;{N?Z36L}*^4wPsU@KI2 zmTI%LqLe-Ju5kBj@5rUg^6baFgrLyNWo3U7j-$pISr$*S)n~x?sf>-3F zD`IvwqJgaXvow+lheu*}cco>WIZyqHOs&F0|F3mZuqRV+T8^%--BlS2p_ zvV%QkKKg>6+_`XGjrzgBUE99(Z-#5yy(I#keDd*Ny=(=Crn^8o0S;Ijxr6u`;KphY z;561CA>j=18>v8VA=6avFrRCMNe9Lx#Q!Ph#{{A$0 zcmIUi_u|wy@86HlN1nRDSO||H!~-NZIqBzTF10>hZzx_vfMJYse*dW>Js3ic* zP2f@QFHc64+ZQ^H`+V|RpE>JSEkoCQJHs6OomcS^rXOB7|JMBmi!d{1pkj6d9HqOB zAq&9ifKA6d*t3eZz%x`gzl9(mzUp$rOvG9z&j>){VOuTR>kEO;%MlZ{}Tz5h(Q~X0kw(I?*d_sF8{h2o8m`A`ioCW?12%tfD2RcRit$0kC^Oq+! zy1dM3LzOSaduHFJZUa%!@cmH^M3>3=7>HTLaR>VdED3Nl3784V=krIN*f)|gonrYg zmF1C40;+e*E+I3!nr-WzPV?1scVFrG@)f|UT!6$*pJiilam4%-9% z59FA3LG*7rdAs`A&sI+I=E7?T{`$~>1+oq0Ft4UFekkNs;SwVp^t^5aw4Dflue zdK{vE+9JaSa9$F{;X{zOvK8IwFP2*r0^w-)tHSoV3^ex-ul)SND5ROiVWN2taO+yg zYfa%R2x%C1urr~wp>(lDY79IzOD9hIX#1|B&i)CuXWz9e$VxOAACrANF`tvpk2A&6LL~gHz_D-P(eV@aQfz_98afiF~r1=t^e|+ z|AX4USE_5x*1)^Pn04<#$fG;?T2YF?#NA*P>=C&spcfFJ0J&6Q5cN%Cx#2ocSDZVM z2rc(z&3EYO+%*OT=ijq$j|YzDLWIBL5H?a`*xw2?6NDjt*o_$oU9J$CcVT}2Ia97` zjeT^^MSC?hHC9{NDq16ryOYNzeL8fe?c6p$6JC~R0<(#G{_(uVW$;=huYtqJMDoG| zv`DT{tf4|7Y8h%}nZJZ|+7Ety9_DA|6mtUL|4HTUB_NwL{|ub62f*I`awpeu*nynD zwrud-V1$qX`a$|yPwLG6&Wh)cPx5yzrpsE*3m@ z6&{)tASKZ~K-X^|%v#|o+J76r>}Gv|n;k>SM#zgHqy8@>X8(#C77Xt(kbh9+hf8DG z7pZpHtn)o5+=(t{Oe_x;w{Y)Q`vWkTtOA9^!@bD#cF<!gqffy?gIeX76)A9kh#~>s%DCSPxKknTD+)I77U3n^ci~kE>|r?nYJC?d3p6EZLMo3{iE}FJuhj6?SJ^DKL=xdXp4#R zc%O@d%7*ut>+7(gg#Gn`gTA{Wr@RIpP$cmjeH6R*4{FOAU^tdmZ4>TW%tKM^zWsY{ z)K6F^R|)NMBqZn<^iYsK(T6l}CjOT;mxVt~5xo|wTik9YjGF9O~{Wni`F z`N8;TYZ_AUtgraMDV3C!t$5?>l)`)L8YFa6ptACT5i?K0y;CadyFfhrLg>~}uDhF; zm8m+kdP z(p0)X^$X|J+Kw*Iq~T`S3jQ96O#2S#OI|&5@AVpa=H7GTBpwpi8f+Ze z<=6nw*E|vS*J+p{mTJ0{4|XAvhm5#q-h*<3l0~yxjS-DdHPQdRKfW!bb>BcZCQbHf zL(y5MQ%wo2_Y_2SuWM$1xY=Oft3_|VgjhECBG3uwZa&S3pdzUwwr95NGgaPx>0eiEYelVOf%^pYnHxO{B7C}1>zGG-i~5L;k0RVWFEyIu~5FfMC?>& zJ7IzF>A=d~XskJW0gj-Ul$6vQtyZ@uO*}TD5gXa8is&`QTcg%N2=cKtYSX{V;7j?% zB;b>P^d5b;H=d&sn7=jsag@EI{pMxeLqp!FW`KfB zU@<%K85n5D0<;kOG(qJdaeu~*cA8LqT%EP;o3M$ONOV4y^Qf8A28$%|^lm+IVFaZ~@$K|tM(c65PDINH_WMxnatgZS zt$#nOMnHaX01>+sR09dzR}O~S_d6lTUR+wpgC3=mnfUWd<;-ULffk3<`PnyODYHPe z4ts19uOS-*-Dh?5b}%#*Z$Fh?XoO-31P(H0R(&M(>04s---KA|ko>XUe`PI!O$CVb zBK@Rc5KYv7J%C2XfV`tNL zn^caGquMA5l_!iU2{ceH8ZVATG6o%*TEEnRo}zK3t8L-ejKDTFHirKCdS%^KrmP$) zbbTQuUMmQ+{`;4?i;?2jSep=+OkHpp|h+9k*C z3DT&ic`0Te3T(Ltp>TPNj8OUou7U-5dFO_nI`F9k2;3A5;|&D$)IA+HCSEKa>Y9OK z90$_GAbfj%@2U;pde?rAFD^Q6JmHcHQwJEjs>=D&xhGn{Jsi@B65;I3rnGaD#i_} z%V{XY+@bnGQYPf#=GSh2%m<;v_X6_~Udn4|oJuNh$EiE*fLQ8@=1 zUW{=Glp)c--%3GvWkF5`dIvZRr*Qimz-MCh)7l6VpH)dlb zA*zrrXmc!F$-RdY+Vx(jLrlPWHUN)6v3C#w`N0M*IY4_pIv=hYw;%nq<+l+i&d6dZ znjK(XCS9nh_>}#40TxkzF&1_R+!&4zw{F(=za0!mFG0Zz`9~-kWh~J*-4+c-VhN%XdnjS}GhMe!YObV|1sr~;@QrE{J^ zdC&&MU;s0*OdB`xp~UXSO80e_lgUsAz9Wc3O$q6@8serx)1s()_;SnV8pQi4X_;pc z=?8!=NFA`V+#EQ5Fr9QX;EDsrjd$kf3G@=gc4;0A#E@Pjs4V{Gr4;H|-hJUf~ zOn}Y{B z*$IGRKMdamqhP9sk^EZXVTF%+I&9CXm8JQpg;sRthbPtrFvNu z(dRw5&iwV83{9cz?7zQ1UqrP6BA0>)pGuEDG(|cN5FAc}gAMx>y(ImE_hbVGhseZ? zGvo?`=X#EN4d6l26b#!IP_cTag#qk zSW2h6sc^JCh)iLnZ%+Ywmm~@&&?UbPszue3SBJ@kH-&oOkL{C%Pfu)8A!jYEHh^36 zG_*p!(b?qaEfDi(kWR}9HYn-&>?w$j;akn+riC5KV?Gt|lj4cXY9H8C`4@54KGTD{ zv=)8sg{WG*CE1wl5?L=ysd>tpxF1(iX~7;x>nMI(diUJx=Z2&-3q-jDswFhI{R|*Z zDNO>>7jn;geaoTCeV1YLLq*nzVd_kW59oh0=}uP5h#XcB2@MG`XOKP?kT}-OQ}qHi zGi?wFtf03;B1rrGKUqMeT6?W=3F~nh>JLf5zPXLuz2<=;~^Gh^@+@tDOe!ParAbbU9-e{i7Ruo9FNiozJ+s2lawVEhSv3p0p zu$t+H9?twmPft%`6IGeL>~h-43$B|v_#t-tq~glIDPxD|LDy~}nW$KH_FEg-xkFs3 z0?&j0l)sNNirXVt5m->P7KWgoFHGde$Wr!me93EMU1T~=yDa~(N!3Zw|%k7ia&E{Ze8gC6{&058oGRH;?}AiX(2qDI*wLi7jM3t^{K|6(|@ zW9ZIOH&Lje05ZN+Er&~R1xCg|VKt6ux+P?)g4o2nTr`Lu^f;IX4lhOvv@&|(jMe?( z(?C#VO9;_FX0@efg7*}u^ERZjq9>>nQ01u4Zacs~b)PZMD^_%Y0ZkNws8^5G!&>UA zB!lQ&1pdVl+R=aMM6fFhul+ixJi^7Bb#d;!x<}j}*-r*L^NVnPL8LC`^ktY_dQAp7 z>Kl~ZY)wQ<*~zVQ#Dq)gokGSvhrvuCou{T2c)>&LET}Effs`*O#qdY*Cu;T@x05F@ zyUbf^Hq=H&`EaA`aM@XZ^i}jJLL8+hx&l#EaXxpV(=vQCHyX?AVv}I28%;mxH%Zem z9E#(6!>uR14nq%-HHd4XB=OFRG|OlqDY;`H;b>>yZ$EaVs7YLcKrodN4#o?S4=@KI z6DJT`NxM2Zqk|X+7_Awfh(%{_akzBw;rM`3O&4A>RpmKVC{Cx&-gGCK;=B~`d{dZe(G+9p)eoU(M|z(eYcqz`!iGcot+Rb0(Uc zu?)Anz=!7E#)33~ji^wxGB+BBU1eOg2<|0ua#0pqXwRX>@Y`UUvKCdAETct{VD;5g z6x~lK6pTI0NJYTB^;8LTT%wp@%P1a}Kql{gNlK4rkQN_8=Wv?A&UwF|{B<@>LYfOa z?YBH_1FyBNm-KN?BXbWm-YiA|nmUx^^5j;eyTje$PvAA2Eb$CA5I_{)>Jzc5?5Z4F z$6(MBQBWZae~9@b1VJ2*Ba@MQm5#}4-tLupuE~mU zE_1@+D~?tWxy=@Rt-5XUHz>Ujo{v;7B}89KH)a4%&e`ff@Fa|498L?9VFgBSbdPu& ze4*rF)hm>TZ!2G6#`MA^L)r2WVbq5_c)i){Q&Th@OIIMq8MPVX6UZ~GYD2a8!1DwO@9gPCDJN<~BAgcj^syiYO*8qxf+ zJD5R*?Z2aHp{zjBg=S%EyKQH4W5||%SJ;$iIH~%!ew1F|Gf%v&dxSrZ&B!)$5ZYj@ zO(x0weZS6uy~q!KcDbC@Uu!vVbz|Li6z+strB)9_D@|UStYy#l)X}pV2u!N3GN$Nm zImcTS!8M@1vF-s}hlPi4An@9&-LS)((D`zV6DZu6xQjd9ZcZl9>dht($qoHjFGmM4xy%pvvWLDmBC9&f2VL?TeMvq_BsV_XTrwgb z@8H4z<*)$YrbkzUa>eaPS~00qKkHqHvfVYiqEAwFzhmf@&SyRFP;-$S6l{zi;E!;- z<_ocz?}!k+fiwKP$cHZa)MAIt!`n|UOS)kCUG+BWLy}f=8R$+4YLhoZA!VJ`;Lz5* zO}=to!aEB5K8{AgsN=7rRNY3aInk|OL{90Ca0mY_7*wB|CB6z7^x6_l3-vCKswUl# zFSXy{v%AmvRWE6D*Hgzz+!lP|uu%d@>};Kqizp=o68~Sk9p)@19OJURwt~(=tAYcZ zTXb6jrHM77+b>fIxD>s5-K8_Z*kYnJJ~pdh!z(9oY$SgI!@zfd36)s?;u=jqVIYB-`8a;N=@zGD3S7F98sOVf z#|`a?1qlaf1)1dkVnJC^<_#-8T0PX!_i%#ML*(6xP{+SpB*ohkUyWsqeK8Yf4h}^m z_c6q46)ET%v8`iSx#z_vBRk~!7C65AF~0GYsK2~IyZr9O%2mDa&q|oM7kMWpdXBZS=&$aX4n+zg7k{0P zsH!l?I_(jFJTxV92F_-%8<&gv`W~EkBP8~?7cseHyx%4!*n_md{5~8i#Y(F8;=~TA zRV_pF`O?#i>N_$Jyo~9RnvKnk)?Z~6UrM^!@d^S@4`USSAziB1bLZZ^JW|^4-NTa7 z^s;odqk_qWpsm!P-r~$+suPtPJ&f~y#o|ILVh@zoL`Mirq==VfM(_+;;VUp>r+n{8&8ic zI_DSDZK)^6A~;u9K*Li^g@_;}dgdZ5~BZ;k$!ij3!-I(Jsf1UM9=p$g`cfYmdu-Rx zOWKd&+aXYreCXdMi-RgK5YH{hUnhm~T45xfhdQx6rIZo9 zoicyskb$|c-Sl+3dXF3PF?iJ6!1*IpO9}NvfMR|c8eM!X5u~p4gK9|#f4vOxG z;ofD6JtVQ~pOY4#m_>#U4aSHSwTnGrwxyBy-X8=?QqgUaDZOu(!GYRWRnmP^87v1c zROj<~UNeT~z0Pp7y?l(O%6}|m=t;wUQ>>`15ii@GGW_;au&SbrG#gAk2friwUiAA; z`l;aytKJ<#*E3Gy%4uQh66Hr#>&zoi5l228C-IL;>r!A*4QOjYWZ~KNF!q}^6b)L1 z^1{PAqGfD6bHC)~Z|3NEI zXy1l6OGv_!I?beAN>+vO-)_|!s(;-k^E;dTb%UW}Oyr29X&u@_KVq0o*}O$@MnTBjYww!_OUKlwIlb#mt|Y&fd=H~aO?ppO;oGVAHNdo- zEKgaD5}Qr%1#D*w4%T=VR!~=!y+x&C_itcDn=1a*xSNvh--_QKId!X4{Dhjp842kvn9)=5Z!w{kf7gsA6`fJ00_TpOs4p*0DR(P7m=}qy8NJrXLyumX>}5OALK8Rl?da zKa{RZk3F<_?nsi)6lf1*0lkau3+e3ae66E%T0ak;Wnl_pi{F83+wYaf#$ZhJv>g)FM(ifn7{8k+#aWs zqW$j(fYs)o%s9@yyS1n|SpZH9ZjyLc7Hqz3-yz79Vxc>nTr@xN*5a`67K?M>$I~A- zp<7dsvu{`}odGm#4%9VMz^@gSmfk|{#()II;4W6Agze1fI9+QIxjV4Ibg;fZYMk#MSO}<2^{YO5D3)(DxdEFI$t;)K=+I= zK`D!o{nd>4VfoaXQt_AOlLkaB+Fp!Awb(#DcG^;mDP|`_ z^Jjnm{(VH=-zMoKuw3ZFN=mg~)<7Ph2zV-}SvmoV@y)Av^5jxq)-iajCUV`Q*ZJ=0 z;SX~Ygl4+`Z0K`_lzhOBOH98&H^g$jGZ!Pd1T`K=q=MbuXoUFit!fQhjS#hi z>zUvT$DmiM6FVPH!Ogxb`EQf_NRmkkr!E4A$=rW}qMgbEox%|y_mRHTTceEZKyE)x zddk;0d5z&sPme1=KLBiA17fc3%QjGRgx|k|pViiyG%%tOleqV0TwQ2f?O-@$fdOPc z48f4!=V<_K#*m(#4*2OUfb&kQmvKjVmkFKhwFUN`!^H22-C{kTpWHsAlAORHWo~U9 zVIlH_PeM>5%&)a=9>i1UA|togoFa&Ib#+(UX`Mm&6AeH!ZNUql&xw0Gn-?rC!}^AR zYElB(4+3R^LeJpL85+3Hs!Gc&a9BsD06&J&J8>7D|BEsND#(J$=wjz)Q?%A zLoh=u^^_YFl{1=(^e%G8m6TFJu2mDeceN?(7rg+##93owHvoC-{703ii@RU=4k;KJ z8)pNZ55hn(HMML+<_4GYgxCL8wfv?RgFB^ULiLF`27SBRAdyS+1oa3XK>pGbr-1db89x&_!fiWgX_q$Af+ zWo2am$F9BRFu?-$ysUuVKKkM)3gI|B;mWyLRW|4fKtG|rwF`P8_?K~neA9H= zOH}>htFK{Q_X-9jN#dU~hu}u$oY3QB3$79^x#@OTtB24v+!6M?`jTwi|AffFWcCRn8gP{y^UNdpPju#PfyHlKJ#-TOQH1g7P*jzBmtm_ z4cIh{y4xsI#LGt3`Soz>I2`W$gwvJJ7mG_uYWzkN0p-_f0+quPs~+SUFKA1Q+}+(P zXW-z-!Cl+8sZx|daCKYw3ezkI`iASgIj+;Pcyth}9p-@cKxoJer~fLeC=MW8TBU{MBIcj5a- z)^pMo12N#x=wjfOe-(jmAG2TV$Ox7M+5lW1q6?a324=uLACF0cwy*-D3^<2%_{+=7 zk-$X{z+klaF%P&x=L2v((Iw#0oOxBRRxbW#>*MLU5E%G9Jv_jzzERov&YqKIX|GqD zqGH+wTrtPq09?8URNbF(SjHV#O8_TG9d6VY{QmY9So}?#HqC8!+1tLK^MG5&W&g;% zo&-#%zz|>je5%UJ-;cN!s=QnRoCfL#j{Pzh0GIu^%vn|!6ddg7@4vkI`@7Ol3nJlMO8Pt~(99pug%z$Hg#fs1<>EuKyZ2Bx{b^4QSWXAIwgt+aC4vOU@btQ#19 z=rzSm63GSzUEwso7`_^&HCB)A{B($a!296wvfisjcdA~meRFejJFvC@O$q@gTx3dB zOylF3PB+Ln8U6|W`Y~R>uK!G@#&ngVvw%&|Ik}8(OyZ2L0%5u;rt_E& zTs{D7bgA`pPh;NSZ!|^a=uBXkv1kMPX0IDGTlklscbVnk74x~l_CW1{^NsbaWp1-P zCMBJ5((pVu>DB$o4U&Ou6NT=WJ-FW>SYa&b*)<1P=r6GSV10=FYCeZ@?#hX3lZ2wr znl`Im-48q+W21nV&>yiMh6a5F-hVzf+y@z+vpnp-o%tn^NhBenmIEp+wHF`fZ zPoEAPc6V>=@(}i%1Z-T3WZ&h`7Cqgtw_SF<${LwCwm%F%mb_f-$@yHr;b4o9h2-%~ zsw*Rbos+K3p;!0!JuTj|&oj$sKJ#;TS*J-Mvw{5v;269}Fff*Zj4n-X@R?&rrvQBj z6i+$@30MDso@O1TaS?83{ F1OSk=@<0Fp diff --git a/openmdao/docs/openmdao_book/theory_manual/images/Par2.png b/openmdao/docs/openmdao_book/theory_manual/images/Par2.png deleted file mode 100644 index 1da84fa68ce41a26d70faa80c7f8175cb940aa92..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 90636 zcmb5W2T+q=@HQGk2^~X`qJVViP3c9Y8L9?QkQPBe1p(ND=8xI=m=? zbWwVZARrw?dizfB_y6v_bLW0DcSZ*0&3Vt+vuF3&-REq?Uo+I9qvoK7!C-W#tD0yS zj9dr?BP*t)08glfj?;<1+^(8?z+ikh;t$EN(K{#bklj>LcGdD+A_Wog_eA;omKb=|dHaQM`t7{g01vAW2xzvl&B zpW_sM%XH=9`yx0s&tn!ODJ5F-95o3U|NGb9_Mb_pQq|)xy^nrbTD!`Dejn+zIk)*?!=7V3D{UIKqDHowi;eJ3ZMY+u)--J$k2Q z`@UnqsLJ%Heam*IY=e&S^k>=W+Dh4mz_fKvnd#aY_Q{IWbx!cCT(-eDZ5^4ozWR%O zvO0A=Rd$*W%x9RkZY49V491nDu3O1YD}!+sLamjqjse@9J8Rx8+g~GG2fKIDQqN9) zOkJ;+nH~ireA3oe9a<{8WsjD@h_;B}ywvr;jfmBFcJ4UGfIRT5C&CqMuZH? zIZn0-bm`L%4M2Ft9uq;1S`a9=t&{`*m7dY4O`-M8rKbiLk%qoBz3M{ii& zoRp_Y|3p-(G|0}S_crYcRmxhY`z5>9)o(aP1%K;WpJjXzoM0Dl?pt3|6+bHa|M(AW zP|Ooa9KoWkj=6`C_?A$0`oC8$-cji7^^fWcxf*REoT4a<`XJy1IYQA;0p95qziCbs zwidf>%*sm~E+QF-ZG<7RB{cuf%Mm}#ynmaM^AV>zP>ooZ99bH?6Z?sFVYEGe| zMM7kS1W*;3^gFa>j8zvOibrQI{6)I4t_3YzIoZ!l7Q-4V0U`?bf7~DOQvLn^ZkPVs@c+j)O=GpNqcW8nqlyl(`F26(88!4hHM7NxR%JH=Lk5aA ztID&!F_=-A$vxyenITtuQ~bN1JbBPS(~drYk04B-C$#bQUw2u+K9JdtzdM(3*}Sz= z{E^nr4)ejfOwB~Zg~>8FfNBS>zN zXq+E+cZiKTvpdDOY7KpWUO^0ag0nQi2$ILmG7j9FZ!mI_+NCcRr;&(JKf|UWw)o6! z){n13b>nh>E>#tWuZkDX%xtG@&z-L?d2%~mnl!aUa(3*jPj7IWja%~`HcUnHo4%f) zDmt{8V%UD4A5}oVl$6O4`_6%POpQcAcyam+>Eu=Pb^BXnI}Zo;TVMJgcbDC>3hHA} z$olGq621UOkOZTw!cI*3f~?R+;w3U8@iNg$lB2lks)=T?8dF?Im_6i5`s4R5Z4D0M zfT)|@2U}(4T=OFDa+SfaF$B=o#Z_kwu$m>CN!aLgM*2TnNDO%-Cl!})o2HmNbrg<4 z+RCj=HTQf&KxNF?>|OUYeKwDpdmx(EbI#qVVLjX!k&?Pv)keXsRI;IPN}2Ias}s{? z(P~eO5l_M)$|h_S7ms)$HHZcBZDF?3iya^yN@bPJxf(_!CUK`wY%M%)Y1yF2u;=m|Jj}tZtR`xVz7rspQUj zAmIvTm&}W0l8e*!}bCs!lK$!G=Z#BNpMOo9Z&FgRM%if3JTzxi@{<*4OtcC)W zlu-CF`ZLNM@9e|1 zhb)AfP~a~H6wN(mX__?{ruXc0_^GA^V@Qy^YND*U^D{$Ok%g{@5?d{oLM&yX{|wuz z23hbiS$C&Ls=(dDYV(94!oe!^NHNlieaB;VM72U*Dse~ADsgAGCO>&@L~8L!BURxD zm%o3;>;IDWbE!*M)`<9otL%VG*&jgylAx}cP%YL=m}bLB74sr>3yf9-hKKh^2T#dT zr=N4+GUSWXaMctW?3N2nDDe|T-fhqmi?@E2VD3$ew0&@C$xcpXQpgha4jV4Bos6%C z7x6l<$|2tl^lBnz1@J|adw=tr+A_)|7*-~GaasVSL>CWlZk#=qkit_&%wgb8PW7RC2@SkY$qUypKD2%e!S%$afIqu2t(A1lp-DPmb;H*XJhB zEu`lbeJM9oadI595;xXj!D7y0 zmXcq!W&`lcfd!x?24JqM{EZ0BLjuC3H?M&j@92X|%< zQ^Qn6MB7TCdChlmr{kHL-27eMOxsVMvMKA3*}e&vTh$bQp&65K{EN+rw?R_;<9zdO zX+P6X2R$jp*aY^m7!2d-u~Lj&sz159|Loc))w%`~mxZuknnojlIJqMASL@aeEvJ+~ z>|@JHP^^|h{=Q?uxFgcL3MC~%q%JH zExC%T#b)`l^kcqbv0iP0QR4I?q^xp|&raA@frir?)v|Y_wLg_Lf5bZ<;=6wjAANaz z+0AH->}*=K`s)Pp(O1cM_T1$}Lop$nXOt?e*&bF+@w^;j6-zXGuwG9`Z6U5?BX5P_ z;ViAP5x+VUx}-4HFEzExkNHuYe)yqxmeKrPXyH$7Ghv6E8`bcgAjEVaex}pu)NR92 zZ6Wa&o)o8WbIP14d5H&?O(ZroFI94s=+_h|YAOuNT@%fGTfVaGt4>3KT5d}VtPEl( z9QT&&zQUo|dU+@qcZjc7ueD_-EH-bEJUGK)-@|KF!Zi2j+%ZqxWhdjzFKyh0g@#Wv zog5|_Bk#KPC9XP^{JIZ9Y`8YzO`q0Du5e{RRY+Z6i(VyaC|MxhK8%8Izmy=>5h5Ss>u{zg+>O( zIjW3GK>YzF%9}#w7cV%dZC%XB^K4k$Ze}#!MmV}yT`%nLT;+Grc39N9@K!N>M@3$s z2IH!=-{tg(ly&2a!-W^+L5*r{0nVoiZo4`*IPP-HUg~UL&i5)^Y1CRWGWiOOl9qKJ z*<{yXo;ZBD;AiGZDkhSIei|c-8?9ZhI)Wze(h0iqC>g11t{l?4@;uJ({_Tt{=Uuyb z>q(nBDJ!FY>LCB)frSlKLfo9RsN;Cg zrdP~Aslxg?YLrWvq|Y$gZvBe%JS!q${gi|LomckHRlNNtPT^VimD8(^tVC3KEDVEhR`>ANi=(i004E_Zg; z_F~w=A*@jA$j!YVf1mRr44l~a$9qhp=eyo)bWsYRKHRI{;EwLhHs3zhZ*986jk$`v zKg7RyB;XfmmD;_mGi=;rKYr{$Rd`dP@SMZ*D*>skGYSq5Z#}8VnO8`7`kO6%M5o!g ztyXrm$e?r9vbrAOsj)S9`-gYb{R(sCs91g!y7d`S^rp8iF|lTOm-4x7sh(xS3(wX4 z*uaKbpq9j_fn!VT)22t_l|?n|;winyPM2kS*>b;J4tUtuRd~<|Tk8FhLH#SZxszZ4 zR=G^}I*;Y|fxN@`WO(EmvIke3r1qr87w=2PmMLN6q|aDKXI}At<%>M5R%r#rM}#}Q zka~MfqYI*SlQig6)iZk>_vzecA7>f&tv@Ox?{K{SM9Dig5}qJ@>%I?+;m*n+iBKhb z4?X{2_t_t5huBS;o42C6c;kGH+UpucN`z)^Evvqvamn`L?&!w`Dk*FUnQseyA92(D zaN3}`XUi^^2GmMf@nQA7;~YBCnT8^BZhzk9@2d;?+*}C3<%QMnJ-7N?gwt|ATLv$2 z#EeExeetAN^PW9&Xzr|@u#0UUr><60m+a;>%$yIk_*jTs4vuq zS+6;8Pw)Zm;I)=AVjf?Z?m57%}auscCi2?r}&(Q_iH3sGX~kWc?nZ$&?Zj zc1V?Df8Q5JdC-9Bge(+{SD$IVF>zi+>O~7V7OJM)<=9|&!y`MvZT2i-T`^){t^A!p zeJsHnSh7QTcbdutrvouURN}r6ESXdu=6Y(p!RJjy-u^Ds`$e)^Zx+7KBrD7y=rA*( z)i)NCXg$ZM*IFrfq&m)EiBQo!pnZX95{F0fZR!2P7?C7!kaM7#5Z54kI@$4X>_@Zv zkUI0*XO&@V>mqN`D4z#KPgk`;IL)iKMChPvwtiDJnrziVoU4JmBvM5?SuEo{VMU6~ zOKmo%F3V}YsgriQI(yhJV?;AJ??g=5#n+1=F}#8t%EGPw0iDwf2fkLrOqs4 znT!zQWk*hs#4aith`TkT&cO@+Si{EmXiQ-)v1l5oNHEu)KKYC?I> zZjLC<#^l?jbsCRuKon4+=2|&9P4`#|m3~*BVxrlAs&cDeT6fLtBB$kQdbwPxU+$A> zrN!a=MZ@_|(=MdlZ@q-~13k+S_pRtv0?*Cvi*abC^-ZaS&Shg-K><`WwLM#j>$V|&5%4p%CbtxXQD~z+ zB<^69NKbsw{ed&aFNXu<5qj;RaTW|Ozz-Geln}{-58peWd%5A@y=XQ=-RS!$J~uZ| zz7iRNCj0S!rWPG71uFFu(%|*e|Kdk*JM z{{-MZyP=~^^cJv+(9Xa0JP9U*;eVvcUbe)g=uF1@@Io>!gMTR!ZAx4XCE!2sV4d7J zP_E@jTp>!qZ26DE0A)1QXVi%-{3z7F^}y%4pTII;81*~#V%}@%|0tAyqO!C|5$G_j zL{h;-nkX}1RT4-e{-tCFh!%Jck+sRx?UnZ;iRO>J#^gr6c9lpvn9kc1{u1*G6V$)g zkP7@>Na8A)gwhiN2mJm276I&^@mUOUm^(Z?FHOt!E^zFlE6)jp)qhx4}6P zIVDTN5^)TSjdCfry5Qy9P))wwO>r0VaXC&l8kOXe&1g;bkpoGt7(h0-P~xe=F@jR` zW@i)_6h1z&^C2nZ35qYW236GD{}%cmW%UB$fmn#A)8+GUcKIsiEq&yfSU-Q7wfoXE zK*%gRAOuiqRtL5RtA(ovJc?AY!R%3pz zEvNamTo(~zhzL=yhtYj>OgIL!Vrj#yi=Pq=4}yZ^ym$b$Kn-jQEq)Jv1XoF^c?rME zzlHHnH@o7i_)S!H%W;b8FtnMLf7>y}gvx=0NoHHeeuB$T#nF$cN>S^&)n6GmrL`;n zqzE)-J%>V5U4(~bOJrat$mq$A$Rl8p0`T^6tOMaxdoWcdzr8X88>#aZXQPL1v(v0h zjh{g!6Z;VGaH!a22SR1yej|DJ*D-aPgx8t8sD%brytwE&hwBnXoktSe+6ut+l8Hw3 zAwYiKc?v2(TNu6rGr56 zXY=|4AR0{qRsge!o;kx1K(~_uiV`8Ir1sy=c_Q*2v-;Tv$t@A=vcJb`<@Ll$$eaqR z1vfW0bM=gOlMrLtOY*xMZcg9t*Eqn6FZAX+-yl+WA1H~oZ~Czw%Lmj$dIsK##LX}h z+BCg#pVj|q7aeo^OsAi80Uvz4KY2k;VyHE>q|Hcn|?ADGavyvoD1C5~mw^@%i;TJ5`;nj--?1T%-fAZW4Ob^*8r__^;N> zA0I(D(&2!OuH6>-h9+v!{7XgF^}attSgW>1oOXMHk4T|rJkxxC9xJP*+hzY8lLzK! zcI;=5vY=Wjr}@ZuTsjfpG6D5V#DW*l+uZQ;FJ*CWo7H0cNz)x()Kf;l3kA2FYr`q+ zv!dB#?ohUJ`kbv@{&#yok+F4yM?t9H|nkZW@01GDn@t7PuBHieF&(d z6{8Z1_B%EpIL^1f0?s}~#4V8?%3F@QTt+K_0!1VeZDZ=(Z0N@|{Dvy)np}^5^Jqt& zO_WU_dtX#EHx(}Fkm@y%cwB;K5{mnv8JqJZ4D*MNU<0{{xDEh2U7&CTh;QO|MB}Wp zl}p|FT9Vg40P4+!{XEW;MTqe6+o?Q?Y4Y<{<6ASg~{I!D-K)J*$gKl(wUgpYY{_J)xkpEvnxVx&LD3LzXZ-B-0_b5wgCw z_ei34sdTqYmuTi5bE!VFDmRUdnciu?m`e4TAi->p`^*)8rWd`^j}E1PiIVn9v)lrK z@FdJGyj?oA^j%M~%l`ma&YYy`1h6x9K*d%T9Rk_^^y+Z4EXlb})VU0PH;oZAND6MO zcTmuHM*9=creqClmmK!TSA$xue8$NkLemq)xfGA**GgUw-hiB-$bTmOSJw*5j}kCJ zbi%tyTlfpJ&N`Ef(MZ|1Ge4s(aWwSlN>PltIA|FIjO!`bOSWNZJD`K(Hil+r9ppyt z@&R0+fC9J2<%4JKeVpsoTk6(6(ToQLSi}e!%_dJCI5c$+_IhoNc^JX=HWyp@qEtQs zNk&zJ`rco+4BYI8Iaklb5m7)ba1XXdc)}oK+e|szm8^4obeO2(C$m`87}Tqd5Wl-L zR1(3h&P6bNbLi^eWkEN@P<8tJn7TeH`a{CM3lgdX7(nV%rtvsZL@78#c<;+B0nLsX z9D*x>Ti>O)i9(fHP_(~xwV@+Hx}b<5f=jjLGccQjo%Q^Z77o}LP;Sb@7WPYGuhf@{ zKhO5;s4SQNpcRY-c~Osa0)WmR#LJWaT=SZj5*SqVJ0twAi_0uHB>TU=dljQPthWNc z@`V1~sX=i0o?ETowvOugVH?77dalP%IRf5oTczR@L5T0G|7@o=%AvUK4?p7-9_c=2j=ml6to2J9s-jUdt(Zy@ve_z z{c}ktb9pESI{)VIUNea7RrwYm$bQu7NVTpHs4qQ5uUhV9yJ8=2wH2~1T2$vAl<=`f zZNJY%YItbq>+jxobBGu{iE4iNF*l9)$EdL3Y$9|*2SxS(3#xxb>(L5!bkIPi%{Mr_ z3*-1v%O^2wD<7dkeZcF*)9(O0$YIz%o}93^STr{srJklfFg8|q3FEicgov-I@k~le zBC4wzU@nQ-omW~r`;*5L%Yynq@0Nr^EAQbfeuIEMfoWVi$v!Cq%*N0H-1&*zJPCcv zW>gBC3a2W)g`Wor9Zgo0X^HxN!H>piQtU(5!W3_x5@#x5wDSR`InG zRE*#{F!oY#%v)8~EFh=Z#3%%M7rdw%IjN{Ci>ZQ_-+0dBb7jC8T!P|YDK4VQZlFWtDYu9x+ zO^e-1tE&@__ZyF=!PN~SBX752(7L1LjO*C42Nwu~bwF2_mSlIq6DijT6?)|*aDzo4 zi8HgsKkh5!N~9^*w?7n;w%8Z3J#H(&O=A`B>5A4AU@fq!xJAZ6aAm0FW?p%6Ww?PQ zhGX7993Pey7h@)s;7rktx^7ZoaW5-&$fb4}=DRiQ=(qDTbQkQtYQ5zQ&g^^L{z`3* zSjipY_0AVo8`?{OfvPuc0==RfwE#F!VQ1oP()?GdLy>&iu(bgT*N4qW9iZm1 zjZgi?Ww=e;lS%05|MUVJC2miiyHY!Os!WKR?1rB4tzH9!Aw}BBRZr}vL}E_(T~kal zCA#J%IEJ`|{|CS1^O%pk}1H$z0TUq(sEQx{dDX zA-gE?uXAr)lIdu9;qA{lTW687Ror9};0B!@QD|_@M!G&?e^)oUPvGBOs5U{0&e*7-fH4?<)BC$wPsH4=v91n*$rQ*KB;oy*3;fu6QOFdQZ2!n9eWjhVv%W zf#}Aj7T{aivpwvX{tzNTNIBhb{^E^ek z&QSyRi1Djp8$oG9!GE0J*x(m{d)t3KvAtqOH=Z@AVy#)b>Z9&$c#xJ@naQiaE93q_ zl!{IRVj$aKfjy(=Y*$*$k0~wB*zYe2x~QH&rf{?~ZSFm6@As!oXk%hT)}fQlI$&K3 z_HpFCXwctx+~QJdp1-ddg2V+j+nM+ocvmNIBJ|T9!_*6i9a^@L?wv(QieO3bhYle^ zw22@px;-*g31qs?;iN*aR;IEjrM=M!vO1c8L`wRPuodzI8GC9F0EJz$E;@jOm6l!R z&XKzdmPWGfB{53fHvJ+GR!j>%+I3OgdKaaZ;j>CEyGg|)*lCkp|M$C|$MNBI834n; zpCtwc21^b5D`S3hsj#J*#X@lCZ!Nsu`~P$>ylbLra7HBxa+}uM{TE{Of8SFCl4(~fpZ6|~D8Y$Nd z3hh5j`YJeX{Usi%-6w5=8UFdkZMwfjZFbua_JLnXf+U?PPxu8frDm6P@LVjZei5D+k{GI=iW3l;O=N89LTJpjk zkEOhF?~NlaK=d_0qhCY{!zf@K@Kq97rV|JtW-Phx&h#m5kN2xC{@}Rw@oR-lhWwg_ z?R=*0o7`NyXknG@(SF4@N(2ESWe^*x){bE7ooRIjftmP2o_BeCAl8kM;DMoaV_0(} zOhP1(xK?hV??>MI6HHYSp!0#>l_fn(M^0Rh)f)AGji zJDjcPryZjmvu@_-ZxOE?`vgKOEmVB_^VkpLidl7#`pDPrR4wAoCxZF5(p5-zgkv{8 zwR;loOnJQIN2Ov;vTp;(zP$T)E^`;u+*zRd8DBp-!LQ^~8F&yCZk*(vDCDC9bmMULAx_5n|;Qx{KlibfJKW*UCY$hnxt!1ghj~C zD!{ozW>p8L4mX1H;YO*=-vG*^hv=m3A63_UcoW8+QNnLsV4`C;)Vp@Gi(U26-vj$G z@rA`jV*1Bu$}cb|qj#9>shZ-v&PqVqCT@2@J^0Cc2C9XIpvn)iJTt2Y=K98{uckv5 z!(EIST(NFu+V#=3x%FW2M-Iao*E;FromO=-06oU>d|7|Uq>>9)m4^ptpcW_#$^|T< zY7zgTB9cs#vMm6W1+IGDNwJu;eV>42YUV_;UywAnC z&ru7$y_gcfSoPtl7YZ$595$(^LF3y!7dD&_45IKPrI3U5z0Gp%nuZnI9DUJ<=+72Wr05LPg99dnx%n3Tqt`Fe_FFpa{hy5f$ zQ(3<3IU1DE-315=_f5)@8BHE#Z=YY%m_S>wl=REcuIw1FTTZg%yfD_`+r8qg@N0V6>6UL~(bQM2u z+pbQ!Qwdp&(d8q?7%cV0Hn@hmJm_FImYb$QAX6mLII#mhxW=gnFnrFKpx7>%LgpJk zw7u;J9RXpzY96K)ghh@kGZ$U}3Q0auNFRv-q=+vBJtX=Q!NHRHz|X*qU^l;4*iRI}8p*Ox=MdM8`tKm#VetYDD-F#NZOWi*GZ}3!5H|Fm zfHxQ{%B=3=B!!5o5;;-TL(VH?F+Q+bTVNv}RT%=V?uN#|Hp1*DKI3L^t@ywvz#&LZ z^raFC3x>OVW{-&``=;-?T=>+j^vg2XdJZ-ZOM?qK(d_dROFbL; z477EkeK=A3eX8QJg03wiy?Dq?j69cvnAmiIsf^-#Or!ZLh}1lKFhVfS{g`tbz+2k;kAXrwTd4u; zL%6$em&GMU%dU0BnxZJSVo#^C+nuHik$=eB#0QJCsQ zi-QnKp-G;t^G6rI<^0V8R-g8t)$cH^VdS~|`N7S318`xqQN3QLOn$c8A7Ud^pB#5E z&TKZ2$_?&zW-aS`oFEGAM}*)0Xzh$wUCk0|E53vPf)dd_UUI=3Nd6jGU8U;eJ)+>$ zB#S+!_=vg%m_K`nm@GtcG908)3T}S}s-nI}jnpqp2IWxFC6N9{na!%wr^f?!At9Jd z0Cj(I8;|*OD@9ru_2t@&eM9e-jpn%kIyVt1b~8ERi3$*n2JNB_>xLg`lf`Th0nKzC zg%PY^HH05TCAz>1B}a(pv?LF+mi~M=d#U zd6*vaM*srtG5>*27gW6*aePXF2^ft?*8LG5cMdnkHBWi2ec@sTDGoM6vbf|Exccvn z3nZ+h46|(e>Y#q&IC&>j#x+#1AVbQUB*@_D>v1n+vGBUjZ)*bCo0~m)G5onIL|`D^ zl95T3bhREc?1&@vBSDmk)}4Pqp!;Aqd*`hRMZS^P>e0nDAIy3t3pi)~6Z_ca1B+0D zMRc=3qVgMPUgi%~vAml~N$3odN$V0C6@B_754BgP=sx$jjZ_NQoe2jv1Y49Pk=c2H z)RAetE*a~Ol|He2=)`yyS|L}Ew|9e~SkOWB_&Rx{@nQ9+pFJ24?4f)Em!ln)f42NFW%lO|;)w40VZ$AGlK#F_B$6 zgR#Io5NvoQT%Cy^8N(&#>S^! zrcY~EYQjN-akhj8nmk!8@X=et{!4XG`T4bnZw+tpf=F*@rAhB|sw1-qE(o85H|Y3D z;9RIUSA}XK({-Ku;p-kH!LvuVn+Giba}gcAPma{Kru?gGsV6Y}c&Em1W46nJ$F=C2 zEW)9?>;Xg1JIMvT&P-=AF47xDw+vd3qIMMT@;d|tu_Pq!4k=nG^put#IdfSNIEa_q zcpF9|1qldKnnx>C@3(OyVpQ~RXHw!uO7!X_uhDLZ*4AOc8t-(^NRq(a-1A8pqDkD3 zA~~#eXOROyM`8qU;M&;M~I+8`i4HY8VPZ{qZPj--oDHY zJ%ak7i_LLDt=Fyh<>i>l?xw?T@LXv*CfUE9*3nB8r9Z(2X=E7`1+vWCk%Mr`^M{Al zmVtOxndC6T`6oZ(Vt!f)Fe&{{Jg$QJ;bXbPVgdhWd`*Wi zVKLtntYrWVU$Po;Hw&TEPsY6yAa)OBgqFs%c)0H`mbHZwNc zzOO!VOH+eFAhi%s>X?}Hz@l;_GPIW|0x0VcOD_p~j;De*)Dd!k8s>B22 z0$b;rCr{B($yJagesALYE)!CdPS@({i(5>-(cdLECSDB$3O2^u9n2+`C_?fc5k>kJ zcJEcE>NzR+IVG2H(pc6KClMjKZpEV8)b{a>T$ zUqm>bqTyS3dfPLI9V-v6jl8zDco*`mHRYEbrrM(TbG;1&)4rNMe}wTOS_d;qRa&$p z(d<^e<%s+9`^@vl$G^LhD;H|VBO2cLEWr6u?cc+Ps*c!fx!-fM&T|l11HfA6Csjpu zk3{=kb_X!S+$Kh#B%H(80iHpQ7_3ZHwo{fMA}s^1fsEsMav!IH`wmO@Hi6uZ_sk%9 zixPCuPrLTi=`X1R=^Rq8MF2HZU_U=92<1^N{?5M#;h`V7G{U7+&1$a)VU?#i%Z>PI zoNuNQ@8c%wQwRBS_al`S&aVKeXA>3U+D#d zHB?GOpK04FjvQ(!HS*^cbA;KuY2fg9r}>DRVU2bb1V7#j4W|@XhRzP9x8Hg;DQvU% zJ^uHP9HW!(L6DGAWJcyhf(goG6J{MVpfPvI7!(I5%OVFyFx#9%70l#t8>MEBp-u$sqjzp$@ z0DK9g0}~{^kJ}SSL{TC<04~wOcMvf-upFTe?~qlesw8_i1yJM0!uy5|)F~wTdX_rX z)nUJ9$H~S>BT(S_CO3r`q@EWDB&fWk6bw!)j?=a+>d0dnIEPxOLCCsxsHT|ga|tLF z9y9&#YzS>m+!~89Hv!F?ppsbL1a#vLl5O1!Df!Oar2ZtWqPPZFZnF_A(2h0{7kMjesJdoPbU%}@ZColS zMg5uf(Hv@e=oivNfzjLqcW}~FGwk|X6psJ^MEBW8k(Qlbb#vFlC=~~S%;GBG2OnTV z@a&~J{Gik|K@)R>)X8drrEnI%eTU*Cu}cG9i!==E0h8{PD92 z+b~u*lN-ec{{9oTmFF){nkGQ6Mi8`WIq*U%+gYGXs!yB_T7XJ(Y3{Y=LF`a@vzN(- zZ7e62`0Wn+2^#~VA^axhcB=i1`YU=%?52fH{F(}W{BB;vwwQ4lfer7w-$hLHyC7HW z1DqDVMKP2}Vwji^#ZCK73Th<5Mx!NmP=c4Jy!LZaFh5-3jq_879pHU=JTb&#LW@|N zEfmIK+_`OAGKohxGy$8)sM}68`B|rTfpw3)#H*7J8nN;Xu#zSQp*W%oC@)|edpY#e zVnBh8rn`5}!Eg)cLN|lRNFc;e0ms|(zw8DD;zISJ+Y^W{_rP zRUE(knBp?8BgT-|e9bhPnTvGxTAIPbY`74N=tj@G0A;xf&!<+QUm9;(mn4~lx16q( zLZP>buA%gfyX3V$D?vd-HsL9ypd@TXkJw^2>VVYqmq%G$r7waz!hSFgUZ*)id2R&7 z?WnhhG#}p_c_h8D#cVsNi$W(7jc+DK)$Wf1p{{)9?^QlG=5>BxvDF;dBA6cdE+DG@ zvRKwJofNe%%&eeGTGC;>5x$-NIKWT!;iv$W}g15)Z4Xhnc+7{3N4*& zO?}GQFOcg@96GXqu+#%zL2D&C^y?qY8FoI3syVrc23oP#H7_~j&aja#(dUzlkp)c5 zCOBYti*@gTbDcpO27eVm%~z1c@L}KOc4JBzN&pXHGMo(yN4|L2{v{6bOo@#jnbIsv zNhk5ZkvwSeCkB4)mhk6lsS}hiLZAdZ_drprH!9)!L#~pjY=A%ZtIE3p z?N>_m$%ANBK-I)iEVX88n|;6IffvSrH>ER&wU~X4hVYcP|2%4e=$nj}f^A!VMM7^O zdsLoqnl`Uzy(DN}%j8P_bX?{$_tW6U?tbeeswQ5GS<~L@Hj-S+#M|vMTMId|@i8TJ zX@qFU`)M_GmR2jx+n$=*Ze+1qoJcaqhg!mLiT7fUY0tm;n)DGh;@qpTHhdQ7=W6t>DKeHMnw)mlti%Mey%q7BUZ!BL5)-yv;y)c07Iya4+0O41kGv5`i$e+(x)kaZ zx;nC7Ix95s8)iZdtKZ2JmTfdO`I}5qn2{*!M2OcUff@NK$%%YcwoB0)@#?7ZNNKh;>EbP64FmLo7dI|l8`>tm#s-!w9H8nG5%dnEz zLX)WAQGZ+N+l^()VXsBV4iIHywTdA56n0M!*_Ab6^<}tP!uKZZsto!eT9e<8nMJUN zRQqL0NZW-@eS9v^5Asz)8wm$oXq1yoHfF;J;;4OcwK|A|i*y85FC~hi=H0F!*B*YY zr^!EmBF5QLtdHWggp>!4Y-d8SjU?$KoSK@Nwt_{ISC?;^BrF-YvkEpb3%Qcb{(ISh{8aWMW}z&H4)Qe8BwLQ( zVPRL{ce&}1uSUda%$}MTdU8s&PAKs5@=8~7m!;hLg2HEU&M*Mk&AxS_z^FXm+kbAI z9N~y4;y2cBl?i;tq4|aQnQ(G+NZa7j3qD?6#+!r@v8eSc`$rS>Bw8js~LRh4mX;TvP71kt;0G^8kT1IkOY- z<6qj+AO>jFxf?t}+YuFo_P;|Kj$46D-%FV%T{uUnU!e@_ve|^qjLNe@v6X^Ak9pCh z2vm2UmPJTQ^L}W?r#tc(skONJhaU<~t-;suW` zW8r*fM4K#56z8-2!q_eZ%ZfExq6eq{5DB|MYV|5m*v@k0B%6w_6gfG$oKW2T zh4b`cGdFG%k0wBC-*BX&+yb0KBo-ixw0Ea;>)JOBhLVTV+Sa~5$tmd1Uj316fHE); z5D*CYVW|9~l{+{EG+Iu3CN(}zmUo>6O+ICc)g~HRA#L=~Iim9*bHf0L)h0~=5x*j@ zJw)@jWa}q>dP>Fg-Z-cGcB@su`WIqz*xttXEWK$?gZpt)A#I{Kt$zS09Yhjz1(*bI zyvzQEy0H%?>JmZw2pbl(>v@3IvKzl2=G`oE=;HYCwHWlCc`oz`yRb90{d6<&UD9H; zYEE`-pn?VpEEuXMr`JKFQ61xS$KqQ3oi@gx!_Av8Vi(Ax%>~n+D(D9nAZEc)I|qG< zkMfa3hlnAyq|UTW`o*t%`WtiEpnVX$N0jJ?;FwHr4$g&g2c46Z zEgNfn!L6=#E5)+(4yw-XhTgq<_j(tzST2^8mF>x})Gf-2mr7Jm823S`InqtNUBaZC}3@DDyWifBJQ)3$^>DhyTPKE+MJTI1QgzD=w2;c2 z7Bj<+F3*A{!KoytLd7M!hJ!yHcW;*(B+Gd&4u~VK=cN)mS05c_#u$otb=pO#wzS8J z?tis`?2#ge3)h4I~D%k|A zljq$i;FXjt>U$_|@$uJ)Q_&1Kox*SY*=D;yII}7@p4GYydeNJO5kZC$g+Mi+YcmNn zg{Hk`j%{7aoD9U{ufNkS!8E~P4qZv6E)U%Y^38h}LAReuM%Fik|4i)7A<$Qo#T{JS z^aSDYiAtz>$=7~;HvqJ%p6N{w-maaDRg&Q2djgvLdh>2ds&FwF=NM)=_N4t?9F$ZN zWTuvW^!Kj!>I5Csf78Zl#pztX;q}{pOiU6Ya!#$O{0HKN4b1`U{p1^~K^e{@X}W}6 z&HJ&|^^ewpdvkf~pF4%=&Csgbp71}IHwvBtjcllEN}xfI0<=)D1~ncR??yKPAM!g` zZ|&f$wC^}g?7epAN$Ub0)7B=GesDwj zh>g5%S?My)h)xMuDrp5qfq-=-U9?*xG}%W!=>NcCoUMNwbTXR7eF1&XP^o>#-S3&G zzh)hA;-qi9760oq82d!CVtD)?}?zpYc` z=-}}Eoeo2d=lm5SOd1(E0MW-SZ0^WU>n_Ls`AetgNGIeC75J~Xfs5Lvsw zVvKw@`)>mHWdNPsw&1wMEIx|u`CDD+kWp4vUg^1Pn~lt7`(ISO2RzpO_djkcvL(rg z%~_k5l8 zJkN8^>+-wyQc$ZLCp@RG4tJK>X4Nfr8>fbB^=wWTbppTge)|1+t5%c46S*1 z!8i&hha!S?du-4@Y;00`e(sWlGo#^suk+wp6=sj2_5UYymTyA)7S4Nc^fnzypS2~% zX?Gpn|NUB=U0BI0#aYea=HW?-*0Ng!C9tH(VjG#2*2> z_r2H0&SFKnWgGszojFZoa&->c;;okl@^BO^Y*gtCYlQrGHm`Hs(q|bax5Age6=bwo zWuKIvG|Z3+TQk4P#)eT>LJtX(&C`vhs@SIsV?EKJgj4QSj?r<9;?860@HAhUDjx4T zcgb#RxqEc_{6v~Lp;tS~NA{o3=6R24%3UUq*aF@O+L2AiN*Nuurki^2hQTY&gC_DW zP=$HGaKQ8|d>5_Z`qYt72$%urZ z8Jal9&gi5vHH)afLnAC1A|ldzEDR$!W6lyT^+)0mTKR?N^NZ`k={QF}+9@Rp+%>cF zfB_eSefqExKnF#Y)8P*v{6s7Q%=LVrzl{xNsZHlJC`7};wy(?(O&e4DEH0PkJkjp4_tlVjob@u`t9hfqDxW)jye}QZHR6~ebL>&09hLVD-JB=^C zx+N$8#tpyFV$+rC*Lu|Ca!Jhoj^0uB#78@t3ugYp^+&%i;T-+hv4r+fwL>BwScabi z`W(9`g{?-K2%80sPFFr+8!git-$>$Ajv1;}apJ#~Z&GKRHGhDlYiwmB2IjXNN1o5?kRKVC@@IN&|0$z5RLG+H%i1^azg_u1_k#%Lm`T**{6wnrREY_Al5Z`;};_Yy?4qZ9&*`QGMXmSUQ%7 ze3g*j{obkJFMq@DaG!+hCLAQi$o~mNIv#OSVM51qv4^$jVt)R#$R>k?vw!Om0FgPD zpaWyi8yx_+_qJO4PDpEPyP1pN$rf!tgW|i*C)+l&dbz_aPR!e(o1=q;cKtaud(#oU zL-v3|5!^mtxi(&1(de)P{o(F|DBHWz##;(%;C{HST*>Tfp~gBg4W8w$RdUy8QEN`m zlWL>(Kt^dI1nY1*wA$X<@NX}4$B2KqDCv4xjmhhdR+rCb!V8VRj0=(R8_)GIDVvtj za;Hm)Chlheu@P(8ZdZ9MBvjeC`nkKi@3os9>rEr`3Q+g*{RbvRs=xYi6or$-aZ!(c zv5LJQ#DXm%a?Z4}Otn#*Xx+JB>Hp&uJ6ZqdonkD%xGf;#y>?t8yfw57jceTwP#LcX zQg7M^udX^GwhfBU!_7UpGh`;{Lo`=_Dt$Wy4dQ<^Uyqg)BuTj0LD0Bi(6a9@|0C$+ zXt7)F@ozpa6BvU5HB3nscVgJz-!}ooa0t$u=Qg^8F9LdmS^xO0VQvacQ3UNN`M*}A zMgg3K5(Tjq{*bXtZP|bu*v1UF_qnQt^aeoPFkEvC0};$WjmCXvqI&l6nKB4JGL$5_X6HQA|$c`VO;4MYB9{# zb;1#ACP{On{KC)LVy0cACD2^LQQk#wW`D$4Ubz@ZNY^wqCZ=-6mwB%d$>K4TQnBp< z>^YKE=Y6Gd7h;X90Q>Y#cfv|m!W_j{zRk7oWf4sgopkR2TUhpFTz@I=a{c=C4%_?3 zfP&8^`~8LiuhO8CKsX`-8w6FG~1`{8+EK zG7e43He{w%K5d1s74q}v!q=PG)>c+Rj>AF#8KuDHyGLE>wX40?_5fAmO2TlGg|x_4 zIB!v?Kw|VW0Q};dEhAMPT!=S-1du2LkZjG)pfNmT6@O@WT`%vSghfPK@^aJP0+b%} z9(4jduNuSLY5O`rwc$CLW)ZkkhzUEolB&C4OQ1UOf{9xnSY0$ z>pqb?2(Fzt|R zBlXYzBEL|n-P?FVRQ>b|j2$5*Z&f89Om4Wz*F*N}HQmsWBK8rd}GFvWker89P^HkJXW?lp4Y$ zBm-Z&#aN|Vs%M`DoW}O&(}d4Nfaj9lzNu}3wOc!Xg))~e%pJLDrHB{ywdC7~TNs3i z;r$@a!Os`gzD+adf_*J?aLbf{Pj2z}&gA#jor{Y9N_XCsrzxFvRUi)R|N6=FyuhTK zR;)S8d(!Qx0HU0SJt?j7t`JM#0{Rn{w036zua3kYT#=NLj0|)~hZev!`te^?;@<&y z10Hbuc(Di4#x6)rB_YUw0AQ^#kfcuU9spBf-G<@lZC1i@u1H~)d!(EgqwuRo2^s6K zzol$34JeX-IqO(to@GJVm!-6R_MDbS>Mr^?wmpI-9>}(=PkX$sS6t7{pPaCr7be<% zlYR;tn<{cK6C1gJ5qhnASB8oeX15#ESORGI8@<-Yl990E)_G_A5KQ*h#)QK6ft<&? z>xfmVi9Y@nEwet^bfNS23Q!QeU#wsRE?FTcFgx)~1pv%HD95QU&r%`ndnv7hA|WNE zsAgoiOnFB6v8AD+nvw;URr?FHgENRhCqw z^k4Fv@UIVwh~K%&ojH7(JpY~hXFk)sL<_|sbv{@8s%jL&d43%_T3Q?-pgN51ClPFu z*=z~SAqfl2Di!uvviMF0EB&DK@bIA6Px#mO0kxLl2bj0$D>hSr`~-$v-x^mqheXmC z@6-4eT@Y$&%Ma%HmkL-S_6o*H&W`YZf<{tRvhD!iV*=Wbj8RyNH;2s?w7trCXl(>A z2xg=#VB}ZL#N;syd!as95Sj%Yd3_xA0&XCFk@o)m^J^baR4OW8m6rz#M_DVPm}C;Z z)HHJz1g51wlD z;n*Tgnx3^z;_nKVXuf!<%AZ32b*A)rE;F6Us$(>2{>(f@vw+qI_bWN2Kk}?)b)pr) z{n-r`T!i$^<5xKlA|pxM!sI_regbDvxM>;jyNBBu0azm?glHJF6$(7u(_=wK*R~PE zKiH^$t#}y=gQoY&9Z@fB95tiL&XoG7=}$U!L~gAm1swv-i_zZ7k=zo{vBB*W+Nj=l zlTRGDhRm`i+orPuN#ZbAK;6^^M)2Wb?2?`Dz-=`o!LGL890Tj47c$2I%OK8x|9|=8 z8n2vO8c?&!iQ9N%p8eX49v#BM?UyKYa!bdKD1A15bXH8fsxyPxZ0RTlv8%gDVLf^J zT#x{?HC;#{G(mDKSp2C<$b464`uh4R5w%4GP%#cW3e3qDH~7uQs=cKN0bUBG;0<9i z&nG*^hdul}*DlUU*!EKNmx41o4O*Cy=w!L7v;E3D^+^o(WchKAiBr|vnk6|HrfA8E zbeh_flS4;dc+ku%TrI{wnOB_qBRk1#jQMsAnB@lz&$j@?;@M0z>7+@Iq`tF3=Db0@XT5i{VJhJ#b_=&pz!=0=zkY1>QV*m)D@ljZ(*YV?!>0|!+C|7U@ zRWPml&Odc}zWa61@VuznRS5}X_*pf=84_?Ca2_wy(H|gBXAU~tqOL%O;!WI|&Wk#I z|M=B!hC@v3CHR(w_m*^mrX8RtWLOv!zx$?Ek_Mr_KH7f^kGdp5eU1oK?s(ATp(t^HzFC#`Px;e;l^@w7&u(Bu4Hxy=%rB9ZA7p>HEL|3;;yJhz86%-s|9|h*KR>= zd6Er5rZ9g-fdg2McUOCO%5@WDTmsN@5$JgC^-KCU%YIS2vJjKFi2$ANl=a~Ey?tKS zh)xs~RfPG>3454I2l&&qaR>1$EDbEf7(aknsHB*8!9ct7_@|~CDxFve<3JbF$620N zuql{bSny~%Z@3Agrg*N%yyANGhC6Ug27i3v=|Y{QypdnR#BmAuW(0p=IBx!`;)ynb z<#D2~27D1IJ8*t>S*G1CJE!e?j}w`#l0M%fg;r!dxqugCx$6T${Qk_D$1DK zTLD}%s8rfK>fHXe*4Qh4Fva~g%FUq2nkz*W zR(&=pXeLN8$`xos1M|8J&BnSow!7E-_gwq>`*B59R>plM2*E;$%KUh)0yF!Bm}0*F zmI7giW=T`Tg?h!C=L66~07OuM~*_c?Goh{EN{5c^1T=ixy$)lu`1fM{=-i z^2d9YfRI~2nDlawL=h7aNX#FIbQom(lBsWo8ZfJ7dG!@QV8U)951V1>ILh1*wL&=hEYzfxo^mX&E92SwpEhriX>WwsU&-L}bL|{L*?z~0^ zG07@Ig*=&%lx`CyP>UO#AmJXYV`2+)!`o!a;Wh3(F;XD389A>Gp!HQ7ZV~Y^&OXo+ zQU%}wg`x1a*i9aC#U{%2UH+FZHxLqhu^TFo=S`affznd8HKy|c(Um%nOJi4ukVi}W zv&W;KgLuXgyM9jj{aKoH|DPJLuTGJk$nWKEEALU9I;p;`POwP`d(d73%hjKE>>~kD zAnybFxqt=$Z{GuG6(Vw5clQD$ZhW^t64JwFXAsPs@>;5a;zd<^^P4Z9IbF3GB*K8B z7_M}?0<`@M0IQgF9@Om7?u1}(svTHaN4i|~jqa7wZuv|=;pZ86&x^<4H?=L%Iy}`^ zZvVt#1H4&FvMmYicd!5T0;Fps3fxDw&4Ed0>YIrI95OPa>krx5uxCYa`h+J`YtNYA z-YN2`0YzIY#dF17Xxw5%DPGdTy_41V$$RaqU2YoPQ5B%~kIv)Yo0Y}7q5OhS8n{TR zu@9_@sg1-sFb6{5SQk46ka8@fSR8x~+IRf(YOPU@qvcUR_`HJjwge&&^|LlWFwu~; z#KO#b$v}<I)Wo9qi=cIaubtGr6IN;nW=e zS@Bk*)UX*9y7?*F-!92zg?ztR2T(-W91tbg5I1SHTjo5j9Gf<&U91|A6VV|hA)K0s@n>7Zb6^V<5Um(*&EseyeJT}qP(hfE0A`awnLd&-@FUzu>pipJ@FYD5% zW3N_0?JQZ^>nb)zal~nf{dOPv#$@ro4i%y5C2ly6(YMg;GoI0O*vsau?|F0wrC>qy?<5QBEsjS)M1Rc1$7&dc(-zS{a06LWIIApBZEoWkIxIkc z?kBidz;q)uej?opC$d#lRXDKuQ7YtRBdNN|v!NXgj*0X*-nzlH#}4jp)RT|R8HMmG zFYcZ5Wm+Vo-AOwU$q+X_y>1C(X-%~VjN+n{aiGF;j$UZ;#VFtk<@NfLz)RF&p?5S) zXAPfyo3$QSIVP`Rs5+LNH|`n2xsaoG@Kn1BA{Th4MU{BzRD$hh1$zBZPuvHS43iQvN@1buW%1Ab ziYcjLXXEj+@NAo=hUL&+iuIC=#`Zk9{TkqQ>_$qmN#fWr@%A^}s5^oP!sQ*MaXN!8 zP{lsUejwc}=ioz0MfCy6i$0VySCZ{KKU@kvQBK*0W5{iCbszv*%(LbV#oScVc_7{v8-Hcu`1k zjfFNSfd5RrkhAQv5JDLWagS388 zX?Tm(N=lK?Ggk#(*{pBfxr>#HT?x8zNe}5#32nvYWlP+;_Z?bZ*wivOFL2$$PmeGO zVlMqyUD<|}Nm|&uI-hbIxh7RCG>#yBKFW>;`Vny#v5c2V_sA=2ILnktt>vz@gIu88$w5vu#eCc1-)X-3R8MITQ^F$jnRyxBBIp`sUiIvY8xc?hy zQVV5|6V1R$vOrWXPuv~t5!!jda0PaQm}s25XYu+Xx_zDU|E|c%YR<43i!`BS{b~Ko z*CH=R)bgD~7T+=a^M^*}0JO&%bhu%(J{D-h%4?jd{Dz#XS4b)$TDvBJ;A^U6ma<2VG@-Ia}IkH+_<3lOKH`Ve)DO9wZ{JRdQ33`dfmETesvP2RhL zIz`0CXZ-c&lrh{5!YbBas9`opFVVHLxcb3=`@X()kxcq`5JbCj#C$mK%V`tF{$ttv z_bYgxOLp_8OHB#;YMP%ewYLGzj?iOOP4`L)$Lu}#8B!BY5#=II-9FH;fT?NqKWs&7wvs0!>Z-69Nr-yNTvTSMRkSvA@M-* z>`bZfV}rpc6kAx?9W4)8kTfT<%&}H+xLLzU58%6O#tz#okE{E8%vhrgmkw%;f++m& z-;w_!hqf+6nhcW-t0ElG`uIMz)!I5`x_}r)<{CMp8}C_82A;O8X}pp`bdvok>g&#F zFJhL_AKYQ3xrjL9-V5g6M$G;_!~{UY9Svgl?8#4s+7YL%X^CrQ9v&Cn+N~#(N*4JM z>GfdLZ2xqs9?ImcxY4*eA7nxsBk_Hs`xH+8TuJ=g*aZ49{hy*xNoBk{(9zrxIk#x^ zb&WJwBX;FP!-NrJ{C^9I?~WF_sxwIEg>E7L|ZW|9Sa(hYaZj+M0rT5ek|$;?poKK=jJ zbiSd57V!$|W#xF=Dh9b%{KpxHC%r8=0jj*;gBCX3Uga{OcfOO5(X46nMm&B2$23Nb8``qt|F@PQUq6dNlbi>6Sj=0F zah3$PlRwIYNc{Mp)-qn@avy0Lq@@;|JJpBl|0mBoYv6qu^Z$)RNGAkDh2qQIA$&Uv5o&Ect(WYxpFE>u;cDG6wvdlCoGjDST1egdUy+4 zij8r*$^PNA>s9QnwG(&{ts7Y1Q+1EJUBlDmF7rgaBg-OTVna>DL`C@>;eXDZif5bw z`$XD%h;zY$;X**I^Zjt)Z&v=z9ed!34Z|hk4oSm#+fSC{?u(FzORIz z_z|u)dQIFmHe7~~p1_3sBmT0Aa|K(Xw$MS?9~u_$ui^bA&WNLgqYFb#yo<2Ij+3Gp z&WF?LgsEgOgWAyXsL*g#UTyX#`J~6>VxuYzz>L33!bwwlu@00@#h`;0b zbos<0A;63lYS-ogs0TXIOIak!bW}XXfq`}7RPV&jb5S&`d5_lw#iPPnO{Ipks-St`+hw))%INrL01V2fFi zcs>II<@?weyL+B?vmfX-jS3q6hLdu`>J#!gi z8yV)Bj%wKpYRud6Hhy)_k^}sPRgp$}`1+9Wd|&ZeCxTt{W<*o#f zVFKWgL?pB1ZF;zDk&t^8o@0%i2ahA)C*)<58MQ81~UmA38Ra!mV%Rt4}PwY$gBQcV_4i7y=;(1f04lf zPH>Z+{Ct>xi^uz5L#RoUl-{t}QU<45Qpn+F&R)|gL_v%%tX>OCQR4{5zaT4^!^+1l z5s#lJ&(`5(^k3gi!#kv_QlPD3#QViFGsNTOM|8SO$X~+cB>JMf+TyMZx)N?~_JydV zw-(GjIq?Vj=x#Pk@dvNwX)ksJUyM7yzw=Nw9SuV(k^Km?V>_Ebm~|SL?F;4r^rjX$XroaPz@AV8Tyf=fR8*j zRjgjt0~>X%QWD&eIyYLKzLB)W`m zg|5Bixlw}GJ8M)>Ok~iPl7jUF>zz`*mF@)VsvB>5;HX!q1hM9l0;&Y-G+L5pHuz6( zyb0lN|AZ|m*oFhH#Eu?6Kn?OFSl@R*4YKgH+fz%a+H=!>QlLNgyMiy@n49*|)oaHb z?Ik*-L{F6949WHq2G!=Fv(Ih>0cVKM*DlRVOQIbW`Pv3$&|)qmDMU*${s3iA3qQ7$ z6j;I;d+=lB`}do_;5!r4Hf7g#52^{Ce4Yro@#ZmqyJk$J0#ujA>CX{`7$FT47{ZyGmtU$5*QRDAjJh5XsG zXNE+0!3stR-!5M}{ueIL0>8cv-MZ)MCr2A`PUn=ue*2EXUh#qNJw;mr5=Ks>Dc7Xqm(`WI;jNfY`DM=GhN-y z$HhaVWpK6~8lEK)YC^Yo`S=*&A%p}ZBq|Pha_J>{U zYvM<>KVq!#&(F@!GwZ_{KUQ|g1^D=6I~TUKV;E1CTK5Atw)fXd`FO`yc96%_ zmSyfHi%l(rZ@f6nF$+|$yN5;>w(l=)`EsgsNwq9&r!8*pt3HxV7Y3134XvbEq7vF? zwq#$temya?u)PB$c#K-~mb;TyQdJbZ7C-9kZklh0$>^qaM&8&&Xgw`y}e zoE7wZXr;DqW#`xp9K4!e?GGs+k7TYYD4fLj`S}^nFD$%{O-$UyQw$6|E;TLQHHm5Q zu!En+V`omKk(!w3;{pHfY27!Ajtr88HBk!;44kN`sUZe2%+1Yp!s2#9wYddWA$qJn z=p+m-?1A(1=TFYLg$1*u#Ka*PxE&Q5b{jPubN0!H=XKcr`O_KRMH`Nqzn9>(i5dOw zNIzejWyZ5B*0LJx>wbeq17(V>F*vbBvIidH@Y6s-%}pRn>TmD!V)-&H3E2le2N!PI z@2}F+1ItF&emq1CmIW(jr1vj5&bn6_K_8f{fx)0|@z6niWMpJS!_?-n;^p_^A|f71 zqie@qB?T*Z!HP7O+vnWVmUbT1LdVj@kV4sBz zQ}175FnthhH{2x|hiK{O*1e?D($JCSLcV&cs?9yHCN3oBbb=Kx3-a?9zJ})_ zKgY|>eI_d_>*EK$l_bzI)r%J|DuI|j!DBgq4WHlzV+Ems!SK`gdM7vwV_n@BQ=ZZ( z$SJT$7okg9X?uG+NbA-u7E1CEa8hNxbM9mw)#EGZl^bt#GhHS9b8pPLiw?C4-z z-tw)xsjbaqhZ=mY%l(l{Ol;>`#c1t$dZ=44VZ*RzK^J$Rs1gG9!lI|AcPz+A?m}Gt z$)qTM+<8|uq(Kq*S$+QL=RE3dF-H?VYOG!p%oYmh4M+%iDMN`fdkMBz|)(J3@ zvpAp(Yq|KWmtRr?LDO zo=+a`@YEWma4(^36mBx_U0s3G09s4>@N6o`P;aQD;0-JR{Qpi6HriFPrfJEO+^c$A z0jJ@~XY&BiiJp>zLFbCmSPbTxDE&ESZ4Hh6G;359dB}{6Ibt+)m0e4o;$?ii{UrsZ zV7ot=V5MM*81r^D&_k0lp19u<{Ks^k? z>7=Wqt8^Lx9xiAwauSsMT_qWrAaLkf)HF7PtEQ`zfRXRaJ=s>sb=p z?@mjd@4ojd6`lwbmZLf31}i)+L^3|%I^W~I<_qICD7pk|Sm}QzHo$0J>T=Tt=)jm9 z+#TFf-b{aA-x;DO9L>2|S;{T=$Q_(OcJTcnjM1yx+h6@`PMO<}n%im;kxuJ+nEuw1 zHb_UMg_~i+y$VhQ-x0qj2TI<1@KrjncjNKa)|4xKNCQ>pBK-fe8@(QMa=D;*t$Wig zf}b`fXD(Rt$=5Abs)RRRs4w1gBAzfjyq7)4^QqAH_({w8aWhY|Ua1K9CB5~yp=Sr1 zPVV;D>gwvt?4xS^_CpeX!e13IUc2OLPQ1yV<52HN18X|u3IWB@k zoQ>`7ZJ?MF8>Y^Trd_#CXYpHEmPkEe!KB{=zoWSzDajm}QIt9-)(D_YL+7<>g2-r^ zlCpgmIM-D8qrcV_dzlxVi5`TU{8}1I)nU&dEx>PqQ*IU09|?W(r4b20&)#|2Fy%`) z?Bo+oz*D!8b?4TCrA=AjDeosG`}gPlH(|I>gHyvaz&7jFM+d1sCO!lw4ivKPm~qrC*#AtOH-TaOL02sEhSLcSn{C zBYCw)s5(FBY-@M7Ae0o+R=Y5YQc2zVwil|`xF)40)!+)TKNKnN*_=kz*WRCji?^c2 zBO{(vFol2SiGF zIH$?KhK`aPb9cDnWpf{Lsyq&Go+sZ5jdvebd$9LB(kgu^0Ft8qm}RV-^}H8mk*Q+Y ztx$vWQ0HS@*gd%2;!KrC0zpR9<7W~WXX3aYbt_&SR@b&o#|2B_?|e!)(*KpQE_h-NIk_XN zMBZdhsW5d}Nk;Ni9WHR0g+nV&EsN$^0OgMN&7_gg6qvuAKk2 zdw(DhCt@!cV9@lGa7Eu*gA25m94E)d_9~OPDYMh@wL`$<1Hb27T{>(oC3H?ilOz3#!H3xN=UdXBS=HP zWK`lbCw*HLt{+R9Ri&}~9ZklA@hB z5&zMfl>lux>lb)qneXgsUUYR_&Wjao&Au1zMisETC+&^BYAD5JM!fa4Smwh#H;asg zj!0Ryr-NrpwQV;sIWO3muBpNP6?bl3ocm*f`DlYm5Ec{j+8r1e zSjsD9*TpfMY;B8opENOl_0xe>gMm#Pc42w_(%-Ro=#n-g*GoA?0%p8J>dldlZuTv(V;Pz7&@0&GOaqFB zdjy+xjVMgGu)Lj(%{ydON>d|4!vPsvhYF@A&hVA#>%*s$}V3zZkZfq{#XgJVWW=(9+Sjsv(I<77c z_)#&FdA)e$rdh#GCN&WcyVxk95^4(p#NDA)S?0z(H#rl`*M5ezrnY}cR@N*1 z>AXafl#E^YpAWFDwNe+dOPkmo+9r<7%*<{_@szwwbq06R!NS_N6C1a(J21mySW$5< z3B3sSsP4zyJ{pPaWUMK$G3qzn1Z%Ea+S&pCjux)eoyQR6+3T8)EO{=ErLNu<=YFkG zMf7%XVgyU?$}@QfM}t8R(FCv3DEBOt!Y29?KTl6jpUhr$CY3%qU8ka*a)bQ43wGl; z6?iT93K(g1?Nmdiv_5_fr4D$&JRKt}`{wt7>w3K5(#t>zzbL*Db_mv$zwb+j;jr1%- z_GPO23&RA>!MW3lFLvhRoi)C*A&PoO2@*gv=1Z3z2bPX#!E(GHWu?4<<7<P?^H0)Q#)Z3TYK*2Jq187xd%sqh>*t)e^P>mK3nrX;LY21 z#C!a1W<86iynGFt9N)iKA1@ifZWs0UeyD`|Jr6~ri&SMa`rbXGY~4;(&D`n)4HVH^ z&~TLLA(Y0V!b%;j7x1(vrw6XHYdF{sknb$;r3&-S=PT#A}nADDvRVqtM z%b}dExZ(a_UT%pdx7pc0ZnYX3-*5cS2ysr0P`3M|U}EE>LdNT+Q9dOeD&aP5(`#?( zG#a?&>Z8dE!R^6nIBMBZrPd><@mc(hr$YaOHJhB?|8;P*WsvV>nmJ`tf*Is&RtNX7 zqH^0)AI`LyB*Ur&qxGwY8XiF5We8oI6UC;%K=7aJ>xpUvy>21+p+@rnr#ke%ZhmUjtQfn{(|J%lLVJ zk~Hor<(X)Z&NjxBBd2A38X^Cv7oyJJy)6Rs$pY^135-vuEoHB^mwXbmI`D+nget5p zTFOy}9D}R_ofRTesXeam79 zzy6FrT@sOeNN3x1X^K#6i#~BIJ6SZWj9%I=o=b5Ta>+49w`uFYL$;X=levhVM+kSh zgn9(#fBSP|SvG9^Tn+A?=FYAhb_RXFjoOJx4L|0MBhExRqPgh5+C*_Ez3Gzo#1ZdS zSBayc*mn(#0sm9Q(h)0fZ*Loj4;=%3;>op&IO(sXTj^Zo=Ny(0Gqd=s_QK6vxYd6n zkalEjRh3i;-{U?Ak_-9^BUZofw2W)$JCYSi@MsvvZ>Yoe&u?r>_gBC+hd==bZnKkB z{in9Y9bH|_ai9g;){`k%6g&=YH!jI!31TTv?(UBNf^)MWpgN4M{r&*wS6WtNm4ym- z1xenlLWQkg;K}vg$VtVlu9Ql*;KyW)JzPu)D^3{(@5njE=c0}T49up8wkH`VX}DGHCDbL_Tub~vo^r02Tn0) z?4c<01_T(QH@y!jZ_vL6n#U15mK;+hk?;CH-Bs(e->ZIPu{%AeyAlh*j#Lt-h7%0M zBGN2eSVtf!nnxp_+A?csMfln^3^ZP?g+rvNX5J&5Tqb-yO8U!fysgYve9mF7tx*c@ zj{VPC9fBi3$q9bbs<>5vl6$mo68v*cN{=qVj@GmYy*O0Wc;guvbyfh&G=s!5?f zIt14cZx=CHBMRrxD;n?OB@Bhxd8b1Ao z^=U8VjZ^QFU@e-WEuE%QJi0a*@Es~{8rTOEfBXsLpQKDPUd{EqsYf;p8=igZB+p2b zR<>t-x?RdA^L;)mtLznv6pFT~3f}WM$U}yBkI(hj{!1GIw_5FS6h~@zUX_!b>_PLOniT}lTJ~` zOt2;Qr8p5pmj`428h)@D@iG6YUM+yHzsV%UV^?SqdFLdd#@@PH?}cGtn@91J>$FA+nu=Z=I7+8aV=nNeX#XIEDsd}x~h zz*r^cK)+DRFKaLpyuuM3_gxBW8%#=NDpR+T_lYQOc#^0#s4EG14s` z5ZJP=2C1`7nyq>y{uOMT9`T4(I*}v3K(6ZD~BUNsRv!6)!{1F8v`x}}ACdIHSNHpA7+&wVk z4`%}yp(OBBF|#*gbgi!RFWPT(Ci+1zR{4*|E>12^OM%1NsShb7*Zp`X`&B_D*bP4? zg8ygPiyB}R4~ux!Isf4(kYy+>{v(~RYWUmB?Im)j_#~aRq(VBK(Eqaa3+*pFh&)d7SIAxJ=~Ud@Bi#&<^wl^rRwuUs=D~H&u<}*F z4hz2Wn|zabSPB8RbsxM7dPNF*b-_=^{hN8rVu*AX-mG?KjD7674O^-!>A*hOdCyDu zQV8bX%8o2=eMoLs7LW*5&1JJ@tLF{pAnUKz$+Uh_@2!7 z@Dm3E`Kiq#{snBF#|OUOO*?^;ue$!u`n4!n{_0a&N_OhU7dtx{Zg z6Q%)&gJQ&Ek_$M7_!kN`59PkO?fz-J$gti2T~OoI4NUygQ-}HyeT2=|C)RKR?>Pd` zHKTGnE1J!Yw6!FXuRWb0vS{)b^QO-%#|NXPqMDZ?w%awXgRWe8>BPnG>a`WM2 z>%ettB$s$6g*XJKz-n%_Ddov)U#U2Wx7EH$$xSZZyf5^0^N{k(hax1hl<;MaJNK{b zz>YtUa#_A8El4!`;ku6f(_Nn>9Z0+P_3|}m_Za|(3DW9p+nsGOT-vM}n#}+WG*TvN zu`zl@H@yWme?CdwcAVUO7jtsKbzkqoIaXPMhZKBovc6vp|3@cR3F@WUBaWsXXDE)& z%n6KD;VWXIa4&+KDn0gOzYGe^I)!`FnXLQMXRQPk7hm={Cy~ z?N4^&XU49}`K$)g%?*tsMre6@1Lk7fAGh|84(?;OzO|evN^tdVN z&((f-Te0tZBLu5|-g4KO(5fs@vK&{$r+krR^c6rZ?)}{1N~)g>nhQUKX?o_jUUX`~ zsKQr7mEVm^C}?2F@m2Lg?q#1jy!ADTJvzrx_t@pD#IJy_c@q`Ivp2xe-UgV9RKQng z@9N#ZBjXikJ<g1W$kJ}F=%mXHOQwA9#m)p@LXRThj|1UR{!2!( z$xBop2iM=*+`WZUED4`N#aP)chM)(;cPN#PjD52&BQN72-sE8(?zjZ=j?Oub)g1Z6&J|;B^c0eSJyyJ}=MIzhS(7Pwg>xuGEGWlhu<{wW@>?B;s~L9?fo7 zS@YC=<`VifuBo~d+b_TP%o~PQo}Rz#;9;iU-t)F}flpdH7*H12Zo z;lEQQi9e4<^a0D`wA_yVbHeCqh=iR+L`b4ge?M~2u4DFPnG22E-{CwV@cpAHASxoV zy>F1e;{-6?j`F80@g4wy$9t@@1%$HrmGA!?$e>4wQyl;8a0xF3TovVJ>g5T9RU2Y` zN*0$kP0i0$((SDgmMIVj;4)wzqpujc)|#cJq#P6g_A<(+zXHJ#Xo^F)N7H%vRn82h zsy9Qu*rDK)TGNXXJ>-*~lke}!{w9SFtVH2!f0RPfnE@p#Y+VWb&e%W$k`OjThalQ5LwJFxeWD|H@nI@Ro)AVl%S#(IJDlZ^`J!DZ0 z{ra7JAkSRNK5_NSlIs_PH}C=<?|KYgc zE9e#+1q~^TJ8vcPp5Ns1UHL%4xy4<5;)I9WQfGZ7H&I`S%GmSf8S?PeF&~5fVDr)K z`ovPoEN}g3Sa?$MBld+{CFcZL4sPCm50ndF=Qr^(0XGq<2SDyq%0Lfq@85HdX{s~! zW9Mr8$YLh+nSAigOdqFeFC8xXo?=ezhW}DJmu&J6Bs3we>&Gus;}`aAgDoT8W=xos z#PGm|`qm@CHl<4n+zzFQ?s?IrXdjNQ5>teH`-Ss2l>Rw7jsuJ-a22o#(6);86I{)E z%wNgwe|&Thd}8CUF?_^l9zB)PbXmiQ3z5~kH?rMk$<`w@CP1SgsJvnnClNqdOHNlQ zVI=-7f^slja%PRH`B_^uK9XvPpZ$v#Bey=$&jiz3p5ndRv_Pc*$MvevUl_0hatNB6x-T*23(k-k91ngU!(`xBxYh+Bb{FRTQUWq zG5RXr1GV=qr2!oaP$_uG7WLaI*Xt(3Q`2I#$gL>2QS<-`a!uFqAhgcWQ#Kk^LfTO=Q-#7KF{-1 z9AfHPlp!iv#4A=9uJGw>uM;b0FuCGpcC;>RE`L%PojA1T5_#DVVGY8he8MLP5S_2g z=4Pk9t`iitDB`V!msg@-d)G-WlT4ZJGi6mWk8^&+6NZGs23!O}v?RwilqeIHJ(Xm0 z7BD{mMc$4sWdIs>ri|i+WBTj^q71SQx1eb_wu%sgs(X4qrkvuJ{1W>?Uqt0d#up(b zyD=|k{ACXIyCbA={BX=okNQiFjrEn?g zBS_>2{v$ep=}PKhOzjR|K?7JzX!`5H1wbPgf&p$-Uu47F!t4^8IOuqe$p<^+B#>fB z;!$>o@srY#u_xIeF1f}2fN`@^u$^P+5XEo53UD7-<_fT@9|%whjgNp!KT)ygm)S&aCW%XLW+h|5&v=IR2l7p%N-85D74m#z2@8c^1wc z)>dPGKs`H{R?lmHT)ijvXe1*<8q=mSFY?fXt~oQn--MEcZRVN`s^lqwwjBCIr+U1fgO8Xp%+c(!?4?xl(+00l%OU zjCWa?NW|OZ7a|$4fJ$O-FP8@F6O~A7Zv`xblh)6wJiLS0tY_UcIaH|sm<0vQJwg;U ziFU#5U$l>ZAp*N5YTP+BN$M{^2*XGNX{sLu4HUX3U!2O>T zV5;=OhEC?@X+?)zNqh7#70H%WD`BmRJl*%Mf-y$^la_bR9c;^%oKx`?| zN8JSzqhNa=XSKLz#^pz}E*Do`_FdP1cv9*F#ME4bIOzfbzRzZXqki)M@=QLQPvyGL zva{##M+l4fr+qvb*>LT(<0kb}_h1PZD$F));(nemljUa|A4~(zWG^riQU?4Xoz#R1*+erqaYByz+u+p5 z(+})E#|g*IeK7=g;XjB=9#uf-bV$lWpT46M~Z-mqSY)uK=e?n-hW_Y3VJ?!w0NVO7+hr{Ks>1b1Auyrp{Yzedka zmW{5%(MMqf!wod6^&$We-bMfhgPa1WN^e~+-?I4&rl-=g7$XUMu2dMYPA;4DcK?Sv zyy~BiEZiw2AJ?eWgg$9r>LzBMbnP&@=I&C3-W1O!SD;PP>YP--7cIrc5pdWWM<4nC zY}lH06sJ!5jh=y*HfL|%ks|}k-F`-r=2)9)og@#8DX_Rls{QNNFM|rf9h!f2wOo)V z)a8N%pErpPu~$*=I=W!mHhn~X5?L>1LkT@cpbTLS1OSc;OwaD^JEx`(Lod*|%z-^C zjb}T{bUtza) zpG#E{=MiRKPgFOt0eqaNG^t#=W_Y4qKJ02d!Rm)Ss0g?b&ucHg9%4XpkOJ6`i_~{F zg60FBpsA%ruOxN0`ofXjY}-bhW6wiM?7#D@M?b0tZ_xhUf7lhxO2|=A>Iup!@UQWN zNFTKj9M6Exf7)5Q{L7tR)vIP zFA&J5|K`7Cav@D{K_rhJ~vF;qG!_e|8r57_l#T zsgEYGUJdIg^XKJdF7JBlm}G-;22&JqpYUd_p0w-8XLzGd-va4_Pn?RI?i!{G7D1TC zSo<)KeXHR!?V<;E`(|r{_m_7m>`dlaXp1|I=e91jR$oEC$xt*8TG=v8yqs#Ca~4}t zR+@<23czB08{fjVMv_7MyW(eTO?-;3r#B6W69HyP)JU<(Fv6c^Io`@nLVoeTTmZe` zc-Rx(k8u_%D{`okx1(GoXD*QIq1djlDkal!okH!?DILpIqD-e#qD_9sB26*d9V4Qx zr+s-W)5h?Qe4s?=6$-X@sb?4s6W1XzpGeO)13kYD0Aw%9G0_H}&S?!iZDE>?p=9V4 zLvf`IJHvtQa>v~5UWyHa&e&GL_FCTzZ|kV|e(XjV-L_$3r>&s`WG>z@AIA`@3@8k@MRk?dAD&N_^dGHFV7(^Sj|HSwBv2d6Wy*;iPAL&FsmeSqO%watq-*od{O+cGHd=9+RIZ^vRo&( zP`g|57H2%F^3;HAa*CSQ3%2syw`nTdcZ7W6}iq#q>YUyW=PC82LM5|cm z{&?6Sme#3I+SLCj)@JxY){oy%lF*gmN0pRZi_Q{hdve`&tkEqV1^pL;_m;e5p>z5j zC1jy2CZ%KuPwpCGS)yAUGqCvd$w!q|8Cbn&m8@Pz2`o&PSFT=FTCX^OmE`7rAAer*Su0N#9xenCIQ1`yK_TRo^4UWB?pmRvkw zDbQno&n-2(l1Ae*B<)PN%?ZT~From4LdY zCnpavDQ#L}>Y1mrpyx^ePYZWUR6^+_;MQz&tGq z6x^y{`!e__32*s@qC_UtY6h@PPLOzyg(-?DB#9LLoUYLs9IR28XmhQj@}AKJUJAWJ zue2HVmS3R;=DZfjL9z>x)ZMpQKM$0OUmK)@p5#KFw%%2@;BDDV;tF+ucB_D+B}~Ot zU&SBc2R41LtgH+TQ-{`>o2#b289_`gR*5!E2B*Fz)r~LHLrmUam>3|~vN?$|2Vgs0 z3{t%9qE_>Q;qzTU%<_i1u-fS1sYP14(aeevxGKU^?9%WQ1RFe~`-o-^@&`!wEU=diZ~41`>%;=_8`0ENz8C?_ zn?JlS7rKIAo_3_1GEh}f$+!aBhX80$500Nyi_e2Zn;>w>-;6)T0n^HPY@a7|Wu19C z6A=}Kl#!LKqum6Mr=mj8PZJn;kgP$-CohAxIfv+=nM+{Oaf0x5NITE{#L5YjJ*ozN zLj#2(8TnQU7TBB!NrTZ*OSCxvNXHv^JYkX`W+f?>&3z!q$AC^{wDwn`oRSPy$C#q> zjEeLp2x){Z~EUW1s33@&4ycDO+~2r>=qdrhQu)B)5*68fUemSN`#%|wVOoR+Hi@o-lY z=2w*rEFXYvgU}cXp%Yy7b+s+gC|oEV#~JJ04Y}^7Dp;TMZEaTs@?#FZFyoHxi-)@0 z$@;+zh-GGsNTW<>>E)J9M##z1JN4Bh(Z&uEvUGz5+b48SeSKw^I0Fob4&@ZiI;xi| z)VzIdmkKUOK)*n*T*tfBo1aSMFgxqbQUp zWr39^+NboPTzQHjL!DeXh8Ya8$KV9gY;+*{v+X~Sn!7ztq)DWs;c*ajEbyccM=@3q zPU=)Q$_K5GcO>kK_Cn-2;027323NHJ=O>IQz)Fs*DA;m#XN7tCH*EJFlZ-^lNwf^T z>*~(S%|+C~)+DqsA_#aMXSp_52V0|6jl=%nVUX~}&Sc&3evK9U6*PyG3+5XX0Le81 zL=d*|jr(HLNAd8M17u@_GF!jva@Nw+WDCHcLb9x)mY@Xu-C#mZV1t+CbnnRkd>8x{ zMgl1n87$V0I+~ht4`6!#&1~%eAQgT879SkyARjgk+u6#(B&z+sOYwY*)DtLkhC9h?0!nk6VKlkgI1Y(ozPSxpUr;&#RYF%!Ha*4b#G zQlHP%laE?Og&A}U6m&V?1?r^4Aa!!5VI-R=@jA8Ah7jARrKIUnyC)z47`8n|XtN%sUS6j@?1ePwut2 z1)~7!S@Wla>UCFA{E6vX8KFW_Esnh2*<=4s)*|sckk(7X^S~*5S(qS=H?WjF4dL;; zPEOrfCd)r5;J%{qIPNT~N7FM7z40AJs^!a&HU_{Sf$VO}k7UZF;Z`4sE6#FcHmkgj z2SypYNL@Rr)SZvu)Np}=OnKE=Zb{EPc{CX>^MY5M^d;#|NRYY<4iEf=f0rBIEqDO_ zj-(0^KOV>Ukz>5tqGMGgH{q!0%PkP0!(FlZ4I(rt-(!rt4k;pkS-v6!|LR}fCXUzM zXQ>FmTC;qJ&XXVVL;%p2oJ#vi@a_pk$4R85qf$`C)QvcU;C9L1W@lHlERH{P{w_~A zDc8PTiiBB%T`BIvdHEPOiUrOg%+5aPmG%~~6Wy=<5uD?}egJu)peETe_Qc+#|FxIY zjKrko6MkG{IN864e5@UK2Kdy6Thc3bD10{bNe_|yD>t!eVP{l>!u|1uLh9cO)o-&i&YH>e|+-QAqkz)8L$5EUf-s06Bc>(DM#Tn%w-D*dVnk=&^-u5FpujF&3qQ2?Q2hZw{ z|2CP+o5qH8HHWluneqzh7;z}ExiQc(1o#=w&Mlg{_4+f$dr={bKzZA&>BVJ-5P}IP z-1r7X_47}Xe1t|H7S&U}&%hs?Meft7ql-xP|4USI_#AXau$S&~Vt`NcN#Sfq-avG4 zv80ML_u5BdQ=PbM3K!s@A+E~y&pNN(^LmL~ff|`~;WkJpU;G`-pw&p>7NEEdNxu z)J*}>YC?kuZMwYd&CePUV?OOkLpmE=QY0PAs~$

APJ49wTE*|cX@4$>3z1Y##o zYtSdS^)a#gHo`~FA$pL&s2;UAZ1Ez--J!mvKcV-tj*%bRzn2&6`Bv^?R{E|7OA-gP zdGZ)cznEV&XSoljxYINfq_tN1IN%APy!IL+S~Yd+zWGmY7t5v)>ln{WXPU}S5HOF0 zHbS@wS-YmFON2~7C-qDUvy&z%GL(4d2=|SPoH(c)R7=c4dL<9#okx&rnG*?3FLz?@ zDeSLZBW9B=RX(a^+@op5;5q@;S7ARN0$}P%s>7?^^Axgn4x5$h+7|7^;rk-SkH+Rk zFQ`hkCjspuv^clFSa$B{Gw|!c1HSJR6COa}k<5Hp&LYoHJ1V}Xc8E%jT1w9IC7A-P zi6^unI!M{v>^ z7p9dqLa1%rgAHLKHOQqYN12_Y$S7}K`&Y&WAgP>WvzYMA$ho{b+P(Wrr!#^Wub^jB zQ-zK?-=5M27=orE%vd}>Yf7f!AaufI48UVh*^%1gm|m3hB`N&w{O`jL|0l|$QZ8aE zr8Zse{~KD;b@qhbO7rRtUrADV3v?|Xp|RO4JC8A!1@~tiYPRu(rLf=g_Q#v^lXaSV zWS0tKipuu`?gewT)XF7QojH6QejnMz9dur!S}1{oMx5{Wo_Frg{2w7 zdghMgoiRdsK~u*UEMKj27R*N)I_w>gPSh*!p$%>a^|H(#korus%qy1tv(1|Lu8GoT z4}PinXWP8!&A#5HcTJB|w#|j^9L(9wAGnjPH{H+O*Oi#Gy{dX!8)r6OBOg~DvXxJ* z4)q4`Y(q7{8HF&+-bdft`+ajhs9baJ^u0Z<_P!XS?ESoZd*9&NN6o$Nf^GWtJ|@4l z-M06Q4zb(M+WXA!_0t;XHoC`d7Zhw;bxsZn>IFI7GcS~uGdC}+*d*K29^b+9#xd3apzqOTndVt9R(O3i5$NV)2uWSv#rM1EK zzDcOSQi4`S&ivl_`Jfv$jDM{5y;8O(?;sqF-%psn-KTk7p}d#g-gm%}Y?`Y8KbxZd zGOPzd!(&i=SJzw5&*R81Ku_7FY%9VNlzi1T^S6!?=C6?x4au1~9s7K&AeGP`^Os8h z*0#f8W|Jf9Us=)yifG?98V;@p@!3p2DKkfQNa}Z+Xa6fu>IHvv+h(UfdBJ`#lKiTj zLJ}8EKu5|9g9`WvJ{ryz;f9zy`6K*uEBC^!T6Tvi`h)VFV7HxGjI8t`{C9)H3ca z6OP1JdMf<;pzDbDZT0c zmKsAL$w#{K!2{}lRO~03*C+fNws-Qc9W%?g2-7+tk&7@8QcMqZ5vj*EkI%G770U|z zn;8Q|e};swwG9%xDZd-)y4-=JgmSW#5Cg4afA?>$BT*zHRN+r>BMnTksThq%`b z?yaXs_L(&4woBJ!gK^e=g5D z8{EcSzH=b^x=wkI{hD3)wyeSJOi*(?s)J$O`?B5vg~gMU?LN@(V;7R%HTk7%`vr&F z%u^)3J76pg3g133&D}S{OxVotz!g!KZg|izH>0gG;b1b|<>iPm|3RkFqTfG|Uu|2) z=>r|s)~K>I+xYn>b#Oxhb#ngz8Vnlsi}4}dI`~=XXYctT@-E(-!~}? zl(w;U&ENiZ94;jby&LLjlKiMYMYep z&i1~hM$J73jXpgo1|PWObN+h!*lp|g+c;-}4{{L* z2ZAgck6JBgb-`@@&`1&_{!qNdZcE-4B6aOx!1^b<)$ZGlbsQQ@uw`^Ih;k9%rK{LI z;+ei>)j~fg>CC@g6ZL>4YBm4H-HaR3EN*2W>@5Y;!waBw*%akR-+H=!yv?-CIR9zs z?vwa6NRTT1aCBo~k?b0U&*LMPZY!ZTTk^BTOJULg%(v4>KY5q0+prW2YL-q_&nR^n z+}*f=AK?iLYGyUfYX{`NNvszV|5Y8`%k?kHrMKo!XB@=M@463OhWBX8&*q*zyie|& z18sHJ`_DxqUA_03V$o!3>uf3K=HUAAqt^4lvll=**Hus;Yw`d@#CVP0kqjY5U*Tj6aUoh}&(8nk0iGCAx zYq2X6Vi?X%(jFS}npc1P#y{ThAG>pnxZVo`8V=79A^*RDpRD-@({{gHS;m!LnRsJ9RWg1^v^zL;-g9PXz0j#&GhiE=|5VQ{N9{w~o2|BDXc6tsBJFKQ+$!5A;%(y#t zQ?yeRmrnaV774(2;P^P$c;1Y!kNYI&ejQzMq zinG{K@qXY1o6l}j3FGu__Q$$0usFBzGjH8{KU~nD0`~5`ydP?!xA-aOUf;Q9_~P8L zILDJF|4@At{t;Z=Sn_$nrL0Vg;X<;-N#>K3SCUq&N{P9K@2hjPdvR&P&hws*LFPTb5;9d1KVs_UJnuo017b%c zx~%J*BPUVyNuPw3*2%~acdB*cTSY#N(n}8bhtbkCr$3*2qc^UvRFTL@#~|=Hq+73}t|C)dZRNzgg%HUZcGv?=Ey>eCLo(FO z+&hc5<)-oj$z)ucI%Rle{@Td9m8dPBbAW;>IL znp>$iDB0)c-M?I+LFx?sDZMmqCcUMCnK1qGieAP9R}(2cTj%!YBxb27NH~M1ctsJnS@)=Ip!I3VX z<1OC~$qEUe6gQYqd${a+VeJx4fY?gSfE-74o;Z#x^i`VEj_8*98wv@=+V*=?(E$ht z(+zr=L3YY|Uk)_xc?@Zg%*+O*3(u8XFLtDw-Pay9<9FnmBig=hvtMxd$4u#=#Rplf zrJg-hVU|+dl6}#Fc=PcdRmuY1Or`stXGy~t8pDzp_c5t4K_p!crcIG6qBW^6gC&(1 zHvK|Rqa?fCyHe`YzN@tbqbVzWYgg4PUwlK>v_cu%4u~2b z*>?Y#2x8qdXvP&zN>mK`!r4T-94m^j0go)Uakk;0I3*o2P?C`DNklQeI4HY8u)TcU zcB3f-Yh(Da$$go#0k0@lAqq6eJNIQHj5cmhHr*Wybl9c0{M(~~?FizGDWsosqK$zR z)-1thxGq>0=`NXVza1-u$V{^trtKBmqY#xvb#%FO?YSn}eD11Ag3KKn3%}G{MX#;^ z1;lhbtW)%+s)TgE$%wbVsSOkNnx*mcg6(D}e6VBQ@a2#0%R^#XP1iT+J{+&-LQEUv z*gpwK&24hF#S-}eRnU!g^}oZ0&V#R}0NY_}OG^}lQW znh-=xWAp5v1j5pbwBaJuP8_fA^#-%1z7B=o4T=m>BIGOrLlt7HDCc39P6v=9zk2eZ!B3CYd(FmpT@syJ~=z5+ZNZ7fOmL>3^hWK4_5C zJ~KG*sB&Xr`|9g%zk{I8#kw5bxzHyKfy6HP`29PgvLz-$O@;ae)M>+<*iu8Bh;rC@M;#2nS<>*S+85NmLH3U_quv4?aZdsFh7he@adb4oE^kzx0 z7Od38A$N9e<@`a0hld}CRp>my(Gq3o<=@>eH7bGyBFB-~hM+#bAp27{%?s~d0qTvh(7>%7Re-g~#lH&s4lp z(d~!BAM4^^X7(9{Wti*ssf|6-cyjER^1{L2K?(|r7*=-nJpSqr-hX60vFO6^b^yw2LFk+3rk#&Mvy-RmarAtalZI%POHYKKYz#s7z#9Q_p zVp=EHe)}_|3L<&c6@E4}O>{^Q2Z@V`WwX?OWn%OyjV6-xuA;`s1Z_E6kT!6Gs_;Nr^Z<_NSM3DfN{HRDRN6G$JMO~lUNRvU`SOK{j#`hI;!If{ zgEl7PrrP_U(62>8(IG*w&cJggV?eG-&`O9t_U7tD=ADv3gA_yaI9`gFn3y0}3k!?! zFJHc(Z{_sV3bv=(r)J#Azl#_JGS&bR!z?R?<%mjo)iXZAX|p9nXl+eRe}2R?Pslr? zB976)!K%kVQEKe%?O_KN*0bvI;0hp|q7~m56=iVV{1Jk(s5Fu zna&o|;uL;$EAK-~CJ=*@OUO~#h_?+u!)`1uEEr45$ZXFwHa6<@iUkw`2Y5ICuJ+@v zn@HOVL)R{987vJlIr-h+_4OiFPR;{x7}0*X0-!C)X56+6;fRpU2>d=-r>(m)1pB{M zfF~8ktujgWg`yz>{k6yuP*$h-t7NlV4r^-{Bj3 z93^5RrhUd?F(;EoPR|KgZ8_kln8@2oRkX^NXTNQR9Sbx}EGiycph&ot;|a2xdHK~X zJ0DzTy3H`a(ArM&6y1--Sc0aJ#!C&8C1xch7hm04=n>n?c3(T{8rpA6Wm^Jh|#H=I`zfS$9r8hqz(9|_Ng4R$ipt}c4bqX^b2BG zNp>&ZlTT00ovZ6~j7}*Muc$=H;P&#Mf&G35$@u*oTQ4_7Qg?E(A9kE<;D610lbLmETTu1BkHbf zC9G%9Zn+Uwm$VgRPwZqFqvaH|dYdBO{-%2!mpgao&Z#1TLjMDmszu0PwI4+*N^AAv zQ->Q-k8pc>9ggWKd-3@Flb9&mikhN5w&dYC+l|4AoqpSo=4|L2s9G^CMSEMm)M2on zj-s+}#1CL$tg(}=4dy#k=mgSzp<^)%^D)2m>GEzjyb~#RMO?<3Yj`}?8PyHZVv*hfSCdQy5YCPsPqbqJ; zDTt@=@6(t5V$EkpOr=RS$=;BzlA^9x_;)^AtY=s=MTN~K%TBen7NsC>+}EO9vBuo2 zh}scY85>x?;p5hPTIuIfe$m3|3kGo?mo$dr##GgQDP-REbHA_``oO48W!k=IR4={j z!R4>c^!jRDzeE@7-Tp}0o|QD`W6=M3?fx^-EA3L&Zs#pWPf)>9(W{e#iyEQF&Z9;t zTSw=#dHUiU>q1txqr0zcSTuY6IX;DT4pO?@Hn+@Ei21uXX75V3NfY0_AWRkBx-+}4 ze`blRdES~*k>Yb5^Cj*}NPJ-M%BS|;_x-TDr^>0DeG8r!UmdSBik3pGPQ=@sRgb$} z(9i$sR<^y*s8bARKP-1fx+kCQw2jxz|L}X&FfrEVtScK%Y@*|eXpdOV-HTwR!rxNj zc@emZly z&#z4=*q*ux{z^7_WIC^92mvd=v26CK%VMevc5(a>Ua8i(t4p~2fXTkC(N#M`iRl16 zbQNTAB21$*Sq}c9_h$84RK1+Ur<_~-yr)GDB1$)>yMuLL*15VO^=7w^wN~tK+L^6z>!H_w8}PjK z3vcM@Xs5L$+xqa^msX2HL%;v%7(X^U*imw0%wKV8kDhDJ`y!X`J<%W_@NV@EZQl%SC+5-es0?CiKD@E zo>?Z2try+xG1-@yvoYA^F{wFSCxV6&M(|pD8Sqa);oj!o_WWEv1-sOsToHHq#sX>E z@af*fxi`W28Mt7hF5ik-KNsbOO5uJ41PmsC;Bx^pdd7esc*6=IQ874a)AIXScoG z%Jvn##;@ct+9+8$8*p%mT}su_^;KBr?ZdJiL%?%hOU0%&)@|#h>|GVwZi%-hBD&EP}^QtARIZLc>Gw^0n~E<@_?*9DK`uL^#S)42{ir~tKc|(N%e3D;6H6)z zU0M#ie8)uNGO5DIcGW`6G9SCA zg-_Y_-KVq6S@3UCr8hKNCU0ylPV>5KC$cgxg$NrNeRoq?W~!lyc9U?JSl7ojA?$5O9K)8T;FX}z5huf9!vLe zl%q0ux?8`_<^3~{qCoaHu3e3$2uqQT3yx=>zkE1ajVi%e+gBll^{IXK5=KQV3)T!n zYaZ+E6B!JzxejY zQ~r&=3RlW-HGaQvb#>J^m}d>4svJ>t&(H09M_}6R=MRfz{*><``)e0x6PG`FnR*}2 z0_rdVjj3BjGHJ^mN`G=2YL~b$$D7*{RE-ZtGBNVBn+*f&=>^Za9UF~pCy9336Qm_c zVf%|J#KBNPS&4P|g?4|d`%&_>S5%w-Xh7qa4tfLrDYe>AQewbe4&4^ZOtInM5xcit zG_Vx)eGz-1*(*VHxW@Jun(ER;thIo?C_*#^DHrlGv|sl;ed{xP;OUjk@89V_i1K72 zxVcaCzn4Bs@5OYvf3l_g!w6QQ?mfQozBheW{#|d_a6a*iKg-V!bY6<4W+1ZZ^#|bWgrpjw4p*r+qavX{+i_Nd zL49&LC(y{{o0Fgcrb5AzPZ~-pz4KkcG9b0#mh-yHDpsD;{OAAV$S%ywwZf^NPRy#302h%-d%_Lp-?`9G2 z)1OUDQyH82A~l?%TD5&`5(9H)-C~J@BxW`iQfg@C{${3`8b)NMd>tZ^#c#41r>(?YFpp*QcjT-E=CwXjR|!>C z8Y?T;7PGdt=3he3&ie<*tt<+$xz|2uTb~>Ml;;Oo0GngfOxiX3b`=OC-{NC|W3?Q_ zL9(BdpViw2eUE-!`-?_tysxupw292*OOLod1AU4*`zEZfEW43jdovp%I-UHx7Zq0~ z@L2Zt1%Z- zRisw^n5CG9n#4C`FbgQYI+wBhkM#QU0UtbfAPthpnhiHYe zq^r_rltisO2@IJr7+vLR?%oNaS13~;t7el5`^tx+pk&8++vGYe8>eU{M0^`7<}2(F zg77C8zn=$5jwU68_7auf)s8M!*K7IMl=>2IIP9BfrC6$kpD5;9|D8pMXtXyw? zUok#@4dae-4{pyLtKus~+zwl_xL_Vhk8$W2AIhT)R8bqazM0~KPe8MvG$_LxKK6(9 zXDqLCXkIlI)Ss@G=+30)jy2>*Bk6u*RxVWsW z1Z)hehPPS`kGHCSAAmD8=C1PaJ{^(xZurOLwB$;x90gA zr-Do4R$IvYml`#_E+Us4bk6f~(Q?U!;TiTfYZ6+qKc$*)O@hC*Jtn8(W9n@)-#rz8 zdMX+vo!HtUKiDlY*@vZUe&6PY-AqmS5{mPDv&>ycc_x+v)1VV93BVFVhOwU%V*1 zeW|GB4U`aJXKM{%C?HTceJfq%+H8Q`LnWAt+o3@e%*f!{B45-oGdm1yLEgV)NyVJ*|24hI|+ z_i}P_bN57=U4Liq=QR-`*i)S%pQm3i9-liBAi$lzHsKuo_BX9z?d+?DSt}*cZ#hO2 zjZAM8w%JkR8Xi$8g%r98goBcVdfZNcVBCZ?)poPFjyGRIh~ekTM(c~a^ACuUrET)C zrD5WD1tB??lKOOx=t2K>)!i;$L9yX<<62@B%1)J6gmQ|1w818-b2kS`oyi}!%D$v2 zRKiVj_mEBa)#KAb_V6PPC9wqkY0a8=FnY!1ri6$=`)fgT*0GF|=mZbOhGA(5uh z?{HoDNqEY=5C_Um?FYWHC$3=rAQ1Y!tU87k&EPQqkn45uc^`Pdr-}0lvp-z{Q2x21xLK-QxYni z&D22hkjM_p{*BvaVuylwY3@ucBIWFzQ|uDpj#S~z*pus~tuW&r2l&5w!FeQ2L>zN} z#!kGch5qRgC&-nWa4qX5i%Sqli0*L51T5`l5p9=+V`Zdp@`i~{;uVU-?|sn~O1n%h z3ggaK{x%QTn1O{WzEC)EmoRn8}%1`d3=F)WAw^PBEDbjpu4Bij>Ljd7TblGt~M9vyP2(5^ju1nSG_y{ z)>st$L8#hRntb%A$`^q1%{-nRJwDyBs3xib$`Jc_Wx_f9?eA|58zT<9xcmAq`cc|R*k<{25d%sq~2Fo%5x6)ErjLD(FP>HJTm9c)MW zUZ`t0{8aTBAJ9*Dt4x5v05$wReT`H~7I*E8J-=43?9TQ!&Al6J^m7S#4p9dw>@xA+ za8+-UW4o8RQiiB?NNJ806DRi04_IOZLWkh=moqn85A_W6m2|mV_T^2M@V5_f;P-cS z+E#V>9r+p^i&nKsnhTyubox48mxk-cpv^_jrjOtl&=-QKhvO4jOff;?_zen}*XaH0 zFyrQ{i2bWC#GNi83jWmUg!#$8SoWBXhcy@YN*XuKh4dc;IfzP$ZgJgB01`sBSFBT(joy&B z9v_Pj6mC9aH8+-l`ed3~VCwwmk5K2J)8`@$rzJQS@8;2`Kwh5$XuA*2+vm6ZZSbyu zl<4P<=fQ(St`n<;bXZdFA6xo%6mm(W={u>~PmQ=Gi5XG(gf?-1Jrs_=Zw-jFFpKr1 zhS?Wa5P;#kB-;3*xw*+;Cj$G_34o^YB%;O$aq7t5+!$vnoOrqv&HWQ5WP*b-pQ$nA zn7>C_pi2<9CqKqf2V{rC>}Kl)d*RH$&DfvrUG(VHSXhxHj@x@*KW)e*M65vMu}!7x zr-64Xk{p&e$wt1I=uN3{9ro+rH!kSi!s9E-y z#cZ{Iodm;Uho&;ZjyvM5WZ%L%MP-pO_!RG@YeqWuL%SS!Z{d|u?vzy13+@OTAw_4? z_XKScOg?S~CuN@Mz72Y6aPc&3htO%8cfM0ud8Mmn#;CW{Af2+1d_Vs7psREpXEoDd z3`+muNY_=~^2>ZME168mwVx<^q4voaw>p7?bw%yhs>Vt!6Ah1*f>VB?1JFFz(*16c{DO_kJR=7O6B!b zLpQ{;y3@Q@@v^{W=P zhpK{Y;M3~$YruDtwh-pYAx0^mSaU!91yI?q<#k1TynLR;V35|{N}hyw{}HKJl|lq% zf%B7Ob8~c+kGnlO#nSc$3P+;;ug;dOuw?9^T2Z7Zr*WD`l4r^FiMziYnhf%Pr&$KI z5O>1#R*x@$edX>$UtL8^6Z+~j%8xT;n8O!c_77onOTW|n?&cEvvgQ2xou}+`ZM3bSmq(F)-6gGFf|Yz_S?9j|8Kn7|{tJ!ZChI9`rw9}0 z@+BiBU41QAW#tEEci-JQnl4{~$=CwQdBRb`z#a^?N3*o*shi1X!9vrDj{f-MQ9`Uw z+C&m4fgYM9-{5vM-&dzVo2WmhQg7f1(UL~@gtJnu$Huf0T4%U_q+rQjr7Nf52uqzf z8vwT5t!qp;eqRxxvb==DhLn^qxJQ$QI9IkTk4}jgL~vn_VK^ev0b6|#aN)HsF~|bn z(U9g@+D>ZrdVJY=o5;%;JJa0pVtO;!sA%H`E+8nZHy{yU-z%7Bb9Y1aR2zI0WKUy#D>x#k2W5ibTxLttxY=gt@~Yfm za!Is-BnYCFQIt>`O|pI$TPnF7j>Yi#!?|0K zmXoKDJ2f>bwDo}vc|EbumyQ-7fWs?|mX^h|+^O`H^=V)}-VR}+pI7#3|Jx!Zd_gRH zHS|~ZF+b9UVDkz&{lDJrG#G7X;z@2jsZ4@~38geX=fX!{pD z&GYYOyeAkp>aLn&J{QO36|1AZS4wmFSO3`<*i$5t@LH9+>3!jieaEd4!X(6Y;^ZvB z*Az5Aw2y+EF(f5?Vz;FjH@PfN_r_3FwvV&~>&>rk%RhR-!#(~ocTOL-mr2quq`i>h z_+%{%<@~X@*BuY;~+H;Vt&g)TbHn+j+Xqg#PK^Z>w>MmH8 z`GK6nxvTKd`7#Vdq&fuS80i~9d7j$y7D`AxBJ8QHHNHv{; z=E$o(s-`t(Vh*#P$$wlftw^~%BFrb>{`@Ow1`PV#8T7p&Xa36ti1$7x>La`27kW=D zhIEI4MO9Nvc_4(4#E^^+#H;diUy5)?@TzaXK^*2}7d2d7v>PX#wLEVlyh;4#jiEcj z)vi)Vcyi|5&$eKP5QujE*THJSBT8zUm#d23Ed&QWG(9-==CjkScfA`}y%uVcCbhU< z7DrAifyy&{^}U|g^q(8M+6#XensJk6D?FH|izac_7cSKhV$P|jA8)uLASg)`Acz_S zM#Mgs2u@KhGf^Y;U(%*y`wZ12n=T8hv^_??R<^RYa&&X_3!F52bH4Gw-mNQuS6T3_u#Fh z!>%n)8X9X%gmBKbS*FQ-t-HYZsBA6IVyRn@z^ zkJBNgNQs0X(xph3qNDF7OvPp zjE+Rs8Ts>Q;f%Si;i^CFApEvtxxC#P)xH@beg6XY8^+ez%}E%az`+~gCMG_$e}{I@ zV=Q1URK4NQf;^4#R0~g-CkS7gxPp4@za7{)SNp_g zX|qQ!1x|!Fl|AQPS+RrFP#h^6#q||hw9Vr43@eQ{x^OL4|dGZVIUre4WL2jw|a^6|)N5wa+ z!u_!yvWX6bhi~Kr-Pf8EoFt#@tacakdZ7n(+Sj9-W2j3JeK+7EjK_-DB!)~kisX!T zT=?!aZkW?s7s=G(i#(5nTYQiUj(eSof0Beka?JF|K$RgTi?|~Aq0v!7+6T_DH7H;KU zwxnoK=hdh!V9S(Cr4A<_UGn&E=fBtIDqo>S&hdqTfE(nwz7=eU!#0!e#uPEg}{{N!%UukoGe=M|V!=jM}ZgsAkcq74)yEAqpdtYv)Q3 zC@|e6c=9bt-PNz<@f00G&~`e`zx1~`2%PY%`xd4O=4ndKBbZ?;!_~gOOY*kf(u)By1*-=qAPy9TpCu zJPXmAFwV1$`_Oaq^wX2{&0mb&F&g*cuS4%r#jdY9!o;oo8`DQZ0{Hx!1Fky0ezsq6 z70!qUQVayx<1AFCy3^Q)IpxQnS#VxtxSPjXKY9wHPt}kN5Ju>fE8}QfT2xH;z%G54 z%Ob|2a+c?u{Euy+Fz07LDpQ9JEh|VvJ=V4m56V|}a#ZF!Bw!A==#C)>a6@0|oFD40 zErHUQkO=bQOnzr}UHgo{9{ubg_dOz$CP7QJc(ZtxzHmBqTJ%2M)f+nKPoJ6T9CBe% z#`KLbhp-(cNuEs#Ne|UpEiyS^z*@lXiCK`vHXwULyM5+aG=6 zLPWH6+_l_jF|PA_J_|Br%5$~#{B6YujLU7LS+qjO9IbrVeKP@YdXD#H&Ru|;B8BE> z`&4(v1=tePV~)RhNhWiRo~h%RYV5es>UFgi5n@Upy%sLn4!mWOai*!wJUEAl1>8Dv z>0c*i?DSlW2{1Dpk`$;nm-uTub!(=c;QfRVe?g3JB-ZMFTUT#h+i#~}sr&aGGtRPZ zgsG>g4(9xNX5EdDUB-l~&RvK89jGmQ{MJ*PzXV?9asEMgExLfh#s`?-`JTTx@L66X zkMao^EekUYW0H5wJXcza_YsSpj-T0Ro zo=oi@rHQwwTEyqo*NzQ=iXcg}Tn4o8=;VzhK);-sKoo4zw=`5xbgrO>M@SK;c_Yel zBJ)gtIr^9K%S^10m7jdXBt{QA!;H7n(I5?FKxb~dmn6V3$B%3Fr1p~bvYkbonvJk> z$Fr565hb~eZ>YB&5A|K7Q%x*lRUs@vlOxWv*{8c=AJ-YWL$`>ZUa6*#QyG=N^da&8 zvKn`2UxjW}&=f)KdTb^`)0`9UWOtvRr8=uNO?R8wToA+h4lSUu=b4bl2-%Av|IM34 z*P;2b=c4TWs;}*Yf4W3N$Yoku_OIadq34{T>1Oh|cy|S=0#`uLPlEv3&9)}K?Ri`T zt!n5!NTMx7=uTlTSa@UnQBieD8bSR~&yA_LuJk!q} zn;8+aCrV}unWX{cY358wSdbRPu*0?OKy*!9?$FPF{C_q|-B0C%thcBABzS!?$k~j@ zp3q(t{Je;)iMNnKNbC%McF$#JfY#Ld`uY)T0l_nV_Iy`jdrVSFtBE;5INUxlO+9=o z0|7hnhi88A-^Lip!-o$?Xqq<|kpBMj;MC@IT7HKO>4KsB!W|hOUvGTv9(?>m;y?Ac z2RhMVJXvv?;Xl?l zR8&+#jEcW6K@K;zK{g_R;=_9_z)z_YZ9F0Lx90V$zcgoGd6R)v)6%Xnd-g+7rod|s zK~}%Q3($@2Mltz31*Z_-CfxAV_8pA)_h)&C*FTY3J&?<7ehxe^dQc1ct}Q^iOhNJ_ zHa;*k^!X7x4A4W#^s2{ZcKz$6M!P7d7W-xwDrFcL8LJac_JCa&e7UlRVXq8w)pr=l zsSz9+DoH1ONjoYTD-A(M-jZ+a#L{mmMO-M7i`p^*WF14&p4p=hJhv-uGnBj{bxYx&_d?EH4&5XS1*pM^>YjsN!}*ho$7E?@O8VYfo0&?pG6GT^Tr zE33FIG>Om%kyW6(6KeggQl^-@B=uXwWtrmA0>e4xx5x5Zx&GbVWr1>14+5zXyNR4_ zi(`kn_zCHzlS3d5Y%`DJS~K?GMZLI!qlDIa=Ev1R{~S|#K)}@&kr8+6{Jg!7T3O!^ zpI^o)#IdrbJjRDQbB0#=u3~}UzRv{DFNK~aDrP3;&@+vFQ2ZYu`rpGQ*MPPSqir_@7Al`&UOEOFO10*`IKcM^K3SKzG*b@b~l8fGvMrK)|AQ z>!4i&mRiQ{BGxy5*489}J>x=yut)RbJF@7wXiF;PTef@Hyk}JXGb%m$@0YZzT~{D5>P^%kvftE$-w~Fj2k&EQv2B(NGT~(&Xi|O6wp)N5 z^5e$m$mE1Sy-=3_K=9~U=KJ`XE84AUpn7`qV)Fy?el{rywuR_RgdtCe+;Cyb~g!&9iGKCVs|=TVm-c< zqBj4~ixPO@P1+=XF`Lgg0w>owZja){()tYsWCIUz*tPhL{!7g==W+C5owB?*DM4bfV?>6Vle*v z2>G+Cn^Mrl7GIa9TQ!X%)xF#E|11OHw}+r;xTUZTNkR!Zb?-ClXUKj*WYlaB(1}TM zt!-&@3Vt%1jV_psB##`4=dS)6j)j8;fpayVKTZ+a{RM>QcW(3Xe~U??a_3GC_(=o` zcNh_-EQZzASz>Z#i;PX-F@rK|KG)B7Yf8w>$Y|ox+A;Imu@Ttex7R5433JrL`nUZU zvdew1G}4VxNDIaX^0TCILH)%FQwh954lf0mpvKPf8pzmn^2&(ZDIX!e^UQ7`F>h?f zD_M`)x%>g?KI|8ZS*DcK<>@SmGIgXT%U)N?dw7i;SuBi0T5P&e%_ibzHm zR4yv|(z~-*^7dqLp^h_GQ=vH_eU<_V&WYIZxZW2k_~*4KmM4Sg>6Vw4hB0z?#$-X( zXKd=Te8qQ3`WHV!U$(8?^02H~tmh2~Qukk$O!<(wGZeCT=1bq#VDh=ejxa5m30g9c zlMA8uQyE_0ArV8W@v7byv9}~7d?nXOZ!pi#k{O{@VBBbP4MG~yWDq*~qx&DPn1Ysf zA~l4iyy1~2efjcb2$~GY@S7nSXmg2RV51o7!_(s^jdLWH`?rz&4N=54PBc~X2od{e=%+sW)TJT7f+q!%lE4{!VUq&?W z-ifpJ2~EWn#ELb}ILQ4AX3$9o2UCukn+3&Wl_$J%?(|9`jjh?MG0~n=!acy#uPV`h zr%OOnq|3NSq-Jy2qq(!sd_MH=J_?tT0GrY3&z-9bl zsPKGZh4`?~WBNGXq?pP0xQ&w?J-$)F;hV$L=5HJpO@6OgHKQjtIvg%#gv8OET02D# zUPii*t(&dK6D6AeHm8sGpCqP-CP~O14y-g!3;C*wtvJBtEROP@#EaC4ryuhpP0UV< zds)rzruXEVABN4ycWFa|dI!HtNY?alvt1}~qu0|)l4gfRFG6skXI=RluMcQ+iB#i> z(EJRWl(ErS$ujpW)UNBMi}xYCjzWWZ5?Sb@VJ0|!hD4tuGe?W*cqeAgyl|Cc`E?AW znYaehe+oU3=%U)>b=rmcw0nZXmxEazJ2;9;2@DsCck0Igp}=5*VqWiZ1!*QmdCXvf z*^ZFC3&&k2%=JR2Ro{t^W^(}?9oFsbt7rn37JumvQu~F+h~tSpH4`igGt@-Cjxyt` z7Ju`3qi?Uh-BzIFu-!T*M&Hg1%1a4Op)rFL6aAf))b^%i(nLnjV@6HPhHH2GzkD~G zjvux)+eN!NC1i`HSys0SpW^DQY}VO&V%9G@UnVlY%2yJX;G}JJR1q_3YZjNnN{yax zdBfTEc1cjY6G`l#V&2TJNENbRk$HibLE6sBqCT3jcHf2g>hX3u*Ur(O)P62B&7q?6 zt5~*gKWARhyVC6NyWQ5IbJWTV);!d=x7}GOo5ET(+-@5@T`@*&C|F_$XG4B;T2(Rf zA(M7rD{$%*?~a|1Vtv!C@lKtuxwpi)_SZ^NV%+vb7+i>Tv+Ph zn+yu+Uj+cJVN5M1D0_}cSGZx_X`#BVc3bX91=urpj=Byo8?1_K{?lN1sfVn2r%t>TjpqeXS;ZjE1T-e^=I6Q77bl+IGy%-b+gA+vcm?QW8# zMs3q1Mbw@=+ZJs(%BHL7cAS;Jhr-!f)f zS>@v4c@9cH4S10t+=bDJDSFc?0=~+da|ltdzv1bL#YA zX)AsJZy9O^hlJ$gN?4iQmPXXPopy7_w77hW?QOG;nhD%G*6hkI09JD%;V?0^>(-QlB|@X>1B@n#Fu zO%y$PTtFj!!ElEP84(;9b{EosuR!n0(f#mgY|YF;@HnzfQ}y^MT+Xfd+bwPo(+nD< zPa2>VgdaX87(LirjJ)wwu&}h`^$QX|*yRl9F2fx=e7SpK81JOH4gZ0w=;3>lXJY#O zgqPkH;bX*W$Ag`fnqF&~qVp0_rh~ze3Z{ysI>&V}`thdlBRh#e;a#ABg}!#YhGC8_ z>%KM)E?8?&Ko4pjH*d-I*x_kX?ICe^w%DQ2Ny6xhv{#v#cbg0frGE{4mMi&oU#0K~ zHi6dlCwEmC2A+RbQF%@Ku1?_5R|+BpmAk((FM2hRk{8~`rl4@0JD!fAbu}IoDKR&j zKdj8JOiuvp)*m(Yo%c;N7qEAFOTi^zSmVBtfkJ6-LNv zqSf^DY7W9~JTQ3M^dtp8r+(M!*JtVZ?(S~VPki?$^Gn~RE!w1J!Fy-o491DpB#lcLw~-IdnX){_J|Cfip{PjXmD zU#DCkAlPN&;Lv@fp;6{#Y&>a!Dux@hyLci?o(g?=x<=a8fK$ zg*F%9%#o3iJyc*KC4mcl32mM(fggxO*}+G@7HQ?0Z@Lxo*>kTGBu{U=(BkZ4{vcO# z;fR(MOX*G16EFA$Y19~MBT)FV9RugBVv%(k-bCKiLpq~TEzTX7H73Pxk%!0P?~B$K zS`p=a`FeYa*kvK!DV?*CBw?vC<&wrqow%R5nUuw)@m|^#en@5hZhSUOQLSW%ctOLg z1}7K$FNUw|pVWWzW;PqF0z2jJ6@4#};C=No@8HJdyrHkh?S`uVUGg`MmIB_79(Upl z$7>csZ1Rfl8_(xkdFlr+PQ1{utrsYio@;Hnir?_rmIU*8Ko>rN7BT@vwQ~zAfhnsj zp9Ra5rN{A5H*8q2OG|oxLFrmZLU>qD>R!{su!vA%%vOKN`}(h#*jOTXUhP-6c$dtNOj?eT&Om7r4J`x-qy{Y=+K*7^K8yx50*2A4Ce6J_m=Xcg_5`BdtS_Z zO3{{(ohecmqsL@sN=hQg)oeT|YZN{k?cdM>d$@=*SmaB!7KmV#di7 z9vwnaEby?*k2Y`m z-%{$xanhVUo>`JqGSLr9#UsD3@6RNEiS1n#n!IY?QesV_UUNSYOQ3njZym)|ugZp% zt^mJ26MEB@rBZ4G;ACM;($lZ1QBl?#I+iZ(E579%Q&iYCc>S=MM;WI5-FkcR>F|82>4 z{Xn$b`~;uuTyK?REPJUJ&HZQ+4Z! z{pcay!yJ0P4{Z%GICIzcZdl}=O-p2wf731#h_)P@Nzh{4 zgsR7vY!3Ao$QiKZZKJFhEO;|dsp4@dL0$hO;rq}H`QXg5ye;I%B6xP>4eGVJ`U~z+ zl^r>FwzkNNVw@@%tm69WH@Ws-(%?759Dotr(5c*!?LHcXN=98q1)=<|8Wc0JT`0;V zc!tIDfqw0=>a^!8-UzwMocBM|805R(tYU`Xo?waThW3wR#LNGo<5V=!hc_sUxOu}0 z58PaaF3#0Oq&l;wEV6N^$?jhqYHA7HWy^A~A@siDaw)sK(cb}uj&i!x4-Z4a5N!!_ z#ouso!(qS%PiwAbz}Ow9H9 zzg_^ZPaS>DF88FLr9`BS}U8yZAfYp$ToP_F1(ojb6`_3RAN?`+zf5A>rP=o)>aq#WI9{E^wN8OsE@%Vp3j|u!_(w zU@D*;;*u9;x6?q5(ELw1;1}xSnU_$DqFG_`T*PR7>CWJxGUtQo9gN%%C)|*#gy&VE z>x+y`^4d4_?^X5g8DO8t?fF`jXEV)G1;Lb9Sn1?bZZD!|MAnopa3)O^-`NTy)-D8a zqcA46UskyOR-H8(ZT#CNJN5Q(A@cn!GC37*6M_ zGt*+F?@q|yXC@Es@~;$HgH59mJ;UzV66Y?#tUom}f8zBvyjBk`^}u1WH_o5%eztJ( z?^W3e$xHp8W1IX>0*&|7J;5#-l~-*%N9$J2dR(1LFP%jtpYA+B8%Lu@ ztIct*^B!vvXBTt5-BxXKLtgs_EENE(uC>QI9N99Dqzj2S5;#DCNwmrvocQoLQa=WT zulUDNmx{p2idP^EY^uB~WYyz5+n820p?%7UdkuvSY2z;fxm#24^&@=nlNO*}s7&{9Tx%ltU)6r-HuAp2| z0Vw=SJl zlQWd$@Jzw)B0v#yl}w6%Z7NYi^;jS8GgYm!9 z3;5cbf4=rBVV(9*Qzn7R_ZoCDQkP3%sYCz#mxtO)Znx@AGa>!G904pX>a%~ldG3Qh z30Z!8O!fYqz&qn+`7)G`a1ZTS;^cIuWfG>FjD8?fmh1n2C*}lEeW+^G2qFE_%?WQK z?U$h`>Q{+5-yuHK@z4Jz1SZb6A(k#MWdFO?`96%{HLLc{VO~$+29)FbA0l0LKi}c> zqPaM=oeDZF8sz>K|2I zoVYvt{A-qrG~EgHQ5b^vVF?z!$X$go@w)`mD&-S6Tc?_2nROuy=KNpwS#fSZFnACa zTeo`W%DUrqT05$B^q(~!LNXR_eCECb#}c0dGV7xY%V&@5-7w(*)t=yby^P45`-GU0 z_H|mAOdYRI=7J79XX5gN#`)?^^+IR1-v+z4w+PjzG0B%Ah}kMz;$QL^HlT-9j}Rsv za?TZ=4I5U})@*pwQGifNm zw0$R#`1`^g`RnvvV)k@}xgSC~fIhRm0d5z@xQ-9qb7xq3i#eNNbhi23R7Ds-W21al zl*HZyrh)f{^q9vyZQb#X?$m}*PF@)`t*C3N(|o7%QeW_Y7;zucXl@b+aUe!~h1?Xe zV#^kL)$H&6hJGEjWOtb_lzTl#r9{y-_Zz%DGH88T(zwTfH(g#KI8y&j4CBD`YBFH7Tq70#lZelLg^?c212%sZ+}>Md{^ZzSTO@!I8na}*O>hR}SfT5w9J zM3u&Hsoi*@apTLIMkq8iX63|(+S*rMym%4AsjPb3`igD=hUdJ7c@xKF9jn zD@&>i@%UtzX_tB-bOC6p-5{XgoPgD0eyXU5X1MqA;@c)m@6n7T-&UX9QQSL0#I%3L z>-}Vw+GX^sT@^>U+1QLvj`qL6p)=hADKf!Rhimn?MqT2&GbDa{_eu}r#XN33PJf90 z3C>F+q`sZ0QBNYzFG5CzOM@mxd3GxYw1pz;w+lHxQ-8-yl6Lo&4|9AunaZwhQ`#FX z;Uj`b#uqWZ*p33fsQQ&6>(or^hQ{l?w@#=LU8_Mz6vuB>)-9lGS9`Wq8}9KisUsol zaYi=`MPlFG-Jef#f*CMdZ9Mg8_NT|YS05@YY~kAztV<)|ygOfZQ>1mDnMKyyn=OZl z6kh_K_n6F+AaPmdSTK#F_3j)gSV$%l# zBlh=1To&&y_2pg9TZ4^yva>vRv|E2_+7fmH?mISEVCa=nf(Qr}Z{@5WA74?jyup&> z?Z_Ta!)Nv|c=D;dJU)r|--~D+af0&|(;;onueBFe%Ey&P4XpA?wcwA&kXilljtQKZ zakOIRX95@cyE6Q-C7k6q*s8)Xmv(WW@9{y}e=X>kIQHvbam{y#gL(6YTJSR~JWyCb&32z| zR$DW@A7VjhlzR5)V1YJ0v7a5A$TfL=4xGolw-gl=6tYLsJ+|hKHbVVorrz?H#o3fu zd>KMapIIsXMy0GH z9knM(Cb{%!n@0|Z(o0H^f2MVx2*|Bot=g8@`K>J3(H_IMbbLU*{pVLoxBb$?s;y4Z z$K@nq>nL?sVPaa5+r0?~V_s`F-Pes|f~iZp1!g&X*I+;OTzb`GIP^e7U3f;kjDJKD zP3yp=Y9P8h;#4`G=w&%#+Zivc=lo0h9ld0t!}Rx}xmYU!DVO!RIJ@F5iNnsJOtiOYOQ>rw9G@{=D&rHCvs!vt^(pjleN1&Z{Fa^nOPlz|6bT z9#Dn|P2r&>dM>ir)*riGBP^lkGb3zjYP#Y_0$`ztTD@a6fsguj_sviE@Z)kpz@(jF zJ4vu^!J%?HWfhf6aIKpp5_{L|YBq2#N@Ys;9IP*wbg4di^nT+&;lTsV3(S!EaOr%} zGMzVi+z)oYGhjmEI_~3!;2UxcV`GMK?=2#jlOup!hk%kh^~;wCzAmxwV8OBI&ID2O zFX?FyKBoWIGBx$2%56<)zB2)Ft!KU2LXIKa0?07Zg@Ped;SkeknX*j4&c`h}k64#O;d<}x3Pt!wZPQ=rXbya-ITDlo4R0mVai|Gn+JS41%i?5yL{-RFPCYDEEEBNM^* zuflN}7yOQw3@gEY&>#@uOn(oa{&79D-QA{0>s^Q7czlBE&}#sKqTfoQMbjaojvGys z85CZ@3IG5;2Y=BUGHbwrpB_a|H;0<{>$~aC)&#S}+hc%{a*KQI%zzKvpbK@Hjnv%i z@;l+1E=91uq)Q?Wtgx)LbI739^H;r}pXc^sQ>f&L99;FIo+JKc7A3d?1(HU2b&KDM zf@!j$bPr>1zXr>G2Pmm=MGV$bn>BoVux+*eAJ@H?gjr;N`PI^cu-fAULsLRT*!gp7 zb<1tz{Z95vJy+;Y_hdW8wjSD6&QeftYTbe3z1X<9=lo6&=`Z>w2GRSvf49`^)48yW zxlH1Wr1gBpiekUkQ1k_ZKY`njS~#h$#;kYZ|8NKk@MVLB|_yC)U`Ie zAmFQ#>jaDL0`XQuC&O4eLBwV1V5wmK+ht`eqoFSru$qFqzdN%w0CyslB=(;&E5>4> z;0Qy_3oH*56x?S%$oli%v=y0)d)A*%y8_w&x-T_Y;}PmBE`2+o`UMoR=rRgoV0rhI zuZw!FgQo5A!r?J|=C{0dHZ4As4Vqnr;QV1HHj+=~azl$G`qh#}^0YSPWF_>mHg9c3 z1@zBp9j5O0X_*9ztYm=BjvND(TQ+2w)eIw@nqVI_ zr(+oSEtE+yS}-QQsayZ16XNxNGvHXWUoQ}GG6$lXTo~l2Lpu9jxlF~e60 zcGk1>@DKo9b=Y;G`vv0dEBy00?qUWO4zU6@Lu#INKz29^YHn?pNJ1dW867Vl_kGZx zum51axJlnD__X-k;7k%qGX=C3KvXm--<3kvxyjIGqBuyUV{IOg~um`c1cwwh* z*L~m}zg0D*flPgJTN}$#YDNG6|1z`M!F4T4dY`|SAQmW$^02czqSHj8&px=C_%Qmh zqjbhak<&U{I=j@W*f3>(T`d>QY$mqdr@y_!!vpHkHy|0C9<*;H8uUB0A9I4IAa?xs zH-Ft65~n-f_E))Bg@`O>zUzz5178ZSsvYxO%KPhm_b$5Ih=c56C44MibgibO)h9AC z(qXfOB|~H*M0k!0{zU(!Ovp&|TCMC^$txf8Ok+;d;`|OQq!XNo1oiFpWzEVzHsFka zathg(7Bf>qruL>o3*2mO+_=%6AS&ompM4so#zO0VJ?vv_RYqTap-sYkzmryZc5k_z zVSeq7P87F(EBM=Ef@w89JoC3)WtqK4WTg%r{QaperK=BkY9qcr$IYO2LZF<- zig1(lp2y0)7C)q<$9)gEG7>%6H}*FD%1<`ic>XT+hl0r^0#(aMvz<*94S0Z0kLSne z{5Xo^T3{5~F2)1u@-NKB|9DWEy1;xjWZxuh z1yJKFtR5?Ha$IoC-Q@YA*VJnktuoHrxdp&|tgBbbh=!812uEb8B7c%l>2{R);k-n5 z3J6~Gyat$_1ARp_&Oh&po$O6}A1q`%j1vgX*RQ^1)*js>z>?_xL zNfNYp<}yjX2Nn>upL?#@HI>|#rSo8-_g+UYG4GlKBc0*#+kar$D>9pA7l4nN46MAt z@iN>GO@M(=yE^9=)UDYLzhJoD@b>l9YAjRz4je(ky10cI$XO8Ej14F6ZH?{j19EpCsbK-f!T=b#X2Rq2A9a`m`(T8Q_bi zizoi*n5?Y!h4z>}Bw{!;_x6;kvh9oCUxb0siCJ5EZIv`i6zId3s|-IUK2Dbxh0GEr zOY&Mew@2SxE*rKL`}_0WooHpo1g_1|YB%L21w$YuUOGD^?i)`I{w{}b86dr>hTMt@f zBv6<1CU1xU^&!<8kBA-fm>R0d!E2&V~V z$fO-_7xyP|U!0-xPAqc3{Qd zq4TN(2Hv$nZI}HxKfmt4()Vz9Tp)Aq?sJ@tUQI*vLLk4JaKP^CyQ)j*33%g&6tZfK z&!mJZSs0#sWxTN=W^{hQ>|ed)^j?pcGSr{h>KtVQPY57eXo8)D67B?m4bUqCi${3b@3M}Im$!ErjZ1&6;1Ns{vTz1E zJ=*l3Z1}*$W-nJ>{C%UDz6%!A0VW1{3U0+r*n{l&e6|Gr@QW`aGPbIFG<0nu%Y4*L zAz!2M`THwN7{hP3dOXiD+FlwqaRd%pL^wRC1)G0TLW{vLp-!R9Yd0o_227*yOWxa0 zvyia?qRc{0J)fPEpY33}|E4|%$v-!y;cPef#!R^K=BV2^`p;@NJ0;=S2%KJ*>0rTh z>P%>d{SMDzSdAk3IO=E79_u>JIvOwWUe+Zm_bYoO>$+0m?I`v#HoL@qN@Lqok+vSK z5!!%FZ557_pO#M5`AWF?SUQ;F=;LT4m?Rimc;Szl^C1)-?5@?fCS-c1$cD}}5b$6% zoGwGT9b1aP7ub9XCszuj3c`@$?9EG_;s+XSE@UK`E#;Nx?@G+WXCjEsDCvobJ!5-?;Q(3q$G5Gy&Sh_f+>7HEVXadaUQPCxEO}*xzDHC@vF_XR0uS$0-y4J1xB(0D3xmcvBDYqyjCJ7P zXuPh=1CeuyM{qxX28yl&;G329YD|M>qClkBi97q)Off-(+#ktQ5V~FZO2Ot~%&iwd zsV{{#zixVxMV=}+Y+a(z7__vQThciN^b}L-;vTE2x4T`;keH(Eh1y=j{EtsKeQb#;{sBitKPiUll6F^H;2;H)HKppGtRmg zpQrxF@%fwpGBHwkKHf==i4@B3_J=Fw=HA9x!?xAIZ@H`R8r;6}g^}=}Q>0D0s@vim zVB}nKEbu^;ps1+LnMSXm@=&M#M45V4xJJau9`n2nyz3%uYD2$X&gN`ct4#{oSUPxV zi`)7USl^@Cskm7vYEM*njcZMY#;N@#A-zO=SXh`EZ)LBBZV?!mw4Sz^zT2ovmJw-# z7oE7LbDn`0Ab7rW*-gm7j8Oh+ydHm&gETy%oT1cKv+?~Z_#!T#;&b5Tu^>e8Pb9En zdS2?(qm|tgjbJkg$iegbOY&r2O32NGckm*1U~lIG2g{4{tu8kH2?0DUAx5M|toF>0 z9*WP6sjlWwDQN(|YIwBOl_ zZPH7jX?QmH1vcuSvwL2TF-81r&yFUi{uHF7KqB&W45_nLbvMGY^5|(AZ zIZL=w&aNMj7--l{Dkb>K+em<+$fxF#l|?Z^{4dGUd=Pn#W&G#vV?(_8j3jo!?YhlY zj;zRW%kFBowaA8+`r{QlBu%;_^uT$mLr`;n2x$LJ*JZ6Lmn9iU`UFSpF(IXMONpOV z&(oLJQoMTW>Z219_g5Dc#uS7eM_zkqntZUNNJZ*^wyyWBR4$Arb|MuH~Y{jT-`WIt;ZU5xVi@ z+pC^qm=W|A^4qkOo_CAD&%y7<*pm#)tgk3bTyabUN@bm$%gszK+*D- z-%8%Qqi){dD`EhD2u||Xm09)YM^bo?LdGx%x3Hw&wzn}YJpGPObxXql5-%b;Is;vD z$M;xmjV%w^n4ao(;K%fPH~g6v#P7Jp^I?CV)erlGL@1cIy+({GyW(P}cpz`7fum3S zpA*#mhAZ8&`1dLe(R^PQXQnR(URIKO5e~XWW-t9UE@UK^L$A6fda3AxvlQmZH@r@X zLuH1XTGchA%RyAB~8(qR*dzh!y>#51&JIxp)4 zD|eudS9p!kOid@3`kN>X+W~$a&mYhDKQY2ECi*W!E79Ya<}OzX{^X1SryXToc&ag~weSl8rQm^xR<4wZNcO%c+AuXRykGO4#w~kOUs+M7s)Gu01k9 z@cw|zr4v%{j`cKjx@yKH9iz3}zdt}fd{-YYW6})WTox)UY+O)B%eb!{_~kPQQM}dz zypZ>yj(5kqZhX`W)5%TI6RBpFJlQiD11tTqvi8xiyMODkhQ@bbJm0}s0(fuPAv%sRP9XRQiHC80Bg@R@Y zCdEj+$KEH*I|^0e1z8JbSFh^O1!fpV%I*KJ7r=yHIO{c)d^F~aj+E$eOCX=@ftDM(?C`k5%n_5y9=y^y84lmUYSpRFmqtIPr$IhBs0C8{nkV7pm59k zz%da`8EqNr3KlkNA7lNYImZhBUX3zCExPF?`=^_(&*wJZGiAMAkcsvhW9VsBKos)q zmT3^DWe-{5_9;potl zsgS|Av}?G#!VthhAHaew&ANQl_2^_5YAB06S?{R%*|+r|5M+1{tU~S_@bmRPa7BU4 zU$c<;ro+?7>a2tyTeOsKJeRWhsThhZcc}Jvfk)n6*PjUj8BnP7jjJL;!;z{68ua`9 z0T+lIJ_Z^Loa3W`CmUocqK|~tUS1seNU~qh3YPT;(U%73d5!L*HM#UWi@F$Viq<;+ zv?A^{=ZV>TYPK^V`{vPB*JF^nL2py_!*U&U(LDK^At|kw5K)?cgK*=7Bl&r;B+=3* zYvN^My`-8`l8JgEHaOamIbAG1p4x((x;M-kU85V{jKsjCHg7sPCG! z>;1O!$=>>8WIB#0fTDJlGg~-IKTJ{ud`ZuIHd`@;*g1%0o-ctL(VHjfdtBH+xErX1 zFQ!8!B}#oVloB+AM~;z%c0P%tfNK-}mTKDtdR>}lzqUDT8R!Yr>{G0K^O|O_v?TJ^vbgdv!^DbqNHOIf!$ZpFqWc&7)PH!vi@n75Q2C zUq_%lA(bxqaAx%b{o4JxI8L1}tS)|ix!N?2KNw0qm4USpLFLvguS`A9M@B2l3wI(3 zMY%AL#TK3o{)&YK>4Fi|J4M2Fi5fK!!2QOx>Wd{41IAZ9W+9 z7!}PApv&pU+&*AvlcnT+WQaizPA)K%c>?PnLa<~nlvtu&AJkwRP+cP+^4Ak55L z8jgRKCZKblKsWHd8D9-u%bW%^?rFSv68r)(jY_Z)mI2W=WrNJq&D9VTginvQVxZcG zhzd?~?MPJ^DIp-$@iH6TvQbxEq}sFtP|9BYsI4POQgXX`2vm_jP?0Ka=7uT)QV)AU zX@J(SmYtf)v^|ajJyLu6iEvIaq&_g1D7$Yka4t?+zEMhOR%125)^qC4GMGpPn2Sn3 zBU_L-+yRa6-sG&CAWmIwQyVu6;Yt;O5P8r=D7ACbW9X8;bn`Q8ENiukyR7u^*8io6 z3*@2A2p!mi0C`rq8MZ0cPwBhH<|7kI*8z8gXFL=#^{UO{;b?$4{@VWShNEzc4_u8^ zyXd}XzDzY=vv1?3-zfJW*Q)f>3M=m7owzVT)bYOs>pZ-Zy|ZV<;9Cj8am!=EE4c`V2dsnA7&l zB2$*}OLR?AjMn;adVP4Y!&EhN#zla5Axafwvvwr@RMk(tX$p&&jYW?`uV=`8yM7Jg z&=>BVC`BhF1H5H8Sj-)3k9IN`lLn9Q;g|%ad(N+%@YI)VNrU^{Guv5#m8>`9RYy&? zyu(|akgyN6oC#>DnEfD2Hwn)Sd%~^$kQO{QbFw!qLcI>HkMW`Mj#D~unBwG&>L(Hg z_mTe!8&*^TDY}aSl~OiPHY((-@u{Qpf-NhAC@o3Z)z)Ay-DY#JzjtXXv`=v7D6ouX}f@XPht@ za~_6uPpeK?HjMKHxyxqw^SU{d`4B6FS(aTYWJ}Ism^O@FTu_Bh=bHUila=A}`B&FN zkQxx8rK)_qF0aiNC&JPujz4lCUV3ke*qHP1Qecxk6I;K2SWR8D{jE9Y+{MFR5#1LrY_$aV+CQchQc-E!#aWydH+s72>_+;YO6L=YJg&s;8#c@hDKqK_Q z-s~^^B{=#f#{OpV^UcpQK9~yfVH!3M8vcvn^EbSnFCE5|cDIZR>mueVx&YQ3^zc-N zcLn2=g=BKex27_n@Fr*`UcRydX)A79&cJ-@J+u(1;STl*3*!j|7OL%UpacV7IJjcF zqS2wd;wpS*e-w5rwZ>_)VP?Bhg`81dm7)z+1O`O=^zr-*H;TVy_`3gmcsV6zGum7K zag6I%>u<|jlHK!pw-8nQ?H zv?ur70s>lw&Cz594J^WXo{{Iqf|($CYxQlX}{Sdmy?;m*76q4g1##d7>+O1{M3dsr?D(7qK%YO@`{u1 z=8*p_AG0Fws*CcfJ8#kT@rW)INbAAp6;iD6pruTmgM`&2(Ek=XVaONgO!{1Oy-3Mb`pf&&-b@fues_sfassAq0MVxAF4y{y`DPo`A zT@qn=;NQlW$LdCo))eT`HifiFWsh~Ds6k`uer1)F_aJ}|KZ!pL`TtclpeHbOBzk5SyK#eWTNj=CPWrUb}?&VPqIitY?(6#DqmIcFmm zp9Vc%K}!g~q5j1+tZ@7ZmI<wOL z>04=!y4NS#b-Fgu{T^Og{JYaFcW~KBh7Eda!W#YO13LH&;H8u?FpaDQt(%`7qG`(N zsO+#`^Ip)GnzbbWGuspYN$UP`4kMRK(}9P)UAf6>xDH=-TscI|1S*c?7v^ti??&6S-)8bEy9kE_6t{hQO1ED zrPG`w*K-JN$lD%r*^_a9W|9~Fid-lWEK&xK_40sl;pfnRIDcWN`^)Q;>LuDaDr|lp z(!DGcw{905tm$s6-dLs?aV;=B2y;B6;I_OfGxS}m{#KLuBjo>6I0o&6TAP0m-Y!$> zk(LY7$RaPi%1g3$ZAzy{$d~a**OaD^ywLQ;1K9?#ME=+4|6gTa0uJ^1_CJed>|3_R zl8l`iWDQACV@VC7>?&j(OO}XH)265fk-efKArv7dibzPd3XK$n5ET;M`>S)#d#?BT zzrXAKpX=yy)pzFmJkPy8_w)JOk0`3d^Ntyb%mzql@@qS(y-f~(gx|^Ve$5zo)P#t6vS@vwp8Ry$tY{t;bMOEc)10Q~+wI za3(slbf$Bd(UE|N>j`4X+mg`=gg{qpGD>HcJrG*%cC#%=&Ir$tK;-soA`hF@+G)*n z$Qzng8YP`pEo*!cgT!XR@GJV!UoMPa9E_{*p?dA;? z1&znJq*3X=V0o?S4VR~O#a+TRCfMoV1x`LU+D7*>eZcmv#{I zthG4BY4)cTEq36&ptIhUg|wNWv7Jt5ft&FOY+3PwcWyz;h4E2-6NSS9o$L`+@&3St z5oF?&*;DCx5{JP?7Ym}#g{!|9%Wk_(4gOoqWo!VE=C+@x}BTnkqm%gbph1?W;c;(T(@)v(8zH~ZD z&ufCMGJ!)meDBo`!=vo^gaY*k*oRmn76fX|^FV5Nxz<7~CHK?AjjEyLvhD}3aosu5 zuPYv#P4E8L4|kI*+neJQa|kTSw|6=2F%uRXy*CR1uWT3Gl>6xxkmv-N+i0gijXJxZS>gSnCC!Cl@@GSnza@WBURsVN@CDGu zEGjl|k4Uk%Z7h1eVwoolA^8l#g;P=HQ*6EuRH*kiRTN~6k;|`XysdS}7x>&r?}|=Z48SwBa<>sN{h|xYAm3*f}bHm@bmVb`GFA6Xh&+D-%cXl`v3tcY}dZbqM*FNZ~=dA|U> zy$l|cfeplN+v#LcUBvEZEmoM#_ZBrTbS9|YOsvVDs6;9h2DeT2SCa|dyOPAeS4!Gn zkB5w_Xz18ZYGpH|_gNj?Juk@eYuSh=*Z_Lf#e&hP(>g9jSzZ-b}DtTYEWZbGQ?t z(+2KOt#Z;>9Zbl^;vICBL7a?>O?qIk&uo|Ny*&;IHGD)4UEVbOQBy#ZBT(tWOW-m{ zZkDt6oj4V}u%{5*_|s8F6D^W6_B`-y+S?h840(`g>Y>U!LIGN$SB+FyGD{6smdQ2H z?t&=%=#CQamw;9_)ay78ljaEuHwcCYv0$Hbrp5?dM+C_}2F3P_p6THjq*n^;jwx-N8qncLS zLWj%;_q_=di;=?ak>^fr7iq>=mj(APVJWQcaoLVkxS28{MQ;OqTt`TFqg z$O>$;Y4SLkiFj>=U$I+{fBTUC#rbM(1;wF~pp+znVaL4Y5SqY4z;yeAQ8tZHk!~_P zKNGne)j*uG?7vV-ed*UAanwZv9bF^-8e5pl(zctjKG=W_7(b+P-m{IdFzwW;ihmRK`MOQ6DLj^ z<~zF+5_Tp1dRgwXOJ3cl4|o2eMQWAbC*8o$@LZE@a!%>t;OdgYUD%KkuB&Y^WR1!p zRNlS2ucdD@>q9?EKA>^w=|%DAPfrAT_wD{Xs(P`M4P-wibTW?%sGz3;WNMp7_lc6X z%Mc|T*lPJE`M2|I725A@jgn2elwdjJYKbeJ80Vu+<+Et1(qs{WD$1WjQg9zz@>zkp zr@v)xj98G3qS9w{A}WC-np@1sY01`G>0`}S_&qQ!o0)r|^;(Pm2L_L-E93j*@k!sU z4d#O!OtoCobRUG++x^szTQSOv@>>$;dQRVr0_lyHDeVzYENN)}=nqsKs|b1-wrm5d z`y!vcQz%&CUr^&{eEs6=sgl%o%o-`?gnwM$xlOqcH0ulKx z(f7HO+)u<=cQAFBx?GBOypr6pcU7+WM6Y#HmJ3BCY{i7O8_`>SEZsI<+S1|A-exXYbGgCC z1|hiHVtg>IfyCV8JaVY6jKgt4m`r4+RDY_~`;Ykr`WPX_HtnhPj`!;%mm+gF{#Xc) znucJ8@A^mspt(6u%b9Spw;_hx+w=U)32RMR2B4;_-ggPELpIY3^j@ zgGu#V?xdD)S~BZ=ij4|#Iy{G+PEQ59;W`B!Qbzj5`36j@9^z30sUv;9@fyrn9R5x8 zo;&5EbMqEVS89;)=7#r!cE_@74*2*@cTZPhvurAVsbaEQJifvj&;>nJ+&D`!O9S?A z%G_DiIhA^IuDL9oM@MW1aD0`117R{-n{KyPhMyTReO|q1$ zr-NhlYo9bJ6U$FYbP3fb14!axU-I1j_)I1m^(SY4?z?1~2^6@yQq+nMbz5uBi4=~xZdxs|ATG0zL@4mqs{!KL zojR5}YnIEE%V-TujGa|Bus7H+7PbSNe%w^9H zf5g(fAmQqT#7Je<6oJW7%xg$aVffDV_0ibdI||7{MM7D zG&6c{hF`oPwWTOv2K*hKis- z_TQ*?RdOT_ghZ+Ac?{cHFW~ukH=T2rmF}Wbw=(Bz5r(20VMe2L+e~OT5)bF9vL=pB)zq?+oqO!B zoUJA(70_#kK7L|w=a#h}dXJuxsR*(rR0Lgm8T-$2Dj{OP#kZ)lS8O@9R3TW z1tpTD3N!L8+`}%Y6)uNe-Xk#r{TGQD-{ZSS=mZW#fJcAPh$^2nO^hRF%tlJk^_Zd7 z5NCSwx)ZnwHTnDbHD-9G5ll{#rai^6PX&lOD1?M*-m{A)uAGBDv=C+tw8tz$i4F&4 z1FRYWp8duUU3=1a?a2dH#*2E8IVM{_=9sWD60Nyxpn{D-*Y51+zAgGN2u=g8BJ%C- z;Ze+Ct8Ru(qq$2IGlzPOHb0ctPcIc{w_@1PtSBz=2d@xFrfUpH7HLq`%{Wa(sLv>c zCz>xyh(3t_J(y@}a{84@3JZg6dI47+!Qh;@Z@a z?R~PXG-{Wi=g*7JKyk~iC--FxT7)LR*i*gs*3N|AXi7QxxC6QD=8 z{hmlDDqryUs9$zZROZ~ImG_9zwXOiiRLI;A??Re48upljgbjjlK|vV1K58@e#l}9m z+7U)Rt>f$w+8g?sq)g$f38$&EyOHPg%a4>F}1*nBkcivU_Ircr>H`Dfv8 zh%myb@%oqiCH77wt(FlA?xc`)-(++)ozMkM;bz{oidZSJ=_qN70sT)|b0>n5e5$O! z0=_RSG3Gh8mLk4!Bt@W(d4SdOqN91aEy^CXx}iwvO}F;cQk)F!hYpmv*RRbt9s3)d zX5Z1jl_suvR~@2q(_60oM1IRnmz*qYzTz%oSFlz1&%Cw0PeUG~esegaW+A@PdF`{u zd8*R*$-QI#+T4os?KaMI+oE52F(SSe^ZOXKxp;~`40i%?2g&@%0`!yw80+!E*2?_0 zLtlp9cJC1>ENpRP+R`Cui$r!PZNj$o0^-4gjD7MCks`{j-i%g5qE(F8+lWgw{e>@MK$uQ0)&J}g{1{99yF)2h+-O=S7YD0Fgpyr7tK3|N)Bg8+WeqZivk_<#x%$DeXiMsUy#JJBU5k9`l5S^O#bpl;xlhF zcU${wUzl+vHU{OyR`l|yHj zIit*Dv+fv*$}_EZHuOBjwPAl>KQm3ctZ~Mk1AW9%gFX2OLz*(EK{_pCKVg+^k*4%U zP?!F}cjPJ*=a5nlSt5Z)@dfo%oXxClDk0xSXis8C*Wz0gAF2{-Eu?)VQr%(`v|1_eypK#MOuq7BJJV)?gbZw0Ei((H9eqP0=l&kaXim{ za8guDC3)DBQCYYQmEC03T^Qjl_4zg23#{&E4s$i=U#({^%!qUW3#A==3M~}~>H-U1 z%xT(%l_AE-X!~0f$14 z?}s@)wkg>*ez;v*HXlFW%*HLF+I(Lbc7_DuE$^TTOA0k871LD?lfg%xC!Q)Zl*t{4 z<7sU1F;V$gytvuQryn?v2rlWZn_zBrbvp2tgPscPLfBHD3iOY_F{%;6&uvBCsPa}q zl*xF^W~QgPP8yBBA=`R58Vwoali*Rr97j=o*f|eT@}+cY*{x?$4q5j#8ocK17}m3{ z>D{G;h04dje|`cGfEK`6HX{T`z;HCretL0jTdWb#RL1~ac;rplks_x@^zwH73%~&C zA9XfJB-FC|`&fiKGGjS38>P|hGSParuA)|gMv zPOAffWug=v_pFu+xNqgX*wS8 z9fMWOlUxg519yxXxt|nz5x-j+u5CZmo1u!0;dNqfV@+iB{8+pN^s8;nztkUNxKed+ zUq3z8S^pwWLN~KCZ1m89l0D+ka^e(VWiUvi-*N!m>WoJs=m}udpmU?V`SGtpf~%(R>NRUhm!>P!TOyMEZxw6e7N0^Yy`?lJ1d&H>ClR?)$rv7~`3? zx7pMjN~u7-7NP=!;Tyo)yrSFVg;jKc+jyYtFb8NQ0@Z)CseFT)b9Kef80d6Rr!@pH zoijZ6CV=#z7Aj+xbc$RiVOaFGuDibV29N8l<%*0^#y!HFoSad`SOfzcU}Eue^vDW zhN0dp(Fi(~`g%QA&kjCA+o)9lIIi?Y+}(dqOx-|-Z%j}wM{7sWNj^g|U< z!otD`Dy_o*oi@9jg3B#R8Kw#&#-)npZE`EaQc%~Q0ogczxDD}*4^Ri;R#@MSedI$t z6m|+9MV%tP*Yp?HyJIw-kQI!oRY+!8(Ty7?fgTkM!0&y|&M(gi$X1OP-t96PE3TO; zd>jj0a;?P&eMJWv@9evo{`>bYgfwnzYkT-dCVpxBZVB=*fa4TU2(tumnK6%>5tb>^ zas*lwTpAzuVFR=r^g{4{U$@LQhS#NRK#1}v1bm4wruVFJbA;iBmv7bmkbfsIbgItU zdu+wNA-{+izaKiz%<@q%ldOjfLjZ%dc6N__Wnjxc>`->dV#+3nsfb1p! z$=WYow)-{u&}SHZHV%LI7bcKtmsIC6(C{VpW0D^VGjuH-v=ji z77c8=;iI>!Z*&UvpPk9KIn*l%?M&|b!I_ZUD@WRo^<&RA5fb>|avUk+mm81&aAxio z@;o?|e7iciY%uIy5P$^HG6Q$72{_~k$eMKzN$XPFl1{aF1OY!R1Q_%|@beAmKg7t{ z!!YjcK+_ioFfcf-4lzp%LezYa@eb+d;9LBN4HuiE_2e*E!U8mM2|U#@fVUru;ezy0 z%uV2W!aKL%9YWN1#571sWY-+@cYO+Ja^w1=oD>#{ggZ~AGcNA1)%*_e>&rF6k~@+$ zfxp8GrcUI?luIvv-t}JJ^I5#`06^XF}cmy;#pT4ht&w(D;4o&R74A$tV>@Q*h-pvb|Yz)V%6r*x$YcqjEDm$S`vMWjd zA-@$hFX+jZ2t9YNHY-CfD`BKaMUQVBVZ4tkr@lwO>lRnRwm%iukL)Y=vPSwhcy?sJ z1eQfJP{O|qJep%`Hu&}VCZPPps7OzNA-F~ox=pY?j+zhX)eHU#h{fW^^&ZbS0hMjo zlz?xZ8E!mK>V^i2T0Zdj!93+hpz;PR5Wpk{0kpg?*mIQ6{(GR#q#zX zv`FLVPwi}atT!WMvK#vMbgSkM+CXZ4^njH^yzL(>q;Y=dh+rNvPJD|K8k6*_p;^ab%g{ zj5K4A;ch&#auX`PS|^bt^4Wwx(>7Q1Si{L<6O&Wzw}Efo-9G*<;!tLSm(|r2t!>*} z>Z0`k2-Y(0|EV?QW%1qn`r%V;MoW`6OH05>r+?E!cpA|5;w|$&KCt!*u*eTTHtM;{ zL(yx4OqyBQiV9OXQ1B^?2qp{5Vgpf%jND|Wp>)IwHScPH9uECkKzIY zX>+f0R<@Y?pZ3o1@7;V27^hto&2?bKND~wkLT(1u9I7p_Gzg;1<(OzcKjn8W?1 z>#n_^+T(I`5I7R~ks@@+w)w75v13)!o#z30Z&vj_7ngRRA^A9Zr9?q{pnQmc2#pTd zikjh5PQVd$su9?;y0Ux}t{&g-au@tAxCC{TvBE3Uu_|+89VL%*rlvnZCk}*;8Vwj* z6vCi(I`N|i$uOXUM>)`B5y~k-`7CNPw>vd--q~27nlB=&RJ#;#RDbrAf#^OoWHody zUnQz|%M{kww zk|cx~!2Po{)K91sS7E3Kt{ix^b3h6sLgSG$e9}XoUmnXdxOTZ3x)O5N244Y(HNlW$ zWLmPDCme%dPzJbhECu*ey#cgx@V`L|!)KdS)y9yvFuldj8Q-dqRzg4v=vf&8JSa%q zY5M*p!=GZnsbJ^V?h;q`U`a{2z9S5w2qST5c^Qz~hk+FEa{171SbqdT1-qnvXnUdy z7qEI8HjRtd|V6ZngSh-^nM9}Yf-W{ zUs^@VwO4zL|BgD%#d23h+p{bnIQWEH2EHki14{Xj8#awx0o_gl_@8@r0pk|wPqRmN zm=>4-#vBiQi+Tbkw|IAM2&fu9bG*X_;x9CqXhX;jr4Pptw;}(VqvAe9k6Tl3Z)GXH zIzj|b5#0_6IjNNnU1dU`b4TyWP<`wwolhQ(*fH>RG^LOZH2{nzK${QSKqAL=(;E3S z#Q81`PRH~y7a{4Y0;KyJ_p_2HF`za2t5m^0KK>N`1FY?leS^IpAe~k5kXsCaE=kA_ zlD8n-VMGCqmcQE|-3&w={k9`Lq7d2u=;qhz%pV2C#iK|Lo@}!RoeR;^5Bh=}b*%wj zh5@hwRGq5$sfC4wlgm%w*a206y@l?%0wB49+`sJBnBK^_5F zmuj%-;sc27W$>fDxCngF(64WgCFS(horr?gJMTaCr35oI)=p@I9|5+KGudqMGKYb?@>4*fL6h{TmpAEffb{6lBAJtySGvfvMP5%ez8BIt#2p?jH&iY{2{L%o+R+O=roMq05*$}4)L^;?}q!hBfZ%B}Z(%d4e;>G9HQm zM_!y_NK?aT;v+dg>cK}f^mu}yau$iifD+f^cx&huIxOFbk z$}ntKEgXuFWsqYJrDBcV5<`yb50=jDez2Rosg?_RX>C^Uqs{I7_d~7%+Mf@w*IMFK zR=FqFl`E23%N!>L>XW(#aex`~-?;X^FT74!)vH@`rHPk_QM2e=4X6ty`)l<{PT*V_ z$<>LBINIjSe~O5OEOlHJ=NuaMGnS2oXD}2w*s=3Cuzm+{s$C}G$eYE29Ba(9X_2h! zCe~XYQZ4^6uocEe=ACm|zs39E3e5#kH>>e)N{m&D!Fq08{ndQH)Slq`oKYR?+-5TS z*@bD)TVDw~EViea>aY(`4Mf+okD46^F-i-BeKg{Vh`sr38_FJsY3 zO{4_M`B#TtKYS8|L_+?jGrC~E$p1I~bYjh$9LGuQ?#F>}|0>m+Gi$19&qIw5R!7h% z@SkNJ=TpTb#v|Q#34>T}w%(@*JlSOIq5vrJW7B@qumGeADfcD68_e?vvfDSm(HHl^ z128V|K9ZMX<)0V-DQ>Ne!W21`S^G_i6A?7Tj^Zq+r~SKjXUpN2fNumM-iV+Z=kT1_ zKjU0sh%)vehLDmMMSa3Z|KsNnVIZ#ZJvIa><|&gl=l`p`!RsVagG1h9u8(XM{+B_( zU++UsaA4|TUs)trlAoYwGwQ8M&rf%LP$5{6}6L9vjYraL2J(~ABG6+Om|BR>DbP@ySo^IL{ zHKGpGD)Z>y$^TM8Rc7CPXif<4&MUw7S8H?fx;3AVjOZ_&f#3cf<&nNU#^JfD|EhZ0 z0TL?Zk60|Isoz4(=y?~{W*aIoqt~gh_Rdf~%BH_crPU($xh@lwPQGDiA;ArqhebgK zD8kz5-p+FBi$nk43N&SW;9mvtLd;vt|281;zXzlyU37VzFZI!%d7S@W z1_b&*J}<82vu<G z`HR@uoUin!&r@FYZ~n7>ibR~^ zr8NaB-WY~7qutA^iq2*gF$GGA?QtWFi(Iq-IF_{!{^vi*Nyjj}Y^@$(EAXJQft|5q zuEw92aJk=}DnvvAU&Gl!qFrc;`LiB7;zFpif6Pt!&tiRIc(_TFDk2D)t~mi>!i;I! z|KZ*6#a6u>pV4df1<%_?fHuedrOj%ztm?b}3`;>^)A}XE3~lMWtnKL5KZW|c=hm(# zNXA}>3u4@O4owrxJ9aU)G>E$Y=SlzluUZEba@qfw6WIE20p=mA@*l5J$saaej5s@3 zm4X?7#CT7Ng4K~p`s;Mr|Hsm^sj~fR>2C?t;UK*J{fhtaOF@f1+7O2?U3;V=?yB+D zTqrDqNruA82W<0N`;zy;g2OlNyZ&*l{7k=G19*fWTpi!f3OtkT;-z8{A`a^yyCi>E z#HE=nY=m(3AKv-*Kjq24yHkM}Wjjd^Q+stFni>qw9TN)Q+7#FN0 zRxfdn*rV9J#HM*`yPtkj!K=sY)ohC83dPE66JzXToA|t&7Gg2ZaU?5=`f7oZrF`*K z$x=Szqw^zuTRfttJukX)O{Ta%8ae%&|V_j%z7U7 za8!wxSdMAqlfy3~c05sH;2B&;o8Gq(jqnjhjSC_0JErNjcutc0Z~unG$uCXK`Qowc z5%Mn=YQ&qyFssjrM%phI4BRZ!zs+l0tmiYxa{oPjv}qeE+gx+SUA8I5QgcOMrgF>3 zT%1DJ#m(MN7e?S;+01T>uM!#;ZZi&vH%VJ**6}a+rhn_uxERZ8KzIK=yk*1{#g{Mc zn(Y4Dg877dgfCCrm3tt^Gn#bfTR!82c+)g|hPz7P<$}FMJJE&5HeU}yFM5^T XUMfpEz#n}W1^ZdJ*Y}6zS4yNHB^5f`%$0NR!^A_kapY6{%8# zUWyoo? zMFD>Eu*F#!{6p=lbK4&V6Tm<(_*cU`5AaK_08PsPQy-UrU`Ib^Sa5K#ei?`{;J!)Cjqu?Nw{*du!3-&EnB|ww<=d*EGvwy^L$WMZdkvQs2Ls82;$y zom8Z(+z?_5WjkS`vQ=i!#JZGp5I%p7E#}iz**{8`w0Bg_ea~uGTFJs8kl?>nvfgi! zS2Y*3N68-#-3=YE<(fcX%HKA+-00rpuW!G(UyxP9R%LLuLW-P}DoV|IX}6vE&q=s2 zhS6V1V!UH+3!Db&v*TDdC~3DF9|fE$RkOAUfo7% zG~Bhv?0CuL4QWlt;Af$%7v&u!n+0}F497}wU$N@hP2?dKQ4W@=%R%t&CE{0AbUS_l z?}Le2+C4s9sW@qkI~XoTaS?I_E}tds zEB8sq#ms{T@&5KVkD6aU^avv@m8&>A`G`CFVVo>UXqE^SbLsY3>#QdZ3ZNoPx zeb}1w{Ufj9Ef4gvo2~fKPnQklGrJ<+XNHPtCr3OB(oX$s?PRmD7~#X?`yLpN}D8latZ6O0O4A_ zLV9na+PZw#$I+!^R`%$rET;TO8kLtNrS%Uz#O~y4-q}Jg?8L^oNcf?TC^F>3ao#nrsLc*s-K;9q#gf`wO-(RCjwCNq&baJ_rp^IPk>+@PXd;A~Iv&XvplCuy=3 z2PlCaVOJjY+Rp0W;(40c8FBxOk#4IWhB+0ptqW`NU5CYAIrS`4+KQEyNCm7Ldjz?b zvok)(Z4N0!6~-T>42%v>>i+f`u1aaY;%n~SlDbrQS;w$}$b9~0R@VJ&(~L%;X^Z{Z z?ZaH8gVh74`Y(E?r(IcN=gyzxUT!P$yq^5XZS`0=zfR&`46 zbJ#GX#?@xo@4E)rsr|^s-u())H%u1JG19Wz0$pyy+mA=esc9*W3>j7>Ev^7KNOg^- zcVd}8GO=sA-@D{v?JT*J?D<>bAHvQ&Fm}#(UOj8}G1Oso!jO7>#)zYcL8pzoH`6}g zyyGQj54!WUt?EhDK8d64k1B^U8hhID1=!R-GT6(9f2i{=CSHgSdno;FJ8fXpkDK^? zG>9dg%2Uho{*ebZ)WYYZ!ikB4*;YqeP-*VQP6wi{GNbz>BIx6Szdh+vetplE-H9Yq z*?OP;g_^TdGEN8Tp%DqE@Wd(Q@D_PosW{*N;j2*?Ik83V?|SBaTPFcZsXhgiKkJZp zMT%OvYUvkY%nn*VLm}9(%WY^oc7yF{W)0u@kel_+VKpPpVRKt9`3+ij)xwyx^z6p_ zYb4Lb8<0*1vmbxGldTsXIV{-{N3Y4v z@y~VDD3=&gi;t4>=yRIC4ANbl;3m`usW{0In&T(3G2xgf7Om_z95W^4C@Kk$o|;qt znmk#7h)YQh<77Hbd08QqLA3*B=i~cR)Iq^9VPF8WV|uVtu(A1-5*zeap;fCd?zcq$ zq+R9n@8Nenhdloaxh|K{SYOlhgef+aRjd1$dJ;|Cfp?vO*6DOti&lbS4!!|LUA5d^`|DEi{B%DhJq~8{ z&VzDO51K=@C|NA*V!E~U+G!*Vy<8vVd|-*cOy_4^DjayyQ6^Ly(}1WcXu%C0oJ@-m zDvQ@6siJNfAlpu;Lh@hjWM8*D6g^ti;M8p9^`#?HI^0vvK7V>jx;I_v-A9A;2jr$k z4|DI8VwQpD-k{d0={_mAY8~{&>B|R}@7nIIKK{~ux5Cr@`oX)eMXx57N1hhj3lL}= z8b%LZ-a|=R{5x_>&iE1;;EuBtd41@w`Zj3{HybWUq}MP8!oMA0Klmw?jNx(dJ1;K> z-gz`2Td(f%Ux=CJ=;ke%z_f;$w=rX2BVdQ{YB zlN50QB7c^-2VAOiQ}0Sa?#IsYX7+wN9V zb>{&Gy5UaI_0upY)~wL2?CuwkcvB+U`sNvG;g>buLC6e-jE8**^C;dtUi_V4q&y7D&4@mHm_5R~HsZbwb}uZ3;WnTaHgGqIEv8 z#JlRPHVVO}du*}!2Fq#qvc|Y)2;_$W2eYS07Og10mtiL-vI+8+JKU`3&-19l6v$Ry z-k0DI&{FT8oQq5B1lJ%kzT3|?=*&&_QSTffrdERtBNS{&- zzO$xS#-^99Kl{cGbB9O@3+ts2@$Rp#9~nQlH|bbZOH+dh!PSv%9i+Q_zOLPh&iG<{ z9oGR$|2t;61G7x!UtK{t!;ncO(W~GZPB6r8TxD~-R=T93XL~MO-dBytS0!6lnY!;^ zplf2hGEvF@fRZtxg$~I(*_$FBx>~%1dL#v`%I+-6kj;A@Ib_OFC0|e-_4IO3Wql*% zuDt`ThULLkfn`s+h>W~t`K4v-v)@;{?&t(FY0Q7cvVGloeVnT|LhY5Sub3#EQBS9k z&Xn3tCn1RKQ_df3c`ltBuzPF6Xr3PW|S2Nce7f-NAw6Ta!W>xN|w-~3wpPMxvLy~;;RlIB8ux(Z!L^x;z)FC@J$8If|8 zJ!gtWfr1lhyVA(Ad(rojzLW(yk9tKPe)H z1k_-P!(RT?KlX)x;9K(|Rx@UNVDgG;+?sbzg?^Ns+;W>f=j|ry^9nZERZOPIO6?Ng zXM&4q&&>DbKI)MjjB09oV>iRAkW1eeIxWuPb5OE6n~60lJd^p{w_$b6@H&;yXsUz*RH8$Zg}2 z($cf4m4?}B5u4%j-(A%YxE2=|$tz57h?ENtjG^^I{WqGH{gz(^*N4n;*hh3Orr8f} z{}^dHIX?I?-QWW+Ndh+XJD7pDnq2!?KkdyEH0c=e{p|c}pZWIWl$2i^Et`IQ7aNG5 zuQS4liBX3SjPtZ)oVw|n1Aeghe*m`0$#JFtkSAZUZ4Ow#vKkO({D+(m)*DWGe_eY( zArrCdD`c3}Zy2?23bPAem-p^C3hIvMGF|Z#M4C|FHxE0tQttk@8x<`Z&05!oZ+`Dd zTz)K&V0nOA9WCCI)CZeUfHq_1XBWE2#-Knz*Hu1XR8!c%{{CV7JE6xfutzFGlO6) z&OiLy&{572t1HI0*?zDwwLWlpTB2JQ_(W71+MVF{r&s0(z2cQ*4x>fJGPeXoO=VyF z2fSr{FefSr-qy|q!apBZ752w{P-#uVqTK2Oq4R;_a;E<`NuRFq8MQlx|q`cLN4a*TUjCfjQ2`wWvszSoXK3omcC>wv8!%Vs(ts) zIIOx(ELy87ds_elIR5At3;mxE}8Q zlDP#+H(d-wM609j9v*uOj8QuJ`t(N!JIlFolCgmHMfFl89RK_HT*9&nhJF<^_2g5M zFsr6HiCgQv@o0dAK12kewAB_MZhI&o0FLZw21p_m88FD71k?z{keM;<@ExZ`ZiM zt3I$>OGoYp?EhIy>+6Y-OS=Nl{3;3BYwYZo0zH4!kow8qkjlZ&8raQr=}xwaf(u5r zwgW4}c@xJSUpu0r-sr#7IypY_KPvLabifvy+N*(^ROqw-Hf_R4U&75SES9#r`6Gz% zG#fu^*!38O#m4UmeLs!_>*fHwxUP8f$hi)ZIRJbt0pK{{(eS;H8GZaGyf~f^p>i(U zo$y((3N;g`gB$&NiV+R#80icT=Fj}E%OrwN5J;h<%*elACG6WXfB*KYXy13g)tGvx z-fvOYKn!#1Q*_rNmN7^_bpT* z0-m&}0a(OU+~+t4;MHY^XTz&qm32C4!80^Pd>$<}g|Cd4pXHA_lmV=BKW{nmcyHOG zVWRA|>1ONqMD?uohIzF#XQbfT&zhm7PD*~80~A1D>+ZKW=ylOfIq0WlbCB1BcB&!z z9VX7Ck{3OUKm0pW%hbCtPWF@G@YWfvsl>+)H{>HS(aPu!3%s68nkdpF#s$Lb?|c4) zG(2*YJlEL$z`(>me(4=Jt)~6e(i)}Ufed+@z%g@=`OWGcK8ie(g#K5`4RZ$o^sbTX zzk2xR=N!&B(ECyYMMFyz^X2vp5BJIRS@gA&=b^sKP`OTVp*~)5| z0s_(os_ye6Ecw%~ax#apoTnQ&4DN9!baAfY%$oT~AHga~8{iYnZ?;9On9J-sYNiJ;9n)t<2`CIzNl#hf}+ zu!hHxy8qE@o7)^})(rDv50>3f38SiL)eDUm;6uWWy5uwLonJ^&#DM=!b&B74k_7T& zM$W58^?Nt#@Uj}d86009WweF;DcNZc-kjSL(H0B6{)p~L3-*x(tJRu4+?&jWbe?o0 zwlaUoyvt)uT8--si5rru%7^U0;$1$O3VY`(RYC<7%U*>!*yMW)K>MS1;mk`LEf1s% zMJ}o7p6TjPJ*Kp@K{q3c6-Usvm{4q`eUiwx@qNEtap~%mv_G)+SJ;;&3-8Eo7Dq2s zcuFDvanqE$wMIHoBr4uh>+C$o1r>r$xzGs_Z`n#*SzL8O=;iU!nBUngU$qWz-?AA|-$ zi;V2D`+t^XhY0n8I{gmRQTnBOCNs`DVv@9IAbtP1vM!xdh3`ra=5m){6~}zTSiA>+ zl?PJ>ZIawF>r8U?^=E7pj$Twmfhu0J6G}18-?Q<>13a4)Gt{0`qfG3FI(jVZ^e<8x zRuQ7X7O-WQ9A*uph_@sxD8#uJ@pvQgS=c6H|8hODcR1-XZd5k<0O#vLuEPDWm=d`7 zOr{f9wC@{DftOYyo9sZJ*^>9KFa+M zR>qI0EXKOr930_%xAfeK`C$>l@Pjji7dXUX3e@;1E^mehlg^W%V23hw>b&veB=)d0 zw)RWN-WS{;Qu?s`$E_5ZWGXDK#`jXHMBqJ4Rwv=L0VsAHKALJt(9xT0%Y9*hCPX~C zglEPcm!JF3uv}%we7AQ-XS!nCsa}b4!1ZXYGZ2d(2Cz-IO+@YZ3Ck||Ld@0b5_DS0 zLK)()ASDgO)9HZ>5r}TUNjIpFz3%&M0y39oH{||03#BuvxTWBkM$GPEBOdPfNWFlNGp+*Gl*Qmy9k#cfPTynaK-L432e?yarsEadm8UCqP&Xpk=<7C7KPd?D`og0qB9AN0bNffcL!+S`? zGnA#M&!)V)hTp@NAj&9bGSH&PB8jWTIK+p+Q(xp~5*<&@X^1ElJlygjvm5Z&$C*i_ zGB0tda_tI9pPdvQj20t&pz6L7Y`{XHUG;`UQs5@Jw0fYMr+jPH*wx&G-oSAbK=(JwP1}dS#ATR3v zvS>ZrXTkY8UECQC$R(r6k8)N9ZrknLn-jz3f4ls$O7>6{JHCH_R=dwOl6S07ZajZ zW%Z3BlS{ikAkdt=NIJI8XJ9ydCyFVsKM#2rGTAeC?-?2%Cb;;BTp8Lhp~&kVni z4we42Fv%)e>NUZIRI-_08+U3Cu&O5adLKYkxQQEjb->?WQLns zV{^FTASMlOOyi9HKofGFdil~!w&IYRK##k)gNdv}|F*aLPSJ+6GV_v23ZnQOR2>xS zNIZDj(#e`{B`MHYky}N}<@_jD*hxeT*;GZwsm~c1iU3(^NP}aTwIYsS_~HOJK$gcf zF(4LWi!L>7p@7NRqyx8eL1$=2mU?}@f~c_YP8h!#hJV34%3g$}aZm!3U{AYkgA^l2 zI0PQEG-Rg!`bJ%Ok{_O`sI zF@>SCSn z?Q851md%%OfigtSq~)&_E4z!5XyHR(<2gvdesCkF)9sSOU&fx>E@bmX!Sz;&=d$0Vdu}N zCR9Cpgb=ku5@Z2ohhamtJJL2|w5oDRD*8UOi4o?kIC(m+7u$wcU~ZK-L`ZG`xuAB6 zpL6EY7ox*4eh#5z@5C22|5bq&AAVsCdBzl*&<%DQJ0=V` zK@Y=7> zk)%FXGzqvQU56e@MJ$fHgVPH}!||oq%5`2X00W;St90s>6szs z`<6SM?(mlTL}{ILEs2_h1J@6ufLjh{9_8XjZ{HLyvr^}1x|OQNmK&9a6yJ#sFOC0p$-#7@`8pfu;&@pd(2mxOc<;q67yiKC0qaVj4X?oIBWWcTT7M0p*JzTOs6v}m-ev|oU z`_D=6!F|0HGkG3{&eI42cDRvlJqY4Oce)Lqg^$6L1^@!vkgnwUD+Nkl}B`Ex4CVk&V`4ig`O@v2rVcdRZ>c8tkt zt%szA*=mF_t}gfT3Oo@|Xa>bIEN?GClfwn*$XEyhx2@pQ!V6pu;^(Lj8+jxz`f^25 z<*Yb3DCC%%;PSg?Na)~d6nrE?FxxxjMJZvOg=~O%k}zUG4m1agdY)Rt04bb&3A4@( z4}z$M^0y4Cc6&H0t&JAH#=)V31KIg=fMyhZSeJN@3@m3yLc6NClVAZuSm^XT?4jEk(2TF6@(}LD5F$IH-Wcy)`C{7mjb9N&0N1 zE-@ZE^;)@g%)#-E{)O}`tMuYA1S7)rRjK}3Q`$5=oS9+FzK{q|T;`n(NyKwT)MhE2 z#4KNXOy49`2+t(Nbz`K-L7OILV5qXMjALx7K5!0w@4~M!N5?nCFeGQDUAib&mkE%r zrwk;(;u}G>15-syVXlXtK^Y7Mh&0QeKNH!C2agp34f-&w0K?&=QltmnR>o(`XtT}n z#$2>=e_%r6`0M7pzI^6LlLe^rGJ2IQlj4xglG2in#{xdY))?P!6rcdeBY)0weF3Q)NJ=Bgppzf z5R?lLgjOn)cKS=tYg|9=seR5s25nDs0s#K@;qq1SRq~}eFEOhOPU>8=x#|K=`YPf5N(fgYIS=I;OM~p%b^BwJv5kP;%UjZlowAWFWOc_2zdc#)-gz17z znh;Wq1o$T(mkbfCHCf@CAXRe)%1YX7j>cqt5MVG+2;@PhUn~Pjy|H~O2cbZ9ou5X% z3-*h?SHUXscrE6efgnxeK$sT)hsX(+pMmqxGId^G^bI+q_3_LXA?)8hIJC9ckv=>h z-U)w$!1p>BIyACUId&H-$QJI9Y9Wab5v15P$Q%YiYTK#M$RV5j<_Uh_s&giJow=foJ$$I!HU=T8cDl7ZcA;^N(W$kP*Y*?AY0HwuwjJwm zS1m`GCmO4IRGJ5@TIFiygk?bFwAsw4#$ZWb6-T}=yiVmhxctkfhET~nR$~{K%RC1H z$E43i7bZEX$v_O~deR!M>)aO4j3c?(ELv9%!rN*{Wd z_|!0H-NL`V>$W$oH1jCi0>8IcR89WFXxjt70G*an;tTA<^ehS?Qd27bXFzCgL-|Rw zQmJ;;D=n<$!b^|Tcak0&rD0J(IT?`KJTGK2q`myshe#0PkpY75Z%r#&^HoDOyNVN>6i&6i#Um5*|a-K^))z8m)!eND6aJo{ZB z<8_CUupb!g$&Pj1$}1n&T>wN2$u%)6>Pbok8Gn0!0j*NPM}!L5HVI3ebS}xpp#O`f zpXf9hp}Nb=ik;bNO-e9=KZ;^f2(sNB3)wCrm_ticKIiW1X703nt26jQBc%vssAS?Aas+%_{&>QXe4O?_1N`Xsl%Ha)Yj>O(r?0sGj{>f^f-ULW?2N`MsNbCTFMZ z)nEcG;o0xc2KH_*a#kE6+qXHT$wGJ=>HoJ);qzA4$|HB}sxP zwMb9mt3kMe=D!;CZX^j#zZI2ze=f@Pr@ayBbhp{>C44w`xFr0&*>&wdJdT1+_U#dS zy^9_SVixZ(Ll{APwaml?q}Y9^2^-rhI|=8U?-vz^vH0@&-gckw=q0@7BO0`?dAwf& zasTIjK1kjdUudO^x$^l^N>f{}2_j|gHNJj`KvwJG=^WhT3T-ILNDn!#vdqO2p3&@9 z!4~C{xnlW9T6wP1^Uk_p$!kE1R*kIWo216TD$OCwu20v_(06py2dmRDl@Ee&;Y%YL z**CAgrLh|bIq5Z-lz!uPMY#WIeR@?97MxvM9+l~E1O*S5|3$EAoLLC6NCzLC8a*68 z5vSLsGoav!$pJ<-y(K4JFw%1?w-=uC0}p??K;P8FxG36Q+xJ>(VI}OCv`k214rY{u zbw`W*EG4(luA)1~WhHh?D5sv*W}bAZp?dTp)%q5s11oEPse#^?K``M2U4tpaus(Lp zIGvIc%jBLEg3Fh4UYr`As0FD-S$BX(v^m@xwF5fQ?{o-y1XBVr-2>WM{-G|uR)sRx zP{?63D7A^)(fF|;`qk7)ODY1XKgk~Q^z)Gne1*jadJKn4rj++}<0BnTD{%;6=;gU; zE7I)5upta^Kk-0%-i;+4z8zP`4V`*#0U|&TKi$M|qtwBUL`%?LF6}Q2qORCJO9#b; z17Z$zzTk~b#`Md}3TKXj+2ReCDi*!Z5#b<%BRVlUT~-S?*TKP?VG~StPI>bXDM2KK5MG4zeJZs&?3WbXxlEzx-0<)^O`vcxOeB0H9#czavjRNouLFi;AR1Riq&5j3*jtjYb$$h*weIIoNyK zNBIn8B_*+r?;d@y)lx;Sn9z6Rc?q1e(DMDxy_>CTnY=btM61~X^#x3-Gbn$O+fdrj z(ZLNU)DmloBKE!xp}Idl7x*rD-GuToc$O>KNQiFi6Vau}(=5AHX-CJWqHsDJqL(K5 zv;|wKga5(&^Gk1^g{JRy7L+jq|58EKe$L7j%$2QeXbb|?M>RsGN_WVduD_xcDmj?@ zI|n>GMoA`*N`hyyi7nV%=RORg*dZ0a&ulXltLKhU?sM=}%(Q#nVHBXOAu>e{G$Bq2 zCyEsfvjBbNEnhnWq)95otK=i}D00vAfhJ@~8GBGAdRUb(tCCvB>lkQ#`k6Oh@f&zm zJQ+dlwP6_>ScD8pvDeHxE078o<4m)k63|(`I)eS5{BP4=9}W#gWdX1y7U1vWf8ZCe zQP+QhA`g_!u7HPoTM##xZS<+mz5J~~HeZH5>RtFjk~~%wIR~9cf{$z!V-}M;` zNyvk23%x}uOmZL}B*8`UX?U^9FqBG+){$x9HZz3Cz{{1e7Pu+UraL5UP@*(TKK)Dz zR+liP#n46@UySAzCk#^I$i?n4<^21SK@VzaZUqhn(7Wnzg|-t5_2NV;OBSMVpT(Uf zVvwBn4EB}i4o%0!J&k*uvF+wyyb71!rAv}TWh;&ZwX`CXlf=ZJwB8CHOB5f#On5sV zE>`Iq?wX!iIu^?hI5^~Xh=O8EjI9QAa@BcWQO(eoah&=A%16B~5jNyyWRj&A)FN?> z%KeUrbGU{`Gq(a&IzQ+s0?m*OpTAH$&PZuT5)=#4n90x6oq?i3*zuTqvYBMiUcLVh zTg*DOzzqvX8JURcHHH-pNs%hQ2drC6fQ)!G zJcxIK@>TvDK};X$Z>%wc!pRPDB@I;lkA4OHEi&k{k5XHmMB>4~ZwN6~@EwdJQlm zBxC|9ei+r#K+qgjGlX47PoO(M+U|n@kjw!cN`j=^8C9m!H=t;#)jjna#??tZc_Hs- zLVgLn+rpwTya@EWeYJ%E0CdtAsM?XnD$9tMqxKZPE8WmkCO~x(& zPsCzRfJRKO4Izl2^x8UgA}h5LXsvY!)~ktxRiqtgxDC1!A_#6>5qXs)J+;0A;dp+u*Os-_!A0x(_?@}%L@=oG zYgn7#*xLFmm{Uj&w*4-7709^Br=Qh2lBG?y=TGg{9X8Y0>0QUa!pq|gG3HM*mHAi5 zXUHQ+izJPnUbJcplQ&d9x@1~tu=qqc%K^*?Ad)`S^-S0X+y0&QQExw(-Sk_@iRrd1 zMl2=@fRp8Y2<1aDc3N<16EIg+i*o+YA44|_P_xYGmd|?M67i&rRbXm^_-ZpEX05U# zE9&H6{*Q(+xPcXdH-v1LXIH378p^LY zOL!(}Kdge%1^yi!BRlvayIPemgl5+Rv+D8XV6$WB+lRP`_W6Jj{jVOCZIcmO1I0x} zgrlAL<>%Ubj>EZ{SE)?26Zr;e?~SZXjd5NxSzfqx2{7>VE8rJ7&=zICo7y< ze0-i-Vu7zsKpR4tN~3iyif_1 zg$=14NKDvAse&09seocIbaSeBUO>KCrJ5uEn_E$LEJN_XN?&R-y32L`TpMlC2-J=griDn6wi{3xIn`_pD%XGyxdd+9c70VBbO2^!UcY%GWbceL>4Yln*x@<+ zJABV+ASEoagJ$Er3?n>pZ`E~klSG)KvEnk9g0Dl{jy-D6`DTia$;Ro22Gst2z>C?J zBIm$aN1(?8qt2w2$X*9F?YSOwMia~}33t0AL>p{CePnj?@A>#L(52`9U=1Exnj~E& zsi&yFN93j|oA0Xov8S~uL{@0NCtefD7q*g9Hw}i1#AjQA6RWwAy%(V*QLU4B=7?%n z=#&i_=@GT>6Dh*#7WQQwn0zm4PFZx^{lrqy_IIjo(@^Etd(P75`7<>sb~rP-zfg)s z1WF9S=Y`AEPHYKZ-E!qy`qKht)C$l(;*i3K46(z zrU$%hqFu|=2%lt}WN4(*G3Ee^Cm^lWY(i_o#-BcT<~}71Sh_3~wMPramkjzr z2g4h3(Knz$l?tJwUYUarQtrcJV8QTG)1C|(B*?%sv*cwjH%V&lPkS}oAJ*i6J9ZX` zS2RY0VFAUZ5`^du#BcY%CNjmMbf`2>SWh*YVhu$|y!6S+621xFI$GUL@ik?xWoK_ot4qJAn(qm6pRs*>8rByp zVTG(#>xPDHJ}XvtK;IOMVJw4TCYP}ibJ*c-kNQ$Ht)xf*7tYKG`f`maelRZ%n_dRa z{9&c*_Hv$ek~y6!;)PgtMiM&EslJEIH%t(_O9 z0&7j1x_kyqlDW@O?xFByw`&ykSMqMRM<~Lay5kozJSuX^`qQtP{D8*wL67pHs!4#F zG~KSmDJ@+J|Ld8cw(pjD(V7C!tj*I|iTWd+Drr0Qtu7@cC1Bk>`CPF-vGfH()cz66 zzaA!n?fFZe_9i>3eK5FJy_jn461WbA=02hh*3&A#Ac|k}2uLA&W3HoP(7`g0VSxG1 zYXgw^ts4>42woZ@`=PCiU`$PPWG;LYjLdjU1$fQ3NBY}5MSK85_+Y3S3LK=SHyDTm z^H^d)Fp362Y~;6S8$yivO&3!tOXM2&F<0 zzQa)RtLs@9eVCi@zc+G1Z>(41Y2--Prqbl-fx;ZT_3P<7HC;r)sQ&L;phsh)R@x3g zYk)Vbo=So8`((b1#M9^oWeWo|;+R~%tz-PZ?~9p`-yi`A;C5?>%+}I?69lyF&G~E$ zDP{dCUtj9Q?_g5WWn-o(;`b-v2*95R7V7=&AMY%zy*@w=;S2xyJn{GRcOAg$O(0vx zh4h`4e#Mah+r`x4w<`W?TF!F%`}3=ZJuSOLQSjjIukTHVzx!&M=e7AG-^=vyytDRc z)AaK4s#*4Km<JFDtmN!m)-Zu|NLSA`Q`0soyV>Ec~m%<6@8%|g?hPR42BSo z4>pIcQS!bAGncwxT4eHWlD=h~I~y&BBxLsY-+fZ93tbNpNk~u{kUXN}V!7oJ39Nc3 z-2rv|ftQO%q&Y7ddyJRmfFdEZCCba|qO)_wdOJG?^X zp*vBBkwF;e;A_V4mf<4f$LAlnMRz_rM{Rk5+bO)v(1_J3ZZ=yjw%hyC@O$r{6rSkU zMc1xZV~ndg&M+6$kf+XW9emHaSZ3|$GMKuv*1R<3KbWbw-17eC&xpUrbHO_sv!^u_ zir17FTlKX#_dg`=>7ig~%f@+Q16wrQ<*f;d(3&i-WomUq)4 zyUS3P?ef-q)QM`6kp7)7`A8f80r?r3L< zX13ZZNJIH_9Hkx}9*!N`<^RnGV_)Xe0LmYi${E3Z%QwvbBO0XpPH$h|3waNA5J#m- z;|)Ic<3JaXm43-sg_Y6L2}W?vF@7vvp1F-5!?%^*&z^@ZleVpeqRtbg2-va4dTwdQ zF08r6aC9`u{@?XJt+QS?czRla)Pzt1ldx%g_B}y4;ma431eM<(wJKFZf8S+L{Etaj zSGOd=t7X03vuQCkU}NTmRL3C>h|1)>t>wa#vq@8p8ICJpvE=e;=%Ee*$s+AK$YTdI z$6VOjMd;-+nAxOz43~E4=L7+sjTJK^{f)_HrEm?>=RM%t)m~>+h9yx8D4wmk=*uWg zrht>qsFit$cfvAG(%&wmUUriYTpfJ^AbWq*Si5>TEBx32^-k3EjW3{#EwCVSkBsZ9 zKz1-`>k8&WD|RxnBYsoI@CI3qF+PNoQb@axiiG{K`lDW;knlp*nI=*7Z@Pbi?f?F{63=*UdEsBpRmJZ{-94C+Nzyyi}{H%L@}ZY zQHjX=>}50IUU8hpF7Z&+*Ij@-kRpo&^?a)G95z1>BW1EP8~?dJN~FpzZ$3A}YnQk% z1LUmd+fPD&vTD{UT7H9h%(vj*w;-HiV`D=I7)phm8UHmE^#msa^`ijrJJ{}6CF;X$ z@VxEv9RuSt)d0fKt6=#oZmN--mfiQy*sZOVyhM^JFwwgeqkedPq@kQnvyg-mDh9zp zl!EzN%E*Ji&b`T^61J<_Z`fDFx<+3Oe1Tgs%#h5`McDKxnqoRggW=i*39pjF2EyrW z4p>S+{f_U%KG%_NOWh9x+SWXSH*nPWX*L;TJv2>xxF{6`C)7>YmF~;{MkH=Vot#Dg znXu2c_HGi)6Q9xZ8hglLXDL7S0qFP_a;^-}c{n>eUOkp<`Pl_@(xQ8 zY~mKMs!oct(6Ba48g!eoQ2Y1VcfZk!iHSi#XxVI62G<4V#W0Ea?x3!*caAzf+`E}^ z>B(82*5}}uenVFfK3}8!)h{6V* zmlBa!i~ToNw7bqvkCxXIeG0j!O-=20E|eMBYf$NRoR8ql)^S_zUqOVqPsXx|@6K+b zdU{jN7xF0uxV7btEDdGb*V$THegyXhN;QThZJVJI!a`q%jtHFN2PnlK#FZ;i`Po!z z05Q$ z{RH`}K2i#jL=q>=-?z~-)yGNyHw!S-jN~wy2GsolI39j=C`+|zqj_yrHTc)J59ZTD zsyjS2E(5t96BRQ+L_(72=F7`Zp^iYqTv!|_gTm|8nbL~pSP*%4H(oxQN(X`X60k&@ z`w7^9L67rE8dEF-G#LXhlx#O^SvTGK!NXlz2}{Iqza~+5@R`9r&0D^n5hjV8b+vFm zh(gz(k#&RNxXYh8orI~xw4h(zTd@76PpuWD<Dq3)x@K=O(ZsV$ z>8nblC2*FIj@!cuFBPx$ThdwQX{qrU3x zEeX;*2Pp-e8&N)`ye;JsBY%5uQGJG=zdz7N9*=X0Cm4WwS)i@trEJ`VL8!!)tp6Fe zmf}sbN$^+J1O77!uNQZWe9cSMG@EFe__TQNJ3X%YKOs{a3f- zWHzIvt*ZM-h-vC}6fTU8B9+Adc3>|i9$b7-Zd}QYgMIw;X#t8AKtWI2E2^q|Q;s?x zP+kc+2Ej4_{?G?nC5(7*2jIt8>4dw*#X5fI0!3xVao89t!=hlGF9b`ZFdzTe1R){6 z-j&qh-DnWQn5p%6gRLB^$7&Fl5nX^VS-CZw)TF0_!a(?D>5Ilk(Zl?#T93S+P4P3v zzy39>RcC$4=go0_I8~nZ5Q46JHkzp$_4%W!?84Jgfuv@Z;{j#3q0!N1M>JEcBw0Jz z<>IZedwwTJJN^K( zHNB{TP|Y>dt<1%~v@3gdCVIYLlUtyGuJrpkcFVqeDoZI)4BTG%BsSzz{ehh){Aaa# z<9LyAo&snta9(m_3g2wI0_M`+fX35*%gZ$&@dHi~5HiU82)3?+L{h_el>!LOYZav# zxI6T1R5vM)V;4?~0MVmCa!CP+8MQ}D*mkY-sqzoS;+(V3n7(*rO=WAiZF#Tu^4AGJ zKdh&&e4!U8J@w7P%do}~{}fB~x4Z0f2(DE6gBFRAF|)=2)TuBvxHnf@0lAi(oZOtY zvme}U7@mj#+cBSBtC$`?SvI%cs*E~{d&0VwZ4nA?k@cs`IHfoNfmJETNp9I3AeyMQ zF9Kd}c{_tcL(8C*=(^dq<83kUv{B&CpGxeNB4*Bn(|elZGg|nTtA-B9uDVOe^$%&| z!to+gN_8hnj^y79F3q;o{Z*DU~;EG|xSbJqM79hE;q#XQjE`m2~mVe0&Dm;K* zpp6AxJA(R#nTr0FpiqAJ@FBRw^#JUw<{g9bB_+mt+%q@Qr}rL>qD+^bfk%L#E*vPL z)0z;FsNEhJJyr&TY>u{4?(h8wx4U9I(K(lR;y366^t{rTEvj#2#PcerIwh*6zJ2>y zG#u0n_rV>Z%3*m*dhYjw%A5V5i@Li#V#o!m6Hp(YsMhh|CWvB&}j_lzFXI?6gUGz+w?^WqXB* z{m)~x9J&SlH8696y0N+Y*ezqCzEUd_+@veBWFGI2&3!DK6%q|zxx*9SP9<=}wo-On z?qeV&AA;k~{t(Bp1#V7l0f_lS>K}AiOw7#Q_epj13nZ$TuzmddViCQ6{syVWNpEQaz2)WKd zS}M6-CU4QSAF=osXtufhiLnV9n_}>k4irNIJpTPM5kNfkXUNCJNj4V^`Uih?=HQDU zUi-bZ_0pyxzpVK=F{W&ddSS)*-?zK>^!E0yWYDMv&f&o@pG&+tdp30W6<6DF*jjlHNV1>UZrOCUhHin2pM0UAp`NwFS^+=4 z2oQ{gRhC$TTB~3y&9nsdXqe6 zd83__Xu0pPhhfBszxCxue#vM_U`nvl`}1-5_YLSGCm8wzAoHGW7PNhov9UCocRO>} z7-yE&%bYjCa{N1p#OS#Vc+LbusvxZsbRN~BgyMPHgz9`0PHQN4IjDz9-;~OfkZ_@P z?1b@uXuk|~+(y&EDk%#qPyUBED@CXeX=+1y29}opKbw)|8PKxDlKPSIJg1$xx-h8W zw$zv`y}a@PTfd(q{lDL9oc_KU3+%r^rES>l`>zb><&1avc)_ypti;V`24wHEI8%Em z-AL6ZOt@2q9Tnq@e}kkW?{nOtndF!(_aOBj+Y zA&E$KAvBgKS!%4=*CcCYUouFtWG_-e$y!oUMBL}w=X=ln2k!mt&N)t}gO~UF^?JUZ z`{VIMpMkU?;eCL-_;^}-$6zP$}aJW zNHE9`z>&NHdW&A%`Uy-PVolR%zD*kVRp=I0rCVT_3q--9A(=Z!Q4CetYb$^7(m}pbW$hse z<_mc#dX*1el+z*WOx}E;;TDh|0?>F&kzv|o&X>MByXyi4+!qcnbaBtN?TLJJV)@i9 zpmN}KbpTtO%&nZpA#M5-i}&c1Bi_)TbpNYCpTFp`^vH}fnZAzEzC0)r$M;X?DUx_D z9TCmE+5s46d{|gm_TZnd?~Ye%MP*J9#I{h(kILQ zoGtrhM<7jR!KGPv_?#EfNVO>bV*y1@-3jqImN&{a zPz;b7_8nvk-Ez)e=(@3r?-EYL#$Lu}V)P7WyT2gdN3Ko^w(^vGn@|*^6tPOdFBI=) zI6(We^uq0>@=Hj{nBaRA^e|-BxMhs0*-KMOA05t6OL85-6{KAd3f?&Ddg;ltz?tK{ zds`-Q2VQ4b9*mPA)*kC5^I^>>C?y@Td*5y)Xhs*3n2_yX;5 z8L5JB&WyhN8b*V<=mh8Ej^}x#tZSC>?o~`?G;KO~gDR|Q(pupbOX)vt(f!V)jH~;x zp5~3;$g^;6&L8x-5GYvl$|Qq|flG0#_y+~8=`NY_URqi^`m9YEgWla)){a+&c0^{* zcdh>tiTcaUE>7OgPPKYbXY-yv(vZ8xceU>PXc^_=2Q>{|*6S;EFUK9S{Vi8`k(IpX z?lZMn5^OjqU5e@?_2c$;Ua9@h^v&GGe*!^s_aA=)1_2SPJ7eeyTfAbZ`f!7{VZ$gCwbTr5m|624V!G{?c=B~YU|~Fa zbA6HS+2}$xLB9C#XB<$EhZcZI&IZ!hMCR&Lvl!vUQ|G=0@FN-_q79duUKH3{oUSWg zTpx5C!TDSW#)ujnwywJxU&$>Hn3wMwVQnWze^0zRs$ zb7V9Aj!%Y4{^Bu}Os#UXO?pf_H>#qT)k@v7n7L($TF1UtQD6nQgmw7b9#4%n}4RRhd`1bWUZYxIXM|MRAM5q`y=rtl6|?!O1`7C3ck;l0N?7mk2W+vu9 z=fx*S_p0K@ho58X=g&KQDlN&m2+B`es@hx(DFzIver6OJZ!*1Mc4N4xC#e%vt#~(v zi6P%L#!}#Qt((WHeVb&9VPKp!dc~iv;x{eFM=#z7*k}0Q)XuwoI;wxecq_&5-;AAA zdD%LKv8pP=&@MuHTB5RMGeebkZ1#G7MyV4cA4VgmBBCkan*n6DZhsC;=}Lu~V0#z0 zeSeaPRu4pb29T6{3Pnvj`I>qg9GPTV!_+kF=_%Da#jZ$Lk|xuzn|5y7S!26FJt@$=k!&^v z=yfOJlzI1&gS&1lT{{YEmr|F}Gf)Mvaoz(jX&%Z<^Dhs2UV2-8z6Gi|e2{U~SnpT( zwK6W7S&V;RMmYIv(p^`!<@f-DEP4Cg!9QHrmrlZ6XngOv7r9Wd6Y?o~p7(W6c35Ad`Q%eWGjosSr5=X+zw&hmV(enF z6Q<=d)A#jbEZ&l9%pX@io5`%_9-GG(D7?{7L#13r{8uQK;b)(B-+AJuLlS*@GQGaG z@%2{H7_$XjCC3-87_ajM&>e#-Het?>{81les+f3ICt@<(mC=Y&=9Pgz(?sCscs1}; z)qV}>s6kmZ5Tft%bys=Bvi;ac9~ujbiHR+-7b~%c{^1{{6<}@MmI)zE{kR4f!|`9* z-<(0RrV3t@5M8yT7-wAiy%V*${j$No`8*7%!!&ZQIJRk^z!bfAJRX78Ddik}-I zv7BV|4C~>4Nj#Mg&t$86gG)WyeBBf3tuG-3@Jh0(rGrR8Ns=+P7XEQLlPx!ZH!HxJ z?nrl{Skv}3pUeG3-ckIe7nAPh-V=^PwQqSpN?NjVs;Cl=j&Z55N~ESn>q|p% z!N2C9EGhoqIKt7acfglKiV#PyeNaUq>>-h6wG!TG zGG4C)U&QdS>xn<#(}Jtd7SC^}=)r5DD0j`u_AKzsIMMG{%KU2cl^A(p*o$ZZq`>G8U<&%8dYWZQc0W?RcUg@y^l_sj0W% zXFUD}s*g(Y{?oj3vPXh9BBu8aJ>Sb&oZb`VJ(ilakP|6lIO22H_4ieuXy)ROuT~RY zoi(=4l+Qt)6-T+r+m&)G4crYX(Fw)XE{~_e&dx;>#I?z~iI4AAuN|mp{TcH5Gl%PJ+d9-1o86ir zA%d|Kaz6iO^+jhi8_PkxQ$cFOHCJ>~3h^$ltqTqHbHfZ)XWEmsx34xc>f6vq4qv|A zVLF8`?G`X}?4!qK@fuGGOo(zOtd3GYH?FhByQLiI9M*WOA6K`?LJtOL%9^?5LGW0e zOtn%q<=PDg$uMk!B`eJvzEhFzak!OdgIZ2dB00MawG!iMUWpTpjW3$N>9xGS=Jn25 zH}@uUlIVQ}@IZ?2)geL8WO+h&nPqG4{y^>F!z_rPUrEjt9R&$}Ox?Ep_@b1h(?*G!OdxA)b9?}n^;FBTgU$EJmkVSf+>MaCj zTlTKGq~F5aoY%M7amco0>h=)saxE-$=AB4p;icl_)P;JxVps^UX2=)Lrp$#ecCswV zg{VsAhVi=fFO~HxU|$mfQD&d)$4?sTO`+^AuB$ToFFKMw&vJ6SFPzLl#z17$V!fbA z{ULd3#In|*9S!9PE%?a{N<9nvB2*W!(6=cz$MS7{_EQh2?%YGTMvi`E#2(mR!=Gz5Q{1b^83;% z7jz8zF2*YjI6-oh=O3SOb5fol zZl*hl-&{M<>oWK-WGC=Q*xrp#9(7yorNjLX{IExjiX1>hHUG0vT z+hrVOdL-PHvus|>vxyA?l##>-N;y?Laj6uYnD8qX2cQ`X`I>*7*_|w5o(^R+aL1*g zFq{XS0Ob%N?R<67cIxK8e#!2AaPhCDA;&k=S&){id7mg?@+)`m_<5fGEc~iqjHyp7dSTD@7Fu$Po`0nLkh1j@SF7iJqWv+0YU`G$Q8qm#t*D*# ze+wAHq4eG_6d}L91MRh>$xPdsg0at(GpLztnt^HFK<>@y{xs22<5xX`AHBPa%3QgT z=)A&M&9Oa#u0h_v9|;1TJ71D@$QPpCZkdM%r>4{730EJgrZvs_v5dpNzKA?$d~bMf zv6&~X=FKz-3I0p@v2+?JKZ-CN*Fw+!#PIv%ZJu&NQccGG>Sw|Cp-=(u$7bK1tv$t0 zWWZFcKO=@>Dt;LMP|KgXKpI0Mjbql&dh7mbJbu4~q$>zbR}EaVrz3~dZ_>YZzhj?s z2U@suH{S-))y#l%HFM;>jh<7F?2bK)$jVrm7SjB zc*J8k`Vx3u@sd2GzIPf~=xGO%@ZhRIF&C=yX^On_0nN^^Aw0u5!`GqlnQM zfT2UH?V<(UG|o!TS@o}~VcCa@Aj@Ok0J$jX4~GN29D4)L1jnWn;;-J3ho&cIT?!-WT}(N5=h?2cv+{gg zI?2b?;EjFBc2Q?9!+CA{(9QlyGA2d>k7JuKJe&lQ%@NIJDW|%fcWliSVSV(&I!MiWWGxMkj;9bTMTJ6KfAvz##9TaVq;D1=v=)uKQ<%rh%g?hilcncBPH+v zNj_ERc*1F;8yF{8Hbs%su84+&n}qw%S%r>^bg$XlpR=`GU`$NyNaYb{I;pB2we01m z8@&ojf8=w0lv;l?hxLMN*V`aGK|vSKou{tJ^0gj%qMkt+9cOnJu3bSv1SEhgR}Wz{ zk{#4gFNSKVw5XJ=UK=K#^7>=Y@JO=X(8WW7n*0oO4)udW17uAceq?!tekXoSzn`1# z`?xPhg}A~IqHB_|GSYR79~Ua+w1G5#Zs_mxU#^+VG*k86=iSh73}+Y63fVlpP}mhm zl!{1-6}-^dJtEYg$SgToVk0PrvBL1(4_^T(c=Jqijm=aKD>#|jd|2}I?jL>l#hO-q z7K<*nLg~@(;klx~0;{~h+z~-_=dNH0uI72M)5oDIr-l~PE?aSZXeDbP%><BqYDoz0;JAT9u3{W-Meu?48(m&j8GTYz z7tofdHiPwAxHhBZr_v`yX>w@Z8rFf%$KkDYy5vN&XI+jfkqT+3Zt?JS>s`7mGn-Pt9=gt~FDWprU*xc$SW4A946(3?El`p3}W&%U}M z6)8#raoT~nmtEBb*lXuAaxgE)(YrdEL}f*$aVY2b+Z+r8O{Gm^rGh9X5GG2lcJ;2c zP@2x+25f3F`Ia|!t5NhV%|i#s?GYY+r3yLenyt;v&6tXFzUO(ARP=(g^E=TdBQRJ( zj{DIg|CYGF=g!&kbdJzH?+KSY)=67-{s=gH8r7O!;*M-|cv8(}p&7p!-kF0lbR7c1 z$Sak8i_O=;w2a2F!F?OE{F}zz^?By+Sbv%65;sJPEbF3sVLDn-Oqz zKlPk^>Xh6gO!(p$@B5_Ba^GDD$Q88qJw}B)i>}&6Zckf>Gs->h9l9m(G=a zqcIYW^W*vNvACvyUbkZ1g!Q8#&|^S)8T#J{#TtQa*j1`CWedq!bf2&TQkc84*_f|G zY@#yZ|K3Jyy(_Eh6>_*a6VwsYtV49+cvM&$e6RXm3a)tc3fU0Pov*$@yNll}oPR(z z-uJEm1^D8kRXuioT;bxc*o+0P_7tU@o9=jQI4SuW*$-0uk?iG#wL$T67hlD&2xyIL zd95y_mRw%hk(yc9Ujcz8NcT@x=V5nQHhsS>2)(Lkw5Q`fa3Y9`n?1E1#Wn>8u{%7U{DNxPH6`q4Qf@Ugw&MWHWm(?PkPbC-?WLm@^@qs+$P}ejB@}(^ zNRkS%^wOqU9|-6Yl+ByqBlW)t3II)E~cXo34v@(3R$srzVp{OyRdC3XiJ}! zHaVpn!F|{9v(cOJ+qa_b7k&rd!X9T4gTv!{MOw5?P+XM9_Bj9?_n(R&7?;o)Q<||H zI(>v_Mpi4G@*v{7s__q(-F2N+)fobL!5R^^M>wzywkNGzujtQYBTjxPGopC8B$eE1 zT3+VUOxM%TSkEMuPE~~Wh*Y17?se}!Idgo50}-`v-Lga;AceRCvlt5Ye;4@r;CIUy z`ftzX@W$|FA8bZd-t8=4mR~ExUN~|~TH&md@N-HU0)IkN6Bw|H<>US;M-q0Cx4c{q z%&U~4l*MDX(>hEP%$zv-JHdM#3$UmgNTi)d9b@IPHn3$h!`L3U_2<1mT^sF=`0P`V z@^JIp;JQHh% zt4%>soHw*-bGnd+YhtASD7tgsn=0S`Ic_+X0Cf?{=s_S!|vh=f=Mea$sT3bXCEQKR}Dg}j7Xy{?52sGs!?dx6DzSgns&pk21 z`faEjngHWR%y<&-GoCeiFBvbf>SJr(1)Rk%SUEcz*)*vAC{w$=E^1}*wZ|5%z?xT) zH3N!KIkz3Ljvq&^m3k?bnm#?s_b;|O@D|wg(&8;nc&ZZeEV-xV9$8)~cM+&I%GJo@ zWa7!q8&6XJw$FgLB+ce@gAY4~N3d^cM3ZqU<1VoCN^;B-BKhh#{9_50223FPCe=jF zP|&bYa)(K*<~|9&og8kOeJe!Qp|Z01_a~w zOJ^cLap~-U^I~v(vt^Pb_2{7xmc6rvF8IWokOInP@`wFRlG%TuGKRQB2i9sv@wnLs zq-t>j`#gG^avCmo(YzMOxo2m1J+XpLawzvi-<9mDEE7MY$Df^osKiurAK|X}BLT!@ zZAhS6L?p5J_z6anl#Vb=$~vdx*z$GqI#OYLTA~2Ef|=2A5Sw`bCngwOYcOMPF(~$z zv--N$hm@<%5ZxgS_+zVII@{c%3pv4I;>$8cyxc#aXO4-0jw$`OiE+F(aG(4dF1*Vf zOA8%|tLE(U9RHk`huX~7@0>AwDuXAiAdoipdn`%elulOYxl?O_X0C!pCqdoh%K40e zC?-p^QG7%x(?o+&0}c8pS_o@>2tDNCn6jnYX50X1tb_X;Nd^=xdRA0dD25~=x??-g z#`yHtk7pMX5I@~5^N7vDciN9sf6;>?)@Dpf^BnyA?bgSyP<0p2b9l(NSKvzhk-(%h z`edLmvv1shSX5ymSIjMOk||Gu3`!KSe02_Q0S$j^8C3%Q zQI>2t1zD)$l^LSP%{PRk9Q@F7u|u5lV#Bd+w*~7}YhG(p9e^{^e7mJ9f`5R-+62m< z4wh%=^cPq=hMVusj79a9@~Ji5misd~2pznnv0zYRf2e3E7AU^J({!?1RjIa=WE*aT z6x{d{bH4%{_5EjwUt|<0>KLvfibu2^Xx|gH7~qbwjwNya<2jOHSaQH>D2v0C<#<3=RBrukUA{VeN}R@7n@M-c3BhgVI?nhZi)C941YaT(F>-DNA)6ZXg-Y5YHQQ5Gj4q-UXTIGp ze7`TS&Rf!OlC^Z-InvOaphC^#w{-U~2T*-Pdu}ohTe(r4IyCAbe5wS6xlZmx1$q&%*=h`$r2-IXEIpZW6&Yku1%*(`}e3g*Ro7Gp0S9H#UT8*hFGY z_pn!5}4fIZY^<;?s$<2w2FNX55l?k&a9xrfcwx9;?=4n8g9FIU|uNeDso-wCs2ZssI^#=iv@-x%4J ziu_7BPqkmP7fkroo3Xo~Rs`5PhW8h#xInD5IS6ZfZ;lu>^M3P2$5LvGuTR3PHXKGH532 z=4|jLXenLLovw~2oIqH<)BX%ECg+39+Kk->PQD|Es!i5gAs39WChXE4j!rV1t-?eX zZbuzOwuJNXJ9QXa;=e1}F}T6yB=Jie!*KlUfY@I%5E-!~NDTew-y~mN>G-rWWVy$F zoKcLcGaHl07mpFJt~bb|!4uZ~X>$Z?2DEitvvWzVr~DyoZ3HO}MF|B41f`qν<6 z>NYcEW1Wvvo!XL%k|QM1CCG-;eyX<8t9uR282m&$ITbT@81L598eWglZXU#yYqg0F z$1a1k=*#g;#KEq_RF0zEK}8}K2nRg0e447!cS?1H*_rkKv?IImwD;}So#SlP2f<}H zRVtsm7zzd>`1bl8aL^7S;&oI!;!}j2&#wn#`SxjKZKuU_NQ(&TAG-+JRoa+kMh(0= zbMioxS>csujA3;T$HgP!4Q4n&!2Zc`2S^ZaRxCjn*sHw1mQ=)-Eiby+NEx!D$l4{w zAWZVQmZSHhkdvx^KU2Bg!&SJU1gZFsL+o|HDr?}NRLn;;uigyXLTo@{M@T_)F|(#Q z{jr5xe<0$q=0{e1dN_h}%L^-~?(ce9LW50bfvWcb=IdR!LL$;aaV;D@P6Zubgumc!c%tIaz1|lE$4RF@zbE5uF$3FvWSE759VVD`BtV*eGA715~JmiViik)qt>I-6+ zNR)H)M>}57fTRb%(dPL{T40b+x6g_{m_|NJ9$X1r?rR#5jYzr}7(NzKKn~iH6YBLU zYlE0!e=LBsuG`$=_j6>KTXJHvJ|Dq*m;&z^QpKg-;J*N7h!{rJdDj$lWwSB#vZhJZ zh=G94f>48`Qm(7MLl`q71b+i!;M~jZ8_W|zx0~BLidIgS@+VwNhM<3}z=Nj62m*~9 za`(HYE%wC=6o1fWnk5$0g%B)aTG-pYOLY{4CQ5x5*+C&sA@7K13W}VMDQx6{5ZF4$ z9?V)jWy0Z_fwW7{h|Fu(9Xe^wJYX_K-{6?%n3C|{$q$*(6wvDPx&x+C6;U{)@}8(j z%=(~`2;a9ebQlN(_tVCAt4+s=1qtO#&9@{=hDrHuBaT8ABeKcebZ&1Wgx}YSV+UYX#_jR@uP(#X+ebd-& z`x{-rS3W)Pq{k`|g-|pf7!UF*4EcxX`De|HR+0x#0Uyd7g$jiUJD26;p+Jq&iv}}$ zk8&hy1o;W}C3ECkYC~$*G>kwxUat&oN{|3#wZDdxAm0@2=EIlkDUw3h#cI+oq`!e) zGlj5|>Ds>*z}0?-lQXCxU>M_OSzd>{tK?iWTHZL&88Rc;(P3n-=eR97bC>RUZ)BCQ zRnFTEaEDGHXFReTn6m8k>l7p{>l4-o(7OhkdD8fbH-b@Tr_Z5jDF;Q*_p@8cGE4t* zpQLEGu>8Z9*zl4={_O0M?My3QLz9A$Rp&NTVG8!g=U9%Hkx!6CAr3fLl%TIP&_lUv zpSnUS@>_yr@=d-?;!s_Vt2q4-#VVEZz^~AYfwIw(KONKaAsmxN{0F@I#iNdMuCKjn zF3a&!bff+~wa>s+-1mSO`!tw{3*-en7nSJ&uDXM%?Ka62iKnVf1b1p!ciG!Xj2}%P z!-hEFMM`HC`1o1MZMxOQoofr!p+_$Q!4lR_hh_Do(=Lf>`8HFd}E#`dVQ zNOVCbeygIO|DQ%SLtLyw8Cq3C;6m?ED9C{PKHGV5=cSW2|9qpm@NTh0il{eVyPfN$ z(t*Z7Wy+7E3OaE_L`2SZudlE3Xavf+#z{i*Yv%;+Wy)f1fxec0@IsQY>Z#<&*n_R>E)2;MMY8F0ync@qKj7-=Q*$C(vr)C zQl(Q8l9Eo)T#q1ppjG_`^uL)}*17Gzed44K!#28+&T`-YF)lfiO%nIFnK5%X=5WA^ z;{4yBHwQ$#sH=T5WasKyZ>U)d1-*2NWTK$^;C;>GS7#V0z6Vq_-O|S?|P+-Oh2z*uFyMG|Z?u zJ3AX?tET`A*ar%ZYtWF5q{lCno4W zR_W0zAb*cf#peR6R84FG8BhK)rkU#+dW0B6wv1Y%m{D@57fw)(IJVj#>_L4dN8>>x zv-%^qn*hD=PoQgsAu$MO)|<~909(M;vG{I%um1XVdCQa)e)66|+)?3E2N>B;w3;f2 z8{B^)5zVFeFq(tXGX8eXj9KO~wfgYm;g@$0U5(Zg;p8Y6vpVC$dr?WS^d$?U<Ygg zd-q^{P+J>+)MxIMQI@KFp?=~3fC3(T{P-~hdg{-&g@tPEtabiWLcRe|exJWfB?9ZA z>gwu81FmjqXY$}LXhrw!9`5lZ?Hnzp@o6*q+14F08n$-UXL4pvo0PTAFKW*o|@bFtA(MJJK`#+c_Mn!r}t< zfOm?q6Gf25m*E)CKfQzx_&RYPV7d!%ir@Q2^1OI?d5?O2Y(Zwrz(^HaH@C3OvQN+> zW$5ec`+IdN6VA{DK!I1TTv_THkukWkv@?P8HUE1-{aWgPfLz0Y54*_#!N;|g;dk=| zw^}-!Dagb;Nlnz6xxdJkKZ$`Pg;#6P+5*Z$s5Yqd$*c`T^5kUZy@FOK5NaX5h5Y@8&}^X#e?7ysy)r5wq? zmEg8ApC6fr-VVv@xdMu=rWqL-392_gGyy1hadfq&vl)n z4+R?yV}ekd-c*IC_{WCc}YxtRy~uclK}CPGM)uhwQnO$+^KZc$q^a z4ZEZrd+k(O6lFB*>>|T9Ny^~L-f*|iB55E4H=4v)_A9>a{n@bbO1mqbp4y*wzP+z^ z0V;veT?25D4$k-vZcMuxqFXt@kz8MJ&W#pjrSEM+1G`NyEe+y0$hFll2~Ah5Jh z<-vQK$zy)Op+BqC`zgtTg&<8sX?4Y8SC{G~sEcmqjIeA`@>>m5{9@{k<4FSjnM^r+ z&OUVGHd8u`sY?0vi!a2hysWHjM1*4c2gtcgcLG121hfsJ-@bi&SM8{iDs`T)yQ8F{ zvH;_g_8RNm41S(8m{#yj(#ZI{DKU)OQdesg+v=V^c;Ug^QuLfXOd3X|h+wHve>?T@ zm4-(pdZxu1ci#UHjA{Mk(ttp%=aPzZL=a3dlKTfvj^%qn@9R7a4GsJLyre1|s?W&G zd=|X1tn+eZdD-W+()b*LvH`rrmanIRW=|DiQpeAYSvmSxWp>t%XwN(RbNo-wsUCY4 zno4?sZ_+2;*SRXoI{4;6x5<XG{(BXbN6=uWKw0!v;`XxV z`VsW{_U~^z%H=+->CiHm0pPm@&=``}n?BZg*wlHvx6B!xm`DcJnlcRSJv4ei z;)d?+b9Vk`gM)*g0t7~5b0%^+l>PuJuK;c<**pK?!|wr`SKQnT%*{s~?d|KY*==n; z?UPw~rcM-gcXR8V>q&a%_U7@g-@oU9@fao$IF&E-D@#i~V5E8%V0zja&a(emOzyDe?+K@%A6?wfpjNGO z;eC_8JHqUw0nA@toV{hNiFPBxV7CuIy6z$O{ znnxg@Ma9J;FubS-6i^!I;bSnC^Nfv+4bZx@f`FziXhFZ~1I-%&oMpGdS73JAE1+&s z@SX1mik4a0g{vQ1)DRjBU^MM@dgd}X0Ep=~_ReS&H4ZK(O$`7CE`5F({|?aRf>rTY zc_HZh>;w3(?XBKCY*SOyQ?J<$AhnPy$quka2D~BUIj}uT%gepc6rKTS)5DsYnqS<( z`AJVrO+8}CggOuo`VM<9CwRFO28C&Owr*G=uy4PBfXm-MwgN!z62jmJ{+ck*c=WH$ zc9z*4VRRMC4e{C9gdXDouqQ0X>g(#1!JWMUa|4HgIiN7!V%IfMs-igS^R-6Y0k)eV3pYhrJJ=GjV`u z1YLfek>We3SwAU4iJ}&SeV~V1{I9CAQU+K@dVzNAxytjKd%I(Aw$#uli^R%bDAe&b zj#xD6jVSyueCYMzN#JuJ0bUh8;(Q=L@G(`5Fhl;$^8$&5&+%D0Jqq}j*ue%81fN^N z|9^r1`+IPZn@Jcp_aR>QgvI~+6B#?(3@2)$wS~LOEn6ADL&jVM<@!d30;SqMW&z*J z8+watY?(ZTq8y0B$)0PbL(Obn25ST#Q`J3|n9kPE&T{jAkpJUPozbm4=@9Y10P4$l AF#rGn diff --git a/openmdao/docs/openmdao_book/theory_manual/images/Par4.png b/openmdao/docs/openmdao_book/theory_manual/images/Par4.png deleted file mode 100644 index 610a6bc0f45485146197f3441a8598bef99a3941..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 99969 zcmeFY_dnHd{6Bu2V;p;Buk5n3x5x;EkUf(vdykM=R@r-G?-N2Y4vJR@i4bulWS^3e z`MI8Yzi;1v;QQ0(_W9x5%JFnPAM^gWUyti?J=N3EASGfXg27;5ZiI62WzT&!%Vim;{I6{2i^Tz<&$K|9%#a$KGo#-y$t1Ss(g2^!MynSKAxHHy`f$ z>?$1cG_EasvAsE*2mep|A-Nv5c5?Rkrh*zL1!AvhUP;<7oJ8KdEcfw?U-`Ur>oOs= zQ`gTA+Z3WAyCT_elwb$A$Gj#fT$-s|ntYu1HR;CHtnnQDJukd5)HagXtDbmRBgzeZ zXBndQ&Zv{DPse`0$uUWNSyQE?IHLq)I4)o3+p@6|tXUmG&=RSWoS_++vvzMRGAZDx z3+S8-!Q_{ljcEqYOsv_tWSYIhFpp#}WP5N!8~%HJ`3Gg9`tD-cz`L`Bt2ltE4x5o%+(!gx2 zhbLWMPnSfbFsu#7ns>xKo)Ta=sgCzEI1QQXU9XC<|eh(ks$__#JMLT zbHWO(I7gwqwfu=?4o2rZ%R!5sz`OFF4_fi2!G=E@ z=D#09_nrphOT$cFbXX#`iY#R8E7`D{iF1RWADf&T<#|<7IJ9U%=HPm*KRS$-+rzgF z+UTvGlWP&&$NyR&R&l7+mXigT6j9tHh& z_|%zF9`Mjitima7AZ0dDXI8(AB7#KMvMBFYb_qHXjgO9)3^O2X91Q&$fentsc7)}k zNV@s`$D{}I&@3#8U1ZWJDBN&q6wA_L)}2(5{%RIWZX~?}TPRtMz1j$0 z{IC?Xpqr$-+utV0p6(XXeM^j04Bj3~amU}VUjXAOGx>bi9uo&^^wpcTz-ScYb57f1 ztU02i%@M9EdghMamiUGKcL$ElW$6QOMPg9(H(MKry)i}6B2Ao19|A`LRYXGh1Q!Qe zv#$Hj=U4C#qf@xVX|$y1j;(l?oB#R@_~`s+d-|GTIlSlD*!4?S1)~Z(NaO{5s-0Ks zUL_Hptd>W;vPZu~yP`E1mL*~CFrA-$K0B&TLQP)0`&+6=;m=~9bCJ#0tr4^j6X$Y8 zn%#SZCdXMaqA51aRHzw|OA~LSXKS<#1{F23uLmo$7VZ(LT$3X{3Su3dzK;-V2v7Gj zL^=j=kG#Y)xqbAwOGjJJl8_l+UOn3{r#75|@slAnd`wrL?Xy1~Pvy>T@&&ti#fElZ z!8WNy*R!>+0?WZdVoG9;`tF&2>6fpkhKKy$C%J|>T9&8JAH}k zuOdPmN8EhvY{JJDib~UlWTEiP}>#&&T5*-cpHpBjM@eVqvY4 zJvHmr6)K&PiT0%Q=GC%u+R1EN zH)5mpLMUwi9%xs;+C)(M)GZN2&nD?o_4S`euVA8(&8nLQ%aZI{#Bca&^!HB-vAK`q z#xp{O_zKC*2!1!Ni75Kq39;?|7docvL!wyWPfe2&_&Q_l9wN3E??;)lejRDDB!(1y z(e_UnTAD9nZ$lk5GOV#o_qK-$AlQ_%KW3dD`-`7ElIwFD|KrmtwJZ!GM@)3!%!6Rr zdn&^ov*wU!sT26^VcPJW5A4xh{B|lc1%m0r7Wk|Pa!SV2mNitOm#{s<=TD+ooapBg zd*`p^E(~Q2HIGgw)=nRgrOe9Oou0ZY_WTp_yEiF+kNV~rZ59RNPv84^wvr}yOjz^l zqr4nUf)4rQbv0tgn>4=PVo4mDB3^7 zixguM6V8K|Mus@E&=+RKt;M3+?2q$jTPgk{;m`Wq^rKsWCR#z|^YUDtZJuD=DYI=* z*m3uV=xLPz(wFHGYV7^u3(6`Dd@BkI{mN2~vN_vgIy`&yx5%CEQgPyta!U%V$}~ z@y_PvN6W=RSF^SH+~x=5CprFQTD&(I{|VKB&em;C+x0*1SrTWTjc=`LN8j|mTHhm- z_t1h(ea+ThC`n!PKE(mqd?6~`|0;^H-m#!Q?bt*c`E=t$4`uX9A8c&ph*{XdLU>vM zxfvN2>1-J|a??h)&()EoZ~D>AfwLTQ5|ZJi(6JvPtJ{g2QQq_yei+HVg)EpO0^qE~ zYJaL^+ewRD&FfYrkKdHJm$+NTT9)5Gba2p>5SbE57ePSSj8Gc+ebk7^f|>g+$q@J} za&3xk9VFtP-*&3|oWV#$Y2+ZZ{Eg_yBh%|q)MQJ^HZwn-2RzGRen&T({Vkm>`OrDK zhj1}LPFBX4)b?OS_(ONkL-RMC4&JjpH}*mnOhu%b+fzh*yzojEwBy z4^$iuI;O~-+{*RZy4&}A>(8O^7KpYx{)l>$Y%ZNEDDEiOU;8~ zA9Az4A(apFa-MvayF=fe51KtIc}j8ZF`4vpY)5-7f^vQtHyhG%i>-7?P^UOB-bzp^BgMoXQ05*meK4rNj?-gEuR5b|(f|C4X$ z*uxv$)&*}OC~jWlV0TaX`rIaZ)_%#Xj#C-AD<2(Bz8DdtTmA8h?9wHZw+y)~Z^9)% zwx93n3>d*V!P|^G^!m+O7Id5R3XAQ@dtdx$oqnK;m9)iw@O9To>78zm`|3o{t^7xO zdd7+tT0hUbo?y++nQUX7y5+OI!YY{uc2m8=%vd*{Uz$`w)q{jzEHCuWM_E4Uh(7V; zbuCgQ0rXy*qoRnIpF)<-roxAqmaUR@cH5NkI~groA7*C{`CkSuHd~w`r1MelZ{nV^ z+4f9^wnt@h@5xbh`_H;5504CSqZNW`l>eFRDll27@bd8vOCpojCPP3>jcps*Ko}vt zaWUgpANPU%Uq2SEcjbP;aWgI;xpUi+Y-=FMXet!*GI6e_+;4E322D`CS&K0D_iV7H z>Zs02-HePPWVcmy+WMmuB)uCFt$v^4ab|v=y@l##xb{VHIm5qj(Vso$v*j_q^Rv5? zulpvlXktw!Qp88-GZHg9c}B`$*Bq0#Te&m#n);|sbaYlVyLa{rYqUt!BhsQV@6mzD zvrBrq%sl46Kuwv8DjO6mE;siN=~gk3R7g3mHn3YGnw!EmUc5y)b{p^cO=9zQ;jNIN*=vKu*K-Z;@7I12mvCfrZ<;(Uj2 z(lJp^EN*RkgeRIpJZ!XQJeW9gDngUtP`E;Rj`8-T6=^IK`?Mm4#YU+Sjg$`Ycj2rA zxdwx~oFnkz025Q?%?Kfu2up&~wDFHJVW(8ZLd*W_{aVN*hn+_Ta}WB?qo;ACDjLEk z9qzn|%b-x{|FdjWzP~jjOyPj2*tOO=l09W?WSdeoxoik znE79$sQP+JM@-AICe8%e)jw0sv^I>cJM?y}gh~k>M7(FO}s9Cxg|DKKu90GOzcL%TKPJPeri7`| zYs!cLQYh%N+2oZaJfGy;zuN)ROf{5_%}C{{F82TG%=^VM;9J@7bHjm!utUoVoq}9n z2JkB+2|u@{zVg}8pwemtg6wt&iKhM5&f8tdx^%AUtg744Dwj&Y`?db(f82d2{eR!- zYl?u!F26RUf)={`Is^Crd$jleJ^FvW1*){5ASnyY6LTa>Q9762?YU6g-`^);mVWLx zZJG6`?OW#l7jxnD-(^;U$#x5&dml5}O0PXQ!Y&aSf#A`4YFo~wxp9L>w{XO{vS7ex zOd-o}giLM=ZR%L?KAw{EnMqDr3x7lV6KWl4+m&s_uw(46K{j3!P(Ap#g%E(>D)hzB zS^XDDz z&u8O>1^>B6Z-9tF`%}g_O5ut=t0PN8{EZKzNF`x5e}AtqhyD3fpZQXYuhaVccb0$& zD&p(cuVvcc{oF&4cQd*YguAn)lX2)}ytx9t`O&@I?Uw70CFOj^@x{zlkWBXHRYkaxqOe}_mu%yMPkTp83FJwAc_9QlVRbW ze7j_sSUh4{)F5jJYVq;OtG}yx%>#_C*ipG%?0=joB0;|&ewTKbLD^dPe3YnrHNdEu zCA4pHIPXI)wHPV}t1NmfyIJ36)r1Gax5);weQjg4!K;b;8R zygCopC(5^zO8wL<7+VE#4`&Grd&lVSdj0;uySEUsd++}J_iA*Os`mEw697ib zM3+VY+!`Jzo?>qud<(E%9mzqUgE#93vv1hVfN*6Hv4ItZ{w5?>_(K7`V4*W8AAtOw z9MPD~#-765+}!Betl-RQgG%r1*VAy%K0;p9ds=fY3n*!LFN?xf5`&n7z22K)p|_sk znTek83nM%K8C!zKNBwL~z?{GOh=oNSeHGV{+ZI0u^?~PJ(DNZjP3AW9r zl1k1*Ok0H{m1}2-+6|B#EQVjGWL@7U67D$_g=J)9F#C+$FdY;$Z*^PQUm4D71_9C7 z8MYKd*EHw;mZsr0SbpL%XiZNrQRGHe)~@xhy@N+&H|zRc8@uq)ok4p<`dO2Cg4W&n zQVW5cm9M;*+ZfHQt*x_Pg1|GqEO+*g&=ENNjlx!)(v^b(7VVxZ`)=9;+s8k|NLC3`{fr~ z`00&{zhmJ>3s&K0`^08NlX>}%kJn|AAXuGy)W!n4fB*jcj(TM`C>pRcZ?tR`v_kV38^z`cAh1#1|Mb^jfHbjzzNW0mCF59XCjm^7W^$rj zoA{hL{B)TuMM%lIQ(J4y8h*0f7POk< zXoi;&ML@pV@>ypYU{OWQP|uu}{ZF_J36{_^uz>|0pH40te_YU3_DQBpV z?3>4L=YAsRV&gplde??77Wkm&-C;-J$%7@bHFY??pl#$edQrCUUHK~h=+)Dbhde0T zmo$FGj5{fPY<^$xlarPr)dOEs4oLK2%Qn-1P?x^Fe@_j`YmKjibNu>ExWy;_)XBL? zoLa;*oB+0oqYEeCH8F^*SrWh`|JkOOj9;pwUvP>rCH0)=IC6W!t(BTC4Y9V#`=t?% z$Pk0aJbnUQ>lP$;@G7|UvdR|h9X6JyBjh3}(QX+0zOl-_fWwWK$nT8O@=w~nz6o)~ z^l1e)vhH|9bK24CZ836dYUM&zeQ4^o3Pq3-%4@?U-~;YS3`*mKOaJ>zsIIbo@W|Q^)I|S!{^fMioT=>BeKz^S}BxI zVE>(m$M@jq(v)7)?6M+}SIIWbY0XEGzK@JP<1d1DTb@qH8j%WX*uFV_rV~pB>!$1d zmL$@M!y^%MwRu=@l3SCH4uZhzdF<%f=&P_1hdCNK= zm5E6W&`dRoxinh~8;We4)=CdX$}0*Asl$p|udH&UK# z%QAmLHp``buk@NYzS)$I8&&W6jm`-exfI%q*gUrg#Fsw=cvR@*=&r?`Vtap`hLD%{ zVU@6byejQfS$%SO?E{qPwvmlF#u6i>!1!NG)?znv4kOzEZY#Z!BuT17P*Va@SR7k+S`PHzLOmY zm=bhIj1Wp}3On&Yx{L$2IwMHhUdd*W{@~G*1SU)}y?b3u$V(CG`>AEAm-OenpyT1* z>&Y`8!h&A0`!HTg6>Uu|6&5V9jX|HlaI!dMW{eV<1(7@{GtRvEe{i20u$@QIi|;+v z6~~zRZ7}i3WT)s9rX>eVjY~FV%&HRr%PU!t+Nov9TQ`w>i-^ZltKIbN)Pcn_YMT17 zIn^oFn|*c-`0DUo#32P^^opIjU7KP5E@8n4vHd>%z&orSXXiMwQmU7Z=qNOm%P;5B zLkPz9DC zbr0Xu4@mWElVYoqf&gLrM->$0-~rU|h;K-3w3NHpYNKyyR6WhlqVr`J2PvNjb3D71#32Se>g{=V(f(>dM^r73RfuN(v%T{wq=iA z(;2qDj+fu9<8Fk8b@X&(KjDcPtz~$7DeGgl7&GrFie8D2*ha1FZ{$`9ZV^11d}2FLd7tP8Cme71 zx5uh*1Bu()O@Q^t3a<*R?`(8uJQCy)KwK!YALXkZaq4im^phOq`7q?=@|ZI9-=p06 zLcSncmClvH31uBFdsQb>U>=2Gt5n)Gg=TMN_AAfJ^k3lyKHG`Qa88SO6oH2}w2>$5 z3;-_pYfCZ~Alb?>LQ=rQI`AqJhY73pIOHsGy(1TGXn{WYGSkvAJzrWwIU?-_Au8k{4D<_&Ck?t|@4@9)3;u!D^_bFO#ilo8>Hw+pTTl&_!pv`uzYe zwA+ZosUfnzNn-w~Sl`g(5%+762KWK)0_Rj{C82|0?8gMXCN51&2f)|$S>Nf+!mA*_P#=7D z3t6n#mXCG`w82Y+?ZD!f_-U|C2xdCEAepyZngutY21wq6pL57*b!_Qm$+`sUqA}xwi)z}F4%(fj+Vq?5Y^RK z1&+Fw z8>lZ|(94T5hHTFKh!EODQJ1uzMtSjF?~VQEHJ|nOxNdP_6d+1bB*4 zww8--3-a6yp`mK4u6z9h3Ev^?JkL!g)$oenO*R!9Q#Urm2~}pyE`hvy!j(icAmVM(hyu%S1{~%tKt`$j&H`f#mU`qxjYQ23>SbgF=A=pq zeWD%uO;V6;DrNh;w;qVkI_n0T9$mA6VXxs?(JE3nIEv5Dmer=P!S`I3#truh-bb&> zIZB~>Hf=x3XYp(5$6dz$JyVq%I6J#{ss7jwa2_@-N^*XF~ z8}se+-f#wRkQQ+s6Wny}X-?vZV{J9X@Ipf5I+2*NBU}}wQS*27@!vX7B$FYsmmbB9 z@**L}VWR5$K<#R4(P#hKkxfPKFB=IT_r8^wraMqK1*w!Zh!aPdd*NnO#akO=z?t2F z`^9|9XA-?Bf${+h64{vG28g4}+9PahxipLDzdBO!L6Hhtq1zoGP7w2O zdBQFH&{SaP^lVv7bF`8Ow;=u~y8toU9TVE~Wfl2*9ZRE4?vixRY=Xk+$wOHJ1^tfV z@dG6eewSs~I7$~kc6R@p5wYLhl6#+&a5#>{|$#LXeV3EYi~{6>sGwt$Si06_%m;)?4b_-K`=U)6_E7{~*038Et%V z+i)Z(1z_FxNDsL!jpe#YEi?jebRU{6p&F^_ z$D^Yv9DqNPuE*NQDK(i1M0<@l^(Cj)M*c=?HdA zx;ieHkGZIeRcRYjlBh^m{f`0oTo#2~)rn9R2A9+GF=0e8(UT>-M(EMAr7o_`p$FoE z$}5WWAWj+pJ<<+a96FfS%&rH8leHcBGtWt_Al^nzF#ncQ_ny`UmNE(JhS+*;;|y9p zkUuv39#pwR_h@(;Zj9fOK+Rn}y7sB>t5oiIK`0Mo$@{{Hy=z5KIVPSVl%_l47D1RO zeR}4tN1QT+Hln<9hsBI7JJY4xAm`yDpuQut_Ci(xZ^HQ|M38+*=pr<$p)}@1m3;zi4G&`X#<3 z{FaGPz|D>E0txKLD)C$BUAXabqL$`HgKS@6MiB@<_YP2NPx1QLz6rsPxyzenlX9= ze=1i}j1@Gg*^lJlGem)mGh!fc42j#s`;9q{0yf6|`7uv}>C>ESQ_EB#2n^0wLpl36 z=`>5SSIIZfj_S78xjn|UP)G8S7mG8U-`0Oqs~xuG3@m6#nuxnVu!A#-&qrNxNTh&PpXjETpArc9IltVM zSbTFWwwo}ESCfzG(tvv8A%oamBt8!5M{K=WJNJ4=lQdB_ay~)1oQp=A1%+cT@>xWe z2WJk(MsX0$tjo}Op)IxO9NR^x{gpq$j%jI$=ST(0lxM|1!c9RtB`5gE2)7L`f&Pv> zV4`?}*8gJ%1{$J70KPuR$vJKhdwKuLG&2}3*o^#4INUdV8w4O}L`a^kDsDg}*`v>H zdn}q8G*CR-*He-uU>u6su!JHt_!V3hMv_W6JQNP?fL^6O+te(6vGD4O{O1Rk^=&)S z6rBvxxD-dEk@0|Ae9;t6atCPZj6-s_U0zc^5`wQGBZX7(*xCp_8!NFbec}`zOJiCk zoO3PUUyjao(C!-(#SjMjYhaV_!2Pe{wi;Y7vR~rQ+l7r)kcGe1b?R=`ep0TPN>uhE zTn4CGh%YVkM4PZ@c%deuk=!YW+Q4N(jBmHMSU)vYoSI%UJ86+g;V$ybodaY;foM*N zG=;--i+gvo4vB$VH>ajT3KSq);%F00{^ZALew%5RK$#D@vLdk*h6|*jOdv+I^jX-e z=0(?4s9KQH>7e!os6?33Hs9rRs)_h-UoTY2>^^!Pdm@DeD%uUGfyWaQAoO9@LBtw6n+ z<2AdN%n^Um|BeP_i91$K#%6+N@L+LMOMmt^K|5FVwd1}VG`c}iD7HJv^A@f~1=-#D z$gWcCYRwfE@>9Ha1}F>F{Lr~*Y#H(O8`Ol~`mzw8ua$C7K}mxmwN=`5z>wcNx~9o@ z3zG9mZJglpIOSXI5Xk|(IamOdtP|q27XRR>jn_iLND6g?66KSjekj!S(-+z%$}vTf z37{xgqhq31mTI{nVNLXjmyOV?uRZN1_HU;e6+jVC{2-pAoxY08?R!z$MIT1JUHFMd z8JVRqfSYSOkSwwCT)f7e?5k<(4*G(vo=LRaQr3i%suVoZh zbqJI=cvHA>)E!AwYxG8=aqLr`Y_>9tly~YP`%5QCVc}p86hv=JSin^+UJqj(4C9xh zLeRZ9Rv@`knV1=)ojQ~U$z=0@o<8oZGyBbI$u;;ZxYpR0|;+#^}Fj zQphE5vI!Gm$Gn!epN8b4=zcr|R1<7;o`Pe}jbE3td`4DM<fbr0_moJ`Y1}WY8 zaj9+?orlrBA&(mxNuhJzK|4T$dbcA>C{5nZ3OQ5CFPh;*2yg5z;53(87a?L$dg39C zu0CkT{W-mA^44~RYNm{#0=5}#%WTaA5Xz4Vs{ytW3V>%-&yNsTp?TyiB??fX=gL4E z&GvMw)O1J4{hhUO9`8}cbIUZ_0*J=AdgEv*1y?vf$I-nmX(Thsfw>zGTUZH0EnZzMJA9Ur&>m?0nvyfb_BN3AmH@k3DpKtnx z{@~ft!X=8M!Ug9I(>aVkeu$QGnEnoN6hhD{>9`U)b(a%R(aT*%r{sqxT$(GO+=A%} z1r0h5svk1#T9TisBb0SmwY3%vh>DB5soLHktK_O!27`D4f2VMHG|bxXwbTz>h{L(j`mXx z>AhIT?rS(9rRYtb5AGcp86d^@8xalcBkiZx$DcpE2jof(&VjgWPO`)O3ZakHDISu& zbZHH5sh0^o;xA&wKqypxO~N*xGilOI^czw>PHBJ@;!0|@FoHv|r>R0WQw`4)L^BkJBL*-e z%9z3YB}&#ubt5(*S`H1nv>{_9(7ZAuHCsC?^}$D)7Ds}2Mz4-E zyp7X}ys2%%4SrBl{y0muT z08}U5A_%>Mj3d`+r-i(!ph#q`%-p|pQFP8T6pk4 z2W+Wj-k=TlgT2hnJwlGIf0Zzd(eVKws6>zNX0J#I0M$#D(3~v-#Jg#L3YV4rU9D?Z zL@NDfU`f6Oic$6>(Cz0~)pNeh)C;`Fw~%}(gdmoKU$I2QMS9=_ z@K|VjOp;5}Nz(?B(35romKq}glNP_6PcnuNrwXa8*faGpJpkvWfgjfF3DqW%7dr!P zLpez(*BhIU1}~_(-Lt;4ZF6ZVD-}P*%Bg&iL1vQ!tSImlj1kZOadGkUeL_}Y5W^jE zn?VCj{ZU5D+>Nc-sX;X?vq%SRw-LgCSkJ6slw+i-lj4YBi4L41Ye!T@G;&+s77idr z34#$V)&^#;q3h4aUXxh_3JX3{k^Fsq`A zB9`^p!ZRRWP~F!!vim}du8VeE-_Aw$w6XEJ&HWa~Dqy;~68LZz>9jT)9`pzc;6rB` zvQx-S^%4UYM@8I$DRkCx|2rx|K4GrTr8h7T>H7X+`fQd&R@F*6HgXtm8v#!x z>|a%WKnyRp=eLqjawT$(AsdlCndsQ+=S?N?<$hcoZbRV$o{Atc%jQSkp_;Bp!0jS- zsTr~Ir0t(dx%Ty-(oy#v)%I&~?v1N6E_I%5sTsW#?ke5eRKhSD z=&zb|Fa7VrTX7UYh8=cuMx&%Voh1&!fO1!ukyu9YgT-A335v+sBn3V@qAQRygOHH1d8^1oYzIHdb-67K2skifKy(X8^lt_PZg_^gmhs-d#g|U}sKMC}ICX$5@0S48 zZiU@1eL4~%3hOGeFUa`9@$(UhcR{yXpB(J*@LF{`C|2AkJ+Fbk+KGmJo_&>SSOIa5 zq6xND9?M%~9GD9yLcT$6NWi#k!guuk8Ks)&WVAj;xsUBe>E&ynGp=gp@~9y@K;O7l zrZu7!avwDr6c%y;?oI=G8Km^7PHL%TLYE9X4|zn)Ud5cET+VgDp_NnBNqQD zl}9RoMm1HMz?eyx$wR3DEWLn)t%Iyk-jMo5d1V)h@JV8EM9TN>41nSmswT7xaj3~f z`=wOFCzsC&h*incKuPp#5+@IVNEK2I+LuO=ievx^%VXmSX=Qe)eT`NhYla30TO}^Y zq*>LRr$S#P*HzL3*eElj*vLZ@L=yb>AhXs#gcMU-FaC|hghPgkT*c!dY&3_YHn;hQ1xjoO6|L`!9x(s5&aRSXk+id`9ky( zUY&MoFn9PmIO0>|#*zcK5J6e{V+!Y1gfO>dJdx194ljv3{%L#KU?=>P=IDW~HmFkN zmqh6$ca=Qz5viv?RB&;lb22C#j7K$_C}=*41EN?PIUG?CQ3NO5P7I3~7d+>&kt=>B z&Aqy_KW)8ceQ`|J?C|mXl(Dt_QeVuU&FK`0FD6m-Z40?zWaBv{r#hEq1=njR4z$d; zZNsaeT?AJ~W)`7#bPtim3r|6CM+iCGr+e436<;0EJeL?EvF|0;_c%*~t`44`Z;TzpRu77aW0J^xdw^a_llcz)o{LapYy(!SBsbSWA}b8Bj95oE-{2kk(C zrHfM|WqUO>yv@31GEVhGo3y)< zB{enn3lnFDi(^&yN&~^*O4_yisfzCb$fT!hVfgyOPjz8JS|CAFzuqSZ&7m-hAbW#LRAwGOgBR#9VHUt zjgO)fA+OR+s)ij?9yHe7Egmo;Q~0A-82FvA;q8tr7^4b@Chs=Rd!q>U<`&O0_r{dM zS6;1ut`u|qQuiQ>PqXMUCtDTK`{3rE7yA8&Oipmb!HSnjc~Eb-`#K*ag*mzdO^ps9 zyzBpcGMmVT8k35<;kwefH)?mbKfUWaPyQH-kX#5{&S32W-}GDbRjEo$KfK#7jifWV)iG6Lh_M~`1@nsMK|^6HzvU|Q_jXhEKhMCui& ziSevl*mnI82mK0czfy+=@AQdcYXC)83JG9kyYF`qe(JX~!j;1nt){$zAA0ZJJyosu ze0t^e#|E{ph3?lHx$c{9sJL3zv|1+^xt^Jso7c9dKQoy(d;R^1QTnkUu9Rn+pRMt8 zHF)#~QL)3*b7`>;6ito}?xI0TY$-t_4_D`Xl%zh9?2Ve16$hayY;@;4=Jp~l9-NF# zDS8r!h&E37>04S__AfUzHNp5PDZTzU@N;nd)r)4j7bGuK)hRgRt*WXjTSnE6oFNb7 z#F%j%s-U@Sa7CEg6jut-BJ(}YqJQ1kj2s%QWiki+1mCi_3O08B*6N=I!G}aKQc`Mg z@~f$GO;2$0r3{9R8|Yo<;86JZWU}i^{T53qI^?D`h6(+u9n<8jM<^IxaS+kv#VPQ% z-mx$!mRma$MvZrQ5+5J$JLEl|JWj+Pk^%_1 zS1I2hJYeaIFrHLyQ`ac&=tp>7bU2ns#>Pa_lZZYYibi6j)*ciBo?!B?RcYOpt!SmlVG)?W%O5A(Qf#~NuAki zqKK};g*-6u%Yms;Rb`IJ;NVCl2B67wt?CNpi$1-l<~^RC<1x2wB$w4qSf9O0~eRt0Um}u50+Fb9P z361wZYvyz)NssfDy$vKkx{f0DB*Kcj*K5xxw*T=|KKvF`+d$KE^j^6A#NAj+oN=Wj zj`cy$*#U7oALR>mBFOxC6Ur;E(B$i@2}23MYG{sP+IsWrb2X)2k{>86BHnQ9n|;3i zy%JOLf?zeP=ajqWY}*4TUX^1$U@`1Ywg~%wsj8~2r6_$>)uwX#Bc>$fl@^;}N?^ho zSqqzLKmKy=B1}arIYBav6~2+z@|RPDZ_E$JbMK>+&*!^RI22@Y5J+gS20f$zPHfSF zT>0?U4R_U*Pv}EEC?;Z@aoin$zAPqw;r@}}948+?6sMg&kayn<*{-yRceX?xjsauD zet6L;^I&KjjQIWfD|Ab>%<_uSv-sP_`2xv-| zW*5tVx)PHyA@TaJ^an60og->b^u;Xk#yl7-58Vw}93HGzQ|vuXa3Ddfn3{1JB=j9_t3ceCY741 zSvllNSq?xU@?ieWS12h7P3$+$DKI5hi7B0RvpbaS)VZ8YdIQW?&FN&dMxi-a{z(2YAP!vwI;-O0-2s9r-5D@vv%A9;^z@2tDJ7QDH%5>m4{cC;7bXPg_ zk)ex@A5yv9=G$K!9|!aC@y&p--+1*><*YA&h6P>q6ve7u+T+h{VT9&=9`{Pkk@MMgPV=Ozo5!F?#&*8tqr05MJGDu^H;D{F6;{+X)Uku=-v=re1d zWm}JD^K-l;-)n$j>&;qWdS+%*=wI)`$H&fAflGL>!jK&qp>_3|? zJTykF2J}*RCra%ST%SkRRfWM<8K_b8!sv+BwP!}Ey+{$H!# zA0*HUx&+EQjAVzVBSGcAjWL=3&mSuO0AqW%i5g5vI5@iI0qJ1$B~-{I=p1WRwl}ua zTwawA_!O*nokcwK2wRstk}Kg#JDe$qumPsUr8WM?7uTB#?%x0sQ?5VpoVzPq#)hq*v?D&{ z89vj_{1&p?MVcbq@smTPM9=em{GY4S*5L{sUHh~Nbixi_>P$lqW*tK>P8J1v@`oQ)+9c)TJeT7n2rJe2GieE)h}?(sE4L&yw-?kd-Mt z6&~15(SYXi?K$fgGK5NkJh3&6urT27{ z&U^t6)jt3aTVzR4WGBqEZ?#X_L3T{RLkG2wnLLefg%o?$#A3vn{wq*<<1qn-1XGK) zDW)|^&U<-;+-ZDH6*1!X2Xd6DDTAM%pZu?P)bjh22BdFkcflQmL^h>D=n56Mt$=sG z;dUGKQ&oETK#{jwO~m|0bxy4xBXAJ(;!b+aOh)#m4tjq{F)hR}6xsU@laBXu4+8Ms5v(6tARqH4xWs>w9x?YLhsC5rVO-$&6TS!jOeIb<+idy5w zRX~63!QG?Noq(QYaLZ@~e4wm;!>0-Hi`%Hx${liLh%DZ&w3n+JA>kfef@_lB{_cNj zmBM5x!?S-X1nub};+ib9yY~^@5yZ|R(Me2FlJX4=4Xf?T$*zI!i(!ZMhu;DmK|TtA z%Ryhinmu?$52d6@vjeHryyq=F`vhE}oIBni4kd6iz%@L_fkXyyJJ2)FAG%}H^Y^1i z`%cGZRZY!zU}HOQm1(PaOmSA|`ae!k90sp@S8pC;+vOf#CQ0vp5k!we`vZ2xF*}LJ-R_a zQbG_!T0puKkq`j|rPGOsgVCK65tI^;k}^SJfYc^AK|oqsdbCIkX&CX{+x!3g{diwv zd!G9__lfIV_c`amXFbSLaC;hXyfbKA={c|Sf{DKubcI-;Jp>5;l+f`P$)xe*_aP^? zW&%FHQXtQ17F0&?)=_j*W#wSQ|A9?_bKEBd=+-ISwhiba#)c@R#F z(ll1W@J9r3z8{OR{_82Zak61D`L3a)Ywzdn!G+aUv?1T$w`yks9zyW;ddKU^9$X$b z(pL4ew4ojr3_na0QKY+WY8x@)h#Qzf%s(_gmIZ4!Nj(5>wW{kGWn*I_=;In#;Dp*t znY|`3tTyn&t<`w}jz)mKDi!zYS3V+@&VBHJkfW6X(|h;&pW^wCu~dXS1EK~(9e@QO z<*T$ExrZ?r+qnNf+M z*VlIIk!nc^d}~+UmH&Re8)a#`?G?;ROjLy1`%v@eh{nS9NSDd4_2{VfTWEI;w1eCL zqFo-%|M&VkCiv5(S#;s%YNeOFrHP$gW6vlEbNeVdvB5Z*J2@)V0wp0Jx>(fuh=TsY zuD~3@sh(tE18zQ)^X+}0w(&BQFiC>zwE33Yus1HAB`oY2F7Xn3x(QI7;CDTD*`TYN zC;+dSlk8mhumyb1kF9T^T_z!_@_~)W_MSz1yT1xbz7rP?+$VQ7<^Ie~UjzO|Zn;gSiq@*3~X2{ZpE_q`vLKfn{^hon-o}<)ATPXO9~2IeHqb z6YNx@FI2n!`Q^gU!N2(ne43jWGB?WoCj1nRn=g@1{VgCr%&h$1ww_FX;{|{KL9mdk z>=64O@hZS0i=HP#@Xv3Rr9|j@tUh>bThr1R#g*va{J9jll2ZG5cj0|d9;9z>q4 zC!YJ3zW#jEdk}2~!9%^@qWpZ|i;#@Wsv z4o*Y>r=^+xS)M_{y?e2PAOOo$^zJ4TJuw-Pu!_vOmE%3e*P+CJ5ZBhK(iC&S?RRq7 zFLdO=<;k}R$s!={y7Ts0q2pjSXrT4RkWf|q4x6g=tp^Rf4ECrIvrn%XK*KZW7R~HP z5rHBX0h#%5?e|wg$Ozq^O8@Xd(5%#^19X)B1P4wR5>x?bRR)p8+tyB{{2GQ0y{MGG zbo7a!EAIf-)N3O|E|&t==u~|N%J?i*n}iG8<2)@jbt4+L@C6A5Y1yDXj=^y;$A7v* zc{%&$HEy?>bxX1S<}|L@8_bWaU~80{C^9mRR7j4Cpr7}R{sm+stx-8HjZR%P ztLbJB+|E7%4#v0=X2J)EOCE4+4mu}agFa5Ljc?0dAbz-`@(F#_w3!k?&NHVjV^3AHMO_rupy8Z4LZUba{YH5EEQBsUwrJGk}3f@ zVmLfHS_v{nvk5?$J*`^?&5U06@AV}YPoeTijmS1Ak4W(OaKm9TO%iz5J>CIE;8f=U z`#ldr>k^0*b0EUbgE0^AnstB?dU_ORO75i!s_Oz>yp~XMpSb3odiQ?G4l2jFty-P>Fd`g;1w;qUV`Ra4KK=1xBxkWKn*&R%DR z+jcT}hBF-Vme)rvZwv#5xfy8c;OzYP5mOidsoi;+c?1|oAj1e7!SM1_s&1|M$4gda zT>UnoDZbz@d(w4F2ibVUXNoYcCxBQKj9Pbdkb_TkT3W85I63XeG)cp9TEhSZ7fT4~ z2GR}ZeYn)kI61^l<#kWONsrZh$xsPSY#WgkKFuUlJQ%_mLrUYv{wpaW#*ab$2IRiE zxv7oK9U+#EjzevsvH^sXO#CHrHVXnAe!*0)m8Lj0M&DW(o_Xdf-YpSnTshVRyqEv$ zbT=0{SM3{4iTCX6O5d4$DnG!01}zW7;xLhDF!==%Zph-;->Yk&1wq z3+;f`BtK0h`<1tG<+6)@#eF`2iE!9yyDT`MDva<3_ujoy4k)0G8Zm5W1sd-35MRC( zk%1@s1Xle0e|!d67v2-GnEKx$J;Xd6KX>Rj_r#U_!0cXlZ?AAfNRJr*qek?xt~L&V zWSfK6iwSO8A{E1ku^ao-ef^%CF_YK}SVcck6hxY50I_HvVgH`^NY?D1HYzLNPTrTq zO{Gd+oK$);0vGa*K*unHMf$tYOO#O)*_vmY{0{1tR;U4Ymy%q_7_6{rp)14m)ZC(aOYgJrNFnm8iM{D!=@nIf%;REm*!Nxm``PMRIu zQ(fd8GGjb4YM?Y1u+KtzL>jTs$KmiI{B|a(0a#G4b;Jb2@w-yLp@ed;eJgZTG4ZO& zL>kVBMJc5Xv-ROW+Uctu3Ckl#u>f(HnL_#IAUEguu=h*gQ-pbZNPsoE0iQ!Q`O4aV zwdT4wVVeVMNZpg7c)w4`RIprpIy;I%)dgv+1eR12%7q9h8E^mICeF#3Ry4&AvM}Ug$*@n%cK; z-9Zds>?Bv@CC(D)B&7e_3%&xDKamqp3I@lOD`eLdioc6Cbb@d;_V_MMxlHcz+tDN-eWWu=4cMwds4B7pPY=HG9cl;#Ak zH?iU(U{Bz;0}2lAx{i;HH2>s0Rt##|cmNWu4$$D=22cYRXru;W_yAHoW6hH{0F=K5 zOja~V_aNn9%tC4HU$HQdTB7=kV|;Gg^5$2t$oWf6(@CR~-i&D7WA0cP{MydT;S)8L zMig9GK5!XurYb?O&$*7Go1*ZP9G9H+AJ^|uP1vKj_=+NSbJPoVcnIJhME0IWuoG87 zRB0pdMW9K%#XTHv4cTZq3oHG$#0{n*M2Dn_nUr@&^FLhbZB;u{J6^TVuyCEAymd%QQX5i7J}y{U+A8vQfW&COvOFxK-aJ-LAGjxp%|KQ>LTo z_|^?4)hC1u!{@$w{3VIOto0|)3216n43y%ilH~bTBqqE_wK%{buQt%X(W8O1wzd+8 zyZY0kRS2LdWw5WbgI+&ksSNmfL0#EjYN>kkV$u^lI+-Q8^08ynUiIXt1%$}?JAWeU z()9pR+@K~(aYM2JzNj=wAH`V7Yf&j1>5TPfzs$KGCRt&4T^S%waJscZGE&;wotLQS z&Iele9eUm-@bbscX&zkM+idYRwI?wl+kf9Y{Gyr0>ldIQx;bH*= z&7F)Jl({RvxRuxf-len~kOUqsrdw7#qJfZ6(mPC|JxR=k26~1@YU4eZdRpoa1ebag zr`nWq9N6yry&`P;%ByaStU8+w!Dsy3`x|i!&HG)oxDq&xGph`@ceQk(dVXuUVda6_ zOZ7QJ`=LbovVODq#d2xGp4Er`C(}>Qt(10GIBm|w25c{V*E~fOZXA`}z5t%$`0Adx z{fd`;Lpx&RLPLw?Ceu;5&7kLzui*YkAJo_;{Gqc8zb_3-!rYewe{3I z6CW`Cq*&Wg!(kVI07Ew*fM7ZVf%xEHKGCu%EVLyM9PI{y@d+i_+4i66uS?? zE~`fWdRabD%baO@Z=uByLu0$9JFdXaaW{#(6Bkbge1Zet~-i(1G z!il`!nuS!;u~+_`%x5xz;s!!x-9-;JkGBJ)>mvgH7G_7KOk5HDo*N|g_%x*Z_z?1Y z%ZnPt)0C-4Zf!Oa)WpVyl|Mrze8)AkzrRj@a4AmXUDTq6*4&bNOx#4%#;97;94d=0rGiY_+WsJxa$L=H&YHW8za{F6fqN z+FUSy>KJDKjtvA9AAY)so2r4X=DKF24p??NZ-xcOjC=XFY6j47~QHRN0N zeePuC=_xbc77qnhK`1vPWCKDe`UFM*pu8?^WU&av4@NGntw#J+aWhP1-&{{-WTq;| zam`$<#1*k;0|RYkFxV7w|BB(1RpreKWq9HSbNu}s_WUy;Ix?f+i!SIMGnJ~3H_b;& zy8k%maigix!gN!+W_ZX9MI#mcqlt4B)xM%+dETLGU)#NaIOM#S?m)mY!PPa?SXIB2dxG*(#*1>V{tvfE%r7NV^uWSWqvF(a%bISAbnm4{ zp_RPbe?A5X3m>3nrsq^r7}S_dR=aAx*f*tdyd|$$t745cO=LnSJ_&y7i%L(=`3YRZ zXi*DtYDaL>85>7L6Eu@q=zG%k8$Z6fMWvSAY@vQz#vM?XS`!*!@Au)w^ks4$3N`8w z%1IiY?0-SlvmX;D$_0lKY61tPFY%1k`obboW5`<=d0%M;%62rJ<@|`|PtKMhI~y{c zU}T)=$daeSHPda&?bbtx=U+C(Bnd5&dm+-a5+LUL8$inXm1N5g#L~!R(G~hDGn0GB$UOoOcGvoWb->a9-UN>5&?}ch)|7K zMx<6Xe3b&Bo_{c2=YD?ATlwPIe_#?4X^HlyESjkDkGk{rfx;c>c7lg{6GH9{Y{AVm zzbk6epp&_Z-k0BtnqnS9TW6whi4*YaIXVxLQ2|Vo ziiZt=)tVCe>wn!Yaz&;`%uC!Uzx(K&##A}IUG?UomC{6WS;P&rR+!W5Ju){}eQ{Ao z>}~8a#u{_#T6eRQNrKAnohLLWI|#(qZ=ei{QxVRi%j(+oBXIlb*?#zI zy{do`MK|sMzSDTcrNZ%CLjt$FU_FQ#Ag#1g*8;{+?uv>34cou0%HAwLLHTfHIsDP& z3qKd>xyQ0^Mw)M2qq!MR*O%}9TsL;=ZJZ`Ko&Xx z$2yl-zqEG$YI*X$=zM-ad0y0VglIL~&A?^M6Yby?j=eUyb7-4Y_0MC~IxKGw3Cm=o z;t#nBYRx17XX{p@4)`!aeFl@-CPY|x3XbDX%5{n6-85EDUC29IQ&QdBxchflTaX#h6Py|$N?`h&BbcUkC}H?% zpCnVUnPbXk$K&EM#@Ye60b%>eMdiw~U(!kN(-U#z zvBm4!Gud@&@fJfJGpmCcmVdF3YDr{2Nh9Y5N~Uaq$9`J?aJ>?KC%jQS0vwe zPl-g-Ma2HJpnB6*K0t~iRh%O=G?;Hw3A9A9T;yT&{&j=O6Z|aB`v?v&`u#k&cDUSf zxO~%);qk0iw_G1~DqQoF%bDEP4n?eVc$UT}kw>~=tX@D!S81n$F*-IDf#Db2{=K^b zjo?#*$>p_f zUJNXvVlq`!P+jNREA?ZKwb^Kx|EAfK9s(w11gY`aZ2t4SbHr>wd6}&2SFf))!y_rF zgOrhHt#{jMQya5rrFc#0Cu!9vL&)RaAHHMZ9z9gMz)77QLnjtG`KQwbo!3tL?v+|8 zR8(*(TgDBw0%pGgBEkPF8;6Aygs(aNF{r5!eUyssu^OLb6Yf`HIF1@fiV~uDcMLnwDV}!SF@{`U-1Y+3!$f0BOvUXyD zWNK587#TEX%key5L-ZVr$+a-)<4>Qwqt-jv{OGrL^6>AeK@G3t7YO|Rf;a|_QH&6E zt8`F3m?T5Oa%)eESUUe zC)FR9VFOhZso__mQ4@n-UFgMIWE55ITA69h|M|(}@qXTPT?wu`*^70=QclKut>`TvTb@F5Lqhn+A=>C&P6`K+_hlK_?5C4Di{>p{w?@gb0n(^GQ`f4>zb0+siAg0fPRw^WIGLEWcR~vv->(GopUEeGj@eLt5EP`)C2aqWP*%jH zy97WU^_d0rcrk)!=Y%vODrxwt64i;nKYq>+ufc}OIvVv5fS2wX=UAHp&O$|x_|!YdWxO(qZ}rzqQY6xQi30w2K6|J;3_8 z;CU0xF#|#C1}Ym!hf*Sa#aX3FcHWyCQsBtCn+@@HkT?pvPHzU*2o)O(F+jQy>8+q5 zu}SJKMNm7L_+NRHQ8CH1Al0gw8HN(;2ANQ8?v%j@VjoRb;VvQMNrq&YIqr{8dcZ*` z88HU#JX*nQNlsE55bta%327sl<=M%ZW((=#=1h(}t&83EoN0eJ`DWk#b(70X=4}L7 z|9L`{53$He-BzVMh{Ch%uktuPT1ip|qGS<_nAqcR3C6K_)HNu^8tjlC*upHC(H@{*YVy)3GGc)8G zfG3EUUUh>EGe?wD`FyJ!xv(zgiEB%cy{3i zR~wS~knUOL-qaBJB#V63Gf+16Nrag1M?Ja(|1YIjc`4QbYS%?q3bA#=&3wu#x{W&& ze3W5SYLQucwt(J^Z$?+^%2U{Ae@Y>){JlO^8g8DnOaH&pEqE#>oFo!hN#0kuQ-!lJ zg(kJ)I{pGCF@n_tcQVz=f$$=H&F3v~;o6`8jVgq`TL(cJB-BNR<3S>~O zrE{kW7yg+qoC3{OeuI5Dj;0WQQSkuCr~kh@TSaG4<}ykI&r!*NzqGZZdiwK*h08_C z`sAan;t`54_0|MT3J_W)f((&PhWz5h8o`7ZCe(%vm=a;PRvMRu#!I5er{^$&UZ5n+ zEt^*33*TNxtFo7a;3g$J?FU%7XVza|T}q|%D10rDIgxu5^;yAq(GVR2;Gp6tUV%qO zt=Fbbg}!_Pmc5KXV!nf0PmMKV7@suUW=@i#p|ZX1o(hudvNEPxdghhy$sLb=3GWDj z>qtqJt5IffJh}n+L&-1XQZ0tQb{Rbh1`apkvl#7O;peB^#?~}rDM{rJmk@YF$93O# zY}Lj=4V`|`sYf{(U>6yI^5hR-q^kjhZqj3h=7y#- zC5{~TsPG?{HiyXdT)A~dDveg;l(I0^O69LKdVE6mIri-23%CN?5>lGtoob^p$aWVx zt$D#f(I$;A~%A(w!stY?U~ES z{XA>q#CLKB*MasG3r>EIfmgZH9t7=tyQ}SWiJidX3JwHDM)-S}HaD?&NP<6RhQ(y{ zqJnEwYtBv6W<7L(P4cfl+k4<3zd^RQPgeR_y!XdRw+;p21Qq<+u~8+#&5?G!BpC(Y zP4>Geb)X{8ku!=h1qHl{)-elnJnQG$4Dm*D)3N(>UlLru(H-zeR$m4m78JC7!ziIh zLPe}cqDJptP)zwKe*x;@B34rM)nOX%*J_wvg9$}Xbp+S$BgSrU2oS#&b%UVXK8oH5 zbMO9tW}GRE;xKHHP4-8&_PG()N&4)xW$)$Aga{@ZKA-=-VWQ+mI!#s-;YnPi7w=tT zMjc+O&4=Yulzzn%p{o3{1)2Fd^Ipp}O87jUjkRc;q`!XACF=pd8H~3BjQ@G7Nn$MziPE zvJCnPR4l-VwExXKf&u;6tTCr$n|wtc+xjjR;1|Lds>L@2fX-gQS^*cbAH;MJETu9{ zdTG#c(M(ZYbwUsmkFMsiCwsAs<(ayne2yxg@_+I_uR!pvELFh~z7ZXM?d*5x%?e1n z(kmuU&b?y+CBU~GjT1FU6&p@q#1c**z$kZf>?II{KS#h;l1i#Wk=4vFx0ls5iPa+( zFe(d}I?OGzTHz^@!5W4y7&o_o#g(7J#|A{AI5gVf^{TK3nDTRAYEi z-@uS^ecWECNwqe(nrYvzK`3waNSgD|6BBz~)3W{51^L|`biRgieg22bqvw0km!%U& z@27bho@CjWF7?SzjJ`Sjwg=ntUf+>-1J%@Zu%e=(r~lr9{~kFX+jNywzc&O`EdGXh z;irN25ugqS?r&J=4fi$#zgtW1*9bLS9I;AyigP-)X<6ALeCp`!?GdE)=#sn}s4$#; z3s~-7OY-1gw$%8jX?REe^LR9}x?dhP^2W)|v?w%ow90RN+^tk}cUNI_h%P6VuGy=C z%aEQ=FDliSy|ZFyod&av-tSGR9CG&ev}WQCX7Y(l&EIzysKX&mBkcun$><-oy` z>aq(^t&#ED(C*Od-Bx(kzU{s3(54a2ZTXpfV{*#CTz_Qs-b`CYyo`6m=aKO& zwZ02!cP=X)G@kMHZMR*3?#Z>;fOqFZJ) zUTV+jV@09ewAdFON@{9+iQ$adv^S}#;->vGJx9bZ^s*Ty%*iuJFN6)}qAx=m( z%S&xWa1iY~GENpJqe=QVVr699{8enMp{?x{tSuur;(7Jnhq<{qvcG@-&gCu}R*mGQ z(Q>+zPfLQ-uB^B?m^%PGJI!Vn_>4OEOi@X-h&4=v3%XY^a{8OI6096|{-ySYd|N$` zCi&z86O*kdFa`ZVTLvBCq@qd=RTxpZ zRNL} z*mxNfS(_C;a%*c#2K>2kInW+_DV0_t92yr`S5hLu9ow3V8aZXKpgT4W3_Oehg6$a@ z2P;Yde#7u`iurS`_rfFNyzw%X5oeVnjnKGnNaH+>I`~+HQ+4%T2(+6Mwyp&29>X}# zdIabRCLJJUmdLOau=Pu!M>{L=h{Z-QWoQf&=jz|c)~o0t5<#V#hVi!jQ_6{^L38fg z9lANVj25r$EzSD5j!Mvjs&*ot900D8_YqPaV>B`jZd9GsJnNx!p7s{(>C=(4)KrF- ztq+TV^589DDgOf0*yDAm+`)=02`tOMfcj3(`KXrJEihx;7)atp>%*_cf=PN5paA{1 zjfqKxfv@lG0Qi+N|1Ut^A@;6MS0$RA0hcW<6fYi-$4cNYQOhScI0<~>05(C$ZJALdL{C0Zvt{T zz%uf51(T+L7kULgcJ9k=Xp;sHBhJF!zDbbf<=O`RH0a&W?fPWkloW>BZ})9eekv`z z%<_`HsI1JMk1^UGe1UcotiV!@&mIW+1+YfrIGIqAnf;T$9!mAz;IFw~YWw{Q4fj5J z3vMF?2rW=;ZUO8D1Ina3Vtn|Fe8N2S5)Xro(hhRlG0?eQmZasU~bJ;)~w)zVCxp& z-+f{}vtJx1BLKZqSnU@t^Z1Eu=CROu+TH4rBT#R#{7ClO(009TNp)7d41}!_7d{s_ zeHFG|(3asfv#$>Az6|Y_YQ#wc?TJk3c+}0Aa;kmGj)mcP@`|LqD`-T;2Ru z?WYJ^_kwo6yH9QJj+|n)aEu$9vZ|ZrG<}4WlxZtzS?9@V%vZXbbrf2`7od`P=kyCG zZhvri6v}>;tb~e>B<%HiFU*l?Mlz7p11f4W)=|1g%16owIfd9z4mrR7u~AC>oB9q} z7!j;e+$3bidib>$7u01S2P*JNh|3MJ7z?VqrlyfNJSSA|a?0}-$K(#7+-0COYUrHtMyX+))O4*xcw2_ae;HJpr{Xdn6f(kR6eg7+)(vw5`?S#gZPYwhw zgxRHO3gq@|7B%0kR!In^-wCt;_G>pr?}mjtQoZKGhlfcrm-&2T0(ZP8E4auaN}{W* zXYI&lD3X24q2Z|ey1{QCO#t18aiMm8RJe|g_MDtYlW@jaGu{C0BN$j6$ai8sl|UHV z-FP;c99+v6ePH0lNWehJfT{edj?p8z+P3y5aQc~rL=E&;^}-nGuRpm_39J_iG3RsbAkvXh`U3Dz{UvX3|=eeVSOS$Wz6qj zrmx?T@Rm|a&^C44Lxge?R8=*~{|O>3hj5rJEpQaM@YpIpmAj_c6UL8a!>UcHq}r2d z^xTqE|M_F60VMoM5~jp1g1{Bg*$3!;VLX+!(Y|Bc;Us6}O%P-{0g3Gcs_E2ih3_Kh z7%F=eO08X*H<&y#dz<4w@sRUGP29hn8LJYML4y&SY@z@sF0lkAAjYT1%f{gwTShFY z0i>iFIAw#bt}cI?=$0Glmjul_F^z)Lz80yoHyjZTXpuYAMV(w#Cue(qD{vsa?QT-^26%C+Bg5N9$-UEA=EW8Q; zR5}Qp!Sg?FdzgOTC-#YPWLw5Oh=8LmpkU?^@H9hK!6Tq7HhBJRD%Ou&5%XR7PjL4Q54R)Kt|5FG*XQMt zt0hBnz!AS10+?uaVkKEeBh#2dIS-esGU7LY#ovH(E4r0m5*%|t%8!hj3*PT041ctGLnW)p> zQWzyz0i%Hj36R~b(3!J506`ja3&P4I_LJvV5fY1eyQk?_Xbq2IZFalk9cUXwOsp7fD z2;TnSj~T=~u{jvSVAvHCC>ayjSuY|xs}~j$fya(olXih5Z5beWKhUFh z18^eh*)lKA`lfTJ^!?KO8_yb-AB{BnpoXmaNhzOS*I^;m*L(A0>0iKwM7Os2%;5D5 zkGN%{ZtT_lg!%E@a!HAafXJ_3+av!y!~JdxP?Y;kDR3_Nby3yDbEH)e7HS%Oq*+y zBATeJ+gLj+5~KQaqG_LZr8(m)j#SB+MoOhsMX?x$AiUn8h%oX-zhRp?&>=IaiK`wx z?N!98^*cqVWJSMWm-;ZP$&lBgjT7Z$x!0PMl;q7FuLG_&PM$2pMHUtY zhr{bPtez^6*+8;`vsaC%PLt}_@s}`*n5p)=O-1PqJSI*? z$N7?W{aj*@;J8VRv$VBU&v3F1WqtEW_K{^xu&e9ZhkyV6Ij@#}emvMZ|JCYy8I^|d zY2^|xh6#c+hQ&8hk1Q3y z5UekC=k!mcpI=WFa_;Ht>+9|5$$0=oTbCE&Jd@N+5Q4~0rmk;q=c_{bK|)8N>87E1 zSI2|xDq#-Y1Hv1jJ{_su;j>WqIl~!d0IvJ=YTLmjm$wNcwlSYhIiNya6%CeYC$atg z{SCgOrj-x2%obW&I^`k~lTo4J)B%!*8=a>8%8#Odm~AGAZZS&@w%6TF9^wVNmi%}3 z>fwd&hHMru&M|&+3F>KXZkDS-7>IS;eVae?)gL$gzLstbl8+dDjklxW9kAT0P`w@hRZ5jDmpIHNI>HXoGnMVd3hTy8V=TW zB@HdVQrNuK7U1~TUhFl0g`8ev@Kz3{2Sc)DwwMCPYuH4jzRtU76rrf|`)~LX!&Xj7 zO@x676Jdx&6=8y(>ecQgYKeVeeXmxK{|s)Fyt)5^bN>4^!|)(4uPukEYFEhMf?Oi# zASOBR-okFtIZ2+2{g%pnnPbZ~+j`6iLwMKU5G!GgZahp7cSp(c5dDTlA19@*inHHO zuC=B72#4$awFCQ9HLmaKCIYwpxj`FXbSRd5Y(a9~X%jK5;Ink(Tq>%zXQ@cY8YFl|UiV%MCFg_)Q08h49iYH_!GHtX}^L4h>9R$S2;GeUAei30h zj{XHxk#Zj4aBr)QtbSUzzKXTVn>TMdPk`&vTc#l)DC0YKN``C=7#gV#c!1qC3A^JP zs}CmS*-9r|DEKX(k3K;6>i98xQ1%wT`IP%_DiU0B^a!(Mc66njUO){@YlDV`o&NKy zv-MX_KC+2pe#fR2aPG;ljghqPc%vr0no-!Vp6FM7p-J@&$GZ578o%l%M3lvIXpb$rB}*&r!<>21PQzAUVr1V%bd-Bhz+hu>-$t zqWFr;uQxj{FZ?kazR_K~TKMbYrX3d7VVoDnu$kVD4x`j7^vHc=1Eb)#De~Z0rrZ_8 zl7gv9g$0DsN`LwDF3mWOF(DQ`sAom8dX}`Q-)~q6?SA9DWSO>3b$asW&!1Cp(Vr9x z;ixmEP1OaFyP84SWtjr*RAp5b+lU589sU}JG&%$LY1XZC&`}{>U`ViKd z1p!slh440}A=?-sH(}x>^ZJrd&y699WXmJgy>zXd#;|HDgG-k#Ef=_00ew(FoNI=7 zEP|OiRDVl9Yu{XOn+ewqi|AU!%w#X9KmO@&X7(-qWMuANXU18DYjFnY2Kg~EOKA5s zL32-5Di;uoj9a{E7bMHo&Ot*~0gF^B(e=0VmPy`f6#_M}&YzQ!@msZ(KtBF4u*@#5O>X7q+Bg0p2LEXRP)!9 zYdU|Wv`xyoxsKZcg2ZW#G>+UnztUcKsL!IzVb_j)-pO)$dO9SA&^JkkgYL?3V3>k5I!{h zL4g$R&RsH8%qC4kfR6#u$`eJ~gpa_H_CA^?Pm6VVJB_Zb4Z7H7 z;V%hnj2dyxP$XiXPszifVk5|?Up%7D5*USe>vwtER~TFHt}FIfAFJ=uGkK^ScR$mG zI{S3+fXOx7ghyA1=ECNYIe(QQ?OI>=!_U5d|K2%hy7mJT|5v=@;$ZQ#B}3Ld`dpM@{0DiMh6R$}?P3dr0SHVXrXm0iRwg!1U?q{gfN#I zjT!av2f7IRi|+Waq7T&@6jt|t(`wKflO;l)_h_QLXrH8@M!^`bZht1=^)&_;R3-WZ zm?Zj2-@h0>*W6wqVO`?rrk`~uE(!3HwqUDN+uGWWW+QSyROSNuiW6dLDv~u0b1+U7 zdB(bxC#oGVY@;C(Sov1)Y+LQFzd&*o8(|c9l`@h^Lh}}mtjjO%-)d{zH$G`emr7B_ z-^x%&$Jzd|u`#*7rOTXc2qxfOW+-fEL+W*5eGqdJmj`qus*gm3Hrt9u3N2aD+yp40 zKu`fUyFuVreAmi(D8i6qu5c8CjS1>u9KaNG$D?1revR}U^**0Ywojs|H@$b_c3s*Z z(fd03bNc=2>*8mY$=~yHtph0zNmRkL?z7IQ>m^rxo6_gA$Tc{rJ7VbobwxE>Z8iGFtTlEC?F>Le*O9t=u*qK z-d@IG+b$JWOf-mwIC49KP}QwQ#*h?RrZLe;6SYQH8Ky+VEYYpV=OAXD(;qJ)DgyV$ zMc@jHF<$HkC_wJ@Ia!t|a_En{K8yn~x&Z{U8Fc_vQ5!+D=8!y!X&oSnbxf(1c<1sX zGj6leENhx9cevBVnarnjXrW=^+a{3EPzOt2g8@x44}b$#MplTolH*0Ayl)STA;%r3pV9Gt3|IOcRUJZg=_%$ z?5|l|(ltY%zWA{d{C?u_{kGMgH*0ZZVcZAl7#$aV+<`>7Hg3r*7ezi}^3ST_+AWR! zZ5RPmTN6RG^&p7htZ(=&Hl9>l8Pn6#*Z(B*6Y>>*emzbmR%TM5h?&I2za2~g*;*~yB=SriW+Me2O?v`Rkc zW6+-hrZ&c`g?4gud|zZ)7hT7K<_HkqWZc@rg{oWCU68|^J5=krO>P0%5Y25=UaB+3 z+dA7Vu2-80=o2KZA3}OyhZRiVw@`@i&%vlqV4R_G;xh0bM*8|ELS3?eKF{Og=*vDYm z^kDcA?{ZkN!1G|>b)?F)^{pQeT)DrLl$6F*Rn@x6=FfhlX9Qbr>M$N|Y--*u0*FS1 zjlUwrweef3>B5b&f@Xb_9)iM|wD$ntP=@`3a4HzaHSpkPnCE5MH2F9=%Yl~|eU+KI zjd72UkMDU<)xEp$eIW~&+zjPf*%M@uVG2LHY3BR-sd_ohs+nc84I%#Mx%%7s;}8Ol zO&-B39rkvZzm85!Zc}&8){Jp0}#R$8yiaqY%qc6y=O13gljy6gwY-dT{URT zs>mB}OngMClzxoKxc&Ti;or*K$^~e66RJMpxQPsC$2jEZkP3{WEYcAvug?gHUhF6B zvZtU-+Q2wVkdu|~BLCp_ICx($*GrkxI^Z*tSlojtH)Xb(`c8QQBsZ_W z+%Gc_(JrZCQ!A-{hO>w;D7GnnXu1$h8CnBiv_3)DD?c@irdh%Q9KMC=4jy6>`4?B0 zp>a41a73`&ds~0%mCV4?>Rj*!-`b}SAI^PChT|^-*dPji_(_VM%HXQ0em14wZt3&| zApa;Bgn18+Lj-sO7*Pv9ef;*a8dEXI(x_1qhw1ZN7)x_W7Am6_xOVeO9n^HdEJcWh zjEu}#8qB@g-ripJ39LIGAaW@EaIaE%ypd^?bmQ|J>-T6ddYg(w>#P@ZOP6^T!Wh5B_j(K_);NsX$FIR39~dD9C*9 z%o=BTh;-h)ao5VRQzi0Ze_GuV?i{=aW?ARNP~5q%ks=0r4me zueL(9a`wf`q}i2xLVlzc%e%nCb6oSd^04)s)+%|^jvDqRrW%t+dX{uNw2YjaoeMv! zD54M)irtj(uzH@QJDc~Q+Khs-V>#_qYBNQCR8U{+H{t!GDvy7XwTF|0AOj>{p9MRf zoFxd6HnomRYw4y41)6{C|Gfc*j!)4F{Bl(=26>h@9UYx>Ha0z!>Ck1SjPjHLkby{;O*nHQ#fxkeNl5A-vZt$I0^~B=N5Hggx-U4bKuqJN;i_n z@>#LCUGW{$g(la!kAxk95j@90QVA)J!jAr!yIkw^#fF6}b_xS2=hiximgr&^N16;{B~nSg;zU3?#7yJpO;f`TXiG{H@Bwd?ALF_Ul{WFXR)%;xo2gepVo(5<1ocU zVsdNgl7x_iWKO@mg5$si4CZiuHJadOWkhPDr@;yrY&&3z*O{lEzzCzQvp(Nq{7H&2c&fI0n{ZT!b#7 z3(@q~1#!V#=u8#=R*Zw@PzA+qT$XUvYB5MSP`;dj;G;Z<^65aw9^aqhC(pfNXFbE0 z3rT@Op_-_QimhK*ELK^6PE%3HSLFHixA{<+wlfY+>-|DH#6$!?Rao>fuAn{0e)fd| zS%~=6dy9geNVX7%Z*v}1LZ+=Bx5b2AB8mC4iY4;_4Eu14{ol71>5Qx_yfJv!_BIYg zcC!EF4Zz6#nHU)x#h#bf1sviLeI#WU&nP;LU-yj;Z}+@z!qruD-Wu@r7N0ojr`5O> z1{qX*T6#0thm&aDMeH3+Gl5*2VOy*i)s6*B-*I=}@zqo7n--a zJ0Cd!7Cn>PbOQ2kQXUy~_w-LXJF!`Tx4#du`8f|EnYLN968^>x5UI<}Er#b{X8efM z)=l3L{^k}ih1TN`j8;(g8!Aiw#Cm_NWD?)F-+*gx+*WPc4o?CXR@kJ(!gkG2vpvcD z5vD5wst`{%_#$k7ar>fTQ96k;6rh|7$%Ir-tgB@K2{^?M$c?7s^s{Gf*yPU5AyZ?9 zvmxV?A*u-P2%L~R-VTW>YLsEHF>J-1C0Cnm`dnm6RxIkRzfIV?2x3$mgNK1etJ341 znPWpg+_bs}nEg0xoA^nCLkkP$tkQv=)d~KC_xwliMX^xd(IID$gR0osED8c|6T+jg zH`ytQRmLU}RJq~qmpbsOq^iRooP!`hnXk_@o#hjR<{S$8IKrnumJSsle;8LsMM+u8 z%fk~Q@%&e5UB1S)$yR2vCgrhXDxKuQYW;b1xKN%*EwegpNbd3L%lZS9^B@W(-;Z*{ z^0KnBLfwNWCWsj=oxoKkCA8f0%|(!Y%B_|bFomXuG*bAAF{o-6crm(${$HG-d-RlfU6a0PI&%T6)S`{&6GdXo%i}nGNUX+9HDJ zbuUzy1CnuIL^l!Kh%SLWO}?$Ds;ZjlG^P%WdhHxfl4PN(jL!J_o>$zzd5jm~_FP+X zbfraJNuB`;m`DgSL757kygK&TF#~r@PUt4saw`1c!-pVQxNBi&7pB%#@gGa*$s)NG zd5=ERsHXu#tDa#!=0d^b@lP$=bT>%EQPqP6(1HUm{#ykHbh4(Z%E!KP%*Xljr&3}4 zW()?SO+7n>(I%|n))s5Ic%F5~_~dPt04URl>b=qACMpGhDjA6Pf8a=S5b0O94qUfw zS4A8cc;^iE^Z%%N?|7>F|9{*jWu@$lI>?TYt)y|xvSkypPh_v7l!Vl&V~;YkM`UyC z>@q^uLC8MHUccw-aJ@gj@Ac1hySh%#*W>kgKIZ-Yc)p&(0NFC~l=pXJDL?;2r{X*R zDCM(n11@P$5mu$Tng_a*-k(ran#SM6WEEC=n*lceG2N~m1c+Dx0f8Qne{!pm6eNkm zi3;*v-@}JgYs1`9yYdn))9^Dwz4$ZeZYEe3Wg}N`S*D$gDQF}2%%ZGI9_1avQ3uxw z>%a~Sfgrm(s)CK0TH_8wSOD3REU=XLFxZ1vop5kr;FBkRi$UX>8TLXHmL$`@Mww9+7 zMVj63`=~zyI&v;g&&Fhshb)C-mNEP{N0}@D)WOy@KW%aqo*li)4!-(6-_qJj^VrbC zBxk^J%_WzCV)75?d+(&oWCmCA@LI*_N>(1n-zp3Iogv^bK=71 zd=3h2jDAq2{UbBfv-5c8o#)}596O4aGz+P! z&jGj!1P?>m$02D+6=+6cTXXQ2{2TGAvyg7tnFRSB4}3Y4jezg8uCjp8!K^d{n2C7s zLD1BHoVjf6Bbd3cO!v;br4y`H0sq|WcZ8b6PI$-w24`Ynk+HJ2o>^m>W_BO^3RmlU zaoYo&IxPR{<>Mb*vLHcoomB3&!dGyg0Qt#lQ?p53V^Jy>g`=EKlWEI$F!lwR0|(Z8 zY}4`lb&NEIR9EC2E+(%Y`RS*u#QUV&~za1?TI{_}92*=$Y)p0h~@0Vrv`>S}78uV@24 zfg@@jAl^7z8@2#c^5|~rnxwf4B)b#V)aQCxu{s^o-IzDm9&Eutaf zU%>MgS(JRZUKl(2G|S@y>}+e=y|@DJ!B7=zX-7l)N?km2m0UVKT8oFlMwnboT$q=1l(34Ae! zw{%eTf|Ae(f*;Mo-!oY;D0#h+GuD>gBBtqkQ7Cqk#91h*R-pmWtg1n%`X^-nK57r* zuHgJ*J3Bi`dO7YbV23}5wt(KNe*5M~9!<1~l|nk*45cKEsoj)F2I@y=B!K}p6kW5u!7x+47sH_F6vzFqXYP<{^D zW|5P)oJ-Uv+A;$YdH?b6Wl7Mv_K3%Ky{B!s4ve&|^8%PEuK8g;?s}w-eBfl@P4xr1 zesvFDjASc;KB%XaN#Bt;x<3zZ;lC?D)omXB)E4Zp7|jO`KWz{i8uFT_b&Dz+Q5i@{ zhg-R@tjPZ}F!5lVFB}EhhAam>3T#XAjaDQ$cz_Jlude~Og5kq-WgGbF57TsDXR%`> z-dj4vqN1adRr~%`(l)pOWu6(MD*5|!w7uMee`jXDW1>qMEmEIoJ^t*BDN%yHoRI%u z2Fxzv#PZ`{ZRuRWgr0gT3wm)G=V)k>LCn;*44>;a>ERKl@e5*qiEeu&;PpFrQKk0q z0*(Ub7<(gPaUncOOc;*F2VUJLM1#S!WQ!1D4-z(DG3G$f4M*L+dP$7Rm?{l~jlAVt zL0@NUrGN(y<}ECvLOu~JqYrujT1j`3c!u?IHUTKtA&G#eLyiDP{b>#Q2?9r&4f4$T z3lDw?O3nAH0WeIwYV{^JhMFIo#Qfde-s2K-Xgy(uD#14)XlDNuFdjn}2P`C*t$$&PG|Fn7npyiEV+>}Sv zVOOQY`x0r8`(KKyAJZCq0sFHcina6c{H|f^Y<(ZQhjNxBm23NLH=Rz=Q_$lL05?2~ zp;I}RH z8|VwkcP`%)1k8sjlJVhWbvp%3gfHg92|*3hwUFp2)+7kS2N+NafowjXr?)uHG(8WD zBvhR^h`sdr*I$%J>o4V=EZvXW+@5vKPz34|dbXKl^#y=_Z{Y2SD?J&?uKk-Rz1NaNyd-v|~ z8s+_>hluRkQPxdN`V0A1AB@Np?nO4VTbEyaLi~6p814C>vtCfuAYLC1@sLR;0*urJ z9&Z6^p#TPV%|A^KUx(SKxDH6JJp!q5R2;z#!nzVbY;lH20+;zIJzFA?)VOO!eWpi4 z6mk#i54qtXr;#%uA%TYLK(25Cst^X8%M2eq`kk0DpL3g{YBAsTLptQFJZHj5tfkaUJCoKSPnq=(_hWf}so2AP44(~Y_Ksn9yjOMn9*mF4(a+UUOZwjsJ5 z(#>DIZZ1aMrU=y7l{JAm-Li`l6BjSJ(z2t8c?)E@oDn4O_h-;u3ji*(`URMGhnAzN zb*q-V(Gl;;f{6WOZtBef6Kwr|nzUt%q5B{xvzwXPiXHiaadhTcD?Z;|Gk-b*g{wSJ zBmQXu0U|n(uh05daZE_bbh9?Z#YN(iDOB|=F|)Edv@mfLfM)Yhq~(6qST~Y42(mxA zzCX9HzyKYYRzeYlUOp*6J>QK=wm%xb1(-+H-4nOm-&!7#ym7-b5EQF}CShqu(Rid3 zfYxS2JPY}@VMD}zFUP?)aU*Qk{_RmjO#8uA#?Sbn4Qu|@lXtE) z*=Tt9(Cc2TL}z1JvB&EV&4m1*aS<0p^}`A^bi2<;lANMQuk$M3}s{9`3_$+DqCe;w;35 zQ8mm=U#gOX{-|OIeL&4a>@1*AE7^b*6|Q?_Kc_29Q(tp&6m%DCC-UHXJZEQN9=PkR zu~N|CbCD=fM9?;LNFUV;@N1Hds)Jal1fu!(^TzHU9fXOxGvW%@shX!v2&4GRfh9_v zcq!EC1=Hq}i~{b}Vc8=dU`H>wf{$6H3_RtHI?v(k&8r#1N(a00I^Qak6+;26(A7?e zgd1wqN>AfXsl%Xol(iy@68 zB=zta+90E_O#ORI@(tG7e`GmSqSMOe8p|i~v8eEZWke z-3fB3a}!gjX#P8ntzs>pmLKBYPli5hN6uGPpqClHJ=Yg2a3$;}mI-N)1~JDTy5tJr zC!(TjqeDYOoH)?5fN%^5%}03*I79PN;F_P9*DvKUZdw*`?NB!NMB*B5dt@M`u|Z;k z=Ws3Etv{h6K;T4-8sBXVFjD4ENz?F7=Q+NiNFoW~7{M%F)0W$M?96==YN780hq@4Ikvy`VfOMoHLmX_|QPV?#8hGZ!%_H`2z*HDKh(tv2g^~S9#3}vzbC@L8pMmd7nYe==t^lowj$(~cOjo#rq0mKG#xf@3M_$$ zv)|gEH4Y9I{uv7<9WJ-EQx>ge?AFzRygJ0t*1;_;)eV{TTFo&lhC}-NBz7r<&9q zncBs%`v9Jvz48d>3!07F+ls@BvWpz^d)*Fz6S`MjtGzfmcmw>83C+2^Zh6qxY?v;N z*0~+-Nytig)@A}LyuH2ZSCF_!PEJl^GGTa(1Gj=pm&^iK>9dc_L>*~!8S|e9UxkDu zd-_uZfwuW}pvRp=b?kZ0-H3ow-1Syg$TvGBP+3A|Wn$ELKCH??;h1#8nsjv(vq_dTmEKfZ-6=oPQw4#pW&jU7d#Bux;q zhl_R-1QO*kO!OrIFqBQH95^PgWq6(O=>ZO_8jG(W%$Lq*49GUk(L8zw?% zohiBG2&;1_9tMg8<92BqY?UYibq)eOL8%c78U}!)1OnVHcB+&~X$Op$f+vD0!`jQV ze)=fb#Qna1(*RL4iOcW}V~Oq)KkS3SSJe9cabe5C7AF#ce)_~~F79-6Z-2~~H%7q% zf!KL=aRCx7J1dvV50}N&IY3R6X}6bY7bna(xauU%5*Q6@YXvLarI<9ai#u=SXNH0$ zyNI)FzoJ!mmthw!IzHqmnGEmy$I&Z(7y`UR?co)m5|zGx*WoBXg_a%P=D1bxU!haJ zd;8+*+ZP54f@zP$)%)hbZ_Oz_Z^$3sP5tRrGVBo2+ZA1Pg?NQ3GaDe=2xF)|0 z9tx&rkF3DzGA&+Kba#&|;xeFdQjnt$BksOnniioscTVno#=JW4rpAc7R+>e#_7szv zz2b@BeE>ogkXe2NrZ8Hp?Bc+?&ZlBT)(;x>`enD@B{z+C=;69Y+aEL3Qh8B98KX1D z@9~ACe5lp>fOX*pr2RY~cJu+0?gTdby0rA}7Dyp|zy>tK*L1;QJLGmVlSq5kM8273~}|IOB@_^7GQ4rvU*>< zRd^1D9<)t2Ksq&h6zl~2ZVk|-(Lf>)KLahEJ_VyDt5jS|-NlZ952x=mZ~x2V3f=%E zw<2yG^sit(jh&7_T`n&N~BCu78_i;wXN$ z<_0#b_lY=rgy=UOUS1c#Hq;hU6bgSYUecYOo~|pZt{&Y@N}}dkQbGMuF!zFF1YE#Z z6m0ha3ibh03VyU|Y-m`KyOUHa0!PgVb|^3_Nx4D41LTmCVU)jkFfA~=d;piQf?XCN z50HnWZV&<^ns%`YXAF;sIBaNdSGf{=O*^>2_syb+vpCBtp)d_sg1n^U);+))h+MjK z>985B_L8>YHgG{I1HNB=Pg8&HfIw@?W>xJ4c2^#>)M!wi){TN0hBI7pRDRSfG9TxZN_86t#&Y9?g^$Y1(Yak z!Nly*)9|(Q)YR0v80%rsLz%I#vqKC{I-~)>1F3?16PP*X9H>e$360WEgo~QW$;j+= z7)HsnD(%?BWhD0$JANuZN&pX5B-H@0bvAea2<@gafWQ*K)Dg`>t|Hpf7q;jLka<6n z`@jXZQd$IFy?PapC}(aj8Q*nGngG>N-BZ2?;@8eDLAu~-*~tbBfG2SQv4mhP7H)`8HULk#bmX>icq@%hFHs6mQ5P-pD=&oa~ z?k@dE`rDZ&NAHnDLqoW)T6^4InaQ4i1D?LS z&k68atmaPzuY_TZ+N-x+Yi~az;DpL=)>r>Mu+Rqmo$yt?mZI*zas``bywJt_u4B9U|D!f7JB5Sp?TGQgEy_n8VRPXBJ+so}bN9qa$lg1f1qn?_!DX#9L=S4g4=m7*=I zgFjl%+5~HcShD{=zcs;F=>&e)$kg5MBV<5Uy_s`)L#wfwaTehn+}2%F#rFWEqX1Rf zFnDiU9ASQ*R|JIUb0Z6XQ3kZIRWIncu>9)EX@Ufu{7+CYhC%He@1dK4lr?bZQxGo` zdwIhXe30U$#f~H6_IsEvp1}g6IK?t<0y*}iikHe?yF2~9?ajL5xAyPjtRBPi^dJL= zCb_|Uu?AeL?LU1k5}LnVXlUnKH(z@Y?}0?eaFh_^TBbfFe9$V1TqU`m05GEyN*$Ql zqWDuoKv@n5jJ+05&S?>;yAY7iz9+eH3r1J{K3 z`H8s+d@U_3u0ah7L0g4_+0pNA-y|{B>pdhxdSm79RdFgAAjg2uVn6WuOq1@|uSsl2 zJ^TX8Vgcwl^cNDd=ckHt^>BlzWZ{5*$>nz;sbv>63qr5?RmTfsDllaiG$BZi6%Y16 z63de;rB@fO%dB5>-7{cpF0DY{Rz+-{886!?g%HgXf*I^__GQG48Q=t@a_8iHbHi8w z8)<{fm>v_)?aB#f9v#+wi5D2mX6Ggo|HeFy-iCkIWC}bEHLfO@5VUzP{3=7JlXgy7CzO z+I`yj)1B=+Y@*&WJ>><^uRC@&mfow%e?M{Kg4r}Q|A#%bvay&eE(I%F?-Kd2a4ajY z>(feSXKU9bKz=VOC^&bNL#u%}aI~d}Bz{)LH5-pfPV8h9#0+VqLB{!gl z)5zI^Lu7zKtmI^6qhj5=o?^8y2WSIM*k3rRKkr_U?O)+sKWeYPk87{KfEvqtNk-D8 zD;uXf3T9GY*O9X;G#4DItZni-5waU1%PDBsCfz(QfQT#8TnS{HJ(ZE!1{qD9nfzxk z@s7gR^^dSz+uwp|!C;d7 z)#}=hgQ5n;kUCHJI-n=9QSw~GI)oW|V~?~AOAgsh=cZH^HRSv*C! zl9rdnJlx&6e9%bLg{SdYK&9D)n6@-)Bdja;E1cWx(uX~VBQ^!p2`@A_$m6LbGvS)eehySvhQAMS)r z^}be7d?fLIG8xgwkdON(EIJdhUv--#El-UxVrDewYA}xlP>pU60EHoA70V+^o*Z%$ z&|ar+R2|!80_e_pY!E?*^|%>INB$(dYYp2V2jBIi$)P%sPgRomVlkj zgJ-^h$Rp>x&u%%qUEj@`TI&{L>@v^Ycxtd+1j`CM4U+91yt|Xq1&Zx+naEZoZ-=qC z-0_2C)s$6l?yS`v#xh8gS&rb`%wrC|+;bS#s}-cxh_T8@&(6ji0fXBp9XvJ(oaSOi z+wHbS6Ox2mZ5A_^jVTi9j!H?!b!J&mX$uf63l_@0Mw|d=znni9t8ci~-M(4H&lq+O zyv+ZU$FIg1ImouP2OW)aZCi9#`XQKqX>_B7A_4`f2wi9z9xS8tQiO=yYZAcY{4x5> zgC2ScwAkt9*6A?BonZ&?6lhQj{4QIGY_xI1Q!hz%oWT@uTI9Bb0J<4xpVUt5AvLN$ zW!NFH#BK*%-Zoe}*qZ`qpqTejb-QLHW)A630L`9nUOZK|uFZGbyS4dV3Z!f#>{q=L z0nQs!pGjO2Gy*CGuz_uha1=kD00GZ#H~F)S1+2LCAM(`_RnyF`U)SFk9pKo(a0G|p za{*OT%~O64XigW<((gmivdA#L3Q|8sM{;383WV?6rsEhGr|GiJ=YwZM(=gCRfIG*a z@fP17Ba!~QZFc!%jK4@fr|#PQsUiS4Hg}fG>8@sr6DN=zoHb|OQH#An15%@f2`vk! zR%afGDThT#r#m-%;52~g+Ese_T|v+k>a0U`U7f&QY5l}7cSC8YD<{#QxFz^N|K9X~H{%^<#_#D6cqf(bEG)#S{RaK9rMD(Ij zy@)(TW^_p{EQ{#WyZP1M_j~}<8nv-ji~85n-4FYBC&OARs_FH3BHXz$!I{X@2&K#Y zt}GysB}UkE(Z|_3_^v@!Pv&g}nvE~N0Tpj>@?i7RMYF3Zr6S~V4`guaDii{Y8`3e; z&3&oeIZI40Oz7CjUU%n#?A!&}{`|4+YIh?nXJJ3G&ZVdJd!Y-Zu%Iw!ae?*=qs#W^ zQ=jPczbkJwyBzlf#xkAs>F3p_X6LEw`@cLjJ5%i0Q`pW)ahdY@m!+@nJs7o#366=% zUx^txRh9cXV~@rQ=CU#uJjP_UpKagt^xyNXy@%=-Zq`~($^2vA>sYKUAfHrhX3h12 zV_=n|hHgvk8aZ5x_0-&cr{C4lJ}M57BU++adT(xwj8Mh4Gni@r%jl6324+$aOFZ4I?mM)By z{+Bm;p(=Eh=vu#0U3&v%Ijh0hEua-=JO&Gc+lN?Yc}PDPRT3OYa-V2Fr-!CCTF^IK zPu)8Vu)_Tq?!JZUS2)$S#}~qFf{sQ&sSX4jy zw6a)!KrEax&RhJL#DIuRCgdrhaGNFEyCbS$7o%sX1=j1w&p?1!leJ97_Wh4McgW)Q z^%bKEW@>^hvhTLFBnjsvY5lLaVcV|z*TAw4PuI;Fcx_U}dsZNjc(h5dp}o4lKrFd_ zR))Ui>Ym<;35_B*+yt8M03QGG5fu}0{*!euW9ORj;_Zznq>IU~FFz9Ad#Bf;M@E6? zRi1_hzEWL`J+_voCz61Mj9u)*EmhedGefgpd8;t2rKP^B*gl{{2T!dty@X9FI4jyN zvVS$w#r?-m3P}!A#e}DI?Mh^+n7F-3`gMbRVfe!NQ4g6kHu2~p7~JkOOlV~j8iiV8yBtY{t;o=5@jczy5p0TKHx=@83y*B) zR~LLB1LaUzcVOXWA|1#-J!S96IO(X-`X< z(m-v|!dm03zrb9xFt+)wlp8P$CVOB`PcKlbi-N{#<-a}Ye@)=(RY?!NOP&1lH;?R{ zx^`)BVu)pK+h5?im(m-sEVLxdJ#v*uF}8m9_{@QG<%heIZ+rsVlDm9t=>J>`q|eRS zu0PPoe##AZv$Dp&DBnx26ctwFSIudBsT_SdMw>pI)WTZj9oN46)f9G3%xbtGyY zz)14U8%dik=u0aCW9u!yW6<`00w-HYGv16q~cP26x&8Nh9l+k9`3jw z$~a|>e}t8Gx3?dynPcfTN2+<|mW*UF=^3K)%X~xfrN7G@9-_e(QA%9?+hM!AXOU98 zzPk_LswVC^umkk{#Al7U0eMlh!xmbeJsgN!bZjhU+VQDL2!di*=vooPcr1kk7px1_7{R{xz*?yO8=~* z?eOkkCzE?&@P+`VjEz@De9N=Cu3a`nRBerlTqQp^6JfrsoI<&M#=t_FaRj*-TCke( zX6`b`zTIEwghIb{KTVKOw&s@ZWX(||6KQrjy>1IzxCw%XMf2(5J=&ew`+1;O<~@3 z!p9MD3O!n`8ATqZ#b@ShL+>%v?TpvI`DqlrT%b;6P|suZiTFt)JwYX z+rJ`49dSnY0#R%;Mm;@J{%c&;zuZV75u$DqVIoOA*-KDR7*eqe^>{;tcojN6*ZMd@ zA?C*{aPg-*byH6n1oC0dIh2Szl=W`5hR{~Ih+O_n zH>&)v_CTReBxv~MlZONk=?uj$1Tqyl?4>Twjlvx@Hb}86470zo8h zmUcY$O{@LUKy|%)2CZ~;G{oUTK^^qr@_E*Xk*f#cL)X?r`w~Uq=SynORpUB)E2fpr z4Z=5s%=?1Zc*a-A6{rQ1W@T)nO*fvTl(nmW2;c5WA7mp)Fsf_OaT}tjj$QWDPf}a3 z#){bNr&rUvOgEihxRk+^I$({JNIAbXqf8;CpC?Q#a&UY8fHgyEbr<8Vc6itxWJaJF zHyhU))qPC4Hu$%Uo@@WH)i|k~Q=F)g3E9!*%O86dw?*P+4~7xU01W6gFE0;Jok5-J zW_~#_5dAp$17gW8qP}%u@X`c9mFN0z(2k1t6|Q@yyRBvxE%`^DnI5G~(9~&4yr=UO zovav=<(VuHf)*i=9OQ5z4!CpR?i@Zu`vn)=>(COEOstsQGkMwaM$I~#f>_sF9Di=- zYwnsE2N%vL(*vZzBSll6l~!fh?C}5#h!$Qgi(rSXcW8NrHM&bg?{e>RI7)$D` zhW0)lKVgy258prXI!Bjran8S}&7g)>&jvG)zM&}5T(L@g6h^Vb6QIr+wI5h1{%aMP zuHM2Qg5K_59mHWgNXd!xKXxCkT-b=`&jqSI6HUCA^}MG7kC-h4M)C$s`1&-6Chzy& zHHe9H_mbWs8}h%JVd;1SE}4|wRbfzz0UPGv#Swb13$R6 zTg;%5DG9;I^>s<{o7b`^US?G z$2;>+1v*%H&Ms#ZgFC`fCOWKMkCpPS6q=ZB#k+>y8K$%xmR0s5&m)cUq#N+fOBji0 z3FYtI{vp?t_=Mwfxx$+-DaIQAU_t^)^(K#)Rx^bX+#KfDbYu$|^@t`ut*y?;W9Cba zq@n6jTl$e~Wsx6F ze7tB36T*Jpi?EOp2TL)*&=&bl24FfenH@xn+;#+FeLQ!H^=i#nA4%D&f>$3@h>sCz zx~s)uKPdr3)XK5n+z->DmZ2^U_tIYL{kL5DsF;3KmZf7P@_S189aR6V8g@N}#9vWA zH2!&EeZ{jU0?sDrb1Ww9IaMExeh8ZD=O|lmT(6ziB*a=PH`|KN_JkrgzwK>;aI}9p zMJs;*$oytrGgj;Ff$ze%LRSIY3|!$z0c)rHspj2Ifbki>k~S%3+`yg#=aGLnZC%r- zx?hhZy8HOXBbum@hs#oTujI;oq_A#{PXGCuh$L`H^#&Jcda}+U&PUX~IKK1mW^8t4{w24VznDM zC}wP27n=iKCld%4D4O2g=!$6%+MKxN{-EOdme8Qf09u)J&1IoAlfdk9_I=P+q*Zub zD3lG|*R?@duI>1MtN&2lPP7$?>1!2ZH1F=66@_bsoU=4ATzTmI^z2+&YD$q@$fRAj ziSofa=Q7_Xj{k~3Zt1&vggwQss8*Xsx&CZF!53=W zhMvZ`nca7&Qq074^-d4E|BV~N`lZCR>& z)ksZ3>IB?@dU|W1OtVzrt<*B>vhq+0!GatP#n7FZ0&Omc0oYTGXwpS5n$NMZ6JH}? zBq*Z~L=@z@%+Mwn$&K5)r^DPkIbWfp3Cn(QC|=k}-t!;kp9)#Mg}LMu)3}Wk+HB=L z&szO4XA>mL!MQ@!_}Oc*BSHF6Zzshe9Yj>X7@!?qP-x}=%PwpbC9mnOSEwc}LE+WZ|62KL75*cahdbpjU zB`8-}UseYQjL%)HBi0lYo{xFB^Uzl}Xu@lN%003BY}0 zz#x0&h6^YEpo>AaOJy=)f8T^%Kc zV-HE%TYna)y9-RZukZ^_1`VLM=~br#4fcjKFb|sst+|RewzPu^N_|t@5I1m-S>jbM zsrZfi>Pm3IwI5C+Gywy*DfMub?OpEXotG^#|gww{vB7n ziWMRp5Bbt0`+Df#pRGM6Gu~`wt|L^r!{Hj0nyJvvRhIaS+;qC~+1 zvLVi|`L~4zGUkMGe%+~5d`nT^8oJP&HS0yQw(87Z_4Du9J%^OfyUHOyhA5s?uC8i* zDBIc(Drrmm4%1iiS`M zW%m$Ry0Byc{yH&~DDnoOZ5qq7D{|HpwKVVQRiiM0QAtONDT)UVv+{4yOLuaFVgB91 zlts0WWuej}Xb=DOSARFYna1&qgwlm!w)g9C*WVBB z>&+Hg<~DfW@BMU&zL%!l{g;~zk38a=Df;=OsA>NIuEET;htdDmra`U9&u)ur%AVzd zzi2pt1H}`qn_Aw$qmeGJtb&>ecK=vSoHf4Z=xP|6USu&VPcV%O{(pZyjU%Xr*b3P21w~?v@l50(zqU!3IWCu(J(e@a&S=*v>Y}f`tA8ier@&Gf#iS{+E z36&)yKDg5m(WG_?1a0Bv zHMU1lH}>>|N3|wV&)^H5r|4x?KXNf>M4Kw6)%PbofmpfJi@d(-$1z@AqFJ-ov^P}z z1qcamyH@m&Myz>+oT`>6A$I*`yN4xk$S=rxMo+#g_ebY7FVqAW3Ob-u@% zr<;$x9Fi4(EEdSxd8Xu>T9tMD-M!7ivT^C$FwmARfdpxn80_yZWt{}dK`kAQ}2 z?L5vYJpjF7Be^AloX~nY(Vkl*u=9D*VvrxyU5BTvig;`auT^iMm_vA#J6&$lWXHX~ zN$MW2W=G1jP^ka5KZ9(P$~fL@6o72&=OjCErB^V77-zf8{>n89fsMV-s`cD__+ww7 z<9=+9xr6b_CcJ0Cf&IMMIQ*}HjZDl^AEzStV&V_tlIVM0Jqa54g8u1-g>G^3Gii$u z2Kuc@C65h76B_+R)uN^4Y_2RB&KSe8CQjj9o@}~{;qXQk7J2s8vWDibUWW`=75-7W z>ioY(u~cXx19t!gj{*>e?p>OFQW_LsJFg zIz@&ZmFH_V)_iiQ*$%2&b;ZRe46782f!b66+AzzE+m2tLfaq?P7hgGy??2}GGua<; zk5P?upv{i+<BGd zPh;P$e;?)EHmPFDlsBi9kaEjIS2mwU-j@QWrY%nl0fua)tMcHb5-0=y)velBP<%~I z(9jbIvWY($34@;(S84|RPn^V_?sySJKosJyY*6wvk1VQR|5OfIiy3u{P!?IIRqWQ! z&Bl$dUxL-_m;?Y-ia%cs!ab)_-N%lSVYXW{y?YkDGc)d2FzlEszps7!Kh}>psOCP$ zRR&cIvnE^cFRH;+6ToTtpT=aJe>rNG^nMYD8I+yN*N~IMNMK~Z!~aLc+G8%kMhSRB zy!&rAu?csY8)X-%wggEG0cYAf9eA_5_)s_2kvQR@lwiok%|?%ZIfCH?Ap`)YrUIq^ ziM98d`(DE#&Xd1^_GEb;uozs~Qjtk3%3yr$#3$40!HpMa%-{3nm)C}@uQZ)^i+-;9 z-=Ioby&kb7k$yVMwUBdCpO3txUhHc{$cJuXG@pLBboz+_&<+jRW12C)bcmgUUm(W_ zUi}_{bRLYZZAGWC&mDh7sWH>PgH@Z<%2(Sfr3%xvlNn+i?5a6v7X^(4(DO_6}v&;2@P4upAM^# z=~I=TV^WpC+6-|Fv2w0>kx*q=1-*n(Uub1L@xR*VK3geVTW=vBxi(^7oZ-7VLAg+UQmQU0P z_f8cRJ`~I|e;&6>hqT#jL^JfY0VE!q zwv+OLOtr|$v&w_0?kV4BoYzt;Q`Q8B(IguT{uO@&2nK(VPIkahE;@jeULs(76lkmq za^6LZ4V3HU|68L&1sG{6g6JBJ`Ip}TXab|xm|hUM`d^%Qq#Z);!ces2R!(PjK`sds z*;^6@J$X8KZ|A9*W3MVTrv+I;<1y6WmDI2l zQkU}pSp>PVnLf*A^Aif^Pe3`!2OmxC!@v}8b|z;Q%^&ZTW6j=dLPo+7Evj;9k1U>*2GQ7#c zdi?JT?>X0U$(bQHaiS%vCGAaR7(5Se4^9^4oN$v&vT(xaOyF&#O=1Y-+3SBZj&~up zwcr^805I_nbg}{;Up7f6%O(7l#)1%({AH-8xD*9wgB)Z86diot;P*An^qsDJ5PKP$ z)JI5bRXJwdO*~4`mVjT}#D9XTpZXOg3RR>c8#;~L1=ZeHpSpIoUm`C#GTCI`TU<^v zd4w148qT6eIY;sD}&wAAs#1>B=QK!HzhKKJjQHZ5C#{*tNUxIBHe?0oBbO|hagQI zu`BQwmI4SFoLb7G6-w+M72E!rSFZyipn6_tGly+qyvv<(kjT>ta{M!p)7oWtkacrL zStO|mk_sMctUbbHtH2X+RF;N2nxTqG31Du?`B9Ae?D@I8t}nL**dsZG~TlN+Q0`l28lTc*md^2h9xZxd6W@+aQNfll6WZ> zuA`WBK`a4Q3v1_;q*w5TuKCU=1%xAKk#3vNWflxW~Yf!OL1a14nijsYmvm_Xl{ zhpErLSH3BIj!pa6;OhiyOVB1pmPg^IlEg(ngh~yP*HMh_3UlT2(qf3L;TA}ozGoO#W+Ty5Mo8L zB(3@I?4Q6hy;F1iC4(-L1Mc*L@!ksFeCugZJ4*1RJ_MYLpg_R zgcfI+GWbUpX$>mc3Ho>=$K$4VbvE}GCq7$b!`#5FFRS5AncuowvsaABm094WLv0ss@ND=yuh6(sV5s%@F1*4s6oeB#Y zJtz_~!OdPp|H~Z2E_aDRTL699BvJVVd08C~(kFEUiTsqU!l-CX%3z;h`!GhY7uw|o z5+C~ZBh@kJEV61wkIUK1Z9WBN))v8kD=HiY=-rj9#?wpq*!fGy)2M$jpWh2cc?j{{q39P>No6t)zl(;#!sD#a@tax1vtB?X=aA!9g zX)M9GPPg0ZPp%RVA68hP+*ZhrDn5JwxI$&rm*Rb~BEbW43_H5}$%-D)#uKx;_V_4F z0dO)tt0pp4C-32$$`2}v{ux&kuP49e2N)_ND^mNl^&~ItfG4kxOln0I zT}Rzu8i?F87>mu;nn$|eUG}0XCv#!$_;E}UI-P(d@F~fbty|W^JxI-G_TYO)7PfR; zGd5T`A5TLss&Y0abqQ?L&@JCvhlTq4P=jNQ;nLMqM*X{4v-1J~kLkK3=tglG{;>A; z!4HV9WblRmkEr*Kr~3W>#~sJW&dx5g?7btBtn9r<$etOQ2W3XoG0MuwjDzfTLX=q~ zWF3T%ot5>yp6B)c{C zP_JOaqVtXO{{F;`qdpT^67^9mF=BKdDW)`3$XTI9GUz$}9U9isOyjL*4PxCP#@teg zoZ@S}&j#n`K3|aqr8e_|$-Jl5hi(`;?EZ$ZMA63sd5oWym7dvW#KdEJFo1gsN`~_u zTwR?~s4$9oPeYE2yxZBXX!&x9lq$BvCq|BTco4AXgEqE!W-ehUoix2P(WLm$iOJRT z5l)w{mfG(8`;RfC7&o*wSO`}M3=ev?j9_M_^heAbH z+&;b&K!v3h^~C?H1$d0D)FJ3jMCibr$Nr6igLZyx;aA>aQiZXeegRd0#~R?`ho9IAw(||HYla&7JmAu6GU=Sj&woTQH7vPz^>5mwv4w{Res^6rXd zAne+DCm0@OB~)GrRp{zp2H)E8VJ{7(BLEYwzDoUO(^Bp1+i(7Y635o@xZJeha#&q2 zmG^J#W41yVJK7OY>I{`L=|==JfZfH$2aZzU4aDIA!_$EY;*)zW9k6P}c!2$&X$K^p znx$Yx#;SP>D-f+9qF8jXD~fSY(LLd})vO;&EkYRclU%}Wc?zPoUrAb3{5HvI`B!s= zZ{p$kX9juz47IGAcXu^w|CHyL9MhLhor8C>JWU0#NEKLqskt3&W>{+_z2q-aEgw`osDHsVI8x&QXX-)X~J+9H*I)`M6K zav(bXajzz(J?}ZrJ_3svvAfp#kQh^lzNGRB+O7_eO)H|yz8hojh7bbrC1cR84cT+r z^8L*Q4LG6F33X$6(&~{!4fD){0N_HQxbpptwK_*>r~))xO;3DQ9S5`Hd+6{w<_cBA zahul4;4iEkgERxtqeuz6qF%uQJK^l5^7znbAJ5~G<5$5hVkd=LW3 zh?ybE1lGqZv3x{H7?MOyfvvlIg}lsnIct(LrX9yhFva1%B;PppX3+z2vt4Wo%Y0om zDyS@=_!;3aoINCL;saxK|BuxDSHsu7;N^mcEV(*B$fbVn@E<8>vgt{8que5%Lr(&^Bqe3n;pr=@bN5k6!+?nIIB++ z#VyBeSMDTsJdYl&rn*k}0|LfKD8`kwp0j|oDn8h1qt~0VCV#C&qbuSa7bxC%4~JqN z{hX-weIYqIxDPa^u!-Kw~$h*ANYi2ry7{OGH?hkx?#_+jCkUA+;uPu(t`#2 zl#|So@kQRX#Q>@*cq4qS1&9Q^;+^Y6;OIKSpxZmFm_&4CN!?5WzTiqZ{#&evP;eWS>5T2bX#dcI}UjLke715}W>7q=`3=z(;uI2F`DI6Rbtm1Eu!8CKch^ z+Yj-AuGV}@{+}H5I9BSa(sD`skbi>_sQuEXpDqc%ZPqYBn&pep_QsG^AkP<;~RPbRfpvNKx zxm@b5B;5eE%5Hn06)BaL2Y|Z)PZP|*EysSH)Wxx;Z$$dJ*qI%hW_NxFm;MX@ZwB1L z8;E#DSid;@L=L!9Vraz%-{T58t>>{#P>myPZj>K)68#zd^J|pK+xOH+84qcK0Rp2T z5w@|_P4t`i{5@J=$7xPZQkXmD72z{?92$3ca7{+#Ln~kpQJ%Ld9|(_x3o9FD3syd1 zPZNIx#nttsK(Kr~P$!49$v$UJ>2KCWQpdS|+y}eBZ1*Ag{$8)|YNF_5x>NaWFHrX_ zUj?Lx(R_%`m7UKKopC_(3IG_>nx#VbgcT7p>`Vx_nGLiyP9bIQ=9L#fGOL_?*ay{- z+d=DBKOu*Zg5_P0>Q|b#GfEQ=k+7BUp69o2WepPCt^3W1=|k5nha)GeYyU`MoMs;0 zPm{%7p>-Hc4|XaNJ3=oRJ$#vo^*)`e<53m}k_cR(vWQUsFN1uNr?bMKk?%g(mik&F zrEuB$^7Yfi19@=v)$0sTeK9+v9UX+sq~2_nuZODa>wvpH#ib~NCPWRI5RL6TFp&;u z8#~=w>aSZF^~jpJzXfgl=(Y%~U<(y4mrT(X59kDi>E^Cm!SfRG%8KiG{9FSEJL*0!{lS*I)nn}O z>^{!vcM74>`Biil`X#~x|sWmXN}V6==of81%ySTrpwL5Pv$vUFlB~gt2+? z!{!6#f6=}P#MR+2&z8^oJ$pR)%I`O2AWRAa$X2(}hawr?DXNJOo{)Q+f8f^o=ByQ_~nkBeH z*2HjZR1Oo9uJNCAz4CF=OTBM?cRLH?8R`j=Cqs^}cQ zAX2oh+d1!>W)|uA-rAkSM}I&5xzEw#`CHBqgs=6${fgX3PIW7OxB}@JZ4J}<8i2AI zLL>X>3X)TP200ThX6X-t<;VNpZG;-I#jjIhh;%#efFPJ))uai3% zS6WuLIREMWocBF`hs6OWNN;eG(E;D8FLf-o==?1eBtq;?Zq6tYpKu@hm;3PS z_MxrNvKSeUHtmwIq)FNtaL6r46|somg+2of>WpjOOB2T^Tyei{w^8-nTfeCWQ=dlt{vO;!=yg8CZK%%oCCIUDY9Wn z@C;mvZM4Mn8qVTTOa`<<;E8h&R^aFW$GDfqX^dl`ekSHZBUl-WY%FGO#2>-gBG|&$ z-rA`*=#QL6=s=(Zdd7`)#|9Wf3hfAZDzb!HF@rS^!9j zJ3WecF)HiLd3#SqHw*3-8EYh- z(kz_=h(p!?hBP3L;a?;hdjl=Jx+DFB*O#P^ZxT_yaOP~^KW0JPH5G>}Xam070j(Uh zv#p&EkTU{R3K4^F12ufc zaq4T=TVY;hPpH!Vjk+ODD{@s*ZU&r(-vDk!v$!sma}D9DQoo>94(r2Q!6+SclNdaI zgLcA*v!9*b)N?su#}FE6m*aP%HRF!PES!&jenN-598pWd45=iNl{pepMTq=e8G6(B zy^*Vwc0Y22(xxt=G)ORwQ)2%vru-E=xobo7%?5Lp@1Gb}_0Ty`wtXBtuI;Qb!V#V) z?M_#1rF76K_hf(Ny`mRIv5u*k86xaxHUp@_Mz_h-Z@bU7Je~=d_At^Z%%FL*mJK4k z`e`K4Qa*=bOc$MvHR}JJ_ZI85Pd=|AOuhji-P?;}8jKX^as0x+b00Laj^)=!ig002 z_Hp+$1CS7zq&$D6TX>c|Vf?~iY>eq@^BJ?~zA}(5=OVU4Y|ryIDkKC zNG&>J&!|3Jxf0gKp<(v)ih1%-iXgt(qq)a;(jDvSYWj-n4VMd=X|FVZYC-|%rF#2Y zf|)Cv8wa9z6&kVhMU0K`DtI!tWN!$Go>j9mlKQnRUJ%}_1kZozyQ1(9ILD%3C(T#n zBy*~d^~Wv$EUREkbR7~%bePz*7A0t-kw*Sr4rV~rLPA1oJ(~$os>>)KBJ*zu7o!O- z87YBKz&aU_vw6Zy&0<0->V6YVB@W{z`IOxqkT(^mP;$6A_oc zH;yL6HY%VwSM4h%Pc=z$efvEZ=4EqwY1J7q2qaA6Pkf9Q!ZE~meX)7FipT^Io$3@9A_JFSmfMtM2`nc|TM6Xc1U%qhvbYQQZK5u=Ce}JV!QJ z1D{}l9TFUFqU$+bu0#Cz$&fOJPaJd~`^C!2LuZVhmch9+c_I4Z&BDQZ($`Y|jI7&n zw{oXxLV9H}BLH9NZ~&EScEv6E`*y_)5`5$fC8WwS`0+Z3&nS0z0i#oIuU2gndLvqy zP4NbNJvZhg?OpCCh29b$UYiI<4*f!B^>{AG?4IQP=-qhFB`H)37%Uegz^v0I0<&>J zCn8cH7L}G(3bTkDcqdzCZAOH;8B3w{c&lv^#Z$p{7}DAAB#57kH+-YR+6LeF!-DMe z`UF&_Km|eagQ$BwqucDIXIhY-yOqO;BAApPSGU+eB5`EoAJb^(It?z)B%0@_k&`%m zrLLo&xmVd6MD|T#g$naR($e`_5HsonnS*Nmw7j`itgDim?f|kv* z0qPd^m?#E&b7?K(# z4|4rrdgno`VCV3&^K08>9=A*ZL5K=cB_CICp`{Q!fW??$d-=;%Wu$w2EGP4xu8dFF zv73}C&QVBWLAkH>Im+eA>${Q!5gj;Hgw|xC!Cy$zla5N|;?qS-;3ZeXl-b_c&VA?eDKBeOHCP<= zbRco+hd|kT(|`3N&7q64D0`aWLHnb1hlG^XoeM7SG18T96vJ?Gu8xa5&I@7Lmt?*H zdEWBMc%7#unc19VqQD91OPLXek_5@`&_kg6-vnjPNPBvBf9M`6jd`w>HDJi0t$UqK zp*nP!`*`V-w?xga5bL$F>5MK0v8*7r<{Th~N@DQAUX3;n-E2iXEnb2WC|{g-)Zq{T z`7QturwXtwOUSm$+KnkS(=4{Avm}W45LR?(xkeS;h8Ne4l_Q5xm)AlS6OBktwM18# z#7q^JrjAlP(BL!J1v$#|yLt0n8v(F6Guew<-cnXy8+}1>J5Z4fVkqI$D`F5u<4iijfU}CHgO+;H=1~nSG#59kY zp1+VZu@2^fPUeaLH?fe*@7&2f8Fbzo&FFYF8<^J=WtG;F2)+z0Di^@-cnF99q$?W|1kJm^q%VC>9HAR;UMMt z)q&3@TE5|ALE8Oa=yLa7m3V(uBHZnj>PQ7AJ8nj4M9PmeU;NvUSxOR?9uDjn zOLkGy2lvBhq?Lb2DpfVbKaa$Q}&Q|{Vi7Q-{7jQi`AlVu#U7{{0RPLA`Iq|PJ1 z{=x0}z;{UK@DSaX9a!bHt3ZI$Z zXTb=MCvKNmTJSorAak5AElq@=Wn46gw)avKZ_^#h?6^dME9~rdVF$c%1=nkxc|3f> zBSK%Mr4NgL`4XhamLl(~?dv`bb1nA-L|JO7re?cEWbbK{fdKK-+`F#o=VTH zwM0zBI?!elHCld#DxcgC-CWS`FhdieIp^Q=Ir$VQYYCsYp=)^T z>1LWKr~?8D9-0PAeWlwc`H6ZoQN=Q0703b(UFGKBaOhmk5VY#2HL?8ojEw0fFYhA2 zNp@4Ujz#a@J*(iPOwBJNvs6*Z&d=9ZQK8CWIwm=8mlg6wDn{5pZhzsrGVm(lJ)w5r zho?4)3^G%U)T+R)xX(Rd!vx{^zu4ts;djHPTf6jr9q%q>DMly&6LZW`;zIBCv5gd| zzf{u@CHYuw)0r#Oam>yPhV?zOOM{X!UOYl~ilGS)AIZ+179`JRJ z)%Kb%8;eAKO-1TAoUzT%jyrYrAN<-M`q5sxQ zb5H~r(ZbF9Dn-3C#tCt1oW<}wyN;p*n1X);GbLqeoE+XGZ3zqX+dUb+irJ#5(~X~GrOlUE-Qiww&Th$U*me^USPHiz;;Sfp@J8~}v|PX+Zxj_}>fWtN z8p>2PbwY){%J83rnSb16*xA{Wuef$-Tv2B_9&vGcOHVYMmw6Ay1|QCU$RwSa`sxDt z@e*I2JP4jkp~)2V7~tOepjN#?RdQo?c6|i_6bCzdsxEydIdiPb`NiicUlkP<<|Tpv z97Us}rxG6U#s$d%{;M40229k-4KVUgXSj!QgK7hNGw~-UJSfHKZzxeAp_JhL!y$#Q zH4gU)D<~;%>~W7!JKns0h84e=2oxT%2J%cHo4-vS3Zu zpmDR?nxp+h_F0TA^sjR8c}#9TC5&p$#X(VDi$2L(c{I2vD;G9m)QD7E*~BD$bxcAuwmhYFM)SqxY;?Wz2;8*!VVc8yna=`l@h1suV;FTo&Bx8Z0T9g ze;w|>m5Jv4jS_AC$HBLa%D+Cjp^mJCtJ*0KnD9yq62+-qKDM@I&9L+(N~9CO%@5vR z2|b*0X7)zIQk&Z1)ShEaUw4NPxv8rQUae0jrHjW;e2AC3{=D+6mRyVc&|Kfc8pEv> zWHx7+K7kAGGV7UZ4_TY2X4_wVqji=aO};p$qgV`HywFw|r>2H|;cI>D%VeZbG@r>2 z@$ik<%+~4`l5MLwlm-a594@z~Bu{Y%?{`1m1-@*w(aYwYP9FK4m)sAIe;1}a0vxhQ z@D@829ppWWxYyxgB1+3o(Ivy_U{_HgUEw_X(RALC1V*HM9@jF+HC=rLh<;jRj?cmG zoBYhoOWh(&f4Wovu5$XjF_Gdj_73*7$;UzAFY)#;3sGM zBd|x`ODXe0_lUrqDP#x5Dk|*kQ){^3J=`bJ{XkeknElt`@tMqTk4cohtC+Q%?`H8| zGb5CY!N^jzF=XaFZ?TKgAem%ICJB~j{dcrIa-S^>@6MAavP-?sm%!B8uMX$Xb~G5j zH!mtFi7n)(Ol8Dc!1yu+?{op=*o$~)&^qXbtJ!=Rd)@s9Wjp>?s%}UG(mN&VccNmD zfIjn+^TK?6eWh2k{mTFT6trp)cIe>RMlG zzDdUY^fx0F;}K4V1MUVY=JVRx+TZPY#nK7aT^7oCW9)+R;zsvdqv`*f9I=^>(`se^ zj#^dYpJ>~G8O^_@6k0IB>snUN4>l&8|Nj2AAYC69=kT3Ckewar5)}oykZRkEs7JTx zW)BL6#LN}RqJ(KyGhVb3h?%TaYtsW0;lnQPJfu^Bd~DuKVLq0aV=p7B?HxH0y#v9*p|p03J~uCX73A3QkVNB0!RSIdW_OrPAZOkpcfNg>_Nb=&t?)U z7kya4{qtJc|J4HchuBqk?VpMVu^X<^f_RJFBRwOE1+}zasj64)sXc{Hd*KrHg5WKlZBGe zJl!iQ+euz_!>0$W+=pgC-4da++=*QeDqgb4h^vcq-Z)T?skX&y^;I}USV#w9wZa^{ z^fFM`P_tZ1Vz?$cvZf5p#B)^;{5f-1ZkE!3K*b}4qBmXizX3X1qsjrC1XZG9se2OE zHoSe{iFyMK;05(A4~yf}7J+Yj9$EKayW}qD-59$L$c9w%f&b?e!6QIA!{=qOEzYc8 zDT4l0h4Ut4s9m2MJZkIeas&(Ux$YbCVq0dD!r*rVv|I%*Fg2XZMw;C*m=A;7O6|Pa zrb}>Rg^726P;ss~Xi zXie1r9Svc^Ge{yzShlH*-1-DYWhdOAu< zbzM(eeAE~;FPvv!W1Dt_%FIVx-kuLTVtFIAo!F@QobglYebzPG$2DIr1!izPVY09i z9KsC0Gj3blolLJ*S#z8tTzXr&fSa2v>4D;}P+c7v7+A`%3AY2TDM+5Rwzl@sR4o_4QDoobuEtl2IP?k!?>$Tx zva#O!+*jk!XS|VJVf=ovyrOEck1b+mTc&zq;I)LCX`K_9SNj27*uTFE&8xW_5)u-n z|0-?@m&#MWSK|f7;Q0mp1peShZK<+_YM5k{D< z;pSF#?LGjPpH?6e5klobIk1lX5)*cV z1^M_A6~g{`pKdmemFg9`S}m32g)ag`>-_VB%dcO5F7uYbdX#|i%b)FZkFz4U7mG#| zFSGGU=wkkaf{9qn_5&a7WUcu&d$-OqKmM*K?)p9M@y3rD?*(Pxr13K;8bG^qq2Qj> zaa|1(TR~km?4C>=g?(1pH%f@>Y0rze$O3<+)J5%X(={Ws*t_r+jLHfXQ0WgOTUh}z2$#TiyCK117q)F}L4dW8c zM3)QyFC><*!FP*?TQ7xgIMO+Fe2EqDm|d2ky*AX5zo&$`Ve7@v#|@hm-gnTD;N;=) z>muUup7H7P0R;2g2N#1jYoU#aYG*+6wDJ^KfH8qUk+LVoWU%cF z*lQmF={2-RIQ_U^91%geH{r@&Pnvjkxf+8I|H4|Vc* zr6g?SXvO+#`B&i=Tq)rnV~|wJ;I1YRiVM5)sq!8Al4nA;+drL-mFNf^P2LVU^zLH` zq}fhklt|d9YONZ~r_#!tD%Pa0kJMD=yPf6R{^WpHz9ann-=}HoYAHj<5B4^?Q)K_` ziyw6Rvue=Bi2Hs=*v@f{D|xS9#`6yPQ1i|SHJipHenXb$RPEEKs3_5tH_|@#J{Cvu zJh`7EVeo!=ZI#0>{r$W(LF3h(7v>AQ6(-e!bDlttxv84c>&k)P#r9^gQN_!M(5guBhnMbi?iNv*X201f0;`%s8tr z{<8u&1pj0?gMK@O^>)jQ17H&$n|ziMY$m{UR=Tf*NwCAv_JcDaUJUPoc$D_2oXMD# zT%HAW`ADX(zHiWsXayc3pi9w2Ew;UEJg6*NA+m3iYcwQV@IcmE#0mMgRNQ$udvOyu zsPw5H?O?kYR|@A83T7m3&9>gJR7f?I^8c-ytnlv<&RoQ0SXJwmtZ}vVbB}2p8X6j5 zaFslCHjK-j4W`@C`+R&qnXcxbEoU<#T zq)FW|ACfIv{BKbxJbFi3&z}e!PW#|ex`2goTUZa}eceV+D@l)^Js@bE951DuETt%- zJl<4R@`GsB17r#%NUo62P%HTI`Lpf%_cBr@$t3lc+#)`UD!_fIxw`T`XOdb0u~IO1 zdJk+mrya~k%aG0bw=HKU*tqYb?>88j)0ZO%R+5vnRwU702u*tDj4}xSR^)wB3GRted50^( zzx7>PRlg{+_E4QG4e&5LOSn8fDd7dhG1D=55bdPpQlNhP$^L2`SQ?ywt(h;tWkBKD zCUh5{H6<2=ND*)-qc6PIsKS`miw)>cgV{1Kfd$n9UO?6N-l8!XM2Ow3*6q@Yt`|K_y}03=^xObS0Cu?BO`(Xh1H7)!!c+RawUWGaIf{2` z1=nsr5vNc%z8-S)`f~P-gqvzCIg8qZ2aaGH^PAPifGHFS+Of?IS|qX#*^~g5M3bPI zN1z28zKa0RJC1q=EjXABj)h{^ZX%?FFbp%CxN~k6H zmC)0f7yhp-==6Ar@6t-s*qH9Uc|GqO1Sswr^>gM6eI=tdbuF!fKtQudv=DRFo;vWZ zxW1E}6|(M8M0~JtvPp&fq}2%CdK)B{7d$0?lI>0bg#7__ezJQ3C_(eQNHP7q3?{Pn zVl3Ss;5AyW3~LEXPCXi0T3Rn)D6JaRMDkTAl<=!%E!?V4XY4M-So7`L6t|_+#bAlt z?)6DjZ8-XHOEh<$ypnX%OYf;)1L@=(jh3E%4La2lu%oH%6OI)5+MbRHi(n__AgwhS zp?g&KF#betKPt=q9s?<94^R!7Z7G1LzH_xB%${zNig!VLE?YG6uoNGt;hYb_kgU*V zu+xfu8A0!|y*fS_XW4h#vtF01j%77Daz_Fv>y~9Et<2z^kA(Ew)fOhf&LOs#G$hif zoA{pn7(gGUZMXeav@R}M7w>(Y3p=v%_49*?b%}b*KE3(`F4ncDMPmx^OpGSaU_mj& zBUO}p*o=*xO-FLjdL152$3@vo*4=}%xK32Vh5a(*m;iKMt_SXlQ6Xy?z&TM=@8@G# z0w%-zOw&KftY#auDT3hNWLaAD+8D}F$}le6>o zQK%BaiL6YkcKhP-{J901G{5o-p&U*iP6na8Wb)p9;|n7BSRyGmcpjWy-*?f4-djwV zA5#-MMenPDFX=vsMi8=^7CXd}F&WGM%R>Uxfo$k_^zp{Nww; zg+oEVQtvlywQS&+);=U+k@e5l$y4zE{T1=|uRnx2vSs`-6~dnEK4z5i9E_vnP(Eqs z-;;IH#?)k@Yo-248H1%aoIxLlYS$&`;1JiXy27D zMk`i+h}^nEF@N#ImIFod2BlrKO^E7yHok$#AXuUoIXOcG_Yx<8y0NNgWH94{u6`%X-yqPFTJBR2qSD2)< zZnY!=?30?EZMh;64v0}G@DxzZ6ZHR}cI$M0vo0|DLpjgiU*AGm+PA@X*(tvR&izg+ zi)wG56(B^2#er~`2Id67=g(ihyan_~++{T8)vH%8Z0Z=T4`%tGVU@{9Ipu%C{m*7A z3`)pB_9CDqsfi}j+yj^U*mGfr%zRm{Xf9}pj5XrSr{io-lTOg`Gx)}LvdM=7+%ySd z)N=wdM(@Ah?vDYJe{%Ro-Q3(9IWn8C1h%zTep}26U+vY z#4=#YQQYkZf6}%tXM^Lzhc8l77yKED+Y=b@$f?BjyiACl zRR2EE-@jOpCo^~XWMA!v z_LHoqt=x{{e|zgbR^No8|ZHUTV@NYs}$)I^HtnnBi}_c&}F9*z8BtaiI3~4 za26I=(6PfE;KR;J{HFMP`%1;D(mIc6$&P>Pd~^clJ&T_bPtK3#ry4w%SW^fifBkw0 zAdM#2Uccu%6#+O8XtJkFt-V!Q8A-;_{B=C4F0!uOyVNoF-_$eDHPpv%xIhNocG0pu zDe|&`R*q9t>IFP>6-;mOb3j<{vj!Kf{k=b$NoMB|mYBZKU(d{Q|D6tF$|*Nc;q1!Y!KudY+#zJ?$`=i$%gtFzw zqPPQO`pkEt30oYyR^sSxu`jlj2y^h~p+CeN-NTL`y#+0Tjs|%G{q%^lB{uoJ&m8`1 zqfZNyWr+uB59ORTf7V+@oF9aqA57pt`&3h~IR7Nc&6X;z_3eYDJGb`rB~jcq7x_zsmYq+j=s0%<}c1&iD|FBO1V!VK~< zMJUvZTL$mE0ff7suS}U*?jvJ$hx1qW4H;o!`njRlA<0k}DNI(kK7a3&UHYz4aoiOh z+(6v0NGTDUpMU{VA(+N4+CPK$yRvg|M6=$0LbHAOZ=;&rn1f2O(k9|l|9~HG6<0Gr zhEz=`QrlX(`-N}f){Gx9llY)2r; z0BQG<*3|rp^LX^;jhkS3bL9~yJclCJ*_sG;WX)3{l=+=N8z$ z=h4@E7)UPHa6FFdiRrxTl?ajiL2qL_lm(yGkeD^6%Pq<6>R4fPUQaX-p*H^cBAbH6 zLz++;E=`V#dbClMfba6P8%l2KXZ4A6@-Qw>n_ye}{QTt8%mgqh5uvfp)SMiahC~`V zy1fZ-uG#kX&-z5g%d?k|k__|?yJ<;4_}ljS`NSW6z3vYBk$o0wNkZTCXR66*(O#BcXpCWdqrfR@4~43=f<~PQ+igY!h+TI z7tVJt@E+|JY}X8el`l8(npZ*&(%YB3Q+9PdAvC4K%hgfgtYz)SOXm==`9+WqxtzsB zreoNUgV0AGSP3{|;ge>-1j$lh6BN{`wWZ$_MO?_ zsixU|7TUq`4R)&2XQx}V0Ku!t%h2sx0o7i^=sQdh@8Y@H@%X(jKEi&Hf(wELAeEDJ zO#~Ns;MTzHmDRimC!oZh0ax1wW-N(KF&FTnEC4H|W|rfm2bMmXH&eL{GgmhI)pYzL zCMJoNGtsA-qqpe^Su1|~UQZ*g1vNS%W%KK_!{YMxP70@(jiV5TM!6+jJAPxUcA--I z*puGyxZ>SU-3if77lCcdTlZd&W8C1ES1~|JSpjmJ6yfF^8SR5gOVcYLl{eAmOc8*; ziUKvZ0ow##_*q?Z?WWDmMsoLC18CWJ*mvEfSLRpkGT8|@o4RwuTLE=D4L|({%I>ap_=zJiF>$7R zuqdbp06CCs;BuwS($$8Rm+iv#pEJnFIFtX_TegiACR5kwl5--ROB5oXl_|CayPu#x zMgSdT8;apwko;SVDZOf0(19=Jhe`4`WsKU7kK=rOX&Y|Z$+i#w_ZU(oQYRf?CAeeR zf+_{79vFU%1_oEFBmpxW z*Rh(9a+&%n0XKeDZcUkrCrY~CeG>MEgkNIP0xas6g*(&$x1be;VaxS*IQrS{nWd|v zH5d5a|ASxvPy%l9^B0wt#(`A3A0Sw=R_SV=8oZ zj0J#*wBmh_o%YcW(rpqUTM7o;v67|Nl4Ki+LhRI*Kg#j%-FusJDf(}m;(&bUF2RR9DKgCNQ8af!6=7lQjUtVydDs3s5 ziO9~0GH%d@f1JskAwST6@Fqxyqy;|IzH^T?JD0#M=e;4QT$)W}2ot6eli9b0=$+bM zA9yQb83bRwCA`7@IROMEdgIlZ_U})FI>8FGLJE||AQKHCW$4c`>&x8SVU%6>HStHf zczo3HF9^s`AxFJivrN@}(e=K?43A}iMeQvsjVQW=oz(y$`O}S$c zT-`?`2@5{}sM?_!xD%6+i@f;|CIwG{F^Q1u zgp<8uO?w10S-L|%SJTHj091yA`zBSE&1nJ_4Z7F95?fST--hP$Xlr(Dtc1eM%q&&V zfO9V`4K>}^=Q+>%jQo>%Be~O1mqei8U?u|j(%}7=8C#f2+Q8FQqzR4bv($@+6(1Yb zM)c><o~sKuU4`b|%~U_Tm4`@*9pcaprGemQ7F4eBg9UntVl6_tA`9adH@LCUf(s zhU;{L0Dw@-&zL-M1`FhXmT3p7KfVJs0g${B4eEHtfc=%Por<{^=QVqKwp*5#VEqP~AScP;DPmcn-Lc{g`i`q=B6n!)K}JbR5PQ+vc|s7=W~CZUp!&$Iz{ca)~B zdtJ9uq2=G(Brwud zV07uVG$p`tLT&kIQBsH-$BjYJ_81#&a&h5O^= z`nOz-ZyJkiM5t!wX{wrdyHj~z*Q71aR_fF-bs%pws-uc$C}? zY@JfUb4An#VMH<6%-W-Spi^oyXyS4@Q+0Pama)Ux$o}aL%~KI({CKq_16>9xNSEMb zoXRwk78v}j!PGSHEdi*al$>xLcr{3*#28U|D|)k`7{$tcpBB=_$Ek@tg{rgOyJNWL z8zLSSLs5^YWC=kkZsRELhTDdpP1Ht2+BIkZW?By`ny83H*t*!^YBF70g!)4@0*d<} zP`NNln~C!sPJruxUG{!#>1{y{+05_h!<7G)Y5~*x0sj)Xb}IcZnd#f72pi!--U#mx z?4a;D1G}>L#c)lgrs-y1&}nkBJI)x65|Y#xvez=NWGA?scCdba4E=K1e`Y{?fgbHj zsvLx?N<*x(=Vf7}Dp;-{UjnD_epxhaCbAPuMnAS#)zlt-xjN0d)rzHZZ!vk*`gV?g zNZe1Y$^U-*aFiIQMyd;y-PZK7nVM&!M+sQHYp(-SBk>%xM0SFi`>y7vH67@zHGT9r z(~}hZgc3W|bW>s{|`DRx#f{H={Q)x0^F?x3Trncz5l9u+qcJ zbnVywHya-xJ!MEOq1%~!MJgX-7URY6ZZqULc1^xww-Nt-3QFq}Ub^JU!2e_KEu*Sz z-ZxMhlnx0=0qIn_MFck~-LL@x>FyRO5fG4)QY58QnoUa!NO!lJ?l`k~&+qH`a6bQ8 zYs<3`=9#(Yu9>;7Yk;l3b^}A393x5wH`Iw?&OX@vPZEUp)3?(IYw?=_wu%Tn*&?$r zyhThPIO{Fk#)q{$KUf!B5M`VX(C>d4|2ilqqRD?fUdAAWN~kbij$zQC+GCKK zt>^u1dkxmw_p{v*2_sDWU&GIIVT8#cx{R09(Yit#I*Yo0PdVv{Jnnldc2MT2`8c?dwA>6J@{j22HrM;??dA2K(6NuSf|)z!V7ZbOh}H zojy*%_T6zMR4bzWjBATB=v&cQBM)x()1^N`^C$^;Sjx84b12=8;LoKbpPGwh!it#Dkd>|{Qi5xb^5gCiu@p3 zAUENK4CCKK6aRS91ZP0^u;rX_i(V^e4Lg}X>lmKVrGO-v$g8p< z#(pdXZbx;rQg$G?75n}TDdIeA2=pe93QBEp`vz>)f^A?xkY|+eA;Fz%QkdsCWpuE> zE7F67A(Czg9~TsJKW;RH@!Lw^nA8&v)_SbormB zStG~uf=oW-LJzn|>S;6C6GO0H{lJ3mGyZi%%5zXl2fjLlz4q1k+vM-;Y`AdB2c*rH z5bJct^4Sj$*!6`BU%$`CZVbM7enaqMeJ%@CvIi~1f&BSOyu|x}aomC9RU`N=gh;aYG`jFSN zKd}26YVZcO&)8(+d_~7|Vs-oWG*x&`5v(v0MW`@K#|9{5wLIzm@Xjusi%~fp?K)KV z^#*C1pdToP?qKzE1jh*s&L&*@Lj>ve-1}gD9Rr7 zScDM@k%6<1GgkQfTlSQAzUO=hMW4kHi?*t4Avs!5xF0FwrTI&rh2Ov5d@4I$sJYO> zQ^8eDVNKRlXo8(Q5wWDt4|#y(3aS$3r2Rxce$|h%QcC<|h2tB2;c$%nW&tr8#zd>9 z0)&K4G8n!N?X?7%HBWe+;cKnIq$uO8J)4_A&X)%Z7{0auQKE#xVzBo@nT$y8_JMDV z?)5*dMQQ1YHJ}g=UfV6w(=R7e3BlfLO35a39W-HwX19HMoWYF#TrHc~0t`wMGpLvxrAtM->B^)f!(ibFxVu+OeKpxEowRQ989AcaYa%BEaMR}PcX zSI0?{Fk!>c@QyNYr9;@ED=71RPW0w|5_;o_Qe%z-3%Tb7!?(_xStIko7r<|>O=Cf&g!q&cOvI2W#0G9z-mf}A5QXaDDB%mDsbuuF25O(WAXL1wz^?SaH2Noxz8_W*` zH=A&m*xtgNMfODyHerg`YriQT4e3a>PZpL?=~SEtrg8R6@Sr6{qZoSp~h$%*^IAZaLAL?8#ca5<2jfyT=+qP7Q7z})*EZkGz_eg@|% z829m4DfG?3yQu8x{T9aOb$1xx+(nPZfI4>jGzl{eyB~dC3LGrFu98QOmg8L1bmLi1 za=L5f$aSIL557+h+V1ld)gO4*=I!y1sFqRkaU~nRvG0>L!=7QqlzCzVB_d*ttwLYTON-1mONWvL4* zL06B9sqC7FSjTAT4K+{<0YM9+)lx8|kBDtMqyVID)t+Mx;Ek64%TPa}dy&o>kGqrv zmJ!iC($ZhY-h{8`h^1nKLy!K7Rg@;4pfw@HT2qEtFgMDpVKhXhfHZlCJ7DIl09JZG z%?}h136e~@hG+aUEt~4z^I7@eYSJ{kBHpll#S&fJ*#|kOeSOqFfwG)JFhstS?jNRM z`eIVy8i2fT(4{JiZt`;)>cmuze>ozHle=$?8rxQJ@m-R4WB*Vw->ASm`tye!C>bD7 zwi{XsUk9y;LB$+XUb+GG8m9RzV`6Sq&7)qb*yYMXh1t0B?$wKF+$a~5pXl6cZNbVD z0G7o60|6Y!Qlr+vY9u??*%H`S>n=TdKNFgXWb4U-9~x5Ef4FK&%90#L@$@I~9b ziwi}rFr15%H&fLEc?dZVDneqaj7OD;qrfSI3PbOnw%{uU-jL;fR;oG3QC2Z`wI`<& zkSXmpty^P|1U~)d{~?>Uh|jTMhSH~i5CG$F9q)@-?>y`Xq9$Iu1-l@HK>fg`Xf2X# zvD&t0-)*oJ1-tS!geKn8hk?aU?BoCRg&lVsWe817V{1bT!Vq9iDhDeoF zIdN$)ZbJ_@ia0ol1;}0qLPOx+pVaMpCd(Ho;$5Ni9V8QSg6KyfhotmI$4kh%QO;dNcY1O3ylnr&oPH1!N* z8C3%;sjrVP6OGMCxrG*j9)lItFxb(S!DHt}YevWG@+IxGo$Y|mXMe9xi(}mG5y!^G+9zc!5;bV%@=Ycp>B6?sCB`%xZ zlBQa8xbmmfiPn8|<08lYGf1p<KIRi1Hi*7P@n^WImHJBozRyBe<9FA``EiCQZb}tMrz=E1X4cAYV1u^_P_(K5$9< zcL#s@()Km59wWcckCQ)RxTwFUq8KG`&q?97c0sl!|BP)$1^NjE{R}*!$_LpNA=M{} zkfe7e=8}qq$H>V;srrG^pw$sXijcCKn(3?=my+pR5$^_?^}J7PlTBWz;u@vKTdZ$i zw*3lFqG_~x>HPa)Q9ac+vN9Y{)?|QbO+Xmo5g}k~{+I`YJS?TOPMl|8_@gx)nKIl!OE6;p9bMlr3a!eez}JL zdzaGhrx%$(`oS}YMoCqikKqiZ0~LrOe=%fpTD@>hk}4BGCmHX@{=xKxJ+;t;fSox~ z(t0ZzrM2*zL9Tw^ulEfO;c_5}fw*gNTLWj3@>sMtJrfXi_G6dcu3HB#J{vw{qmYYDP2sf*)^}zb7Sp3p?~#^h5Zy zzh5Yi44IAEd+}gu-|%@*gow{)?u8~|CL88{EPm|tH6`=V60?tpkTC4OL7KPD^Bui{ zzeJ++zC`KM!0equPYVI7@7OnvsOZT^9P`Jh&21d{qv+(yIKN(Bt>;~%yuF7?QSpB3u$oZOc zsV@jWFaz%|9X(OsHgw(L(-d7PPPf@~Gk05+UnZ`t^M9f{f@YxKor+hrp)==p?Wd&z z(-%Q>@fKk6@#2fvh$U}}w5B8i&S80AtBz05rim3yE$-_t{9$#6R}ZwBlQtp3gp-+TzLRHd zn1m2+>co#7Z$L-RMghefCE!HG6|-6`xIMeUIC2*fAa+mV?pK~_VpvriU|s5&7icl&U-zpt z&VI$j+*ooqoqrPuCtc6;%@3Q3W8@6O0Tdd4KeX}G#E{<3VVs|!B##FdWh3|5@PMl) zrhF4_sEYcL?_M19r708{G&ka34RQwFk&`vaOk@d3u9XYxOs?!g;u2pX=D>P{E)Dm( zrL9FDxkVQ%^@NUH7j}pYw0Wc%wsH)YALT|a!IZ%F#h;`;cTsWqn=7Vhh^0+XYK7#< zgjwq55BOR{@B8!q#kn)hJpO8s(jkuvq3v*R(HBgPGdN;!w2efy1rsUx%VQm7aBaf& z<`~rV6FxaRJv4o;_Y24lY+}cGRJ#%s{&NU@OANU0*D%Ss1HIhC0 zypHE)qS&qkxeG!YR1rPg{`}$`xifH0p{q6O$ZT>!r!Vw_B#uECvI2y?#x1CL>xN)I zDli310vyE$$>Z$V%XA)EJjzhrx;7GKVWEI$R6OVzyKeK05OyYnZ*u!XXMo&@tMjlZ zB{*{^=HZTx&&N%Z3m!-Q;Rd-lEtygj=VJk#3-3kuMROQ0XcoPxw_kda17Cf5O75BD zno1CVV5gVo&|4v1G8awI%SZRUEPGM=3||g4k`;}Etm9#s87H=m1jaQEYGRz0;uTR) ztK`Yzs!gg_Yevvg*GsM+0jH;&bDj)w}Jqqlt<#!cFGEB;%a zw9NPYecTxNXvz<@?DEGtC+x6Z5Jho7S7!7cJ^SlB$$*)M8dFnyvvah6MP-ZGdBK|3 zCMo(=6%uECuajg-A(UOOa@?QrJjI{j_yk+hxnf2rq^A)w4v%|lZv<=tFf{>p(_Bja z!))S9*qF1UFwYa((Ez?mI|0cx&5*9@SHhkiz#Hbw^^`M067J>D$J9FH;)ul)gQa8C zgoOw6>GnY*sb?sb&c7)TOw^e7usO_ZLs=Z8i%$KX?MuSgBLdBhb+9&2QXk#g)moa< zXz7!;P5>L|4!mJ%4@n;a1i~kXrmb)kLsOj{S$N3EsX}em+n0brngN#Sz_lS>D67>) z4R?9jU?Ugp#yj8)-C_EH_``DSrrd#osNGEX;v>J^BTZ&P2sg#E_Wvk>X|!|`i2+oD zy-?7;+1qE=7(#sb+$h8@`vFHwxc4GXQ#vl2{ekKAIon-{anM-M?SMQ!++^Vn^BUJv z*wLIUW}*tcjBvkY%j^QfTZ=337hWrEke$g}|K8x-F0pPvx%deYxVrw(%}V!EUr?3F zTI*<0kqg9a0?b}D59lssG8Y; zqf@zi@Uyj742L89$-)9=F2Rv{#^UvtxNLtDM4aFiK)3H}w+WY8ViT@4eSyPEWfL=E z0CT2YJeeE(!pQ&5m`;Tc#cet@{QIBLa8^+X8Cr}((pI3ry|l&{G1~HX^hHfbC8)Qk zzma3sO%{3_owT2`LDY!vohqvOqDp)D??GEAiesm=rw@Kb96CoP14?4U1SCL!r+Fp? zM5b}8|EmRDaAchh!S~I_rUUfdSM%d?9N5 zs4@<4Wjy4CL>*fu%2PiAO>dS{J(}G0#ReHO?FfUAPB&6m%ctN4H62bd#OlJc{;Dp4 zu|W9Vl)e}2Zy zBA2d{9u`l0Q73=)j>edK^seVoF={CV*Z{3JpBpM*gbX}KIQST)4_6qAQj3`~u*0^C zGX4L=#2orTyRTgYc1KyL2vI}7BE7xM9k@}%7{ug4zLf%Q;QN4|``G(rm7~eW5SozK z3swiZ^*prN0CFQtYf($9)=kwQFi&bLWwi{*5^&?YmJ;k#sCFgRR# zZq$QtySen^G)#>bpRnP0*|)iO(2KxPBp^6f{#kozZo?>(;aDZ(hnr%P=3>{pWTg>Q zvt?+f$qxgz{JB5`jzOsDWe4N=48g7lR^G7?LhRnqytWmqGH0Uwg7Sex#wgFHd$`C$ zp?)oBLL@@Od;X|1(0m(ham_4+6+*vmJsUXwKB15Zd(;4B_TjGLF+JQT(sT`evy(#^dZw0^ zdB_>OUv7|zX$sh+qu_2BjNoou@8o>)Ax4x$j7C(vAt6fL05YYz zDif6OKLe#;h^kzRp^O1QU!wznUsK*Y#Z#cdT-P$)O(ew*vgf6_vbL zfTL+wR}t6b z9r*BoSO`@70)8}!c2F$4FmChtFX0tRWl)Pi`G6BlFyh->p;drM25)?rCYh|mpU@q9 zVI2>oW<1_6_Q)JaG(FvozhekDPP-4Cd|cVY;)3huezyRHdy*5BEL>M0XzWkjn20MV7laXT~H8mXz219hhhB} z+S~oGSBpZdgTO*mMP7N6v=3Ktfg!INMI6*O5=aN zlvs0jnABoGlrMHVY32p@S?JT!x9jYPAochWeoU8c{Yqr4t zk*~3x{P%uR_4l;rpzcRSI6C}_VTP!2a+NVa;*c^>C8RkFoG6CP3Wnuxc=mOn@*WkW z?z2vE00oaWD7-;s-+W;K&{B}*)9TXds#2hy&VZXO>N;Qq!t4Z|%KNw)02K~NR}Aft zMXFsFkgmV~VSFYJ4-XOyfLYqP11|ca8)>Y(0QwLIkS1lS2K9ony_jp+XG7v=2EQuG zeU{)#{3EM@_{c5jTF9piT31FTC<)jMzb~V>Xx9hD8XbrfDL(`}N6@fho`Pr!DF}-& z65Bwz!)<(@zc^vNNQGI1e=X{$_3N2CAWLw!HAXPixvcjio0??A_#gk!9Q_A?KvbYi z%K*$c5&}*_<-Pfxx-YUytQK%Lc+JerceRD^KJ&~=cVA|py}-hx@uagUsbf+|;#vb> zwM76JU+!fued}3kVj=mQnmXIiw)3R5ZmW=?ZXPRO`VzSCiZcbU2fLJf9kW z(y5zT?tQ{m%}6hFsV$@t?gyuQpwYZ%=djXvG|0ZCBy7@#CNTAu81X^e>l`a=+yE8K z>RzpQIJAd+jWii9MC%v3#z*iA>o#6zwkSRWjSIb5f;@8fb===1Dhw`2+|M3VNwgXa z37r1YSOGbXMaM4k)j~U)(x%hjH-Mz$uaYFFlZw6CDXLV~K4d&8bQu&50|G$`OlsOS zPxJX#_aoFVsqoB~vWAg#0%G3Y9Ds_0REp9JbAFtkhTpJZhPR|#hwj<;cG8nT!UDl+ z4o8H@Z0|%QV^jeK6eNWL8Z=MtvK#WXLURCrzj^)V0I-;>0CtJeukN0nwQr-FK@D)r z%vAhk*PlNGfOo6s_Z@J3q$}NLu8!kE^1%r^;JDcsy#(C|Z+1G^*G=X=ic0Ahdaf;X zQl&|viQ={$zc@E3adyxE@gW*vn}72yBfckyB2S^ktr3r|1rB{O&(-5KY5Nf8#ik;0 z@ixP2JwPmrNuD8L|vorl8{O7>+;oSdL&m1og*2)EkcA#HwP(zQO6Qs$4nH&ga0qK#AcC zf55El1Z)PWY^S8*Fdgn2X23PElBQs0IbNj8aF(Z%Iaup7JYfR({t!Sav6H{*iYd=R zq6FuYsXb<$Rxh~fv2{QOFG1-^FiBbaV+QkI4f?+9;yI}RwJwh((RD>TSRq^WBNA3f z@z)m|?oL#|BLvS|h=BG!-YO>?HF&{nx=IBVtPb0iW4zG-e-kkFUm~? z1814tFj$fIfGT{TFaKnz`PJN&D8(@iW3d0f=@mz_P~!6HN4I+A(CnhWCHvk3>A; z;3uhA%@A@U`4B|<7ZZ@STEPl;7=4h>dxa*u&|Sc4Q-s+Nu)n7Z1SP+W0>Iul1@5LQ z8_ch~W*7ih#E6vY&@AjRJ3LwC29OhCz<8Ne#7{_&@-n>HXz$Lu*$3q=K!;-|T5eDJ zGbL#Qfj|Gm5G|DTvQ%o@LEpaasfy&y@$WkTRtI6p9tw1mCgGG>^5OvkfOFwX6Bx-= zA1>H@5M{Tq1gJBY{=CE=xSgY}E#6B%jkqZD;nL!!&5v6>74vl=Xig2jFiR1H#CdmS zfF*K)P0gffWGw8JWF4Sukwr-WGH@A_dI|#vwS%A_U*hl%#0zmIuP>Lat3B}ww|A4> z5?Z`(9ykJ?499-n?oZbNfSJ^lL7{I z-$-6B;Xm4_(a=HKK(YaAv5er+Q0R{rp51iUCE8x`fcyF0wIvKb9};`+9MFy{Sd{7? zj4J?zew(~t-AQ0A-W)gdn2Wulw6A>v=GT4TxM8=V@AnO;YW~9U=SLR$ej-(4yiT?) z>Hd08S!16Id&N&7dgb?q$es>r?LWTsyazPJT-0{aKp7Kzby#@OZ)E|045$&YVkZU1^iE9hj)O5 zYC6G}b0`#g(o@#{7O3~^=*~3=D-Qp!3~Y>!zQ$fj-fXE{%)x)E;0l7$e2J3+m8k2M z1^=t^UbaG4{s6TW&pjJs(Pw!(2)?9W%!hLx3*ycJZ66-sDpi(M=sx;4SS*E&4*a#% zicdPYraN197LelugD6}r2WR*WtG6IAs@u$pl(qA?KIndKZ(wz`3#PMkL*h*MU(20; zXlSU=<>n9JqyFb!S2GI|&pLcOJ(GbUq=39H9i-7@;pno;;wOKVTlfkbnvQe0%|Yy2 z2539;ckja4OB>@5$Q|p;KJFIZ)ms{$U0!Q`C(sY?O9wAyNL*E8DiR0s`Q=25QH6-3 z=pe{qltuTda(|_Y`?z;zd`18}!l<0FKmtPHWqW;oVB2`mrI;oZ3+k18|N8o5g0xxa zVl_T>sIiJF^?!B>qAi}_ye9*tUDa^D>#uolc7R_e2QCMF5wJb1Dmi$o&vdt}CF%AaPbbof_Yg;z|hA2^RTa1X$vXaMGmm0G*Kf)4?yj`i6p z#AUC2X@@<9j5sd=*fkVTNjP{n9IOYskOE$tI=cuJ3Z~wrmfu6LqLRF$$ping%;1OvJUu(S3tY#lndvwJ;x7H3e@us z!L0|M5ry~wA$CHS1ftoidv%AX84@!oE*_7nYub2HMX?1?RyfVi3eCo)zF9NlMj^XY3m{(&Qcu_0)8kOd{WKQ%hQg+mxZAg{K zd7U$L+AOhmEqZD*ch6*_UWxKeEs%+pwJ6trFr7c$bTZ0L7})B#I1$h)snC!lN2+1y z)pN+zS%2Jj>A+l!v5QItiu8n#bXxL+w~M0!Q6q*rzseV=_LUdUU_71h zx9<9A*KsTaMd3A=x0L79Wy!;(k_~gpd_13QmdL>^a~R}5%NCx(6c=j+j9yBn%IqDv z=giFP?7D~oBLB?%Tdz5cG$|V(Aye*;Ef>}cElKJ(E?<>2I__lB7@BdYuoZ2d(I%;0 z&l-)b))XhFP&riM?ZR<;Zg2BYuXeK8S=a3{X*OQzO(~+161B8E_eo7?KDPSGPC2V( z8K55Se0UZ_R$w5I{F3jp+w!b2H@B|vO@RJvHz`amNWZ{9VhGiPnA*SVSIUCFpp+|( zP*^@(+nYO)C@S`<#`Duw$R3&Irv;pjDC|dA+8$H6+zy%`su%8Cp zt^#u)z`rjXe$69B%zpX6MXZq`WED$>)gMaT>O#N)`!F4iumzT&v z-M~Mq=*w<2Hhf9)+$JNl9fF`VYV13F{|%mYuksr4;yo%Ryeqg=muFmh)eIR z=qGC*5PteSpSJ~z33st>?z+`N^~c!NJr79UB)odQ2+8VR%9SZSdnt0tb~z)xn20el z#)eapI2%BitU9?7FH?A_1+IIierSb9_jneMQ35sO*BcxozmMlP6P2HYRPrz)p!@3` zd_)B3cN357$%t`3bB|BL=Q_MF7cVdlhf8L{;SW1Jz%{*VM;-Z}SYf({W7xTR9<)5@T>=I!iy1cqB? zGkD)3U88x{?%#EFkf@eA>^yq?)0#=OAVcBEB{xmFoZrO4`L2uUbT}#_p_+>WllajD z$KDa&Jh3&$RAt5AR+5F7m=huHo44XwSpf{g5B+eQhQ*?mug2tudJE(C?l5-M81|+= z4H!5R;6BkNvEPYmJ*Z)R=uy4S`xx_@Ij6WVY+?%T5myyM@Y23eH+?*;__(3U(!AJ+ zFiM21Hj=!?z|KDlc@g`IztKr(VyUeKA-u2hZ-zI+p004|phOqu)xUGxE~D0{AMVHj%E9oO2chALBNm66;~ zLFO`mI}#A@5cA{ECzkZhS6r=T{wa5Xfg#=rCCfMcGwb4m3Uw4DN2gJbx9}h%O|Oqn z^>y7ubn2c)Wm1kjIiK)YoE+Gm=!|#&Q4Z(jBP;4xWZM-imP%hxQ%=GM+4Gn)W9P!s zmU!<=NuwAMiXy7)gNofkmL`ckEm5!0k(+>Toqo2;D-6;tNmLG6k=;8U`hv;!O7eMw zA~~|t1K~*Kq!U6_={WM~WzX)DgMB3wznn!y>mV97cc5?j zy0z~iU;s2L4R}^YOK6}IiNz--gT}hfmlDPwPP~xYv0o<_@kl$<#+0^@_|cIVuVQtg z6Ul~!zU*QDw!Z$j2=<^L(lou9?+zir(6=6s8ACH)t zi?rIfgmHyF)t>ss+@|bocC7kWuDzG#K$%xDP^k#D*nsP}SD+XZ4WqU&(GfB3uG}k{X{+$!q z+t)Wke3W+26>lmGFQ=0T;M#RPGQwD3>LXzkP)Ps30I)4XzqETs%!e&;2%moPuOfjE{j5C4<=;wPrbp zX~gKYVgTxao|a6)AaR12isH@DzqY&T`IvZ?=sy?8O;&#swX9J)I(aD{&e_J4JE+;F zoI{t&2Ut#Ze6w5qPO~hZl4vIn61V2*jUzY9pY)`V5sNcD)-~~^KM-LrW~#||OpD+f zuRP2Q*`Q3(wrPJG&{7;PAa`8S&>&4yIYYpE9MJIiN|#tDHD5++x?648N5g8cQ{E`W zc4j+^ckgvr)Abgd$ahoXqJEKAG-X;`4;BQY~|sPzX3zPvOS{>UI!?@;7t^n ztE`~dy2zgD%2htMb-p&2DZ3K-*qUmj6M}WR8u8H`+_hbN423EByUvWh2VA_FHyX|IouJ?q;fDrYoF*oXJn z;JS4*yy&lULeMPK{Z!kkSrTK`htIP#Et#(F)Y!QLu^i2>l2tu-HD%VSN3J|5JB35p zDKfL|7zvVZG5mF&4x~W7o)Ns5u4FAiOP{A<_MXm(W?9V<+v6E#ubT?tO^wZ9=H?bV zmH2oPc6e=|e8Z|oqgaz7<(nB`XcDk?Bv|r-&sxKKp~7Z6$*PD0@1W@hWlTGCumpEQ zYs#O}l-!{>M39ttszI9V(e^Z?fxeiSlkGYHZSKZ?Y_umMVc=^D)m1hgeB~mM z<$tvRqLd-$VR-OgU-wvUqLj&hteh4;XKkj{)`C<%?EHS|)n)&a9TLIEGQ68cq1$KT z{zMu3N|5Qw?rrsu5juOR!pOwq?Hci6H3K6gub}+T*)caEI=}P2Y6jZ$)=)69AxFk{ z`$II|SKM_!(-Dr{#7qj03T)=xK8#G~=PU$_(1A)>AFf!s_p+B>a=>^k5cvpbvMZePm^rJTi){?!ZX z)?TW8PT2tWA+^ht`Bz(KMOtOfoQvxZuhJB(F6dOgP@EPursih1_6ib2^-tX$MTG7i zUi<#+ZRS!O;d@r)6i8kWn-E-Mb-@1xlYK#&??d87!?nwFRqt&*b&*)?$f0G<4AI9( z?F^YOyw*7_RYATrKgDz|QdaNGJ3O2qy8gGad*Bu_$aE4ha;tpl&RmbcX#Q-Vk=0B? zf#j-(T$YF4MGZE^9L%Vo8;+;JMLfhMW^_wkB*;I06<1|B2ARRlG{IIw5jT z5vFvFwDmaU;&;Y99SnAE`4Mw*T8|$>Z!N`CYsJ#ZoAp?A>PT~t=NE`MtENztQ9B4S z77}u-va8`|Jpwa=pPNU!h%-k$@@LIl{g>)qZth(t7GtoF_U^Qh(ae;V#dJrs?PAMB z?psn^8Rb|qUhE`6&7;qwEH1piY#&Mbi9*hL+RS{269(iW7G|kTb9%W|o1_6fZ->u# z!3-1iZ}rDTz8RZiCyc7XnF?w6+)uQ0cU@OHV&csLD%-E@|60rJ?~-Yhuz+h98Gly8 zG@1RxgxwjiQ>rP8#KK1=^b4eo*>xTaw~PBU$9j&W*Sxo7zt@vn3<}6tU8-i zeRNuFd3e2tOLcVuxukF#8IQwYr=>4xvVUO^rp+uHDVC9a)53PO<6d^y+_tbl zxb{Ia!sW=m;}}`DzJ)lbEk`DBiSZ+HEiSlsE;u>CpECV3Xyi#V<`7{Nr*4Yyir%K?g&Q?b9pn)O4OH=Zk&g9!p?p*S zXMIJ6n%;1hDfQ9$k29PN7aL^1Fa_#qod(FI!(|uuAW=_lK2|^aRynpoVDHK zQ;#g1@q&!G6JR4LGGU)DIvkc+mkn;6L|Rye|MfT}4c~eRBye>JX^jpUe5R@-PA717 zt>LH>HUqU-jPe^rMWo>p5hDwN9S-M(GVPc2AJwzdR1OTD76kiC@1gi`ZoR7at?jiY zVa_&`K`XJ|dR2DV5?A$7gSIyfFS0gyyEU%|<`7qvK)O5i)l^~USyr4@PyMHVu|is{ zy+7jk6v8t|t8w0V)hg-LT_cZSev$ZT?E~bH}j5Og!}Mj$-2C0jsi#`8|~mU%Pak^yMpwj!AS>8!uL;;)bZ+(mQS$ zq})_&qZ@kRd1Tyd2SJ@Wk(|=<5wz2j_X$=P-;Dlvsejz4WG)cXzUY4&knldiMr}H^ zaQe^Y3wg@bLk)bb+>ISw4O-dsAFfVWrJ9h5d_J8NW?6p?_6=Wp$QesQ>5f&qT~kJ; zHnY{t7JmW;`;pAIifOU8qN=;-wO@MM699a%8Em5o^qz%5+dWmo`@@$+51KdFwW{c5o=q z`!+1X=BQVq!)QC-Rj6}N$u?2j?O$J*WK-Vr zsgj*ftI1#`VUXIJX_v8?a`56Kwzx)ohKD8787;~n!~BB7bMN{f>!y~}VR&mVSw!G0 zzs$?w;!ha>Y&2k4%BCARg7Mv z{JRp~QvyZVQEdzrQ#t9y{34){Vfb{QpHV3XYCgTPyulxh&b!KO_bH#GC{5-%@lfV- zmQiU=4@v2bOTmrz4kS1MJ3P>fAGHJiT;VSFq1Al4n6Vw6TKEohJpSBjJE_!~6<=%I zrlXM!;v*BX0dyrzWiLZ-OM}B}(by(?js+=K3hUk$BYv$6mJjSb3aPv?Kch317W#nZaiRl;RQ}13m=^AS< zT&;gEx4y7B6vPTdna#=^h}?3hVqW49{^5Wd95?-YYp5rjldxog!jh?2Q#h`$SPkMH zd%hck&eU(aa4Zci8*~Ue|DEmz|Jw>9bOj2(KH#d{_UriCRQX?T{J(%z3aa2BHmUR8&(&Hhq%7oQ$XlvaUWy82RQi70`3DuSj zWrmRg&d0jvMZ~SWRVW4pPL)PwMDjlJP1W*Wsd64gTDRnZoKt?IY9}7#;e8wp&6@1b zbE4w4`O753L!)d{{U~R1L%Kmu+;N&K*1nLk###R~*JNd9yW; zY40dPuvVHrs`IV?R%N?j{mp6RE$!X6TaOY|tFRz@+#QE*&t*M7iiEz8LxfE;d7~*t z(eNkR0b{bnik8Q(Ke)?YQb}*Qa*TNo{9?n+hA%AnB~i+y#Z;__%W{y;JXqPX)jd3N z4S13l#gcxxv@EhUda2}o#rs1}{8L_)QCwc_VSTm$&KqxO`(tU*=n=YU8vfHvtzMgx z4SVE7 zpH|ucar=#R&W%~>1*>3Ca`Tf0YT2~Zkyo7+(r>CSJbKA?ehN6~xi`+lS0z2;=q%Uy6!bLczU!8v6#_R7x7pPGD> z5432~v+GLazq%fL(<>|+R2Sy8QjJ?dGiMTh&V1v`8xCTy6~?;*-UHiU4CVY>p`FBV&VNKs}b8$fueS=~R5Cp^4P#m0HHVYFwJT(6^}`iQ-8DTAGx(^WjxwtH$A> zz3vNlu1h)c;z?IMB&G*tHsnn(wT#Obo60FNRusFudXygq`CY{oEFY5< z_8{LW98~vz_xAHMFO$e|OIUa#`7ibqHiUZirr|pyXC9w3b)4SF0I^(4iefqa$h$IZ zQ@lc@hiC8{@1TJ2cjf&L&EZ+qz2=29PuN^j#v&5`+(bN_(#RrfHPN8Wt*~;wdzO`D zprZE%;)7M9!Yh@6yHjXcq}`>Y&jq(KlT2kgYk76-n$PT4x_z>qk~!;OiM4uFAg(gE z5b^zL@2h6*4L&c2{oQI^`PVn~6l+bO)RtjSI!o#-s_ia}IKJ|iwHUAY%O5`nb`rN- zhsJU~;jJ1g^jo^{4Uc*dsan-V^#(;81}Ybyd9V)euuy6piY$wb{Jt=I<$_8mG_+pr zM?5evAsD^+AQAT}Qr2BNfWtdYaF-djV+R4#miyYJXL>8vh(uBNl?_4mdw4s3e{S6FYU4nrlT)prdBaK#O60ty}2G2qBD^Zy7ga$KU)p$ z=v@;OD356hA7aOb`lT+T#HUxVbuM&N%Y=Qmzj)YLF@j>sTd9ZZ%3k3X;p`(Yw3iuP ze|P;;|HEO$WdQL)wY9{{A`>KDmCY_04(aXu1Xdm4%-X;bX%ZIgAO6S><@I~*zn?m; zSB>3}XA#WzziObyf8+IZv^ruuSCA+uI+F;wP>wxYDVV9a|9LXr9E9C0>Qut3JjASq z0rw5Rnp6ps--4blYyJG^Jig4K>ax3joBYNst97Pne612CG`Z*}-ppUl@~I0S)THu; zEpMD+ov_EX$p3SY^_7T9Q}{{UU{pG9yCxOx@}@X6v0J}ij@@4-ZKVQf&5DGKSw5$w zm@NFgI_DN!-3|fzw=n&ob>}@F66{5v5zj>dIkYzwAw%oLA!FWfzvt0;?%!@Qt7wa6 zKW;ua7LeuUDgV+tQEu_J>8DGFm*=DUfPwkYkt5T>qiCxbxNI> z%2Pg1jTQLJ=gpa^zj1Nwb40YweX3fLF3heT)tGV7A#6u=V5D)0J1@uK?1`k>JRLs* zdtLu|{;uZh6k8WPY`)~sMw;{H)LZXjaKc8TXhN5Mu6fwm>1&Cv{14>>x6&uQv3E-4 zOFjlmw_Gx?+gDj8tBQ}=OLM(!s+TN?PRI%yBTj=88VJp20!&<`ypexUM%Hinn`WyC zJhM@GjIY(4wt0{|HtLU*_IiUd1pm8G-iC@uBmD0Mt2>^a^j{B$?d5~Zsjo6B+td&5 z7GC<|MP2{v8|}|e{<$fUwe<9gbN6GbyjyE8Nl`zsI7$WWnBHG}o+5~q3=#1o=el>ip_^{PNb-ubvlcw-#A^POiEmU-r?QA6+uZnmF(XkJN9u0NX(s z+b5oZ!^o}GR^mEd+^XlLYd9YxC8^~C(0#d?*tm$p=c#50vA45tWfc!`J1<TYw>M;O}MCW;j8AvlL%q* znD2+%y#aKCPZyZit8BZ~vH~+SlPBok7tI}iT)m}#tpB$7Yh;8Cp3J@wt`-fX{Fyg@ zF(%5$&f6aohu)FO*%STTBiF}|_Kxus_$!3E&_0TLlBwNXpD2AS>}d%zg}!C@INC1u z%7cAxyHS?x4P=yG#JTXRR&TCV{ozX9-LHn_)6xj&M2$=xTq?^%WXHVav6HgMC#>u- z?8Th58LgoWuWe&8eY+3;PXHei;OyFhxE3A5=;8ar#QSj>(RkZx8M;+}qZq7Qufc70 zBFExyZT$%|Bcj#9-|mnX=KiCsc9=yuoB`2oBgO78G89P+6!WxK<|T<5GU{t z0DmSFC)l{X*hptqTwa|~JelMeXi%6uy+1gpLdN##5 zsQyB8LophWUI5yuoWG-fl7@{Zf^5eSmE!XE`UT>ddmKV7=pC}F$ZoI z`0K&BEV$Q=QbimZ#U=hp9gAVYf9>Hc4Mgt$^=UL||qEqUY5qL@%>< zQ~W+(#Mq6tX-0Z|Ze&<>KQGYLX^oa&)1G#%HVkCaz+1f`<(YI;PxufqVH3URY@2tUW?+VY~MU-v7M+9qn zCB05IQcl{A7|D}O-*q)n&uwJNuVzgkroVs~jnwM=&b6FIu_1pM@mjjo7&MN~mL;LH z(*{f>f~i>p@BKvMSu?$T%Nqfktq-ltbs=@p$`-7=lj6cRTT_44?sQeOi)pNqlp{`_ z6Nz4C8#1;-Gpo6mm`5p_U+=RNC*;oD2E43re?Qs_Rz5)#kk%yvf$LGf$6>t{!=ZS= z8(-LWt-{S#b@r9^=6EG>(rsqAQsy>_1)BZne6TVd{`lt(@4Fc};&mQFv|g4E-CIh0 zr!FH3LG4t+$I?vT^uqm3YNX3-n%p0z{oY;6vutQ&rqWQ8^!G50`0du3yC_SOX?2(^ zMB%B~9^$pMGlZ@cJ0$lPHgNpUQC#ge8xHl0F665dRv{YAjv;z}miPF1fe2!Khwe6b z+)43Tu@#-`wO7e&C|$|=q7LlyV&Kb#`^OX4wP{4q7b5uDPEV6;F^%1c-{mgE3cj_} zyO!TZabcnTh`HZ-VK*WGUN23i)*=E+iBT(`AS#Z>5OY{(Qwa?h{_jUaL=WsiA~;vC zEL5*-wIbP`=(JF;Jk#j+q!DR3uhB(9(sbqJ#6J>q6hfSTI^D}{F;R}1NtFG5TzLFQ zYJZmoDF#{R5RFp3GL>S3*+Go1A3sU-=D&=_7J(WIY5AaDBiq(Q&@k(Pc|9>oq+5+A zucFu~sNLycD*1mD=RY4Jg5tFnZ|sClmL;KcWtu6(oS?5P{QnRUsBAX&a4T`1%Q|Uo zLhU)ubClMH+9|6`DtI@IuX?sql6C>jEnM;AWZTm!Jl~Dgz#Hi~vjL?m3zyGd_?G0k z`)I$_=l_!Zom?mFCd37&-U^}V3V4-TJX7CF(pFjxe1JH)W@!$qv%hK%PxkR3jSrP; zRJ!uzv^r_KVLRYcL`z-y>W6va0(TPKS(URHt*!<>f#`{lZCh&*B8cy1mbRTCoQ*mo zIbJ7wvvtCL6!W%}%01R|#5^`xy8Y|yGhMm1pN72m!W9(Xi+VxFPbF!W5JQam%{bya z^S_k-=Ca#jqG!aL7`tgfPEL+r?RuPQbv5v{T+xw~BVBE-R;F%BUe(p8%41^@R+(#~h}!e?(to`00a zhqkvOM>I5L$E!7e=c|%**K{#Y)l487%Dzv$Z{^pcdBnN+V8(9ooJPHz2m)t$-K$30 zEr=eVX<4G%*VnHVjx|K*f1@9!_*$|+<;rT1IA><>$0Jm}hBxZ!WM3bylXiDXKZj2C zZkASBTT(lCmHJAO_KGHt^X6pRzcRYu(yL%4z*m!P?kLOKcuE6OXEnG_s5}qOVadLp zp!U%F?X)R+8N35x*e>XMB~4KHHjW1 zS>9?b6qJ58GpRnjWGbcSplTKC z=b`jmnx}HG5=w)H*U@+|pxrt5(&OeSjy9=eFN#6cB{Vi9IBl%HFS*~z-()v}m1!C5 zKdPjG=nb8omsW;g<-Jt@-VG4t?|NnW2a`0747LAelK$6J|9wmVev_oFw1Cq9F&|x) z=Bo`Hzg(lgyFrSbA{J76DCwz5p6WFObi)rQ{haf}sE?l!0kE;2?x4|mC)Xl+yJva+ zJJsLk&4{Ts((h7vFuNNbqIhlZqz_U&{|r;Q2=2;e5;6a1_Srs8aZ0jQ9h~g%QARJ; zmB|skbF;MFnv+yoDP`Lh$E|{t7a0V6_e>@&-XG|S>C(y=(6VzCN9?5 z^WUO+0X>`l@&{C2#9|z!g{a>$dbR9Thy^;bN_LjETCg&$C@vv}Y?M2k7@E7N?AVn| zrL4M?=ZFII zbSn287)zTH0qOMiKNR!&^9s*zUuj1LYf$=|{}!-nS(+_~sS(nGK{GwBTXjZq%x(uO zHzMX)`#RABveG$nHDb8w14+JysNJ$ZnbO{s-(z&+#YU9&oPL((R|)QS6NvNg`NVl- zN`*Ar62nQ;fWV)r-UIBEBwe29Nmu^euXbys(%ea(P4am^@JV{#;4zM31H>Ha_l!@k zK?{iAWOmz<=xuNiu{!Mq#PGcI^O;Zd9Id6CdS7X-`Gcq3$b&CG63 z(AnQwH-eQ%CTZpoWxG_42;j`4v~0Da2P?Dh^7V{vOql{)nxuWYLYmBjNxDY93!6|{ zkN5$jdnv9-=?JDdRG-S`wUmxt`WBTNCUZm?@6xizk0z#Xcqi2Z7^_nH`}%Q}beXG? zv^&>H+YRY^cL#AT9m`(c<+8=bPNH&IZxA?^(y~;Z(WB%t>r?u@dziSMG*UKbXCD7u zl4E|Xjs286fzoo(|I&C_r`<-bep4PSTtw7|8+k8k)rFum0Qx&dmkCxQrm)G(RtYW-JiVXNfaPLh&aiJ6w!Mf5 zD!iDaxwy))%7?x!APNXyBHp8HXu@ntdrejgRzhh3BTuw2eupSDXG58m5i7T6X=?SP z-JS?MmGARn;GDwi{S(E4UbWSLPoOklxrA7OH+$bMBtEBfh)lg+yO$(sUtT9|H@uoC zP##TphYmF>tspdd`%9gD?n`mC*F5n#T|)#8W>Q)KpHJTVlva51MB%YsWfWjnO3xW{i5^v#5rK-dC-h>X(!ZjaKn z&I?p-Mulw}y>+9aU(2cVPlHD)?eTd;owRANvf0~BG!~p~Hdwg|(Lh&zyptN(78(my zzJ$`D!b^zX?|&1Y>E9_(l#2neiNnTjyXj6_v?kf>GgCe^&3GxLTRvP z0Wl=_8^m}2M-YvnC(5mVB+h$F%AU^?<-=+Ys*t|XTdnhUJ!d5QsTUY)#t&)z^)xcVV0kj0 z*58YIb)oer9pSWuC?i}()QQqYlss`>k<>vip;#%tu1cC~DESb)&XHHI(wi;((8&9_i1rubNrRR95aqE-^VMayU(x*ZFzLL;L)30;Xhk=I zl@MjabZU?C+dqk^Vbb5giXN=||LxsRh*f16$MJ8|qA1!(MI?w~iWn74polFZlx7yi zU_rC$&!#qQ+O&!yh`@b|NGPIQ=)xK(MH(6MHdsW87(tTcT=gn8j&~MAR8F?|oy&W= zMmw8z?+0RBKHyM zTb&{-jZdoQQHecgHsLuZV$Z1-=vxg9FdI{za}3>Wd^&oJqV$$KPylTX_!(W~97jRC zB&X5Dvp3+_&1xO}QP;xVx{MFDz_hS@rxhK|MEu}rs z>ZLJs@iKu@!0tz@B(6k#ok4fm({_ZbM6Y#s)ZXLhxZR14TNMR?zl|P$rnEId2y%az z(f_hJh>l|s^EsYVL6e$2==iTksaDs}o$asD__18@f45_ruoLM*m;cuvD9FDReTGNT z#o&V|ji!!LBTu2=e$v>KjgEho++!wx#(F`Yk3XH-?fV*r?xGj)lQ3Oyg3v7Uep0} zeuaFW0njR;_<8OU8n^QJ2z~b;eT=E1bgxMioY{=Zn$*w=txwVT@DSk`oy_>(XVJ~k zZK#CH4z#l4ZnW~NhHh|F(7dFc^ED;;oRA(rhvreUC^O`Ayc!xuUP<144PC zUW{Qi^x1e0%{e!s&*x?Ixo?zvNVAua#$9Coa0GqNCed7c6FP^~(F)$9C`CMNE*WF)-nE#*3+=F#_c5Q2!ZOU7TZX&AFVQNMb`IowfUFU z-`hObLKjf5dlwpewxDz45?WAj3|+e~w%7+V%kU^+6~()K?u%KHMoQK>1}g>I%xu_f zC!|QGZ7^b%C7z{GwC1_>U-QaJ>#WBJW9>VA?u&KS>xA#lux7}Znf1gzG?*@;yH;j{ zWSnrqPpXk=W@eqUAC0;5tlVJBERU^(K+HMHnZdN$Iat|UU!Y@qk^Hq_ZFTqty|-6M z-of6?%yM{uP@MYxKKI3}S)-u)EQ7TXg_+r~T8C1PZ=#euvm$OorGpxQuC!X-Hn<$D z?4D6VF#qNLcE!5j72s90;KIzTZDqdWdj)M=GmZk6x6|7?bY@95p&D1`3A+QDnOS#L z(WYllp#t+G=)&48$xc+$_byb2_nGMLHQA~V)5ttdpbaE;0^db{I|#hcX1iu)W@ct)W@ctRF-1rj z{tNgX80%|~%IR&w`_Ss+;ixUAft#6`4S+JIft#6`nVGffHQ){4Z{RX6Co~LxBCOI} g>SssH%*@K{KOqf;tr{P<5&!@I07*qoM6N<$f~weR?*IS* diff --git a/openmdao/docs/openmdao_book/theory_manual/images/nonpar_fwd.png b/openmdao/docs/openmdao_book/theory_manual/images/nonpar_fwd.png new file mode 100644 index 0000000000000000000000000000000000000000..cad93443cdbefe7223e2b78d9778e855455c2851 GIT binary patch literal 19030 zcmeIac|6qb*FQd}6qSk;C8`-hC_;7>F~(BKzLqU}*%|A*Qi?ER$u?xqzK$$|B#brt zGK?_xu?=SIzOT{!*5`iQzdwJ!f9~IR{^;?@>owPP&biKYu5+E|^Gwhqb){1b>Jv4rp{A3OW_-jSnB zt6%+2pPyXo;pA4>WxHEH8WXu1e1i`^`>g!~TZ9NIK=Y~kx6d6psvrz&H`t$^ zjn%WtvWz+gvD-s4}3urwZZ+YSk86oA-5 z5+lger~dlQ{5i_f$!621(_e7u-G~wVdR;=aV7DE*aA3dE@P7@~`lgZ&hb|+W$Hvy8 zPEcc=Kl_+B_7SwKe!}NmsK(8O@R+~v%FyD-FHJv=PIt#qmxT*o@~by_S`*5MJ&Co! zI($tKUS{Aw7dkUg{yJuy9||?yzC2k%eN&zo!eIA3h2l3?b0J;aitz99iToU;#mSmH z8MUUPPCa0IZby6P6B7}vd}-qEJBf5S+2I4V{bQ3}r`|1i+mVmxv(An2e5<9_K2T`Z z_Up@ow%^}hx9w6!9W7fwkF5v6sLpbdh#6&4W`&6znVqMh&LAK12<*vIpx8MR;V0g} zpJVgfvIAxBMOoxv4QY!L7hk}YG08a$7NvP^PDW$)UcImOwYu4h3cxV@hGC)Vmw+Ka zJ;z_u8Jv;1HmE>)HC~uUXTX|Y&>S_+Ceqp*=O~ularZ|~)fBs?v}aL&#v|uGJ*&4z zC)WE-99aeQSsdW|yOg#l0|#sUQX31)_V`(u49TLk*G#R=1`aD|s}H_%i>A~qA+M&R z;pAx=En^ckfPOGusQyN)4^wOWERy$U&7Mu)Qo(Dz{>m~v&u%r#g-)foQwDa=qP!^y z{bgrTtb8bgR-BI8xn+2ERz#@W-nag!r8jaDf<~=dc}5ttYG?n#X>O_}TvCAK5$Bm? zUx?Wihv-TB`OczU;R_bpM!x0!;+npZ&H#d#vHB zpXbMXDhHjAER&1Qh!}B4RQY^fr7&x`7iYoV$txbe0}4z=t9{z`wwD~a)ncCxS+&PY zOf_8(z4Pv}ld^gTOTO&Fo#HRfRt%<8QJT&IqOizxSFVk?ccXLXds}iUNGkb`=*w;c z&(eK&w=9QBtgUhRm5UVxpW)jJWteGjchbGr=HH`l8&$ZhpfpjJy_dhIt0zcK6?e!o za?0(lb&pNR`fNpzC01KSe&m1oS+u>ASwtMPYTHV16XsvUMAKfRPVH3BpVN%A5UPHW z8Y>Eu+xcbN!YV_TZK5G0r9M*KB3Su(ba&C+c;9e(v2y9ONykbfXCm6?cWuBJ`Rw(F zpRLkv_5QQ*hFN_X#i{kdZ|AmjWuBj@uAm9>&+y6pT$u_qRdz&O&90KKdarQy#s+$w zuwa8ShXgmFMS4!}GptTO0#B`OO$&QrokuEM+uEg;h(>)mA`jgchX+fnyBkXEabi0g z(I&ntvPw_&;2UF>lf63cICYXi)%*o5CI=z%T*<)511C4r{-9m$qC1(UG&vO7=sON} z+QaUN{o1H_=XQxPwqBu{9bs?j7r(eP^NlgfvZiH@Hd#(g1(%HIzh!1N5zD7h_{jlm z)#g;(*nCf7$HvM8hMZaCO;FbCN`9;Ns^7@+=kl24`qzxuQwu|HIWNGjE|N7us>QH! zkX6V&f@uDZooZ8?d;##!00EB+%6%$*u=W#tk3jS1lp$@gF&1VG9`=)Jq~N8ns}EZ1 z-ZK8Mc)x}>ElZ5Mu}FiG^Q)HFBp`q6d4=YqW?i?z$aoI@l%r_e6)kP)wJ^Q@Vk>Kx z^aVtl7^9pZO8Dn|Vr>1qrW!a}**CI^Ytt`ifjax6`8$Z<$eX9_I!jbYhRJRGIl>l$ z-(EHJFdbMAfiQtnUSd!+f-ku0V z6XFOx4YfTA2;$G4qb&SLn7<&p$(Ma#+&$L3Sb>we=R|o>FH(m{MO$m#nCeR{ZQ`Ss zjg7r~9xKIY=jy+u9INz6ZXewvmlc>+E!Qz(&^<4{R5f0Kzd5@6<^*%yWSD-*i)WIq za}nL!{>ZP-Y2r(Q2!0R~6>jBEFuQ|D%KkN)z=)MX$S~t1kCM;LI0Q8UM}kwrY8wBy{LkorqcSf7-Vn%JE_BpHgSRjox+{w<2)jw>r%TdP*1U zYvhHOs5>4UV0Bs=!pg!r!iHUZ7|)cdI==qS6KO7N$p#MPKW(p1hJH$r`h5P^W2aE_ zaKdS<*lZhhnn${)qj$~>u%Rj??I$PCYnL}Kl#Mz_y3Cx(+pgRth?<-TDa!ve==*2o|P(M z<{1H%z$0!aQs%*GU2)k!*!pwCTfUi7QAemocwZS{`{`y!VagKTNos?M$#O2ovxHW%dm5gLi(q z948JM%gkq6`8fld>Az-0&4BGRjsZrff$69Jd2w*9+89%#v$Wc~?XzAz>istxuu1?^ zpsXqm(RK;=lT-m`UAV0hI`ypF`q)se>C@h%e-j5iwY7a>NWkty+i~otKWk$e{kgN$ z+f~g3lT#vjnD^vZwt=HzQ_84lV$8vqNn&$+Z2X&73q*+8Jcl|Ty)QH>r$a&#T(O+p z`qZ`RVaNslfH*^5gVzBwzCE-!ZHny)3KLQ5bfU_Z%3SI?Rm?{dGZkMi(7W{w8$jB+ zP^icoN1u!d%-_X#J*&W8PGae5P}c1Ff|UB&rntOq5it0+oGvMu=JMz z!c(jQHX1?GV4Q4Xw3hXrp0+Zo0Dq-G%S!~1-2Z-$GKo@~_cm*PZh|koF7=Cfq(U}F zNcb?3oPF(3pz~KXWOaqLg$&Bs@yT>#7+^P zJd=YNBQJl(UpCmHFFO6*QRn>~Wof+R3iZ6dao1}!xqU9DSZe-eM|jvj6B-Cdup_$^ zG|a*FIdQT>s(knQ_+jqJ&|vJ9@Aqa?oAj1ouIbalI@`~|wW&@jBxG{oZBHTV|7&8}!y*NH&2`HB}zh(o(>9mMUX8Y8fya&`D+Vu{v= zDx3h~WJD`MntQNsLG#q1A548Lq?|~!CR%JKLihTgcAP({VwKyMhIBarF;am>Kjky` zVBGW%?YK<~NZ(o9{~@a=be%=g-e7s2pt^z>cip$)1R5une#s$@7+pLNROu z@C|rcDeDyh|8>-F9$IP&{k%=e`5uFuq{E_opZr$CPKv@cPo$t+i5lbqcoiZZGgo<; zB~v@dXY4W-jBP}isjUz5zoWPmwFWaaH62E;#wRY5kPP60tPtgjim#*6X^-L>it z{c@S8f4t3Z_|t&iOXPpM_Lh@F+?~r7$ zv3rG;gjZ(KpvvB-qpCB05mkb~zz%0R3%Rac;`cAN4+x83nOooj*@z#GXr{W9+s&5X zYH5`)rf21qC%j3sz~y;QSj#CMcbZFN&;*SbHYMiL?ne<9$XKKdq=VjV{ARgEzLiOx2HdIYOscU$L*Q~MtaNA>|>X=E$?QfIf{VgO3qfgHvXT(f(At>;W<$y z&Nv&IB~!&e&cu3mccSZOfX@%F>{L%L=gB6fFByffZ!f$m7C=r97)KRoa^4oO)_c`B zREtI#a&0P-ip)pXrfkOkW8|kIgZ%W$E=;0GJyO5{?sUTw}7^2htW#q4zWB4=SWj{cn1vW@N_} z!Jw>gVmKXS6t;Dd!Sr0p%(rqnJ^!^}>ZuCGbj9&0RnM|E;77A5n{&I>r-o%gBn|RU zwcrFWjHjgwTI@o~dFW7cxx3C_|2MhZ{JClW*$#?+-G6w6yr{4nT6Z^XWcU#%n*V2a zx+hL~FJ)rtv^kBn5E~>0!gf*NrI>Q{;8sKXOpb`u{K*b4q1*nn;O%LAgJ5E6w9>qj zib1&#N#5$SpQf;ylUrC#=^M<1=+#n*3XN}4C;Vwm#h!p#$X|lUkOpI?m|&F5l|+O% z_Jog=Ragn?m1ZskeHZcKb~p_GJuBz))_Pk79@AD-Xoe9()EwvyTC)pP zS51LM&eVj*?}j_B-8JjDF&kSg^=$#ffF$PQR2(92hDi$!jw~SQD*_AgzF+2pnWFBf zIevQ=*)3=b?E7YJJG?8*J@p*Y&YmkQ?LCo61@i+oI-0;~fnZ3QH83Ra|?= zjcM8&9O#c?ABh@LN>X6l9Vo)@npbFNHIY(=WIZmJll@U zX8FH2prk0maVSTb$tPE&G+&HRxp9%Am$P}*e=}vX9_RvtXTpM-*VR0$5Y3M-WwI}` zehZq>o9?02?NE95b47X6#a3OqGY97=SDa_n(Ur3_JZLvy8P^qISH0=@Ge14EDdLU| zi?3nVw?aM7Hc`u!e~#hS=9@pLRt-gTJaV zdd@p5mXmGbacL8caX*2Z#;&r%ftR6X?b^NczO*mqAlskDct22Ob>v5liix^o@w z%3`Z?1-^nkdXX~@X#y%`b`7;@`8xgJ8t)}vViQ}kY4p@)e}Skj(sHF&;h?v zA+6{7{pgs0B7c5myX|#w`=98SRPlH0cfg&Ei;aJFj+$d^jKc6y-n^SBxYJ2)!erU8 zeNi`(l#%tADzXL=+{2BS05gM4RQALRbE~Q*)*Fh5HT$BMB;Cd4$wMD$Klgn56Hy+G zRJ}gWyF=YlF?5y6_%Ltg0a4NH)&YyhSCFlUlXuSMoIROjiJ16Njp%I1+s{RcPuWaO z*_{rrg2eDih!j`_i8n|YM@Npwh_nSRpha35LPX|I-}V20oPzBO8u5;{(jndWoNjnA zCmUPk$q(u-Df0nys)V?59}pL_gt46?(aHBC_`0eKvvy?vKC2 zF)d8aX&Ikg>p z%AZ|j^p=Wxl=sz5ZKqi5l2086*d2p^9}oRDNrotUbE5wN35b@&d}p58z6A{_ghu>tkCe2C!ACYo@A-&0zh^*eq=(jY37 z-9q~$r*Z7M)Z&Z2wRx+kkSh!85O5xp$XR$z=-b=M+^lm;A#3&W3H4@}Rz1ewug`Uc zE=3W6Y8CfC7Th%E(;PXH}uGWK(Q^;v$>KRPfJa?J&o9Cp|E zJZuUohTQo%KOu-1B&9;)Z+dfbbIn=bUn76&en#}=wguXwv9Yc) zUJ=G!iLD;n!z&1#kc1sxgT-5Jh*moZ1L36LnY4OLQR=(Y2W(KO30tDA;B9h|xl!r; z{u`l$oC0qY2Xmt4(uxdUp<0djZmj;9C0+^Ml?^Cm6O=s0vHV|#3i(xBD49~+{ya-W zdU~#n6ZBL{LcUL&tX7yOD*MPjfq#fAdL(oqrQhu6uF9xx_R(~xplZ?zt$f#=DXv4lR(3Qy4gd;~^d(wZ1G+%Z zB?+VltvB^qjMLNhz3h+11~K!*MhO_iS^~A#k3!W0!_FNtQ$V%)IO7^-(8N0>TuabMP9#>mJwk;dkybs zgEQitKJfDK2|L#8x($)9J_x+HyT|Ff@oY(Av$^ZlgerC_E>3iJJ5K9KyyJkEZL#HV z`TWF7J|m0r=`SE`f-(GTEa1M+bpAuCr(+E+!nG|FfrYvSTUxE`S}-{J_v}kW-*y{J z$q(_HE#)@sFx(qy98cK$I(=)9^+RCWZ>G+Rp3&aIYOu% ztQ}}!TQ6#$55`>9|2USpNKgLM50>dpR};~Y+btGrVZh=Tv`Bd~vaWG(P)e}OUnC1J$YYS!kS|CmUk4b;_Bha!T zn9Ul;qk*!U>&vxmSLdZ2Ay&-iVd-A<-OQ?o3<7A8$HuVV3}l3~fq=Q3o2Wf`MfzJw zC@Usczf>3~-mV$LSvo~(VX3w1vVwanN(LZ}&%f8z)W)~fdo@UArb5&6z=i(}o=IHZ zNugOx=Hfj1CHg)n6B;b=q0FUB2LWW7HIw%}IXjaA_P+UT?~ar^Tdm^ZOP)TA0?t#d zw|%zBYVd^+c2VK75x2Ph=$)wg4hB%x&#YS1c^AfKOQu@dQQYm1v`av~Ghg1-z5YGo zKAM2`87O!*Ms^?d>4&QeZ?!0X>?#_UwK9v5^Yhi!1xm1T@m&1n_8l^mlVzCwz3pd@ zbKh3V_7(RmPB$rPC%Q)Ijh1GYjfCvAo|`DkAaGzWAdG|TwbCdj^H2Cnz23>F^~Jzd z;2@Cj%BTMNsXhb8Q-1EVZz4wh_I);_C|@&B!UFmw$9Xbut`&dv+iWqZD}NApn!^SF z_^hk97p0sBp1pgEGxlCl%=f#hU}a%^+xwf|9gpL47sxV)jEl#IjLeYn(gYbNZIUzTrcmvMy-B z8oXvME4yoGt&4m2<061=Ipkr;Lj@h8{e}}hpA5~@cAf}?i_;!pV$R5m8oM=Vngpf@ z7D$5#!)v zl`}OvB$&t4*6-I$>bSj$P`SXP1+43f&RJ5DRxN=7=;PDpJLPin=SM2*6qf-o^u*|@ z_4l}8G+}$GEs(s>RgoFSnp{gSiP7eVwn1d}MlZPG zRdDL#GZ;6-I$~dCPGt&Ktug<{3>Q?^g6I8d=&Iy)Y^K`cthOjij*C^R>iGgORJ?HP z$2+SKbY%xLq`fSBma+@PW1THy`HyB8$Q+nAr{tUJLWT}}mV9GsBJ%}JjvOb(WDxVz zU{rI)-cad^r}>7iNnCZqYP*!s0_HXAFUhMJzK)BWAg8lN`8xQQF=?dOAB|6H=ibD% zp!{TWM~kA3g{hddU-b@Arv;8iaI00(>hcf``%{!=%`l1%K7pRvh|a{~;!u!0euieI zxR&+j*Xa9jAYMO-q}>3JYbK^Qe*4>_KN`0f?+Xc`Bwe%m#YL$o?uO1p(G%1)t(tZq z&@opZ+nt<7^Q6@LiWPUfLhgo@jgXbEJ1Y0>b?kp$ms+xm+F5=*ih}S?e5LH5Rf8=e zjKgyxFLMs$yXpaDXf8S+lT`-rDUA(yzkndp`METJ4&j{4&9Vfi>v&|53IE@$xG);l6pHtRD_t-?WkDr z4}0-ZZ|7Mw5WGE89nxnT=niOYAKF_;G@BT{UAn-n<%sZfCv4E*)@DpBiD8S;(Q^~m!A+ODGS%n@bxW>wp<_MB0x3^&_o=r5naS5^hF#Oy~r z@7&dwA{M;19{yIixiz2P))apIN51Qjb%tjmPqf}9r6z9OsNyql0Wt4ez?A7Z6AROp zMxvVLyD04Zll$MjHp{t3;Vk>4k3PQRPplWFS!;hHx!-o^b(iJq`7D-O{y^hmCp)Ak ze0I&(;2O9GenfY}hCuB#X0z4A8CZ8EV`U5z9>6{5&f+htI3XRFw* zFGtVgm`rJh)KV2QD_I6kKAC@G9*=?RUaU;yw+eq{>^vT1iSZs;MojupPNi4{mSR$^ zHJ5zeS^nImWcZ*|j}{nfW1dwm)OFNsRjsxL=9ZmaI!`u!aIL*VV$pea21?Tg#+uNz zGtd=Ia^yI%VdNezMR&4s0_*~#*_tVBaN}j?;439{+aHH(bZyt7YslSx>p6~`HEZqyrVa0){T9#rJ)UFA;WCUo}JU^QWbM<+w$~ zx4NBh5sqQ_y$+m8+Nao_Y{UQ}dCo^2ss( z&nu_3%iF)ayTX1&zFB`;JZt6&qt`>78e=qA2qrm~nNlZybc^x!ruS~zS6i03Nu5QL z%xsaMeiZgA!V}><9LVs?M8OMMsvdz>?E+JG*jr$SmAv?pk-u8hBFM-&Px7nT9$BjL zbx5wD*NqnIk$dAl+m+T@Tw&JuNkG6q#W>{Z2e>~Hns;&KJqz+H&b69(ByaQw5Is$n zJj+|NT7<-HUHhhc>|fL4Vv;n`TInD`>%P{T@Wt{HUrD{FN`aXA{m@wX8YY?T?4`o= zBk?rHfWp4+q~xR?RiDX@Mb$`$hLB5EmQCM{2kyqpO%S<&}@WF)%?hhihl60ke!7kAH zMBgo}t;Rz?p7~sufg`3=V!4*?2dtzxfj3W;PaovBHJdTj;kV~}*Q^v5WjK0w{bp%( zO=thG0Yf`^0rzh7qp{?Ithi36t6%92BZRK2*~)M^KoIhhdubz=K(NsY)ccOaJ_XIk z5Jc&YMaUYH>lhza>lK4rh{IdO&{4(bAL0H1?DbS#y);_$Di`PMc>t&$D4U-)s-79D z&13e?>_*>CfMDq0+!sNQd;zaK-L zy9H~Tl1%R$T3qIP*sS6oPr~1{Fw4;8ZR-=yTnB^lBGUk)m^veyC&u$#z?m=S zzOpvExB4V`)V+C+26Xg0urk-Y_CNXxgb0>BLb<}BWUz14Wbac_s(Es1W%n`A`%i!h zJ!=a%@f{;XXsPvj*Rx%H3q_&OK2Ir^K64qVd1K*t>F%w>{T&`4;|g~@&E(@C%blg_ zDvp$gkw(slQLCn5djx}aX#8vuFfL=9)c#it!F=Y0me^w8U) zw)fA;@q=#Mz<=kv&{<$>l7>8r=VCHvi8Ktk0$9tRLep+m~}f(?*_)pVBn zF-mxG7NhilJA=>a6A(`dg{Nk%3pQ58HHmpM)W(|;k8tZK!7B}C>j>qWsJ;Im5YA-J zV^V;5PHHZoZzemCu)Obt`(6@vumdPt()j{{P(guIei)$`)ULSJd>5nGI%3S6y#1yU z36ggO>b{Ugf%HC~TYL?#xx1K*Q$%VD7kv5RPEyY6Z=@Z>DKP{4J*` zZuzP4G*!qyX|&G1<@xFYto>CPo&2e3j(n&Cb0e+p5@iHPhuZRM_U1}^Oa2sK!?52G zrvhYNj$ zcmf)b>_3VcATHz&Fyy$q3a*X)bF*iPrn7Uq%pSNQa6(fQCJK z?eVv4^Pb4V#y2Qsf#VdmqdP;FO9v<0#}Lk7Rj|oi?$P01hXq)AcGyE@ZYcTX5TAmv zoewqyp%<7I4SLEfvFC!w3A@NsVeePw2l;P9vki2}NLt`vLH1Iq)Y11(5ZmDn<10Ud z8Xb?ERaK7&I8@a+GtoSc@B|v(t$mf0ywA1ym=tF2T*d)%Qu`!x0ooTk$}p98GE5QQ7j zQNbDta(Q<=$3h?!Xu8ryIx|R2<&(q$#9f6|r4PF1&p!F8iCQfFVBjbU5zzmv1=Pg) z))s74^vlFx)w_N3O3LwnCQ{bP`_NAiRpGQ=mdGw zy;HGNaR|TTf}`Q$a|#|*C;6KP5RO5GpoKx`4K1KFtNFoi3kzU5=26vqUbE;@pRLc4 z;v)p0Hv0!jxYqNqNSpNBG(V|Fq*kEI3nYK+upZrd9mA=+#Mrm}dR=x*1?iYvPT<9s zrGKEX(fPwZ@f$%>z^h=LxCrq^uN~;ZB>?eF;BDdb>B$+jh+du;|Ej45L{DSR`!@Fv zig@p5Woq>|=F)z3NZNzK{=&Aas#bx0Y{{GyoerG98J^uJ5mmgjNg*dEu3^?tAb&R3QdYyEvZ=^SC8 zu*^J|Z~?oCP#cjw%MR9K-yu&K=-4lR>W?X!t7H6;>^^x){7q;yknau_{A4wxRkJZX zxBXCXmb1)d_SYuD3y4JKgu{ZiO^>>;I_Zwp|+Zbet9hM2}#**LG-8Ive(SE{uo#`~dvZ&t(DK(i|4OIuWQtQA8TRgJr>R^LCJ;;J+?&X!UaZk9G zu*b?E;O!WBQ)Q>p-BjqUELZ0uN4|)~W>dH?jW-rDrl9Kp%mWYt0Hu1w7>O|^yE`T@ zeSnoD?{?@MH_ue6R3n2KvOGExyk9FKK8DC#lCHIJ~2C zE?e$xgnmU@1KUfL7j^qb#2s5!*1EBY+3vQ$PaJI@Di=vx=gv*f$c7jBDdZNX+Sc|y z2k|u>at4knER|O(39EJe00QXd!%7&p^21A&1R8&RW%9lnm`Axf%KJGh3&p%|SuHcS;rTHIFT#ig z>wicj@gJIbguRI^y?12Zt=X~P3;@*jo?YAU203Yvd(ntE2qT1|BKgh7iAx8tt`6wT zzoq~^GMG(XHv@CR(C$NgH&-mZf(ar}@iGqiXJNa}m&}$XOu*%8T>$ZjuKa(@xfoxo z9_E^bd&EYHBkzM(gA)`ongFMiG4~rbCWRG7OU@TrZYdQQw&;w-5zd1SDHNbuRAB;_ z=ff_5mU3&;9(RwVeT`!nU}9X~{;iH^1c;J4(AaW7+R?X;UH{ZpkMX=wp6k!AxdX7K z*m|RfuWjl%`FCqSBgK_94NwF0#y;@va>(|`B4RoOBe(w?YF4Ik`l-$sBk+x<&G!&7 zh)dx0cr*i2d}JgTBc!{;t>dmlx#k^u`{s#G2HY-ry@*|Jmv@4wHDLUbZJ{y83{ z5H9}PH6-Dxeuo0b1mIYbgJoVYe1vLbyd_bKqx>Otcz^bZHNXsM`v87+bstk`du)E_ z`{p;MvzaUACUe&}ACYni=E%gfrW@V>6ZFybSvFG{<{Yax>#Z*%o8ng0AZ3@}p6SL8 zr;`3LAN^`Z+`=&fXx;p`T_^v#3xG7DC_G~h<;XnddpJuj$#Pd7tI?3BR1t$aU7KO$ z9wO6{Z3PFe*PLY;W0~#9gMzTdR=Jl&9f$jX}K#8X3aDV}J z8_}n3kk)i*mj>Y(B)+fcAoU1|-QToCWP_a{p)fb6e}{i`x`NPLiqJzmpptdOcX0qF zUd1!mZclH+zUhY_8ZZD&!LA6CLt&ebl;7s7&xpzwnT@ZA1vM%~9dsix)K?=C0WJi4 zy0}(+VN2CXVY1ya8!#Vp)Bb`R0tfyWoN(6YW3~5<)H?CmWv-p@Bco8U$f`t-VWCoL zgV&=Qu_~wt)sxpMLUr)Fi(H$!qz|l&4pL{7NPgw73C7s_*fsoCwSND#GxHK zzLx-&{)g1Z2ejBnpCwgf!-7v;ldOa+y{CkE^LsWAOn!E@2eFoyme1P^Id2K0JT-*)&nG@po5_kRYsGNk4S8jS5 z0Bp#tR1}U!Y20vwJ3)YrZ#Ldz`0C0=S8cm+jY^j$DTgqtz)2l6DdEh(i6X({*D6XL ze+ZTZeCHBmeT2;Smk$mj{rnW>eQi_U>7PQYTdQBVS*j;{WmlGJz@h14z!0Ffb_PKE z!mE1Uyqgtwr?kiwc$f>Wm2TgNPC zFbpD9K_#}zU+oq#e*nbaPfkO8Y7Ak>U?agMI`7x3;aNBj&((4?BbXq_b4QoO5pBcOm&X2; zs5pzj+!q~xHHMpwXLgc|{W4^@!M97-=Y>6jRmNHzu%zX^XKuD%2P47oyY`XXN1GWD zuIJ2%yD?i0<=6faO{2V%)c~frZ+LJxmAm+5$asT$w!`~{iu#|BLFC- z@F79~+ycplKpI%pdo&h*dsDrOAgA$H~~M3?_g&Nvs3rflQnR>jwB=@-npz#zx{lvT5(tf0_Q}w@Ghc5ckiceQ|B{ z2XcYZTr6Jz5#H)UA6v>Q_OR6;;LE`{VUc;~a}Gj!Mc_->!$yFcXAVvZP;**>c|34M zM5`MJ9zM*m{PU16=M#YzXd^kHZqW1B%*=nL)k0?zq_8)j%ozfFhq|l=A1ExJDi%3> zJ%)RBtVHqqp+u;I@D6JFDvKCIWWRpF({#v4k1_i*BPl)CJcmFBRm#XMI-GIpPn7q! zS?vd6cJHWt=8;2w{n?ZQEgzk!fN=xy?vCpaU)|w^3I`_^h$TJ{AqAF$lGPmf#1Frz zrEqYzCjI8%i-DADKk2u|T|9gk`R5^*wmz&uSvJaUsU-5};jl*^Q;#Xo`%-6GAWD99 zc<^^ETY~=RAJVBNe2)~ITHZZHV#tn-*D>+wJ;@>2bMlf{Yt`P?yfHkU+GslU@Q+C{ z!{=ZQ;X5}aYOP_wRu#SNJl!s;borHn?j0%)S@NRu+Rg^G4{&v=ZFQ^h>eg05VQb)@ zsd9mQEgiVT$zaj5ZIeR&AFbMi`5h|Taw^uI6gv&v(v$|UC!nbkXwH0hGzGP51auL7 zTL1uT#xf(fK?~!3?+g!;c`+&0wiRHdw2`b(D?=iRCY2&lf`1meR2_zWfz?5zA@+Ff z54tIxr#`QbRK&P$FLAQk*oNM%vm0=N6%*cZR#m_K?6xozli)Ul#Z@l=7)72@r5A2B z24dp}pOsmIF^+6U`X9s;9*j-~V*kD05* z&*pR6#v@wZ_AiuY=!u%Y2RbYdnorl~ERLOGCzJw&k=`}pVE3@^_JXm2u#VxYw`|fL zFZ+gFy4CQ`PpDA;YOjuoC$21N=_}B)HrD7Bv-Vht<9u~H6v-V)?py%Md?gYazhAcj zOa!9Or!331+l`-lfg0hK$0kxE)^(dWu}&59^1zWl<~VZ-l1*oD$C?|0h67^W+}yG) z{?mXd^&3>I!k}3OG+;V4T}{9D`k3X`+N?Fs@!MG7vSJO96W7_(uR1Oz1n@8vZu{ZC(S^w;Q*mwc3*v(^H|yWh;Z5`PM}<$A`I#&bC)+a=xVFx&mfXcN^8^#oOb_(N^)W%JFk z3LfcTR*drAw9p0ROD)T5-ZXJvse5a=UK?P!ds@oFXJp1!Bl!A3(g(78!>Vq1PeQ1? z_a^70eP1k5@+oo%u9<`}8jWR)nxY8X)Dy%8Pmd(^g=RhV-lqi7WeWt{dQXNOD zpIw#kTAL|Z09p+gO)}7!m4-|B)npaY$>pMm{QT&+d31={t@L~WpSx78K^7|v*>;_f zur=S_p69vAo=X@?UEM-6fBqfD3xX%Dtoq=uM*yUw8o)L$n0Vjz-urpMBp$sBR61|& z4nJs?=+v>$?5gq{>QVw)exHt}50vW+sg186jNJ^xZefer`TTd3ojhtUuskf?W9Ux) zQUd^_RDMoy#}8I*&1FA3n?V_>IOw>%Eo~bwFzQ1|pv0S}V{N6W@Ix}r9!|x1`@;** zr&M9glWpv#_uhdX{-=+V?zmvxNb1sFHQ1e_ygVK6zL z*B+cJLRYz4R6n5xX@O```6)okBLHlj&G&i5*(R0L2F&*Bq0;A2*xvl?!RH?ZT7dw< zfx6m926#n1b^ya>O2cnUa~5r0qKkLK+5f;64tE$BLA^QiED>6W6G4qWbhxFgt}E1$ zAHk^2mrk`b^aR(1(H;P|wn!;fZ@fxvGUJv$JAf7p3Wfc%21}-T09PM5a*ct=CZ7ho zJUcPM99j%`o1qd<1Rp^85+89v_CIXqq;^CypJuSbl>z~|tm~jTk87%Apfh=z7aC4@ zT|Z%_DP;jHUwJc!`|z_0_az75;^`>qMttGByRMVZ#Op`x@>l(}l6ZCi_NBDr>z_Wf zGmfqh&ZFqxY{(7DSjiRY&jjQpQ##1RL6NFNx4JhcjHSNn9(?nanki04IrgqSRlzgx z1X>`qW0ggML%QQc{vPP#nYC!S*@LuHUM=b9Qb{eVY$9wf?LU6VK+}$A6K1*(N?!1i zyG0$W=PuJR^h`|bhP$qp^TWfW;^+5(k?My|DL_pQ_=whp2MBoMqwhqwBBe)FS@tKcn5klu$6 zA6Tgw>Hq!w|G@&a`;JVfPf#^Ivg@~SirUNk|JmmjYvs%{9#`0%gQj(;qPjxfebayb E4-^vfVgLXD literal 0 HcmV?d00001 diff --git a/openmdao/docs/openmdao_book/theory_manual/images/nonpar_rev.png b/openmdao/docs/openmdao_book/theory_manual/images/nonpar_rev.png new file mode 100644 index 0000000000000000000000000000000000000000..32c0f0f07297b0d8298386d67ad8925dd9208590 GIT binary patch literal 19194 zcmeHvXH=8h^KO(Q9KepEfD|Qwbm;<8ER@iZCLJa8CN1=)A}T#JDFFopkrH|dRX`wA z=_M4AP(uv?gi!7qJO_WbeZK47`{x5|LGtcBd-lxi+4DT}zSU4uI7dZK1p@gp`~kB5W&9NJp~c zb?W!(sh-XKREsQIm{r9>s#kV(?h9bX|N8ka4g5c<0b6K%a}@SdW+qhac!i&Zg*;Y` z;0+5SRWB(?|7q*y&QlajH=jSxPWqEY5?Oc){U!n-`JVJ=;BrmlJ20yRY}M>E>F?rA zDsFh#|GN)A`vA>^hLjRl->lljz?ex(9uPNqDhI#BDCu$Mn?O+Q(cS`#8|{MTu#h5s z`XK`UuSPiOA(qPi@PQZtrG33?pX4dvV_9SDRb4P$Z;>m7kc*WW1{PDprB6G;`jzyo z2DcFl9m^;?s@BJi_^Oy`_C&dKqilr^I~3)={P;@fNMD7kQXBJn%R`1@_HXHPjdIp% ziJxpt*6!nCi9){!w=vQ;SvHu?+ked%4p4EwtK z5oeddigrK$e1|j0f%&H>X&hOH@An_mk2G3+fgXen;k|-&9J&jiHgyJ`B2%gNKj@U& z88`ReoxO%se@rqSOEcs_ot~gxF;|<>h3ljNY=bYOC4+e1rK%HA>?Tij7=$d^uaq1e z5<}$7JK}k}S{R&e99HY(>dwXKnQZKF+}zto6loii`Or38gVP|6h4_4R?@T$1w?=V1 zc4ncxfXwg`vha%v?UY+gA~4Y8!MCK6ZV7Il>&WU=2_dWQl=Zru*!<%qMbZd>kmpwolHx9$Q*5qb%Y*D|2Ak@hhbS4eX|59+e;G_7H))@}ySK}yV=vX;Tz9x{<`Cg!D+_{x;mbS{9vGDNR zYVFQsp=nR5RF^by*}Ns3L2t-vtkh=i4Qs?53TEHuY7tkEwo{WlDB@}{hC0o+FVC^B zxf3CU!6h^KP5$xY+`Q-)_TMIblI31fB*z(}v~xg1H@p`Ni#z}=LK@yZFPLqN5#VB`+bT;@2#aob(sUq~AY{_~wYwNrTtW?S=% zHm!?0YlThcEvr|`H}~cxJSM%Ye)&d7ZI6Tz)g@O-*AFR}JR9y2mrI`B_ub!U^@;3( zSau6@Z;v$bu`gWbda-wJ# zkIer1x|8AFW{-b0%|SI}_3vf8-tla2XsfFgyU#C66iq44Q2-O`DSfP5esA8He)D1D zf_p=Unj~*Z?Z}FuX1Dl!%KDq7Yo@-JCRZ^PNSp5;?t3y8V#K6Gs>nye3&@NfKYoUS ztt=rT5bc1S(MhWx5W{YW_t`~Cewt}bzFNI%tGis4&55|%Ysb&OHx3?Fv{Yh>wzmtR zCyBCDP!8{FpsgtZcd&7@&Hn|+lECcwLBBWOP+wQf!B!WxY9DXp96Vf+o*D6Ldu?i} zGgV5`=(Qs99E@1?c82~(qL5WmjG6@JGIIFWQnX2hI#J{fKxAD6rU$nB< z$zE6WXZx&-UY3$6E5KexYon>*N1>|}B+W}?=Rn^>9nC_s*x*Nx8~~?J)SkbxpbZ5N{1aoq&lm#(R2~>z=V&n@co%F^fc8N{h;}|~G6Z6b4^+^>c zLKuxpV><9l7jVmG6Ia6rLrbym7o?A;jzcL+C)BVm(oa-)nW+a&vZL~A8o7pd7qipa z)og~?UXO2AZUW=I>fpEOt!-uUd1ZIL+6%Dirn&?p7s#)gg8IX~;?^mS;JQhVrJQW< zsp_cS`B>42(fyl+M-={1DVIhvnn_b>aJ#|p@$YK(b=YXFq+m>*#zo$6AJBYyO0RdU z-dRq)Xc8Ku$`#g4b%5qbdmq6lne_0ycyp(-nvC?`LPnE6vUWRZ=Yaa^0a4ssKU#{7 zACjfW2hWQcJxdxL-TT<31o|5L4ARs&mj^6?d2zDV=l;-PzAEWVE4hqPWTIX^ipd4z ztVGFx2V^&FSr)ky`RH!hAVzEC0da5(uO6e$_Nt0aV9TQ06!p<1k`Bht1~sFSHNBpg$ZP5qw*4toq&Z>H2xk>;*}9CDP_TA zJ9(T3hgLyv!@s=j{V}cyegX> zIgD5P;7f%$>lvko=wP@FoCsz>`F%m3n4QHVxkCS!ad0G9Pcci$Bz}2qyC>!xSd1J- z?ANiwQPmP!4CwJX_RGHa2NMsuN5;EV8jJd_NL)R zK*=En^{qUH^i$2*&b0Eu#lYrl3aAx<6Cvq1cIPnkKB!tIP9}RQSA$}{gaJcAUA(hd zMF<$lHqKu-v#sRjW9{Z+_JtH)G0FM_qN!rkq2=%_hQdS34--@br55{GrZ;4 z*JTe^SiBzM%djlj`SMs}CqHyzi<%(e4d?PDmXk=b%dY+*6nn}XNi$S(z5F>58)TB4 zr_8Z1{n8=vsW zkioaqGxM=rXgBmGymnmLt5DMn*_*~&B~}L&aya~Oyx!8tp9aqD@gVta@D{B9?{<;<}!+97R8@upGNlrNxdcizlb592^ z2BEG{j}VDAV%Zfb9A5Y6vF4RF0O3TwMp9g90D-_s9@rOG0^G*OgYts6qkON(9WgTO z&XgsbjZ=wBv7!<(kI*9Qd_5|P8`JfnH%~)I(!li zPPi^f`P~>^?#K+9E-x;}-Z1BQNqpi`iVnEx9SD5|O~;P2wD{dWJERSA9X);1hI6~) zK4?JI#98j0jD+K=Ec<&C!)zf5TxYjLQ%N`B2K6p$oT6IX{w=ST5jK5LUXVTa7C1zC z60Q=vKiGV4*NDw6?%uVLx7mEu_yB~9*WvA|{#v2%ihA};K7|TO57twG4>n=CnZGh- z=FUSOEaGo@qcRE>8F0l9@f3S6vAGN9_8lE11KXoltgO)HEhZArOQ2f7#yGHfu@oxk zAul}JkK&=IIS{$&E{;;IWS|6HNvbHh^^KWP%!7r-64%hZzlXry0eqr#B>bHTum91H za~`{SoxPDdM>d`?kmQW=5ay>=2dkx=uKi#w!>mx zywCOO&KH=1VX+myMQr(?=4AIjYF2pz99}`jWTj3QXO=tg6pQxDHr?K+%^djUiVJ~B zF%G#oMJ3rd{8MJdL+j;FXLtLQfDL_(eakXeqF~$hpHoHKKV<|@J>qXqQ*`UVMGyIZ zE=n`1w_46Iio7nl2{oQu$@|AnZVhX=IGarsPV(3!jvD#lL#*wVn{_3=#;Lgtwylt{ z__vst2gG8H(SMIY`u6avf!PJoX-6>V8FaaqxmJzy9-h^d zs5rO4@C@zULfbsO7>_S2EBf}lOg%cRF1rrIk$9ri&?m>o>dWgrHBmAR5-ofdnaq}p zHi<=72SH!t%j*l^92vud4B=A7;-&<$av30GOj19q%S(UvAG;KI9Zf4Li?1%lc-O)O zV17+nY-2Ol%_ca3DUZmNi9>vM$tnKr8X$6f^16|&QrwG?Ao6I7IKsup)v zWY;g_+=tUBDSAVY-Oao+MXMBeIhp;T3**u;VU)`fKX*+U(f^_3wwt3r`GjGN_YZb> zfuzG9-V2Y0`yoQ07R$Yp)LdP=)wub$N9ykKXpDSAV*Z2apY zg)e=saYYJwTH=O%lDrB^sB8;8bELDHTxi%`Pf6~_`@PPt6vclFpIdGgMrptGAH)24~lp%S40{zvXnx=gQf6 z_k|jES+JLY>YAXmZASu$a)c1}lD5N9DD4b=Eq`sKH2ZehcX2XHCn=1k?CWZhc@UD+L#3;#7h4f+nL>U#^1hR-2O%bN2#&)k~}h z`fg0Q6qzEEdBW}f*%$)By4Ep(6*%V|SiKYlwOkOH4wkO=|7mOQ(Yix2-p(s?VhJF;L)~N$?_{2!Qy9Y zwO#KVC)$gPwKz&_V^rIVy%Fw>KMk4JRtbu$&qAtiyIoiosq@ZvvLD`)I||s6AacTm zR1c9eWKpceQckwZxX)^4*))2q%iHLWs^AL9oFcihgr(^4`~L`o7gK;Z3TC@$3UTr}fUjY|2|-C>fpCflMAIMWJp?u#boTC94v~YrRE3 zWvVh~Q=hah%)RR!iS}d|>V^pGYR|*HW^)@vvC@ih_S)S$Hjv ztV}~+$En5(UqA!?Ep`6Pyt^YK9gN+*elIZm3T_iWL%~{Le#myGeSx4;fUzcR&32Oh zM!=DkZkdCfu47r8zHh&(3G)N^zw$tFwzm5?do4~$1H0v%1__$dZbiC|{~3B*tegJ+(Zuutoc>rbAgV5me)P}Ch-Aqq zakuwaN#GJQq-Iz~=c9@;dHtazbL}T^hEzk-3KOxq<_V0xip3ci4uv834OTkd-3Km4 zZl)GqM4Y|;uV$L*-YEhx9^w5(ANKR@|7ME9m)LXY;UHrzA=(y`NMr)D@6Fv>%y|%J zaisOl-DIquxeO0(8Ze%hY)+6CHJb(f=p(4dr$fY17*cLM`x-w;`d}z0P*_b17Ulm5iG96Z{_Uo?s zFxfEMId|Bv7RUM%TTo!IVDQjKwyaL`WZylwIu4GvRLVC!f+E3?nu)u z`goqMTCl5Z^#Fu#J<*rIcd?iQwvyA;<^{bf^Z36iVq%rbW5+oX*{i-kUy%x=kMe6X z0WM|zYm#d zGP+}ATJ$d)KX6OIkh27wEK~m>hmwE%JqYH>yPW=HNw$8~X7_a1cSnYs-)MP#6H@)k^(QKehU0mA zw>9U{%+pWM&(L1*Pp^TQZvUA{Go*oOWiX@8n;<5NZ*~+aUmw&;wUU?I_chL?_z(d< zP$4s)zxK-rAC4v5SoI-Xz9hAmJjQGnzoh38fZ#=qN3U`xY%kKJo<_7chA`pxEx^j;XG(2go$tL1L?i?p&Vs) zk3TEAE%6i)hP1cby8-ygE3nk`r62ujLb<~xm^{acvFr0kUW>0NkB^(=^W~tS2mQR# zTD_+Sc@6qA&a4JfrJuon2Yzk0@$@$CXH%uqtR$A6eJ=KE=^rUjWAFLvel_0kSuV;l zu7+iq)Ou&R&Ueqfpys=ym8G(=$9|KPE$hk-uMd{_x~rNRW3GXgu8ck{ha0gc-lDU7 zW`r4^UO`gk@J`eJ@YDd21<$YK*&31xpax~O76zr(Pxc9v)-^yBNMLn>W;^@y%UFCp7_>) zmMR4D){D4OA+)+SRUc4kUI(BnwXIk#Y7?(-dA7rRGj)B;0m2;@#GCWt>kpRLRXh8D zDt~LUpS?H1tC}n^s|@-EipoT0-Uo_74I|=ICh;n9`%NQ$R;pe^y9Y}yZpqKZ;3&0F zQ*`oi4B@HUvn{a&3VSuk`tu&Tp-w2ylyzBZq1xJKI}=^SleJyri~U_S$U4DX(edv9 zmf;A9s%hf4rLzfC2EBIdCOv1u^BkO^!^R#W82gfNk0XvW-^;7-x3sNmrR+!UZ1ySh zCobZBvUUK#pr?`8sxwW~gD+m+Xzf~Z9~Q~Ql%gw56TUw2U;morwluKrR3OB|$0Xe5 z<9oa!PI@K2_|B+tQqFxA^c>yoW`~}D*H=3X{#ZdmEZJ4wo27a$ak^I2M0x^s+^*%n z{5}-y68hDfE_A2)ZR-o!^nsBGsQmA1kFO1G@{)iuaa$yGwUX65DKHT|;gbghXBb~aXiS_^(v|Kq8k;7;` z)R`#M>NB2%A2E*%jy#x`Iij1a!@`vtF$a5F+_(%mC)`lJYHI{z?Yv1GPZgnW{pCxx zYbFa<8E(jeR>~uKdE^S-1-k9-=Bs{A3K$*aq*?N>E#2s`#(mdIUzmW<{W4b>? zj8Lh*0QvzO3)l)Hw_dp{00g2#P9Y@R1=nhKQim%>1RchN`%A2P!saJSZM0lFcq@Ra ziPPIdU695_<mca|FW`W(Y+8tA80 z+*p~N=M?dAl&xldaP{O+#=z>%XQjE08RcvhN_Acj>L10OsSdO?`z}vz^ZQ~i)l2d2 ziG{WxbFn}L5Wd>$C-Q0dEI+ls#_R0UM_Zt&4#cPT0ng|LiGsU0f ze_Nm^rGm9**62Dn^M$KzfrSD@(CclFDX`<0JB}Zz1?pH!E?N2iM7fr}+n9NHK zGa&}t$8}+g&uFKfztpEyeJNA_E}I9(k=>OnCg!6ofbc&8;5+7)6#(j5)Rmx&KU~!_ za+Z@%P94ikTUf2x?4Dc7MV==bxew}0?tFIS;sP)ucWqC`ZKd7OO>FR>gSxtK>Rx~y z94icg2&XzPTpL+KPU%ftn<#i3WT1}^7mMwLkN+&UVD)ncB`7i_)ZC`Cb15yQ0ci&T zDX*wEXCFJ>zY*byFUPoDP2TTr-Pox=lG;yj)4a3HitnVtmiVXkxv2*uNy@y4R2UTJPj{Q-tsN& z51BX`-Vs%C{hOTUmf?3TBy+sd0vvVSm!R>B5%;`<3|w|+r<8|0GwW^DGF^IVaYz{Gw8gK zYZfWDv;1UIV)V5qyk&z8($C`0prosG$dSXJ$hj+;p`e0I_-wUI@7J#)ex|TU!7I&m zwSRM-5c%AD%&$V3NpXz=bOs0_p={pGWgcsjlN9wybV8QzW7MTb$Nh;bCWc%W?yU#X z2|^WztgDId2@Adqv$c~)fRA(f@PGJLWOCU(z2>~FbK<0W&vqZtPJ%fk>&uzKLqCGI z#g4XvpZdoHXGZ7dOVbX1j(azHB-d)fcFr5f;$EffRaY^Cu-w#X{$qJ#o-@z+E(0Yg z_u$n)>2vm)DgIn;l`s43y!9$O$Mg+{QvSKXr7lyhp=zLCA`LMq| zUg?>5S$GzS#fHA$M2>%GN=b8Thxc(t=gB;VN>gy>NJz^AWW zE&VXKwz=MPAz>n*h0f~L&IEA0SnMF&`XF}u#5oCXtOsywupHfJ)dK1Dc&{>o-c9{v z=@4nyl-a+t|LD14MDukF6ZXn$w

{s_YNyfMBw@< zHs>Xm3!gVCNcAmBv&SLp4<&qgyJ!JTmSipL)9;~b`qd;Ye53q21zKPAXdy4D87Gx? z@KtO7+UI&ln~O?%$d%La!Hs=>z>m4z=jM*{Xz^Md@%1ntvTVp;F3GN7!+$8=(9tGI z#LlvAZ;4w-`<>c>es-EaYR35-s zH;Mg==W5k_V}I#Zzk$@Ve11m>bT9^q+MqH+!$bMYm6gxYJxlmt5OA_U3CTy3!s@rF;uF zMRnH0ti&==&SR2}wo!TZOfl*kQXU!Ei?eWRY2EttJ$E&`N(aq+^h;sFCV2Q-RoW9f z!Z%amqMp=J^~Wn2dp~n`wpK7=zIi6smi?!eps6*1{1NfSN|FARABGS4BxhsS|1QL= z%BRH$MM#)$3ISy{hm`UqZ^1$t1=G9F*ro{dZ%zK1lxpvQTY>9c_q?)Y6?Sl%=2Ed6 zG0@7Qm}k6V8*1d_Ae~ttD)P>!`Z)f&UrHP7Z5T<8+Z=i^u#%h#FnOeAZhs*;bqw#` z(@hCn9t~~Os(V>Z&E8V(bjbI;aa)P|y5kCgTD@9mF^s)Xe(0AcsxkEblb_O_Y*DQO~GM}Jp>W& zhK*Ku?Lys?!7v@$eMUlyw&0JX}_C&T3_7LOV1pm8)~Wh z87;KPoX&*Kn6n;x7Tkf7*Cp@kEp0o)`TL0=*;AfG+_O+x@4b8cwLn@{&UaKX0mFD5 zj5oWVE97(d%>2q@b-b!E1J(5U95;91j8XV~npTm9oJ`#?Pawt$tq$+15&RDr4}nwE zE0SBA$_S{}9y0CuKbkT>xcL09>x#loR&h9O%ay6h^lX`GAsrN*v~T6^Y=UWHK!S%- zxxY*D(tP<}Npf2pkAO4336X|ei?vZA66Sk;btY|8Zm zL-p*#h7t9br#?gh7j^-8b#8zSk>B-Rz`XH-|Camc$_#r1Yl8$?*okhxba( zf?cr*YM{=~r)0{X7@TcwCHdG0pEO53Pyq7uJ-<wT`Ufc3_(d#S-XPNXI7z+WOUE>$%v_i?3z z#Lf8f#Sltw!}@HJ^^*hFT#va#E5cIhhsQLqIo2x;gDt%#A80pVx>axLJQqY{a_nr^ zs^*!H7jk?FP7CLuv_24a^6lHUGo4vKZ`P}DYkj+RZ4qgw9&0$VZs}5c zC5Lw>tyJHaO@Hf;pA%#jm(Dv9l1x`}voGVOHcI=|xP5B6n?CNuc`Oc?9^w<|O<9c+ z7cQCko!|Qylq!FCNL)+k(nTEBJm!Xs17tIc9h;5Z9Vx>1htm$c@GauP(iKDdBDqR= z7wNf2y+EzZNhx3tT`i;6J0 z?a1di_YrS{a-xvk!}R*Apet7a0}ox`NJiG+l`2c4F!EoRNB|J`G`2x)@&k<+2=sHF zBo}ZY#M2KS^3E-|cX~tJ0isL0@~M<}CTRfch(BcWh4%IpTe0j^Z`&4;es4zSoz;n2 z|FWC^6R^aclYu2e-+@SGp95(KO&+4IXAJL)+6{H401Pz~!^dy~14X^~Hmg%9p$m!5 z-f!Rg1US5Lq*;|e4GR8);&44t7Ss@w>tijU1Zj`~qrArhx5Z?jm=K&jvm{Yt-WsvB z2j+$F2f9~*5i16I`Oh)74@uYN%b?qUOaFsU=r%FIDdAt?WdJ`?b`U5$K6{r6%N-sD zsDs-U!+q>h5Z=Ho)KjU`^`n-_iM<&L0ELh3_mV0d6ZXMX!l;9ah3)!w@8*t^qR(Jy z0rn~S2|7ZJ0rbHZs3xEt;7>q3Ng^F2CXprSiVe`;UJjciryt{Ffw}Hx>Z-=<;wHty zg&iZMiZ~jM6A6c+4o&+hix0VS=1X-F?^T@xhmR9aeTWC{kU%;7*{Qr?;%Yg@(Q;`Z z$HdSf^tYmB#TARW3MIqi;En)1fjD7wJ3q0@$QbSKycmQSjaY=)>d5v&(7^so_R{MpFS`*h6>@oE1^d70oeX zSSVFTmG^vv4}Nu02-+eH7y-`w4`|s5iwjE_=eAiY)k(UC z{s?! z{tYe;5AT=!)^@TB;@?&9fzTA_we0-IScqtJn8vsXqSbUwhjNX@z1O%d?&Lj)k^NhA60EbrkmzgI9 z_fWC%7%XkJ4ue9>^h&LnM&6psv@e_j&90xQDWWVPEG+$~;IdcoN;;AaPR%9+wyHh= zvFJh5N^-D#<}y*$LnAuZe$K5RODuMo56yrp>}A>pZrH>ikIWB{kVp3oB=b#31j#xs zvl>@Ds6T>e5~gY@K!x&uun{t;y6RCf!>VQcT0Q03UJ}u2`8B6ri#_;A-Z(; zENg@SY7BdA)Bse}>fnfoxJ*(oiW| zRY;ky=6l}K_pdxvKTabNQJEZLd@hS`epIZiJqO}e`|Jsve7d*nh5L|;vE-BCiv_gi zk>op}lZD&Pm{Vq|(5a$N*h}VCDQiy4&3CH?^aN#ot#nlsi~-kGZl%TMx0V}&ddU{G z-ibQN_Rr@0_Hu?pqQD$uqArjMS7QmctjwT(a?tD_hdbqPn~u+mi@eWQO|rt8w)c04h+j|NUWC@l3=f0C%9@ zmPZ3%kpn`jFie1_!Krg?Q(7xT!UDFAE}2tQ~l z2N3GUoy+D^7w3~`>%^4j9+`SjFXwzG%C7)HxTcd?dYp&p?wy&sPxEO`y35tb;e=xh z!W#3&Fs6ZNcW~`U=i-`+CCFC%#Eod*GP0SoU7!%Tcc9__l@Xl5#;+v$9CK8FHKlyH+J;zt+Rp zqW{iO{htTS90M3)(ot@1V_%v9L5HvVjLirD==qpj`Q{YF1ipVuJiCp0>WWiGmbWiQHRf5yWgsn>f-P5+N_k5DTiZ6eBXTUx|C$K$ zAru;hrG&U<=PO|ooWNDxBX*~JP%9v~Liwx|8xq2_agVhJp_SP5N5FCF`mi zRG@!q+?N4{WBNmI3|LQjNzy^*7BVlKO-zX$5&@XU6^>Dpow%@AU;af0+e-e=nMPOZ zI89E;p6Ef=_*3XmJ$Uy?ad5w`1)i!}fgrzkDW-~#BkCfZQj2fFw# z-vt!NG9oW%fLFQL)NANfHqkMHqYO!+SF1ep!4k@F?-BlEQAq_eH>WcZD_ciB6{{cQ z*QEY;w6OiibAN!?*8WKuF%I6_;k7;DHYY=VSM|cJ(ec*%!8`m2SphrO_zBHayWV*ek3M+8V9t=o6vw3t(!p6d6EL z69AG}pG~fy7tN%KO9H@G0#z`|cG(m$T&*?OpR<#P9gjLdZ}k`K1QC8nn`Z)o%*S1s16_QiuQ~w( zNssCd3Bo8@joTFyge3q+VZBlZtn{-h;Z1(`<@^#kTwX6Q&tx(~wKkkDu`~g5zTq9! zR1Dh`WHK6HccL%-UV{xvt+3!)9*!Kh_f>@5Y3_(+SH~q3>QKS>ITz~lnSYTNs1o;K z*NRUaf1gD)twAlyJp!Y_h;OO;gpFPZKV)*0)JSL({Vv$)CTNU@%U3-?C8~yfwEthpW!VN2jCd2TGz-(@*DRW< z<6mJ7(Euw;1Jic2`5G=1Zp^-{gI)y`%9+)um+*Xvca#fN3vrV)wWB8yG~=>jo~Xu- z(>A+db>9W1!a3nMAOVE2EE0~&00x@XqW}nA5g)_h(M1m*j_+2qoC?O+V>rA0Ud%pDU12J}`3N5ijys{z}@@;7j3K&4Nn|Qqh_h@Vd2($$PFym(|&e)^XF*tRYRLQhBI)WtmcB7bFyLWESO`E8 zgf55N?NKcEAYv6CQ%%KM2J!m3->^~^ZMP?Nuf7{LfCs@jgAhO1Y4P2^{sPF7vN4#9 zua>~I9vpN`V84ik2*(-SN@VwXT$~S(3E;ZFvc|8C6!~!P?V&PFyh*(jA4V2Ybu{3Y zWIpfA!_ks6=fM0*^2)I181p~O^E7M-lC?^h%a=L9eq|dSok?4#WaTsD+%Kd+Xetp) z$K)t;^2PYFdr22sy$8kw$=MmBn}QJ2BIP)y*@+F#7N&D`<0EW|KTn9vLd@8Z0`5iT zgEZeq#h*yB8U4yi;iY3)yfU$RapKU&%qr3J)*`!?E?#?pBq?odnbumbqPC*ZOM0)4aBLQBG@ z;pY(LwlV1$>e_(>q3tW}Nhf^oL~8s)1Eqqn!>I}W(Q&|3-E-DeJtClWQ0@v6jsLc# z!H4YxmAD&=r8>!1SXO`_FD)ukzuI>qLkc#z0uIKG}Z!++$vR%gqA zRsv#{2KrMXiGVNbvM`o*L9qn_a~}Jm8cG9j-YMYSjy`NBoa8*NU2>wLC4*$q;2s>8 zfqBWcGI1#Wco+z+6Q?C_>T+dU9<$Ub2b2Y+3QOwZ?>Y>!e`_BlZPRz{6{7dQO1p1T9^cQu;TMjtIUYumzrn@WC*}>n94J)7+@$;qn*ZW>W@-Pt*xy zkb)^_>25946D0Kz?&X%mwC@S0yNX?N!;ov&1pg`%3B3kkt#Asv?MvFG*xtmC?ys#8 zoCa25-5e*ZV!K-xMz~tPIZ7OdHjw_J!B>=JVF9Djghlw5_LD7){%}001}B5kYvoBw ze97UL6aMt+-vVh^Y=HyfI-_o^m453wc`)cWjDtQG9cOLIL+)cg|M*bPZ_%1owmE4&wR`jX$zI7v zf)JOo%sQaNGi?S25{5T_1V(=ifr+EoYGdm(zQu=sN6wYh{3vLdidFU%5}V z*v8Uzmik9uj#gK?p`(EE-j-14YW-9vLcAZRl~=V~+|l?#GxD%IQHY~vZOjgNZmpWo zr(Aou+r58@wXq)e%*%hXZolui7YsAS6dFCWyy{cs6<1j+Ce+zS)dRlc91FC8PDKY`PE0o zn_V>tTE{crW=q^&QBB}8ZVbTL^eHfTL4VzAB{g~~tjR2W&gi^P0TApDcdgcZ286jQ z#@c!6cIS#Q9ctp=cqA9WNB}{o9%zvNt!`Q2c%F0iy_s~^Cf2U%&dNoq@Bzh~Fk(ET zPoDl>wv3lw$?T7p7evG9MFeFI2wE)%M|;Edjq}1TSVe$vurSk?TAk31ko1YnxefHB z<%;Y1bNDUuR*XM}9hpZ;(N1O$lV`6gZ!z5SEZp&XhR)NsTrIV+sMuPoA>@_p`>Ul- z8UNlHpi>2{1)mM#O?89w94n3DyfF8cqRkj*4|g?8q~n5QCoY*c@*e(;-g8@0&tOTH zD8}=^HI7sq7PkJ*;>%~1>XkJ-S?#e_xH28WEZQTrlfuc#DF9%9rNjKIB%VmgvmU9g z_LT}Tpk>l$#W7tV^Ehk-Qe{S5@229?2lX&Wiq?p_kG|zd3XYUvV%jwx%nH|^%m^74 zGjuDT9m+RM26|32mg^#=_ipoG3jn4HGN8caU~4#;!Kp^z-k^TKHqgeG+AHmahnDQl zB`3pH#uCqoj*K|7X-i{2|Zz8pYrXcix zw}leve`uTdZhTwc02)`FTMhvBgU{&B>QsIGPUVOofQOG|tGDziY)bDow_On3tlMi} zON(FFJ)VgR=xk~arLE6R7Utw`rNBj*QkKK&R{6Co`e07Jb!4b>@* zCf59|hD>}MHUS$*0^rT=Qv1<9H~5r(f{RPN$BjfW+gSaoE>X+4d#yk>_$xc!o|s$X zEA#Sc6_W3NBHkzKfFmg+T^Sd0r$2jHw?e{b-UH3ng8jZr_333okD(=-5YUSqJD z^Vui_|uyONo0NCgk{=Bmw*Z0xz=wO;LsY!JHD+Tk}NbFM2 zV5tLU=5t{4y3^}xNlqB}fH9 zZT2K7qwC!mwj)+JzFVe73;LQHr zM|$##g5N!=Dx^q literal 0 HcmV?d00001 diff --git a/openmdao/docs/openmdao_book/theory_manual/images/par_fwd.png b/openmdao/docs/openmdao_book/theory_manual/images/par_fwd.png new file mode 100644 index 0000000000000000000000000000000000000000..b19fe4fa1371b255a33f97ec52a2be7ea7aade63 GIT binary patch literal 34071 zcmeFZc{r5)`!GCo%Z=i0K_%SUEEAGK_Nb6umaHQ}Vkk7$K}CuzWiM;?WsG$uhM|NS zYh)SIn4$@pA=_XE^IS8!``*9b`+J__c#q@#0A#yh#x7z4jtiuFc4@pIF z#ts%fc_5{0wNFs+uXnBII5>)UA4hXtv@|ojbm?H)vB$6SIFE@I{>dAA?qbu5o3K#Y zL?1I)Q4!-$`q0~=tlX%ayot^2*L{B>`SS@6cO~EAXyDBMZ~pc;E4#^)RQ~XnB1j!b z?U&}GQCSFq@)SQ;ekO}>dtM-ipOg`hMuPjLL=F~nu-_fuQv$-8XO>r{5+T-&j+vED z`z1NqZw{QZ2Sxl{Xdf;Bql)*8$JE)MXFu25w7j5$KVN8n&G*_!FWuUvkAJqR%U4Q3 zrq$ZIhu%v_tw>o-MS_{;PG;Lx-i{f4pTLo=`v$uRH7#2;uHlFQ$Bu?Xt77R07?rgl zC}q>QQ7_o4QLeCS)WmGW+6hPJw}e@Qrx~xkJ{zi!I5If7VnS9RA0mfJ$QF|Wb_P&aUuGi*f11JHGMEprqz-XPqY-KOS*4~6^1>V#z=|}IM z=K6UY$GLGyf{Lac;DdH>t`8*9#Lw4ZAi^;L6@D5v;C}iQHcaZ^NQkw&Kh-~GrKfws zM%V7O&vRlw-xu-zjAL%m_1R?`pLOh=Km?zDTE@ab`jN(SE0Awa#YSuK?Bzk$AgwPV zirrG&QYF%(F)OvY*01f9dW%2sebKFDTt*Gkn%}|!Fs1r(+hJv5PnUJ=3P=5+`hn;j zjsYl8-&y5kx8V}j8r}{i>_s_dQZLzVRCT{}8zOW4yold=6~a|*jA`{V(NCNsz8)Zn zgHxz?SLFh4c1aPQVAxRHDs|#ek2`y^&;#z&vr5KjN2p-5K5?or?c*M0Mie;Ru&>wC zXLg2E!Nthu$ws66tX0|uEu3Sq*T5>!(Wzq?%0Rp=NywI9#`k5?HjgP^n5a=UVBK#a z(XwagL)Dg{1boxOL5+Rd$yHuwwfk1#o`uBJKxy%7ayB`9o*z9sDh``)cvJTDqrd^- z7KrW56NM37a+xVoj7KYHjcIhmnSZ!uhNxb(=9xVYL(J5xBR8cAJszRv=Ebs&(JuOd zt|%RRo!2g5^!&DTp$AmXPN}eiHk==J>c=sHEMXff*s|eOEBlF0_?PtB#z`GWxmyLV z?-3jvRm}BbAHs$cKoMzotS~g?Q+H*9^p!->35EPSBw9!fqqkWOQ;f;}{1@xHfda~j zkc|smVNRANHbNN+tM66M7b1_#2MK(bN1k(yeX}fi;t$q`D~AAP?e|i9O%WeFux$Hg zV>Bt}a73XwXcz0PT_^9CQMhZT91>=wAJ@0)K+?|xv6)<{NRGWxW@)qTMywx9AL*C_srxJ^?y=<|HT`I1zwewoyzz}t8|*`qYYIt5 z%LYXuI4X{Dj!m?urzq++G1ioD1Rc5^_591CTE>BtT{`p$8jjvGi9@IO>(IAp)}6xi zUTd7PZmLa+hz>m~_kD4l4zX4FW#3J{K{U` zC%cO%b4_#oI45uv0v=x}D-YZzfZBIq5z&|*&~5Qv`ZDKOLNvm|$L*S%DfxBMyq`DM*N4%byJaG&luy1g$v?j3SIS3Fw7IH9 zE4=%+prn0+uO{eW9Nt}e!lKy?!h!etpkjlX1UJH-2GI7mr_%0p1AWF4VtEJUBUDbu zevD32^cb&vH(WyFi4~TwUOsZIdtR6-I{4rdojQ_$(NHSvUv<9Gbf4Fm5I>&K&{BSJ z$@M0;Oynrr=F*IH&A-jW%$ zQ;)jb)4Ue1pnM`YBw=wUURNjS88bSH_g;ZQ0p8!;$#DB!^C`0+YL zBf^L3*e;75NmEGS40@9! zhGEuF!uxn9=X67azJCVjx>2$eG-E#|5}NboVB5=^)EErR)jkaI6YYqIcT~j{H@wBX z@2hk~f>8ah-kZ?fFxdp3FDz66RlLffZm8eS3-wU#OZ0Eh>EXQwp06|`D0wX)L4BqN z@4Yx{KjMD*>pfeBqU}o-ijMKn5bz($edFKx6yv%l@4>jQfg(;6+PfiwP)xK=phd>U z1dI23`e80IYRe6deIGQDlhdxFLVfs}Hri2I`7DU-9#kEo+6ME3XFu7HB5G+MzF=74-Yy9AbL%1xzPd7meGqY9lr-f1T{>~-y}y> zYRFdsgz06Za@W$-t-W&&(dUn7WB>;6NlOl|V(y#}`8|cXOvEt2k@3)vEvhSU4M5dH zGrhjwY#^q|-y6a!3@~mo7x=ztMQ3&y+f&A-Tk1Sn^Z`ArF0Q@rdo&Un+@KQ_lO#5z zrsRcYqNf#6B6s%>&M`(@FBfV|Bp@ACz3Pa02i$5X+p+kq&GE+Wn`?2=g*BY`@#X24 zW|W?>CzaN;_^s+SMTwp;>c(QPwfDMaD)=KYPv7t15hGp=sq;D zC;5!OGNl=WLbTEv$Anf)x%)QvuwXj0n=o$g3tkl0ZStxW#2!an-*E>6`@5p*$2N;< zqifwVg8k$%oeQ`A@dsJdt6LgwS2Kbu5!r(DHrCG1P$v4wcyA1hErA1mhL1A4jFPc;a`@Ke^mdcW~Hw% zD9PE+@Ji-GAypi zZnW>MAMTG)rKCZ1eSP_^G@jNZkd@hsOoQ_-uG8EH93^?YImCF1$Z?j(m0;m#KKdDy zj)c7LTTN_O+|s!ev*yDVxr($d&62{dk{> zHr_8hIE!f1qAg)R?gefNci=u}M$(Or{|>@`S^L}!A=|dg*4ox-!#@vn>gM$tx3}^q zv;|Y#-7VtB#x~ykF_7{b)waLQK?lzEjGg@|yg!fks-of!1e~|7dTtyWejJ}IZseRK z70Gkh-8=IqACDPVQD){2njGhf>cgJhe%IM2aO#Zy!M3P9iYTM?=s!vy4E7l9&~Zl9 zV5Z7-LDlMCdr_L&mKmKKDc>*Zgk1Q4F0kGlS0Tubm;By(;7U>OojmoxImBH$m+(7} z+_2hDdiukz-BA$zNExZ#p28zLTulB01xk$ijo7Wm6V+n4)bDSuWYT`NFZP~HfBM2; z${$T~`XO2Ps&O2zeNaMhLbZY%8????>|FN&lPY&U|;O&I!=c zg~L}X?7lKa-tt`oC0E7E9tfA2IfxGo(R5_>HJ5S(cZ3+&ho}YSWqba{hA1#{YDf43qi9yB86Jg6!QvMs71+6i?Xjb{nysD3-?XI>EIYCua3;icRj^cg12MAlW~$c zD#ww?78qh3KKuUolz-r7^A|ZuGLnur?F>y-N|r>(vi7i)^wcfBEk1C%8$6AWQWV>N z|KcDwI@ri2h<4Eel$?;r66S#H6V+TnCw}wbQuoRHxDnkKH-r*rcPE-khl0DLrlm%u z1w2T*mcK#TO2FJty<8mMD6|#6OqR1i6iIHnqdsSsP+ombCqlq;Ojj5agA1%>YPD+# z?9LOQwo%$KU`!F{l&Cn%l>D1-vdRjDxphR?xt8}EsaCE{=B~=*Kgs9FGURX|Zy-%} zCx?-P$bY-3xxwA=8Q}t9*J=(^oZKL8Z#GTWSxOKwWZd9-KNr6A!9rZZ%Cu>$AuZ?5 zE5G<96a_0=fO42WI3!-}z0r(4v!#R!_0Zh4`?C1>JWdlAi|01tFjCcInWum2t(_c? zXf@aMT^cb<{BFnHVlu)Fwu}6p_?0X*pEYiDY`hNKrLC2quTG1Lh?DFKJe|z9^C}k@ zoO8T7^mi;HRR3bTE!V{^7g6E;Vyks4@?NoDQEl$oPA&e2g#y0Oo4TuT$IAan_AjlD zrH^LhP`>NpXt*~x9tGm?Vy=S5rC#1^gkxTiAW7*MFh)k)+}is4YCqwriochyMp1B5 zl2Zapp*)K+wg=oA@NRfGVccs?mZm!jsbgw=(h_=jP&(2enr)Z=Z4{SpC8e2-m}?#w za*T~A-L!7J7<_^;znbTV@Ix)Z$=x`~)bp;>4%r3g1ojVhv#kSS|B!1Klz1nm_RffV z*StM}ir=zLkxVI}YaV$h^go^Aaw~o7a_>TB^xS?s?d{FeemDkR&hdx&d zL8k;`e~E5DTTP%w-f!Dzrzbw|prE0JJEfq(5l>ABD)%c}5+`$!Gc%kXeLF@zK^DOY zyM%nE`r#_KgTa;mL9QK;;~=9s1^ZU-OoZWS-?&*Jh}tuvA?`MgkSd(2wz=$_bd2jv znHk$1%^B^P{jXMex1RXt*BN4gUXasT;zM2&Ii(JHpUaKnXVZH^T|$gM8{lpzw4DCn zj>Dt|Z!W_c|KsxZr04vkXNFYO zK2q;L=BIGw&FQjc2;L44xnOd^rqMk3bTMYnxEv1F_zcpq5`@ukZCZ7USMf);?>K;e z{LIbQE+NO%E4sq0PaD;R7$Jo>@3{?6PThY4_z+?iW1&^cDd=nTA~IsjD~vu{Vz_0+ zraH)2S{6dg>Jke*?G(O2Uaakwi%V2r(l2Yh%JMsSzLjF(1kY!;JIVzS*3EYNygDic z8>~OuJ@e|Q5^S;FwA<(1Q7zbENT2q0cG1`Jz)>hY>J&2P*)(fjKLI9#CUztd_u#TL@pPASr=LuJ zH|^V8^U0~mC`bJQcFu7j*1z|>&E{8uh(pYc{)kRP`Id9xGrF+GH-DL`&K3^3UO77? z7JR~_^~(d5t;x8l`Yv(#)ivQz>Mz5!`m*fW)Uu6(xxITXW-7rig4-kAu`-SK?iw*L zF@ixL-QE~XrxdnG_+Y50qFKFT@?xRhVU7^h+}5v;EJ)5SNMgL3-cVfj`C`dOwK~Hl z2i+u})*jHq^Au7pS-^Cp)fVHjopC1T+$6JJ92z=chJR8@iL&(HnC^Hi-TUwxe!WMr z|J@pPt))SHG(mmoeb92Rau*ocd^Y^saoCgZf3-4*3u$6Q1~nR-Hg=vxD*fuq@1_#9 zG$%sl5R2cEO{<_%!P9Ca{9@=H&5}_s<|&Pc)!`!So15bYObeOC#9$8+R{GlHaylf3 zMuZ2>-W3vo{E9c1oA1!@4G+5^j87$3;Z`_Xk8CS@?N;X2!kTAUHBC~^?xYq6IZ@qKLE3KPFpR_VqVdPQc5Cy}o zl|fC|6LHL^M8~Brxh>apP^}>c{)xS}FuB4dr0j3Z&7>{f8|i!2M69>p9V|2`!UBfmHA=JbM|s8VIg3!y?YKNBM~&lQSy}v%R}mf)*Ws&&AywsO zpg95A^2)L>i;Y^q;dztCGlUvwPGaUhX5kf^W_X*8r?m;SQ4uD9;)+pHvH~Z98{9@= zF`PS)iP#CL~iTDNotjURD#ix(3EjEbMc~bIy0nI^k&b!F?^Q zo&p*NA^3Zvhv&Y!aBMDJ4lts?V2mt_tNn6KGG2_>^W%5%>_}S zEYH!<(X%y7gjW%{!pX^caqWkPmPl?dZ<`HvlRU1v#I;sn8&M-4T-U2)PgS`!Piphr>(Qb=e$#>mU)b>y{E)KHHCkEeOD2CG=e^qTctrNE6GuHeSr)gR&q~6 zC!KqlZWYjF>k*IieSJgkU|Y!Z;M7z>f``9N$M)CELWi=?70s{I5u4XG3j7|pv>qM9 zU!?}U;K3bYHqWRQ1y^5N>e3qbK0K+6U5vLNsm@g%NQZ=PScK+?GuOKsx|DD%i#Is6 zI~F=ttwHq$R}<06E-iqmtAdv_)!Ydpy1#^4+jHa7ZZePd-Yn(1(=z7tK;-t;P2_p` zZRqj7$vAl@!g*EsH0o#G3M4ipD|Eh5WcyEi!k9$tk*syxJZ5y>)aHiI8bo#SvF1=q z?b<{Gj9}F?=5x6Qiig9A_YApyf5rz1HEzm4Sknv!V8JTO{<_MBu(M zR7SXIwMOXXTGWsC+Z+{pZ>I^nHodec7(4k#)$PC3(mX64ZmJhYpv@_xWYHtLaSmb2Q#sz<%Gju$&fL z%@u4Hx{$m1P0|p~ee{kt!na>z?NcDZ){a($G|w5r&GdE%RlqaFD8g?7u74yyZxi~G z_F?29Y)`SE8mgikI#Le_HSD%&FLj1PmwUbDx@>4;1)oWOw{EW@y}GNYKIv$3SFUYt zE;hS{C^Vhr-#0#TxYer=YFs<@0R7podp!L|QJP)($vIE8mG*=J6fqZSNWa7D%veI* zQZ4S5fzd~(DGyVoQkPR-rk13PrZ8OTzU!X2eK;T7cU(Dc)5rQE#K2i21>75xowBLN zn7knb*l%a3t=MecVZx(bj6M&0bEt2V3u&L;*s(*CRWhaDGh7qKousVyFs2WAQ#8hx z&&MJ>wx)h`@V~5EfL9<;cGOjZK(e*7a7oQV9W5)A+F(PiV=4XBenpF#SHq~D&dutM zc;$p4v2YcWP)g1CR!B{IaBROh5|mRMaexvuIqphn6eOgMGvSP*E~JnObHp$tei#BB zE-n6nO#6X+blXy`w=~m*F6NXl%flo7boR#S;RzXU-|D`Xv>E{^&%YSXTQQDL_v&k^ zY<27ouy{E3^p4fTZ=)Mt2VZiyW{)*+P)GXCZX9M@-|WdM83t-yI^GfTS_TiGtJm+m z;hHMh9TZMD-?$ETY8ksH%fimOdc)R~dh^U`mAQu`w3_$|`obDM>vfRO@(poW=*ivB zuh%F0!>xZT$RFr-HT?5%7{$g3lN}!EL-kG%KU^UU{^CTqnLO^_?2b7ruo!e*EqnyV zQ?CnNQXBw9(x}t+7u~sD5c|=*gkZDH`i?QNQ{nJQS5n936x}e~i399C2iK|_ zZFsX=Y;;b)V}K&uL5 zYjfHQin4Fe81LSloHP9<+=oN?~!hcsgsN=%QGU0PoK~^L%=E}POkH3Q4a4m2 z?WsmYEniz&cs-R+1UV76LGhsek&_RjGqz3ohGW$OyJZ_3jV8V8I)onuEk#jUb{#6r zsN$CF3Yh!igtNm@MgRle^ig%Hp;xH_AspAmDR3Uuodvb6HiVAJb;2k}sKL)6!6BS$ zB-BQq!GO(2Ux`T^OTQB81WsY7T^KcUh17csx2XH}s`*lv>ouI6j=l?N&Y1SW$ZCq- zhM0Ws?m7g07FECAu)`}^*b_a%z0q(p%+I%cUftD>gI%UJMimJhsJC|jT`M>Cz3kc& zzbAMJQHsj{3Irr_s{)b;mE9`+p%cqF+&4WqZyGN%UyiPT=Uf29!Y^bD_vf;7YFu!zl>E{)J6!V(<6Fs(nKXJhV>AokI+T z^URz`nb<9ea&`UR-#qm=yuoISmo&FN)aLkyKaL2#5>oTOy=c=MQWp4V+QoZKkWlW` z`qb_sl@a&Bbj%Gy9YauGO_g{W)LR7Wg8Rljnm?RU3@;O*q-%#l;W7KQkMG^y2i#Iu zmj>TGX&?`W*VoGL**NRvF}X6N6q~)-vHkr+-i`wWD*{)q%Nxoe?jMXRo#(;YXaZo^M*2rRMXlh|FDs|K}xr@KGdchSX zIOY>K{r9cQtX2dGaoDQD1IzYYd!F>JfJ>PWBOQB{13$>ab zhI-=J+uCzgbK6V#F(cu3C=27Ot3ad5if>dyOVH=LcgG``_5`mIR2588rfv=5pj|FU zV`^|X7XKnOW(F_s36vsCk7|X7QAbRNevJ9_i?6qF-v;CF%emded>u<~5wvlrp>>l{ zc2CCA_G2uQb78MD!#`eb=tm!fYle@9u<{V>WORkoghSZk(o%WznfI>Uu{8)J%t9M; z?iuvr&n>Ecc-R+yHOj!xAWT@c=iF_^;LC{oiO_`>!y$fUTEPRjLY-mII`+*s;_RWZ z)qB$`4|(e^vZ#4o;x-8h!@H%wez)K7Q(B;wv0ClMcbGo>u~w-+V$5YJnIhY=GT1;Y zUjt(L@hOuF2DvB@{JS(EjJpFUUP9s%g!;qP zq@^CkS$rlRQw%?RDTguIiW^eWZ{ZB;Z1m*qcO)!-&=}D~W8?lb+gp@*A(493T+QC$ ztK}-pN)rylV-`CgW(YLL; z`u>7K1}WWHj$T7ZZrP3b0hHUg<^fEvxNo|2&8U~h!_|I`7d4}0%&d|yynxKWZXlRG z;Ck72Tcg6NYe6|S`&!%f2o8*NutR?P5Ga9iEP%E@a72&I+rRON>Fm{FZWehRBIJ9) zutLl=J5j>=4SO6l$Wv{urD4nP^3Cupzix1gwr`n{60u}u|EwX!_X zky;=a9yLLY&nr|f#KRtx3$Lh_?EQM3st$uYGtiU^Ya0ZAy#KhrY3t9ogHxwz8!L{N zB5wl;FKQnvBpl})ni+3JJauIDZ97HOOf7gfg{@KGYeaPaR6lSDunR3POnG{xb~JH5 z?T>^O5unCZL9A&%OX0g+;Kf9peR#pJ9Qvj@S#aE|)-dIufN70kUXL;5! zvUaP@y-R)?V?qe|$g2-d+ah`CQ55TBhdM^R?L!jP=3gHk3Y)WsaW$AV6{d)~LaC_H zJR@Ch)YjdKzT)NtAAqB+UdaAU;^f&)5OxzBn80I)u_(@@A_e!EJzeO^_DAkE!O(7oV zTh=B*z2K{TKUD2%YAW_R|CufgxiiIEZwKhiyD@76@KJ>GGMxI@QEv=s8s?qtfdoTF z*QejNR^ZcIz(=9|n9Zfk@Nrm2pTBcVGhWC`&Ab94FfMdEG=8`gpLbmsJW;a!SiB>{&;BV|f@l`D#?aF}$P&%>i9dtTx zG2Q$}+V5hPKgPv_T-WL$dG1!g#@@59$2?}a53thuHW(Kz&T&#*UF^6JKgslw5aYDR z9+yAuWg>DfRNi{_D{rYL4Z9K?8V@@Sl@zTYd6zo?eWnux?>b`#*ZA{x;roY8Vtajm zRp-!Pnqh`k&VzIRsoIQZZW`s^T(7Blsqk3hsrkH_lJryAfvW)Zh6ase(yf5bsfDIF z1loyuHZVJ&FAB!4i2S=U{JR1{-*fX%W{HEj_bxl2GA*>21c(41Pp)wIasoH!%>2(l0CS-}JPx4*(HYV49=(KD)D~22e9p5B|1mXGrAeN!49_gHI8=6-vx|+7miuwLfN+C`Tx`?ip{6q;0|L& zjSV>G@*O*FA)7~37YsV zf$0P1f$Q);^_;EiYK7fj_@hzxYEe)=sJ1wSZ`KHwv$RNI`trh=Z+xzJy6=##=Rto= zJrje6zcekDNtBradK5nWE*SQdO{w8b;i00|GvmwNIYfvCCSv}nbcp@_ax_;EGAZG! z8}1SAC5|c~n-Bw(lW6IlYZD?6&6R=mAJdw`5cT=j>y#nW^kRu$Rp_k@n_%lYnjDB*%1wCfzlWHGdQ)Y{vorwiD;b6 zwUmyq+~w^g&3;e(*6})q{kH>o&*lD9Y*lQjtQ<#Uqyf=&$^o+G>)&1CE^jTak8tvR z@a0Q(5{Y^;l zJqFe5@&aG-sbSP67W?taBI~|CloRyooUDY^PI=F{Aju@+Y3*GalQgf$W!5zzk!^dL zmbiK+kgV{#0U`UIu4-?M#8xWxJC>+@vgLT8eE=nnx`ca+OO_zM1rohY`M;!p`JruL zpPNg_>7Sd}{tIV-{>fDe?KPHInkU-cvtWcG6lI&by#^dq%h(7B{XJ9CBy{DY%GO)H zsd{el)5f~`%52>}=WHJ$ZYjn)n=k1by+R7egMg*wrI~!QK6Xm&qL8dSgWtf-Rm&bm zMg`cZ@d%Zlkj(pZ_}Z=#y=D<`*Q)M@YCe75_bxH6G-zHGqt9(7Ndtuz@~=!9c+EtedoHq zxa-{b90SGy?=9X`ENo2~tI0wQhD%{u@jV}2Sp37jI(~xHL~)(tG}Q2LxtoiASHI84 z^z1w-2)`U}yV|4De?+RdglD2iy{w1K$lA>KNC@MW=`dLda@6ngt4Nxc4Ge@ahj19( zrpsfr6OG3UwZivE726P{dLL>I0eK<2k6yt*Q6aG(0>2mLpXoMf@82(4qOoG;8KP63 z74Y%Ua~=r|>1++3HLRmHgfA+9Z-4=`3eOcc>|HD5{|^<%&J_-AeF)6k&&soGkC6{7 zk=wAJ`1Ss28R!W)_iGp_U$R9Gpo~V`J=Re~qAA~O zqC5K0MT*9&OqYil^U*+A%MeK8cG|i>x!aSmPWkqs0cAtOmstO0sOj){ghlVde@N)A z@!}Sg{Pj9>@IDM1wWSmCJ&a*NLt%*;{jm^678Uh!EDeHDcW&31t(_coM~!cOp9U1f zm7MCuwZLjs!7y;dT7F|6 zE=7>lVkdhh*BVr!IM<5`63PXeSxpyV!ISUSN1B&MOrD;nevmXA+HXNFjQfzMv43!u zDYl-!ui$ruzu2`amQZy{Te-#yHEs=2d*qvwzm~C z^pfL+lLo@67whgJMPGrQHvRS-|LV4=+h@F~Ojg8-YIIaxyuBO`jSo3JVd5tQwg+n0$6fB`VvlEe zhR;-=SQ!h^U+$#f3UKAv?*1K^x&CLA{m-oA$xBD9P)oaauloCwySNq{9V=g5^>?8uJHi-PbkBv;gaNYwBYb$?ncPuI7%-nq;i88NRrCuRr7RRVAOGvc50NVF} z(UX$6S=(?)aQBAhid?S%TFQPM#%d%iJtwHDC#AJaoiTnPtaQ!iTIaeZ@VFkbKV)W1 z&AG5_!1j7(`RRGaK=4|D9JYF_ZX@HY$7}X6~;hbuz<%IWcz2t>fkB{xY00TWD}M zf?eU_O^KZwXUI2Mutk2n(c#MW%|t9`k(POr@-9|P0Pg2#r*({Sg7|U0WE>M3a)YueZ?7ITVhV+^Nr{pX+Q50 zGFxB^XZK>hzzvG5$Jgfivge!*mfle{xzLGTdwjw&jIX-W zhl?!j=0jq?lQR2SpK|uHZQEM@vghaUs-xf81OweGjieX(mHO&=3}wjf8*&^IUSbm} zQq@wsC7F1uLT~C`{nGIr0F$F)S)5DG$kF$NkP`X|YhGv-W2)q-41i1n!8>AU4w$Q1 z^;bRL83V!ROdvKU%TyK-jSQUm&J37anmJZdCJC79`(<8>4Ib((Q_rvb###E^xtq3R|=nkag1RwLVX~PacltuJ<{(OK18-A7hnYnz#=arz~ZzW+)kCYjaxT%qFn=O?uh|X(*$hk zx)AAgz2!S{0xO-40D$(mN;uq8v+eK z_Y;bAykKN=vIfu$bRz%QoJ4#(7tUu#QU*9>Uh|uUQwf;IEcR2chup7_NyI1a$FiF4 zL9Ex`&2ucQf5IJ)@d%D_R`xS(a2J2A)sboDzh*+C9y&~?NJ^Q4m+5So)aEPuX=j#M z-rT#EzqQhsn|GAF!%t`)CnW|a5T-V6n7i=`pC}N=CXuD#ewjNI(MUT9)J^NhvNMbU zO?q>pW%v))Dl-l1%}$DtcFfS*5Wkzjq6+lNE^u-xn!!5C(v+ogmXCOA1YJU|%rwFw z(I@kDM|J600+xMCEY^Y|_vJmH9CeEzIo)7wgmRtX>a9{Qul&Bl*W=n;_#e3AIBzWS z)$@V-_K~X_^W_fTcHlk!o>vh(5T<6Qn@hb;ZpCr$2)gqhWoPUmjW&m;8E3-i4ruEt zwiJOP&GR^^NSp`Iq-mNv*NYQklW90DieH_^-_~ZC2e;fv_wD*Qqe!%DG(J4Vg1`*A zgI>pVj!N%&!bf#@j*U$28VmGNEO7TeRC-$QcCHm*wgr=;_-zGd>m2lg%`!XK)(v#! z8E?bg8oCDU{ z)?dvU&}2Zs@-QdzcO|s!Iuj=Ih2JXtE8$~?W*rw`0|;!B(iF!wDP}ipB|I>Ta{ zoTztUbVqY!w7H1=sqOGkZo{`y-W($y=PH(}Ytz{N0c07;+eA+NfK$f`C<0S@7aKqQ zT68Izh{thYMWx|Nt|@A%@oWG=c{-M3h+_5fo_U5DBMfEf!xowuAA>{Dt_z_0T;68B z6k?d#?c82ciYbHT76X)7rkgJ>?$(>_NWa{(eQT>{;bKqx&}T-28Z%x+vrF8%xS;mOD^w$uNOIlG~u;!xH3<@sE2LaKp;nLF4A?fnAGbzoa;W?JO)jjV*&CT2zy*cia>shD7Roiy_r;C))O7Z?m&B3unX zFl2{m10D5w3l*Cq=s>lW>5UX=+*!6oXgLWq=~ z-!Z13NV#I2pY}LuEIh@Q}(2) z^O0hC4}RVvtB@w3D`FnzVrCNyJ9?_|H|_n&`zza;B@VxgiN>Yqul*^$44O~=n$ndG z^wK_-Gl`r?jZzW~<<9djJL6yijE=9tiP{TZ{$vPSEn0eGNf|`W840_e{TBQyRN&p* z5pvzmJ6Ser&w1Zj0j~2x80k+#9llZB5Un}sHJPbj@}gI!znQ#rlBD^TEtu7_Pvp#C zY;^@-FV?y1NDg-sA}K$3KJW@aOuXV<-&pxx7G^W&WQ33ws5Pop;o99|zv>;Y*cRK_ zn?guYz#^}%(EnvZLANc>O;m<`M%z4bVw$Eb&{?AQcY2f%3$-Ro6`!KdUkDXCNccXt z*xC%X==<#$lElHQ&XsO>Ub49+F|@0-Jd3&`NHgiYA6imL8#Ek>oKJ1JDaCtKQ66awSa|tk8X|HQBNpxy_0w>_Md;uim)pR$lvf zVeXF*$Col_@J{eDkjt`_+M+UYrj`jSRA^F^1R#@5Rc71$JNkd-rpy^ql@ZVZ4VML4 zgL{0q3wcNt>V-X%JMwiaHnvAG^&vbo?&OU>0SrUJkzpM>O9##gzNnB&1pBXJKjw|r zxELX?si*H)8|Ukm-rE=P?dZhaEeY3BeA&bXfgTxiO#@SflTzaT6|Jq7Mo81bWDs)>k8Y`f$Fn(2TZ}rbed2#Yp0VxG7N*S z2JZyUpggi=WPx^MN@%6Q(7qAA^Bp_m!p-g5QAO7btNywtha zgj(>5c_#N701*VrxX3k|o=RfStm9_oMgwI;*lsxQu)!_BfrEl%^rtOz%qFM|JA;0v z0eOL~SGUsGULf+~$E#CwI8SU#>dL0FRPG|{Xze;68Q~1lEqoiGN%nNJC5=X%c)gmG z6S#})i9Buipjc-Z#xN}!xIEaw6b3m#CxC9WSXI7?v8Liv&b%Nr~s zs(|>;I_po?+eq_*W2?aB9Sc`k37eJz>dmaDpeymb!#oNaLP46O(LEadOMn`P?WjRp zsYOjl)6#j;1n}GdhXyQSUW7>0ipq0qH($3H{=P1qkRGFw&Z>PX`48I{FVEaF&>y`A z;{)jm>0UPocE}uB+Z!#`8f26x?KB4X1G)#~+c62p=XxpJYAQBhn|rsaEKl9L6TT0p z^*a7a74K}bJGz6WtFyt4Ho+<7Z$7k_bfcPWzdg9^Jv#ru&NXYHWw%76D=o zT;Gp_{Ct!w97G!j(@%k1`&pmOXTZ{6zlva{c~_ zGj|tnkG?+Gzi}yAmHP0*Xv)xB{eeN(n!Snwush!EBUYkE31?J*6(ZJ>s51rt9~|Gn zKIRO(sd4Ua8(~Par zWf73_S8j#sIhz>2kDm-`bh}w3e!>#a0zJ}u9}Cn9HU1V|BSj0i<8$Lb*s`GihBDyp zGU0F51^%s4uZhP04Sepra!IfiXN+ypdI8JwYpt!?QR_U{vLw-h&t={e#*dEygpL5T zE%0$3^?B0#6<0LqjJM3K);x!K!raFIw2U}ghqTpp4Adbdgc*=0upC*0U$$PCVY1^1 zZ47&O)XpfH>ZG7i;otfxBEA;ttic}4E0qyM`O74N!K@LV)f7g`-TNBmz;PguRdFoM z3pWd-R<}%X>9rWvC581ouiWxWUx~Wt&|HiV!6spLl(l$H^Oe}%wpJAIfFKzhHm|fFM0nm&|yKJOZfqGH1m{cT3 zeU4x&%BRL2bgS{LHHqwc!|M#l`I~o@cWJZZM*h-pk{HCRQ%c;L-|CdKo|Jg+xIdqi zT2uh~n_$Yr)Z`R-5BO!E&TpkdUl(YC`7)Ej6IixrRrmAvgFT6 zK5rT(M^Qh+N+^=nCHMFXEb@P3(Nk!Bt`Rk8_2jv4J)W|aRf=zqdfX;&t zbvb}xVi^O7GryVGli3MBC-!(TG*!4QoUc+4YyB<+12nFDR>9;7x3y*}T)thEq`83P zg9dXnH=VuQc-gjeK=d8=cikK!c_|s@nr+b2Y>I9o-uFi!Hlpq>VL=yI&{$*#OzXj zv{GoD9#ub3e_bIodnuiQm5&s_%?r2ug#&)RCA7uQSD){zdtbOd8=cD#hfbM5usCks zxA3zdBgf$&Xuuu#1q5X&WK{io2kZb=4}hh&mI3_wgSgFip4pmCR9wJfl-!{PtJv!Jm zemEzFC47KRdCdu7xFXXAab$6vqys?`(K5ChbPE~6_4B9%k-U+gB-Gvs>&Kf1A1{Rb zrIp+uPy#|3%HkKhU=BacV9~{(-2?nAy#W$Z>1oC(YOe@H{JaNsBi{{Z>-z%)wL*fF zLSq7i$+I|;ZU`JgfCn>^gmU$64(%-gDdq0j@`cU|Z~?6gjA3AS)N$$ZxkAQr3j#r$_i^ z{m1S-;&u(wFM$j;0Jo@sgI0B3UAxoOEf}Y9yKszMfSYIOxmD3yTHxKtmXS}bF-zP# zL6!E&(o$d+w(xBF0@#9i6c}Tb`4v?qlsupvB->+K#Eb*0{fgP~<+udUHO-xO*|JBj z3<&>!!dGuK1p~Ju0{bF6aGM=6f*xH4CyU#Kg&u#gtxvwhKBwM}CA)cC-W9&6)j)ll z$}yEFSqSgfqQvN*p1&f-F7vMpS@J=W=AQ>5Dr1<^QC@I%HU}hnIeGyYrCMHAgOmO~ zF&A}*;w7P8($&A9#zt%Akc>m?tIa;NK(tsy`O@BBi0PSqRIOM_szoh`@M?s2c!T}g z=ePmt069e&%j)opji1rV?WkkmP_`4#6MWugo5yj^pyS6OGQZmawj;O0wWF+ndMdD{ z-Hn|Wan`S2T8~;1AZZ>A!5XGyhpDmlb6`uvsWP(`Cp-=NDc0?9Q+> z5OUn7=L-pz{|Y#KZe6^=j^f*XmT7O*)yDby>g@lVBNZ}{2v(lnxr-kXlrgcfVc+8r ze+?yj5sSTlL)M*}|_BvafqsaaJ?VO)8zJ^xlggBHZ|OBA}_@vDf`ZExSjePoAx zpvW|{j~7fDO>O(v;!kXC6#@50AlD0EaOmlYJ*-^MMDYWEd-4&I=8+Xu6LT#4P8a!x z9k+taUJ1TyO-ZEFfN*=!b-yUqS-$P2Mw)QZ z9Y1jSKvmhMX=d*ZZ&=a)sMWkp=LS~D@a$wA z-v9BDC)&^^MMz4DF`kIb*s>%M+H6BZ8v9VTvJ84u$`V<|k}ZU?4@S0`qJ+sZwlQRj zv5ld|z6{@cMo&-g=kxx2f4|@VzxjiE=ALuzbFJr`>$=Wsz-*Ey4EH~I01!5lmF{)~ zSjqxROJM&}FS%)~%HCoG`miIApq^dB52(JtvnD-VvMol~D}NvMBKQb@^lqu^v7gWg zp_8u~7b0(spQox)Ie(hB|C`8loLiKus~ju-0K4Ne4g#G#b8N|^ojK;od+`-4e-o?>)2RFxSx(7b1rp_j`4vgoOk!dRr zs!`)>mcE1N#rtTo*&x$0D08RWd-d0pz#oN}h&6^-jKq~*Udn9oZPcL$x(K#Q%8?b4 z^lxB1MLsGN(hyvK(mRqy*`4gVk_BPkz7fxJ&=os9P7F3ply~4VonA8QV?-gXe>OfQ zbj-!pyBC{QCgyN&a#J-2jTSCF>`1%aAFpH0A8qQYl4Cu1M zkNqsGiU#R?nASBn;PV?MiSj?b!a!6j&br+iFAB;Ix8=h+cGPPp3yRQ($Hy->Ix;1}fME zeK;u{4VGG|w9fqCnxmqS%=I24j82SQ3yXEd6|zO%P$bv>{w!$&+=% zo>+njXOsR`{EGR58R5X&MGptZr4`p0XNm_mr7(ys8iKh9uNHTt@tARoT6;HOYYKDs zuQ3I1)pyK@WWZ^s3AR5vVwILp33L@8*5Iqv+wVNZX)Y@1mnMx9*;5DVJ(u66-AX0| zvX{lH;@8unX6SDY2eU+s+M0E zYmDl?mk-zmKE&PTth{e70muCC+#V}YS1OEcoq!Srevb1GXZ8DuR2_WvdsauE#gLm5 z#u#19RXjUHYhZC&) z1dID^>13cYNU@udghV5PQz&*`1gZ`GC`-mh>I#9dk8b%8o!M#kKB8MT%O=!)b)&77 zmw46PdxlUq8X#ly9Ejin%D6&1!;sj`9td4CXwb;_52I_{&(x3={c#zy({r!}w;Fcy zwM#5A+e;qyablAFymd2mIRHtngyWQ*C?8p}27z|H=W-S;TNjU6I#bU!$s(-cy%_2) z1e;1pkWf*-Bc(4c?o#4GE5Ib(EqVZx6y7q)!xt5ddy|mNHTk`45l3$oxD{T%3f=P@oi$jTG|3W{k&DB%NcUOLd<0LM zzpYYBoXz)Xcgq1lr%KuK@^<(x)RB|y(rrfV<%ZWS* z$(DQU~8aJ65CY(dD7nskS2xA_hwdvWC>1j5PL>y}O$QnIR_}<&^76)-X;Bf6PdL()y zwRAmoYdG>Rc&cZLr^kqWTBo0DImAC6tltuH;(IQOJ~NS>bl(k!#27%1c|5#3l7iW5 zseEhB0wM6)irXF)r{L}2R#SxqmURK(Vnq+TZ84knLsXiqavu13>3aPZuedI7f_dKg zDLsiO`>cTx0Mk}kvONe-j2Zn3OLKG%EW=Sx5^M3ib;RCLAxtRSoFKfVKU?|c zn>DarX0>N2SS(w-_g=H+j^Up7j@yX`iBb3hp4YlN%Qd?dvf=(eodDB{Gk4Y$KTGV} zlAhq=lek>8ZZTUw0I0v%{_QMu&K31G>>cE6Ks-%ol;R3kT4A4{-b$W?0 z^8j`IJs?L8;&lwzPdKExcUqA6iY?bq=`2^j?eN9!eQHh>pW-r>^ZEx|y{{o|FSA?g zu%?YYFro9|MauS+0rE}t#L^F5mk_s17k_N|axR6ty%e`XNZ}5)OD&zJKBkJH6Na~j zdrrZ=vICn9DSRsL5Ol<sTY#6+*e+ zhqw&C`IAqme-d$9hH$2t>#~sfhSQM|nvp>_W)1E#&le!e>S`GLJp&1>$Ui zQrEzubJU#r7m|y+Z=saFuvJPOUi`3?7ZYH5aT!{=xu?jqkI>49L2_*QP*fe4O1U|p z1v!V{+%7o3mY;2Mn%7ktzlb^uFBeg*pR+?$Z5Zbw_nSjs4Lf@(znc(U*qWCgC&7Py ziTALKLYsFvbvyAqBG?h@IeY+91&liF(ZvW66_v)q(My*xFT{25iD@Ac5-ZR&ozk|R zVjDXN%XyV+(*U;A*utH?)7t*FE0qu(Ps*xnEh7gBLwxo>D3)x~H*{pD(cR%P@-6X` zVZAMi5@*D&#B+$8xk?cK6w%wPcn3i_V1m=fNzc&QbQ5EoVDq0&1Q1?!`wzTZoO>v& zH(%TR3=LLXVLYP$A(ou3It<%sBu@&_YuZDn;RbKvg@10`+et<*@4AWP?Z6wC7QCfXh*=IOd90n$F zFMj+aU1)0D)#|k94*Ri6jpO!p!LUw{cOAI4INJZmZ~OtI)^vZy(oCEA{Gghn;$Wo8 z&XrH$DaFp;Xl)7}{Py_*g6km#V|l5B-eN6(S}x+;`d6^;w>KD3`12%61pQ;DZglNB zG7bsS?}d5`{6_e*%klA}s6w4mz6vdF)b`W0rEy_&` zH7;}@HWHZM`C@YhUwqS{j|pL6xS<_H8w?69>=PqFFA;7BEz1+1>56i_#}!ChlR^{> zgG@ga23aD%~)QjqHwHe(zZQ`Y(3$7jIXyuIP))FcDm5Xn<_*C++(BBr|gVKNS;dcd?vH$KwX$CrTXjhec?}-)x z>7n-K@ynQVqdmIpz>4=X!m+t(O@jh@FtAJN z(Ge3%{;fWFVxE+Bb73gJ%wYMy_8N|1qLt1#<=z3&>J&J84DRlbcNjl`{>ruVxTdWI zgD`g~@*`s7a|b`Wy{4Wa8iadJaIUp^l~m{PT(bh{U*7vKWA?$l&k`dnkp9B+PGs8) zPlFTpb-!x6p8DYUTk*{4tEu0U?;M-f=FJNyX2CaZu5A*`E_O{3ETw#*qb7G!U00q4 z2z%72=0Q4D)suIB1jWZ3cZ3bb=RA^kAh^$loSC|xxqP*1<;lGBLj?ReP(lmRq8$Tj ztydq-p5U~GxFH$}ad=1iIL&FKv>7aAI%Ymo9)@$aujp`lwAHhvbg(IK?73)9pzd%6 zV7~uFiyO|6S*yMtG6(10e3rdW9l%HLc~sK?tU=Gc+L+@P6V0CVxsv*Zx%mu}53Vw= zKc^~+z9ae-<|rz`X?1OxCB5|hz;k-ASSE0rDkTI^;V(t#E&O7 zlNL-pNzhA|Vlx2$+GzD?9b+RWT_Qc38;e#MwZZ9P_`#_K%ysuozxm^`Ac>P9|8kkM zyl;Dm3HYN-3}9(w?SFWQgv^T(zeMr{mODwRl`7c6s$S_6*Go3!Rwe&iKc999@U2Wk zf*W2Xq`|sUElv_*_ zA6^Lghd;HqMh(mECD;b0IyM+%_K6WLmrey(SG-=_)LA26fJ|$`-(wLu-Qh7A{Bbej z|G_S8o`e%4(h$=DDlPE0E?1>SqoWL;v~UhS=eUbH(!Xmm0PL>eFFF)0EzWe}uzdEn z1+rx5KKtbImVXNsJgd9$*)0~rop{Wg8-R%GJmq(aBShlwiQw7gwjq<>{2{IL=?cj* zQMaK$WS6{IDQoJ=F8vye50KM5ehFj9^)01(67&Z|1e*-hXGB~Jj8hR8r(8vjK& z&c=U*h9CSa0BY zZ_oYxY4+*-<=Oh^;m6;*7yE)0@rz|S1vx9T0h>as(@I5)y-vZOMFXlc*2#no&%@^& zZ}i;zESxbOBlb%H=@_C+m2@A*e(`=&%xN=DVpPyK26oly3 z!zIivdb$_`%LvcN>O(31tCK|&>7h^V*U&7JxvF}{!j3APodzJWNbcqdFI_56zieKh z_EU>Li|n?k58FBGJGfTMf+5P&*WvO19wfRaLnneaLUq-C{%91-47X_Mt7cq|uNhCN zZ1i{l0N*^N?M`F?R?)&%C0fa70RN^1-~3Y9+wBJV?U#3py|Q++{JQF!#yi~=DJQ-t z&sN(f;I#OSaXt{ zPVH#YCoFW!j%$iGsS=e&J_k>h4~l?0;Pd#Bdc7!`H1~a%YiW~g(q#a}%!5pk$SBmB zI<);`T>07Df=b68PqJkOV}{X@x*;z#F*P4q&j^*Nlw7O1F`Ifl=1w08Ro=WLr#SFH zeJw$*phCiTLtn8^g*F;p_w_yjw&`0w*J9nvT+(Un8XsBRVRW7p(s$E#{YQlL%qq<; zXz<~lz*Yc5wJ5|B!vep0yH>A=e;%}tTYJpe2j?`+DT z4z=R?c7D zTiof@OSl*yN5`+#u0IywofCjdf7uv4RkvPjQ&)|QYX)Gyh>HiyUNY@Ci2-06zl?2H z4&?`FH!cBCTzP|zYW3tR@v>rh#r1;207_5DR1B@4N~RaHygqZE5Sn7rTF0lB)V!$r z4R;-M@EI~B@pj*WTI!X2cS7(_|#h*V-QHz|v56ZI_%462mV`tH` zz7&PA5UGL&{|x{s=tWH{C!carhg)S!Sd(EHVc&))JmF2Nioh78>_)tri$y$>()z^P zso+TM(|y)Y%IrqmI$SGGeo-Dzk}V4*@O8LWfxnca3O8_*t`;>GoRibD$vlB}0b_EM z!76XSv7q;TUG_t(;-h1P1N3?gOTXQZRg6$R`vvYzl_e@$7jz5#T;Xro_&n8oJUxys!#eugoyu&<^a9-+=-P8xMKEWT{>l)&w=ZrYw07h< zOa`WPHCUTWOfBZ9x#jeyP*OJ^vt!TGfkEJy`u>_0b;g(gJ~F@yr}-we%zbfeI5^k= zCl2&{+Q)EFRJaeVKR)({`EhgZ(EU6bU2$s6NyGbR$Y}=IN`9l_qR&#UN8n)o@NXl8 zy)?ZU0IAG5!05og|PcP$@0k0}n>RVLV~? z0mQ3mCi#?&$1CaPyRJBj?J&Tc=eKfWNmu21pX1H?WLn>?PEC(1K3BtcxVehl(=t}2 z&j!ri*raPNnicuhxKmxsAS6|t0Cq=ENgivLnSryxj5lsTbO5-MgdehpI93(Xp+^X5FS zY$NLp5L65J%mL@cB>FSmjtC?zCVj?UFJ$W6q1|G=C1t z%)f_Q6`p;dW9m(fg~j#*IA)}|%BGWX7VX6-3)=)_ZE_ihmCEKSnSkY}lF*aCmo9$a zV`Gl9_x{Yc1@j}#6OLS_!KzM}rDjgPFLr^2HMkD9exAVbOos<^LqTYnYznk!N|qr9Tg;5E+|5OWEy=n}BN z{e9<#^7}_siP!t@cr|RMEZfWGeLjN8;3OwV7MP3Q+V}0@-A-?xCM&M@3-0$kcdq*+ zcrBNHaQZ4+4iyY^>Nc6BRt+*;ZU6=M6KzLPl4W0h%)m=*kpRfn6K`B`X@Io8(0k;< z!T?keIXk^`_u~hgoo}6mzwww$F`n&pJqCXPdGx@h@2$Nf{hL^WtLFOWH2w;ej{&CU zu?b=uHeKY2hy;;r_@YtV4`xAjwf~P#Vqr%Uct+R7tZTI8{tSn`={1<#UB}6F*P}r_ z1F~9BmVBZgt2L`Fr@Wy7L2NMMf>Z@iqGg&Zz`6_Ull;(jfI+l_IYRgN09&{_C;2Z$gH?&hv?HY0ayQw-C~+Zn$&tW6vM^ zD8nD{8^9Fo?sel9dXV<v`{#5V|`#I{YXDBv7YG*=z8Op z*uM}Q>$GS>fHRp(9-{`i868u#Iah@P`lfV_I5M=yXU;(R4Ly6ir|g7L(ctUwQyw#g zQ$pPDo8Z8@uP;Cjwdr}W?!+~f1GEKi`Hcr}U)q27c2XpzEqs$nzcBOBCH=z>e@m?( z2g0mr1lMq`XEaGTS$WfPpwymwU6*(ja}tKzroErRr|J4mgnn41ORqG;%ZZP~JMu|I z3s!Kz_nBAb5>2cs_}xRnjE9e1Pox?I&p$3#<81a=Pen%i#3JtbvJ?Tj=MG@VEi(RT zsh+nIy|5mJw+PNQT)w<_Z9LSVmM@G-4y!GH;7eci@wkOm6+(+s9;n_eNe|_kh;*Y* zH%||v=VP8Flj-;@2s+JdG9PkIs%D|l7Vz{{W|Jr(Qr-lSlYn;6Ud1*$8dt)SD&S(X zLaXBWYA%U_Zvw6!UZ|Rh5^lDJnXZnZS3&r;cKxu8npK0hUV9hj0L0V7;J$dsQ>;xY{&Y2@r%f20qf4Vd zG42spG53UdB0gLKA8^{zgb=QZPze5k7s-&BCk~rGNs#dvk3)U2YUOYVv(@mViSuYw zRUUIt`Fzw}ABUn?1@R>MZ(sc&vduCAf3Q>Crv?CNno^xhXuUIw&}m$*eAU!>DrLtU z3IQ)E>BV$&=OxzAiJ*5W1uQFGuFG>?1YqUpRa1II?^!>)_^B3gvfU8ik01MO4#i=9 z;0wNM4B8TWUY4~FbtQrhEOcX=A>38$xK?GSO9|0gq=gG#Tr2KA~Q!=ZM9P5-&Z@kj0+$KI@`Q?XIYu8}MafoK`oi>~9 zS!7Szp4jQ?dEpFtrEn=P)Nq9=IHxHzZC{acjd1I-At~-b;(D9d2Nj<9GM>)&CWX1W zAp?s%YcFki0{MlF3zjGk=F@H8nG}7hkS?4EODx zcrDzA!=vGybGF-MO~pi(yS14uIwlRRGKaAr_fu|$q0s^9?sq+R3;_rtFVQPm6ib4B zNrsu2W!tAXC#zid?;fp0pM-+sG}Iytl~$<9v9Kaf+h%&1x$U-`Q+DHovK(`LRa-zC zScBF;1|!4s+)jorT6t2FJHVB_J)B}{^GqbsQjGAldnX1hJzFk4g5b`1*8L$}drjRZ zRj_mF?VM%vE!=VFUxqrXlV5uVWknZd-^2)WrBd55~ z)yv~u1XnS1kcz+MWmHz&&E@=A`2H9F?($e@wx*B*VOl+XE!)9nIHOK?9b2Q#3U;z$ zk}Y;E82Ck|QlHqnX)KU1a$BQA-<{go*ZVvKr@67-XRH-`o;El?UhD8ZpHUBgFoDrI z(=kk1HS5zxi89@}1?f4H-NMj9{UuO~OR;r;Yu4hN;69&-Wb1RZ9T&bVTsDO4y z@z9c~1U&2UdeS*ziGW1voS52KXYEaBP7JWFxGg`?wP5J6>|SDsm^ibrThn2232^l~ zHwF(|`l=&VyJioj!}mh8sfh{Q zzFpeaXvhVrZ?wU-)&=-U2aVT1DiW#PcBHVX5RbSwixo6uzr;6ar#MS!LAGoM&o$3` zaLdEF{w;-uCY(bFcFD1^ljjzCR~0>`LJ0I!|0M&JKP1!aU%PtBujK}q2t@{q_(;BW z^}+*~gm#;^nCUW>+Hn+e*G>}zdcisE>vA{{prFU85RRCU^#Yh`{T04*6? z!`-~$7<`4;)w*+|Gh4OmhXs9TDkKMr%`$^|bhy@>>nl#IYLuXipGScL#OGCEiZ<>& zp)w*2IAVem+LE5TyWiS-fboQC9qNKgs~N15hM9WQ&hOYXx0fP8O+O}iMB!U?v6`={ z61$we(`9DN-eiq*dN#fx5Ak`JL4>sW z>s$d%hH_YLX7Q8bugaU9>l>J}@HKRwMo$Rx`zH*Xty8r_1=+oWocC;MP0BBCxdq?6 z=oNQ($)XHC4g~9gtrQO0d$s-It8Cv*1-afgLIC_%1Zl`*{L{oHv)(v(XhQaaFjghT zBzFe9wdsAPJFw`Z<)qCl=z24o$xG$e5(Quui-1k|ou%?lLVjr1#I5%W9|}dz$z@!q zP!raIdG@*}v>fESKM#B7$OM31bZ2&O5UVS&(SB^XB$er`&_;gT;BnHS z%OUl8q4$qZ0y>+vFs;G@ zZXkEYfktbJWJN4qzNi^~BJ{)=^fk}^_$Nfi1(T2&qV~t8!})gjXm;o@Kg?^&!#|&w z)0*4$?yr=W6LJQ>HZy(qLme+n`mH#coz#0ld1=|juqKKfN*2$AU0APUR02tTgmj$w zMymFb8Ko8*6Ob&X@Moi%or`20d<0BIax3(M^l@Bo`_Ik)^5G7Bi$xQUrH$J?(wMe=My50|obVg~1j zA2|=^TDB;H=Rl?cAKSMV0igAzIR*c>yr*$E8Z=wve23DX{<1ICsiKh~N<2y&!3@G% zutyyz6ow4(uZ^;{TzlaY`FdvD@1gBF0)o}^?RpuNUZJdn2TRdpPV01f;s4~2zxj6( z?pe7*BwT3Pu`TyQA=d}NJx)e>ABwpe1*M(Z@;;Pskp%S#b%aA_fvkaf+*#g0Pi-12 zrh3$Wd6=RSL7Sv+G$6zO)%(?*4z(5cHBbJXebE}E1eafI;EU_F?tuE42VbS#brd$v zugmp}FXJumwkAW@&4aBWlZb2$Ah1^|QyH7(0vRQS9__E{8QgZ6geEhw^u4U?2P;hd zGu@TGUqZsWkG3!WS7JC_X*$SKbyT3YeS5jbFxWCsPvBcScj0dL$!(UEMsYGdWdRM4 zp|YfaWQ7C&Eml5VTdAvh{hfy<#jyfJTTLVIdtzsMYo93>|A8Z8E!Q>p};0?)!581_+xxI3xgAI88EZ%GU*{iZs0$;7Z>N`(Y_Hmp{B*RvBF(OPj+?i_QNrDV4V{yO`bJK ztq~0%xAijtKK+-B`PsU&Zx-ji0og)=mo!Ida5&giVnyLv&=Hla~&zegAAafTDbIi=paZXh2!Ym|05cS9_(;Zly>?BA3|a zx7;1S4Q^t0fq-M2%`TaW?@voh>|4Nie|=OB%hmTF|a?=Ck_LfHJYjX-7fxoM5(f)l4B#@`Bawv z&bc^?vtJOl3Ez8(5Ibavfzrt@)kA+hFw?l4QXRTL6l?738yT#G-iUwxY||u)FL2OA zYRxmMP;&4;>{%o)d6Z~qFo16ZH96dlDt|5Vq60K`n|PWBIhYfR8Z_!g{3<6AC@rwB z;P%`8w%Iu39%cb7TJHDB4M*i7W&)X28mTL{ZVxdp+F)1Vm6W6Tp$qy|vv|M7`!7+o zc?S%3lq?Ovz9=sNR;1A@=B+pt_er`WYTaA(zNfn0FUMb%_MxB!^)h~xnA%YxcrxZc z3nq4-l2~y=Ebvc6%fA`&YA}wkz`Vg!X{y4M{q=$R0XZO>Bq|>v$Ul}}=@<`WrYNnh z`tf_gdY35C_hSKIR$7)>)(QP~tEay`kJvzt>zbaOO3vk&KlaNl6gfo=?lvuNOms2J zazD3vDF}gX@yQEoTX3)=K_}#xh5=1n)s>*`PP>#w+tBg+3R`wqfAw1^iSa0gonfCJ zI!+wX-AL-N5NXhnnFQnYqIb1`?u!FO z7FjCmcNIV>RV`3qWF-jGbYnd3ROS~}*+C|E-DzDgFL+(87{;!W0S8P0P=cr-z6B z$&~-cTq?hO^FQX&ZQz5yT;6Z#WUZ5|^8Mdt_Ku&?H4Q);D1?+Wa$B=$h2RspPh{Scbb0X$sgvD|76s6AwNFdcRk8a`{YTzd^)JKq2Yx% z;!5R&Z)VS%G9lR$JrTz<-Fvv$HRnLhF(`8WS)SmKYQI380wMPQ=H1SqhDqjxM4;jVx{M_M<$5b=o zy)qgSUl(b+$Rp0@4$es*&sU}d<6eJu809f87G zr4#HA`QLY$usO^w=i)qk%c0hbeLmcX9YS1wGedcMsSGP;L8PdQs1i9|q2S05A3JcNcE5FW7nydSU_;=P?a1LgFkN^D(N!G9S0pJD1SlVfrE@bxeI3(PHKGjofFfMWiCQJ2hd_u2MGK2vYVHYV+lR*kz^BfGLtmN}kV z_%;dnd{%HFKcA(UiH^F&d%8<=T(i#|ZDOQnL6nLx-1d0G%y?n{yZ%6XP{w;|AHg;*Tb>Xi{cnS%B z&F!PpwVKSMr=ik1+%HNWUw-lZ$knHHrG=FOS6{@)9)5AkFyHL2kuZnbZ7|FZ%>(1H zpxM@!Mg*_Wvy@hX!Jfsfbc+aBIUTcSd*IUFOZ5rSujcl%H7+A7(yHtsk zLJz&?F77n080Ucumr9hemLdoyD^+lP*3}7a5g`!7)Tc+dChptQFOzPO>_>Q51Afdw zW*|zCUAk%*SktmJDY|7*+f9W;*zluCn(Nkh8+6QRA)Aq|hyoqyVkeWDP}M2!&+l}C z$=2M^ip=AK{ujdC2kwjW&$2K&@PW%&Mklam(fET;c%DX4%f&Wgz-5Cj6_4l)jvraL zBR!Ei-y5iW+|Z6a>)hgNVybP;PBW3`Uyb96aJrbotU>y?K1(2_D&b$A^|p{j4w-$~ z?A!{g1%Yo(9*O23gcVOKaY1|bjKV_D*wC_mm1o3AM(j8@5*vDw^?Ejcd&6euxHPT= zVUEy>3>Xuav6r(Wb)Dz_{Ah))r|d=VdJC6{sF3sqM^Vf)k9tH}6gDhlpTQcKEC_Pe zwgM5NUumd6K#DASMV5#NLz>SKqHEHd<$bagMr0$jvWNrAvbkjK^7S7z%#~XqWk8``??x2)AId3xJBLukT;{;?cG++z{@JR%{vBK zu6!fl8$nk#s=YXwZ{-$RvOzk^_%qS-zRw+Cv$&SUiXg&0?$2Lsa^1V4vu`6^T=SFdtkMl8K-k@oIie78{J- zDF56dTc|XZc!A5Ac~hHklWnWqr+bX_kNp0!+41*l@f^SI9IyZhXY5Dkue^&yb0B%8 zB&0l~iPqE)gp6c!cS$d)9Fwr)gY{h*BeD%$Seu_@aqOwP}6&^ZEVlsN^~|Sz}d7k60(oUDl~$=O<}*! zWINu5=X1PL#(d9`;NfP3%MDrClCfp$1vjE^l&tiHhz+Icy;WEZi0XO_* zI>{Kh73|q?>$$aMv=gZzd42^$i>xE8W#Q&wS2{w)k|Aie^x9(5U8&yc z8Fb8EvM_&y>y{VsPaiLVc4KK2Z`o4^-Js{o&z2kd13O3EEyWI&ML>_2gB(=~N4z6n zYWfW6>)7_XXJ?taRBIXBq+4@AnGi*=-DHF45l=^sUZ=E*^WU^`UO4A7cmADcvsQ@M z0nZH;5-o$Hia%J->J&z+4qQCy*cHxAR_oc&Rw?l%X0kX!p3C2_1x5ErC&4HRiO=`)v8!lUH=4@sQ+t;)28IwoMaws*>Cw8DV*U2L*@RNm@X|e`uY7jeR z&jb!H-gC(Ai(BLA!lauQ0v}L0entw|TXRhOCq8Hrw#PQDmPD8{ zM$`a=w+gUT^u*@kWxbQKRL5czGv>~3^S8s2T{CW!07t)iU7chzcA!4+?>i14!dR)M`n$TU z1F!y;y}+C@SL~fsM9P6Oq1sxp5ZGS#`4GFXwzfT^?l&u1gUatEWjWw`U)yUG-v?+2 zvqsYi)3l88KN;o*9=E&!MpF*AT-Xoi4sq>MLhMZ=X)$7-zp z792~fx8rZ8IvyRn+6HOxCmcM%ocQb?z{C=f0aka^hkREqTFbmIKU@E`m#bc6h05Ui zJRO~HWLo?UYIW2xyFn|pfw`uHe@2rVbbcHRBDmZN^R;Q|T`=MLtP*uz#;c=to4_@Z zc8js73~;H(-)KJbu{0FL&xjhvPZVoL-%AqVhVlOS?%69a)<-TXIfqG^2Jmee-gmyuz7Tg`FumAc_ zOmh;#XPVOUH5ecaq6x!CMvm)q4MsKM%WQwr2`Y?Q)Ks#r;wY>2zSny(!0!!@(J=`%z|A>E7m1ZsT zIB@@5>ST0K^EM&7Z`Av}jll>F+7V+A(jwy2>^JG&e8~gnx3Qf9tj$QT!|66B*;DcS zABHDGF>@WgnOPsc^JA*TsPyZ+D^4LR>+Uk>o_z6bjS#U-*TqMzqV#tWtqY7%QZ7W5 z5KnM_4^>%_?Is>cYADR8cBt#FSH7a6SlXtkbD@4|L~Ni}V| z6MSaXK{d-(q^w(vkFDh6ZBq~S2r+NxrTy2 z$aR#xnA{3-mq)jcT^?|GP7Jks zspc6b6#svH_TRHgaFfIKF)z%rS#5Q02{2D1Dv=!S5o~cN*&nJb)i}#Wj%F#-+5I-o zds%ogpZnJlkuQ6X%-YkrocE}f%sbk~ANDAhxKwWC?uctTCk2lDjUnpV*UKjpIk}q z6Co@UXqGJ#&dyYdQdY$({%SloVy#uVnEi<|NJpQa3r*{ZmfcO zF-#u=w2OWFI&M#nF7tPUXnVFK>m>UblVHr#|2iabd$5x~;r4LiVBX!u-Z|9mL7zDD zw>Gb{@ln>vF2R_m|8*R7d*Hgrx~3UXi`q47SlFM%)U?kWV&2K+tdd{>`IBJ(H3IWz zOig=rAoB!(d(4^zlqL&%At#&9{wyZwXCLKb0l7;s=EMJe=-GT8RcBt*=KV~5#XO1F z#s2>{?2_vzU?C59Id;-~2&ZWp(eWmW?4zK&jwJgpDIeaw~=f@b>klb_1W#kf~?Jhbm3I0SBf&Wmtl^2z+`{d{M(6zWD74AznoMf!2)-MkIvae@xS zr$1~i84#~$UR-(uvak=G&5JTP=;}Jd$)|biih-+7!))fDvi|qO!s=S7+6%9r4JXbZ zvB>&OSfYWa$f?2Ud_HTzeXyC3d~eLEt-0b@jkJcjG^=XYsdD z!33y??BPMhHe;E26145$T;Rp?ZEbnW+FuwZ$l*&m!K&yR>+ye-rVqv#?t%)!7uWSX z4}Ra-%XRbIGgk8Z0Ll2#Wh!JypKxQC{LMlXw;RTjnKYWcs8=B+Zc{(j!uI z`iy76Rq<`(&cV!R$CGV4FSZy4$X`nXc{7}N-KMD@Z99n$%{!>=^qXXyRgMNZQL#ty z!@UFng*vuvsftJ7MEgL8(6A1EO31esjn=7zBGSIOC!@+=R9;l)i?!%EmN+cZ&pu7% zng~*A+N3G0@RnOkigoX7tCDsgs&0NuOCo-L1~Q*U5JiX(gaq<_EO)%NXUAz4&vy%e zg$Dif`SJ25j{6FBruO&uTqB$)(}$3^oOwR`i_7g)>k#6_w|9_xgmg`9IpNupo+VcX z=_#-)6qNX6vi%0q0}(F8 z=OP_SiEou_tKV|4>C{5^2v~WWWn0VhdXhFT=bXM${A*R*xzUpvFSk_M+6tHB{5o1r zM+66>{H_{=A3d+ox0q9T-BbQN*S~1yXRzb6tsAxjcu$kVC!Hm)&FA)w`@m(u^^2IL zJXQbWw4AL^CC7Q>oT24ObH>XY<_|YHd5WBne~SGY?`Rk#TgX2j zD6R#t-f#XQW9M~jsY0X%z@t0@Pu9-r-sh=7L65Xr1lb2E7t+4X!<+SsehHoFKVe*R zigZa8$y!}C+TOEAfgRAM5lWj_8tYN<>1Kc%cNhRnnY_w&FEq+S;BjhmyDCi>O*tGP>zss&+0BaP>a_Uesw2gtpPFi+}X_z(`c^N5GuX zi;Aq1>$WUz#0>nF1*%UukWmO5aZTzgc=LdcK+~4LWU(vPh{-t;JI(`vlJStXmJWQs z#rq;m1cHoOY*7L?FLlO$rOjmpCh%@Ah8MX3o1?JDBXk$e7y8uco{u{d-A%YtB%&gJ zE9~Isl@&Lwxtx+7o`ar|n+Y{9mhTxz%vDWIUMcvsy3-&Y9?zgII|nfnYuJl?^|xF1 z=|3Tty zNs2U_irLw|sd{ZpKPd3JHOQuDBOpvf)~z>lom`xu978MtFF#-Jf5e-dB3@|tLb-bB zFu;vhd`7=+a88yBRJ-1T9;3%P*BYR`IKn$7@9o$_>r$6nHF^v$v>2w9kw@(BJiaC) z|8vK-!*Uu(i`-;<)WXuxvtcjwq{S?p{~JoZ(tq|#lylcJQEW6n5DjgIkU(%E!zBeJwVsxN7*#d#HDTnCx%M}R;Y&FZysCw!TbQcZ{H2eN zcw&shrj_C~HcPaS$)^h!Sn9>~v>C|t>2HY8XD<})JP(&DBoTt*PMyt8$3pRAA(YU4 zu))8Y!wv8BRrdd9BRhNG8u~t@iEA!IU?xhr4)z{(t51bp4a-^fBzfVE;rM+S|EMmQ zRhsphVah_S#!iQi+v0Q>jrwJEJewMAZ@J?4jCw54FxIxo&ze|zAP#B37AG$KOXH8d z#f!^%r11@&){5}1YB{I5DhGpZqtSx=cwlS)RKn9BG}nsO$(;gOe1ov7cya5ZV;-c4 ze0(UbT*RGfNAkhJ6|?SrqINc0s9LJ370-uxqgv#Pq2U^c;Wyx*A8*z_tc^6pWgUH0 zry+hJk*Uy*8B%Qy4cTlz2>ZQ9A-8I3Vr%>jMnHSdp8ppKu+Iq|-+07(_7?PzV@qqE zFY}OGaffl;03FbV*YN)+i&}YG$am9uEW!DZSCCxU`J4uwYx94Kx1L~1jy)DPeu?($ zfN^ZGI@&C6{)=n*D1t**FF}jlxZNa3Aksxr0BJeZ-OI^j!L-setRk~CRKD2Z1;sB( zA)cao`oEMNr!Ela`y?50^zq5E2U4it|_GZ%hH= z-&kq7%%Y`oY}v(yZ~Jv#<=62X8SMIRTLdsP0FmID_iGi-Ob=z$w$$V;E4wL>VihT{ zEqbc_%K9BqdPBWUj+|>3-c@;A*&*8H%4h!%0Bio+up(46etyXPl|D(syq+u0=k6OY=`;HZNk-9RmjRmfmtJ#05?Ud6cyoF^$79ZWlFrSx*z5 z{Ho_I9#YI5f3K>@C&e}lGS%+i zT~Ru1l&ztgDn(MfurqkyCaV(-^J+f;ukeFDt#7lO3Nh5*{@xposaH?~&1tajCsR0nCOgzG)Jp7iUq!uD=yzSQ7+xaGZmwE@X}`0K z;`^>JBAnLg@i9AO;TRJdm6{Ppbcxqf^@Z22o*QcK9h-%_t_{UHC>JY<5RN5My0KB6 zb%g4y!(+X15Dx>i(rLo6Bue+kVGj>8w32k&$zmm0|Bs1F=C+(4b(h|7ip2>->rAH- zz4TA`UT&FA_xjo)=yZa(HLao_(udh1DWr&+7mbJSY%h@%!h zBqOylMc?HYDJ_Kqi(l54h%vpqiHcTiRWnNUA3?#yfFhFVnqrWT^a= zL&0#RrPoM#z1Il-0z@I+6FpoB^%{9y?=^wH;6uSTD7iEVR66jDRK|Oy+|{`caPG2+ z25sEGpj9PjquXP%jzFvU$I4C)v)-W~uUVWFMDcPWjMjEE&{A@?zDdN7gs%?h(>%47 zKb#b%cx3usnNzg7p4xMP0L+V!QgODGdGXXT^Vv6WUw@BdlVN}!imw(&ErdiOn>vFc+cD5B9eLMK$ z!%;;FH#=M{Fs~Hqsd%leFT5$DS}#UZ8U+PxCBJ8Ql!3y&$~Z;$RkXrFPZ3)?85KTx zOdIf8pDXHP{OJfPVDGb&o(ucLDafKntGJ!%LQG9W3yE(g9~sIw7W$57wDRoe^}B0BtgjfBxa*4U zeBm|!(mb8MRMtw3QIM^@9>*I?_BBVi9^lniFPs?5De=&Te>RNBD{nTvTi1pG zsu6b>fpaVGXRg2(Hn0~~fDI?@-AGDas_BxKSsRnBq6k2Vt&Cjgm?M_AzCtjix|LQw z;qJ6OrT@ax+Tj9bo8qHTPfHQ@el0Eyd*R z4#P>6AhMg2g0`?)2@0~W|KvD}H$2%?*wHX@)3bwe*}H9<+PTo_6HuE5+ZuaeD6jfC zYVZ#CXNHT@Md*_$IiH}e7WA|emHdYdY^Q0y((44OrdMsgj`IFs-|VBExxiML-hVh> z!L|9hKfAMhFIcbYQe>oHLSIE&SQ*g;HN+&G1@TGc3m`xs%v}VnBx2O1sQ7-eqC}!;8^|=sv0NGNS$I4JZ|D5!DX-AX1vn1)V!MJ z)@MDQG#$ZC{xP_G;Hvt{CoFh){X{7FAvfSHT%?Y6hHSn65|xq+J+m`0Gms^P6UU9a zgoM|P58^g)%D5=k^yk|T{2naxk}i?6A8!* z*weqCMxF38DC^_lwsVcYaE;QFdGGE1cq|X1U|if@H(DeHO|I!uc7H&@z#D?cx*Ug7 zBbaJnMy_FP*iHrf2y$jj3u|z0e9vpkg~plebwCDTmTWUR+HOqBwrwrPI}~Je)W93y zBpIh;g2Nso+To8Of!UK`^v;+zGv8#`fB~rqqsFZ#ZDW!1J8P9NexL?D$vP z1sHi|%qtj&uDwz;NkXlwCH1lQ&GnW^NXAFs2o8834i8R4AiQB6I+iH+gZnUim|0Q2G4bR1*56AUN9|mOr>yk-^HHOD%zY_Z0WJkrbiG30`i$*WR{$3f4cRTe&nr-y zX%m-CWT;w=sYd5-VP`a;GD*wAZT3A#t8t?aSX6b6wZ_te9t_@Tk9s}b{?%yIWV%=7 z*m{-b_Ue$~_71==$HPC?ra6VQ5;un%u65O&Wan>BboA9L*a_mFQ1d^kQAdToqEK7=W+*6Q-6gmDgz zYx>L9xL?N_z|@AeIEjZ4=q{_EZ>Vr+w+Vj#yeNGQRS-5@#r=86sP@&T$DIqX9`y8* z$M8(b;%%}&o;qATcC5{-u;8B#E!r10 z%xgToc&GgMkT2RFdY!A7)nY7lhBUw9QAcE#OaUf;CxtIc zWO%(r*pm%X+iLvsX;@e!|M8vhTHPh`#+Kjq>F{tq~^q*G|x>lOc3?7_^B|y^#V)(YUrG;`Jnk1J_~eIH2wXJ;HWe=%UK+e!*?h$w zL=Xc}{5@SYt!sPSBRT0)xjjb&J(O&V+9pcuI$%C%tpm18UWsel0`wMi;sJu;^wvtB z!djqoVoq4_)N6F7qoSaL#digF!ucnO-&}*TmKRfDHt3NAYU<%Zx@SXcWgU-|6k5*i zk~>HGgikLQffTw`(=jS~hOzP<+gtrrY{V&jhQI3VaHaYJopK*q-60U~1}XmM%*m~D z@1jNQAWBs|qyBhWp9{f&0dh|Ii+UV-<@Km{vmXjj*YV(t+sFFm%MxFM!ahIf{Bk-z ze*~fB6?Hb*z?mH)PcNEV~1J1Y8H9-I3{~tc`gtZl9wVyxOy# zMnT6a%yQrH@1&I(hvrwuQC#N4c1G%M;TM9@VvQBA&>0Mc@}79bh+F;2s~LRnt3NlkXxMl=|u0c{%Gi<61p`-Gh)D2gq{b7wR_@Dz-6fRG#@OKAQWV2R_gy@5J z`n)3j!X;+z1|Ijoj?kChzA)_6S7@pFwCIXb=+)BrHl>(lGuB(Pb=^Ns@ve=5(*c%y z^5&)63J}^MH+hAdsLy2jA18cpCPMM|4^;6#Sq$n6Io@SAaIt^GBE7a09T=UZ-bqV5 zL%F#3b=um9Q`^$9*&S`q+V|AWituujyjk~CKxv=9lK6c`!FV=gQ;7f>S|`{0+ar?a zC7AjqZy=LQXaWL`l-uO4EcnNo+9!CYP4X$Wur#A@pUjP+-8V-+mXlu#5S_Z29$Tl^ zR)<2^!03eAfS#bpfYW|X!&iGo!R6%?KvYrlT${5r0G%gi4F1vDzy5+2ND5cnelSgn zXCtkjcYa^D-D18m^G!Td8K0JuGZ{1!6huAfa|hDaSJCV}w>kN!!-(Oc#pu`Njd!A_ zXUSeXupXrr%a7|Ai?dQ)Y+1RMxU$h7MgB3|KU(3F>-v@K-w)=c_!EpA@8pyyb5l~h zJVYBezXuG5!Z&JaGr)e^Yw;Qy?I}~|H&)?qk|rpM0fYFsJzXRBkMc?H2#x}eIT+?Z=J-X*SL zPtab~4&gOP9=y;_qa7bHD)q|mP^=tKf1sRKH{d3eDS?nqb<%=Pb9#Db(J7hi-GR zxN8`4D{xy`<>r+p?f@la`dx|1T0_3E0}mG$FQ;QU7XT}iJ*?9v2hM{pkAlxZ!}2Bp zgZf$*TFH13@ZMmpOQPXlxQpB}uIlcOO9NDJ{Y~+^GZ(76j4G>uNDgH}VJLbu$dSCQ zN*#4Igm;`buJMTaIOhG2*!HZxaH8<{FHqf&Kn2z#UhwOs<6A*O5QR(X#@CnLyC^rh zE~2)5``sfciP02|M{DcxI1ONsVa?^4ymin-DJS9y@#MqgXFGE2d|;Ng`4Tkx`dgWR*87# zTOHMf%flC7Bq{#U4jZbwsMu#5o2TRhMW(4QBoj>eW=0LheqD#FmqcurTep1WFX}_O zr#BxWR1TnLvK$&l7wq0(How2Z0^Vv%z(H5_u)i~)XsPWM7&hS&`Xe9R=#EEggL3nE z_mzvJ>C<1xiH~56z4USPe3M-U+#BzBjygK+J4n?W;_<3p8E%;pS)82}(g^E37T@=A znl#m^MrEu{T#on-q{^}q_*1AwUKhiog>UevldE3Wg7@u>f#=%Z^4_6_SJyghk%Kdv z7n;1f_*CMoi;LfmxWg#{)j9DKflvJEk^ytwOC72;FX@kP%eWDiF&{uK&!=+S&-L0t zE9t5lzJ!b7v(Hk~$wy9iC^g_un+VuJsymm08F8gkX7P!zOxPN_DMxU&q; zdb<6fJ8V+BdeMToy$~q8gI)w^PpHOGIhEDJ8XM>9;PSR?o*gwiL0L~*J(QIkLVJ{l zLbjH$eRk6;IXj^R6HaEhglaSu@G+}>i7~g;-}lCR^9ftYh)=3c_$TOSr`kpchlMSZ zOVS11eeUo7>)h&SNa>F3h|v2(BU*~mRTJooz0xsiTs1B?nWkzQnC$%O0X|+ zby_B$g1fF^FJ(Oc;chlJiJe z7m6Bi8vf`9sDksG$1vpXys6~f(_Nb@8VRPnb@)5!DspJfULNQ zITK>ck4S~{Mtkq-6)nEfC112thrEmVSS=f&eptOa8P1F5tJ}nQKV`-{cVk-8UggzS zTnnlua)1y=PRi!TA1;e-o@a+nKTlYufNY1THHdR|K}Czp0O4G)j0GSR7w>;)G;M}c z&0PAJhUd+JTUJ|mI^?IN02Ad?-$06VIyZmbWHc2ma8$uDglH?u<0!_tYrEMwR`io% z0fffJE>6wrG0l&Y?-Ju3JkRRhlrNqV5>C0i=j>UijD!r9PCff1C!QIj5hm1TS$=ZA z7^-#5k=(laC>lsd?&blnRV(2LUUjaXuL8;$2IezB71PS(ttgSNdxarq&qzEFL5*%s zs~pL_`#aoy`~rTBUEcD{IvBL#DLi($(cI=eEmDyIfn-Y|;=s&|@dIX7)Uu1hZm-hp zUFHQ8q4D_Cd^{81HEubVxj*Ya^xMr)0)Fph(#sES&m+P->8XYGD99q+?{~bIHCUm4 znJXKlSVcOIwjUa-Rnc9wZGRw&gcCk`^Z(}TpXtlL?#maCSG07} z>SMDBjB-ysNEyhAV24P08`!Wt>!e3hXZp7uylErMFq}Cb;BxbQ?(Yhw@xN z4!?1kW{RG-v^fV@bmGoe!KDpSN}|H`tm9&smEOUc#{s(kU2n9Ck*-=s}nD-IoDl%PUSwY7O?I)TKJ6zKV>F|#u+oG}5pl1@bL`n7&qy~*!v z0955Q0l6TMmkp zzu@+HZT}SAi9ipiQBRXfFs)+0IbZk_zqogu z>HLE~v8TKfZ#~B3zjF~{QFU8afP`Y0Tk^TQ*N6JOf7gT1`GzWUb}T1n=}%;e`Sxv$ zO}G!s$XcuIeci8ey7=Lk_l$G{-7Nw(s0OR9Y48!#{#`u+M0n>vYX|wYZ6Duh`*!(e zp&hGUWp9<6K^Gy<7xwpyZF(#`8Kjo`J4>nUf+zhgDY0|xnP{8tb)}}B@xwz0Dz`%# z-})8pxw+$L&}iN$UYsAg7@FpzoWh2`F+W0g#hc&CUljhI^ExXgI)?gY0%~&UOY@@l zc=%S`&~c6*=3lx2l?Mgl7>mTlqbaoU9-&?pBlJl<9=6vJQ`rPN|;L}`~xQ%beykkiS=1bOwKAL z3k}6BiQ?Xd9XoJ`RS+cfc(Rml>I3!c|6E?#&R)jWINnQeXI4083`SV*-=?MfX7s&^ znuXCvL&me=%@Mtlzfy;)M}zy>aE%y|`sYA-TGZwFjaNYXMMo$z-tn^>=Q05(L(3}$ zOj#U|nppx|8+*R>>e@be8sDHEE8q2~s>^v&aPYv7BbB_fPk*DMo5y)0W=?24kPgnZ zyR+@6-Dov7MmuQwn>XryRu-XTeizhy1i{RW4$dh;8*TRkDf?gbNKw~#$IllhS>ITJ zYpXn+^K%f~px*k8{cqLwtJ3CJdzQFdII<>xo?Y#C`Nj0T**97y32PMX3C~tX#bPf8 z<`eCp?r&(Aocb%@cn;L8_AVcDc((30Ht+H{6)MJsxknZ_njQ?#7=W6keC-7t?XuxL7!G-*1?IGsLg5q<__M`4_HUj+G3Q&R7TM z^3*Lbj5Z2=#3FxhkUQ!?U=)sKMY8@k$e(0GI7z{7FBaKfUY{q2AlvGLCggt67+7YV zR`-auLi>DjGLd+?E&M~v=_^3EjYESm*S-r1H=_rWvQgwk!h@HCn@y#DDa*D* zmE%bg#$l%r4XS33zPWN9yenz??>hbE`x*Hmzpj`Q{ zTT_zO%Rv0z(?RpnMx_rYXa+70zZ7m1bScP-=nd42CaqGOuRw@t0S^)ef;Yo)5VxyD zPLQ0vfZ5^DXl6O4rgU#x^7jm28O6ER)Tosw>%xpI7)OSxLqTU#Z|J6P*8_Fs5e*9j zpzb#~86yIz7|)lTrJr2@)=l9aj8MC}H`HR|(YZG?wustNzgNov1`=R*l_#bNiVHRUQ%dc&3MuB(lBgeWQ0HZV?`x_7*9^ihZrO+J$ZWvX z$36Cw+(}~MStq>cp+Jdozu{_+vmT)6t>A7E%|$2^?{ZLi-63+%+G`e)%j~a7XSri36Qy<%G>~}q~dSNS_=Qw zc;~Bb=$Z$M?^3&TXd)phk`*CsmS75j%^4xUyJGy*&8D)SIU6!2(E%xgjK=V~;$oaK zimC|NxlU9EoR*84zYMclH6S@AaX-!tmkm52&8>N!y?hOt?$=zCw#a_gyQW{4S=yVc zU$)OiKdnG^iXLIq-vlc^H`Kr`Wc{;1IyYoaxNiU_g%P=XXu%HCbk(sqnG3MiOs{9hKq-Zj)*1-W5KvB`CdWE@`3FbS2`Ihr|se$U{4Wy60 zzRi;G3Zb&V38F;eaZ6uhmMlm37X0PLTn#9OS0pJ?K8tNQ_`Jh;&4i&i`tV=LtveO16lN!`$>|hn=dg6{ZU6yB&6g%sS9WPbHMDFG#{H# zb$n+GF_k!+9u}P<*2Rr{sXl0EW7Ehp7newD(!*?+-eb?chy`v%v`_RjnK_mmFuVrbJiUBNcdVC|7h*H;YCIo8p@TkI?H0`orN-p;APLAw zxL8wC=VmN!4#TS6Z_19zGe;D_)teGKH%{Z=xR~ALV_tVdsN&Qxyc2q^q0ehjyR^v8 z%fDkxI_LIq@m+3>$s^ar^JsOMa7zI|Ix#U+vN(FfqS6Ak1{Xr`w<)$&oN9gDfoMjK z0L~_G%&JN!D4V#;7;b_ONOJS%unnh%nQx$dIQuWum@F~N<0EjU&jTL%z3|b8dupgy zESM$(BR4%^js?`Ix0}oohVpR^xK0f38eqjztYyTQ_pkj#63lO?E7kKXvy3(%&|`J8*@uHNZQfCtG7CQx_z)&jvtPa&X6j0$=1)yZ|Y;^ zZ3XyI-AR#LJs_`gFluYCfqnL_O=&-~a$g%-ma2Br>NbpzeduCE-Z-ip5EcO!=y3_G zMZ9v2SZalfeR%;c_EOI^AlokjAkG@InE`k;q_h<8q)KckP&*G^H1#$)v%#Wj`ym_C zB6}41GyN>%ygZ<=B~dG^<%qat=35`_SHf?H;hi#c^~?0D*~KvK8NUr~3Db&oyLBS5 zb?EOFvqozh+%AS!KE#x_jM@@uYiDJ`T*K6hS9K*M!>z#ISTYS%Q0U?nNaB*gggPbS z-d+SSJOIw$dR`3*b`MknE=1UjkioN>0 zztj6wgWDuzb}Z08vE(s5O!X%11q=;ckJ(m6oFBkY%(V(gjj8rWvWJH7etatgNEupC zlV97!CW~g;QDs7QnhH28DqQm|Q_{c?VIO=DkKin+<{VclNpJBh_E|h^BlPH)b8Fk8 ziMK^Dz{Hvg+^(!9nTUSL-ePGLDW4^{kY+22Tc$w|Grs3e4|6P_VIy~FwP!va7mgI% z0LLXv4xuEq03q;<8MOf*b2Ze%d*TjbL{65_{6*XnYRXv>1r*1}?p}AcELvm>Jp-y94HC|>3uvYIkV_w(P^m(p6M zq#7^w*&%i$&3Ez7J!A1nuDm;DA}G{W;7SlVYcfSpisi#HR~mGo;Ol}2E*^K3XzkAg z40d_-^GU?eu1d)*w)nPvMRl}c#9rWvB6~T_70QYPaDsFZk*2IRqg-~2Z#NR>WU)hV zC)8kwFVpYkjJQ?xQ)`rRWw4>soQadMtKb%=#xs6dw?PHFni2F;zhe@Sn!phjwK#uz zb6e1Q!BiPAo`54lJ}+M(cCNgin~rhJBW_1b8_AXg|0!IGEdB_yXtYTCOuze{$!9#*xWjY+p4jDf zMgE6;$X5v7P0aC+Zx4Aytc6q_8oChf^qmfJK3a43J!?7RPVzqzmaavf4s~i6So}NP z?=^s@=`ZbAdJgVIf3gfEPTp9hLyNYogPsGtV2a!2+O^#Ld)2AEF6eF6~^u8Lp zsu?bGVNh>t&H|O`n-20ORNBwCN?ewh@e)TMxZ1ge=DmQi-pDxjaaTYEXU230b_Afm zscd3ma5}NT_GIWG)sZ6t1h!oEn*Oh>)}{-&`Ac%R8#oK1wO}wsLQ-M_2ATyiB^o!d zGED<2jY`NqI#sQIeo{NFLUs;dY4)FD7GCzZy=rl-1Zsi)@;WLoe_7DYdUt`e4W<9$ zHR)ZWSQd{orz~>8=k@(!r|S!t=gEg+WrXqMW_ za*q|jqHm$Wa35M0=m)_36Q&_*VUsS#~rNy@H?UZCk{7@48?3 z8(LFzobT?|i%`tVUkxCgB~BUzGj1T&5qi6pCCF7rZ!QL~e{3-*k<}LFV$xVP%O`?> zxQPB>CAolX_5~t5sHkMw(}kGh??r$tv=7|UBIyt(d4ea|k^b-ydG5K|UT*+%6D6vi z{E@-ajpk{VU+iDnn+@2PcbDJ82y%r(GLF2x6&Q7v{v*I@Hi%Nt^Gf%kxA#7Om4jcr zFbaLA#VfpEb`Y(Lh@Hb%F)2E6 zY%^@625q=ZaH0S0Ml)d)*ddoUng1+nf+Tm|R9v%>0<8ERaA}G4MXFX)d43RFGTgJ& zjLbcJAV=%*y&ev6`#FOC3wGU$YnVd|IRbA5;lx8OP9`2!7pWXOFSxK=g*N;N{Hf0m z=@=Bb@F~?wpU4<(DtucV{U(h+;YNAy9oHHV=)R!tO*59hdZL@?WYB0iHb`T;;M{!2 z7>l}0?6Ju119M%kCIkpr8Q4=?Hy4&?sj)D>@jN27Sh|BpF7|Q2 zRrfzYvwJr$6D(N@^2xPCO|y+?rhG)YNqA&Aei{7;t>wZR@w6C5x=L7Q4|GBfgK~x6 zA37tMA{DZU$?KnTEbcZ<-Zd;h7BMj+_m}^I9V4m;X-+f!3=!rIiMVEVIHb-m#z*XF z@=mUw<%!LOAsd6{fl;Ov4k&kHJ@Q)|MKKE+Vq)tZoobX@pSLI+@zaC_Hdtusxe23< zsB>eqAlqqmoJuJDPp0Jg-L2|u@hQ9D#5Y3hr=8;@Z#P9i_V8m_ji&~y75A79Ge6Pe z6DGnht0oB$&&OG9w;<3RGvAP>%Q8goPZ*S69!n{7;<mz^*3oO7bDCIf2i8Zz*ugxgFG%SPl_q5BBkZ-PRW9-jxkRkH*1s>~zZX5_&_ZcANEw$rl$5h=pl$vkDF+fV&k{LBqH zW~_q3Ye9_Au(msu%ex=0`O&IXh$UOi3iEI|=VIsBofS#mqzfC{9j@sdlOGfS-3U5w z!?vYB6dJr*A-}JzzwV}DNTXj^7*Vi2r*~0MQxwNhB}bPaO+VxMLfQpo^&%V;da{^r zfuAsmvgaFWWHu~EyZ}COAOpxG>2o8Jb>Zg=zxvN_F@Rc#5QL|=h!>>UK+w4Kg2mBx zZQh~Iw!Cxd$$jU2;DXoyfz9V1c6+D;>a94cC{zf?v`ytis8%7k`NRvL?!;b$d-bZg z=5X&1jT6A@i8W&mBx%wkOke4?o1%m8XwHbUBoJ`x6KDoJMr}dhQ#uMlgYtb1kroGp z)su+Pl2)=LLjq#@g{A7y%p(QZnHnv&BIEgfe(+~Zt7hPK) z;KClTI3<58)z1?6KtcUP3XQjHT|>jk8~9L8kp65qx>1XX&)}7|%ySaGZ3pZOzdISE z6+xKB#a95N#fva*zzAs;Me!plMoBR|`{r~<-9w@7qmat-l%0?-&luMLTDRR%BJoGk zg`W(2+?hC4I8!x$fVbl>pv}=`geNz4PWX#4;v_so#&!XUYx~--Wev*Z`f0iN)Q;Jc ztTHCn;M)T=ZDu@6EOQEE@uKIe^p4f!clzFZlC|lAwbd^Hm%ugJ*YPDw!dX;v;ZM?u zT^n4xsPu0K&72O04%)#;Q5BnwsG6)rub*2S_=E(nw0)l?&no3z9@eLFTy*FveXq+; z=MK0l{PHrfU_AHwGW=c9%tbv_e1kgk8v?((=v(zVzW;?2!4+eCPOWNmy$?9}FP!s7$32Nn#61%Z1)FK`Z3cPfEWsn;`+3P3_%vbiFal<&H!C@&AZ+l zs1woCD{U>=Ovk|bSG;q(t&(@+F`&3({KX-XDS>_}Zkn|HxU0XcfWt%4FCil{_u)?J za~pz|3uekNhu@9vC1rnRl>uI<`Xwz7eUh5n1D)F6(mQwC;=Xs~c;)pKcvgPS{k$5d zJgI+49#C#b*}kmZFwA|xZn6CtQt8~!LN7oGKBR7-ifu^b*dBj}7y{hQxTK#JOO{1w z35b9a zARr|WsvuHA07Fee$XNk(fA>4zdG7srf1LG;CtGG6}?e zednaEOe(yZvaIZFRH)DDOkiEKcUqKk8Bvqk4-0w600adWul6fhlgxRR@^<5vPtii& z8}K2ZMnf`_2Bv`oDXFbZ{6Lz1WX(uwk$l*iWtYMpQ~mu_mw^fjaIx$EtlD0O=B0zH zN{ePaZyU_(%R5)FX>!81huIUirv>&Y35LI0#2jwrX*~g$ptB7!VXFkCX*Ml9B;OkJ zItNqavaFJlCo%>sJKBFaAo>Hu{}lxWKR0TmLsK)1u{WBJ|FwX9nxn63s?=UdNDM^c z&)L$pOH(m-U#)NB(nhwlfL{d#&?1vO%vmRLf0+3*4x~Gs!Ld)~?;g%Ge@8q!zvud$ zLwEMg?74erKNnli(@wF=Nm04GP9F=}SA6vAUw?5O`*7s%L+^C?ukG6X*R>AjTI#^9 z)y)pFxA#KZ{1n4CU|>k8R3TYyira6sEd}r?O@0*HxGNaBGI;I?7rUvSEwiw|(#>J* zBW8=Rqw|WBc~$bH7j6e8f5(OOsPY!|fxZ0D=%uZyG|22DN7zBh80g3>?(8ux_BK!? zgKqeu>CA{DO%^1O0p=c81T+$S$AoL^fN~= zC$w;Fur`EIi!bwU*~)r=!jEFmr4Qx)P=(^4wY!K@U||->0gRRNd8+4)bTIb>QH8#? zU|-!U+n2yfZ$QeyJnI#x4IHXiI`1@3pcUDzb8fd4SiyMjGMx2&a71at$YWdpg6t0s zgIfs5)mtt!DkHMNaRBJTeTgsJhedg!8QHJ7a193!^iwEZoA?XI%kTmT>|i~PMNXI1 zuAYegLU{p)!}{9i7?2){x%Cf@`b9X4?*tS~DP|kFkC0v`e;!Tdh0a&J*e3J}-cu4X z+<{O5bJ1iE&7rAble@56S*ZSg`41*Tln-zNSU7c#kth;r@aYtNdz?G_gXfbG3skb` z39ef;t9o+vNxN6KH6~#jTFoPQ8-n-=7rh)h3h@)!PnVdmbdy|s%EFI;-0ZE>K7~H_ z$jO74{6^~xUDi;I_#*FT@UOr@oln6Rer~DsD2i=p6OjzI@)xApE)4KWo?0h^Iyrw2 zmTZmuh^SPBLkS5M=Bt=nx#%F$x-4k=%CAbTsQg2E$RsIHg#$l=WtUXAYSYP5M0$#) z8xKkQl_Nyw1Rp_v)M21|TR~T$)`utFqt@KAAo-c-*V0?d?iZQ;scR+eT3kiw(zPWO zgd8Am;Hp1ea*1e-lk;7Mi|Vn4=%h?@y=1exo1}wJKdM?bxG71*+);=ogMvYomgGN#n@d*#;K8nV_BJ85gu0(@9wvZnEu4 zF_esZ*5of34D52+nv;{x@nsKh5?x!SOK6G&chq(YA@E)?=-azqf5EFUy)CE`$V(@+ z?coS;2%DRURj=rsfuj9eDYc&^b69U_ZtY~g!iGa5+%Vi(LF7~p3_=64`g3~(mr$2_ zk0F9gDwg4&w}M7i^MiU?iQ-{j!Cw#kOITmjwYt-_MHMPADLAH_m+cDa_&LbtQ3rng zon@~2Q295;gOoD(PFOgl#MgM+Z#dGkeg`3ez4Qy7p_6ifDy_I#M_ZgIVrPJ0aP$^0 z5(pHI=g{|qY&r>&aQZHWwnk9Q*tSkZgjFTsK0~Ni(%Mug8hwUu&FR6wwvh?-*U#L< z*$1lRHqN@{PM!+fnw!&`Ic~XDs@g!mCS({vPk$@-HVm4OTr;WI9&6`{Q*l}Fq%;DE zfV3@rA|lWq)}Ug=Hy5{r(b?;izO01v3{;`ThaKnu5ugDNoRo+FNN@+GKfa~Tnf@d8 z?;t}<-aN4IS%(jVUe-V|ShfL#DDxxbi65;tyzU`;NkZV^ISW^eZ z)+tp25wK^2g&zgB{o4dL&HBBVJc-)`Tni4YMq7>o)?9&Vz#k}q?6W@z3-7R9!4yuO z!MOwfAn{v~cGk#PjI+|9tIk?6O{2>lGrBiD_Sfv(%Aw}^yh-cY4XUgMZyxiCB(I75 z+Ll=LedPNE9jkj$3DVkYC0>gbI5R}pc09b7#N(mvT-_zxoRFacHL3dO4isKlHX9EQ zP0=lZtlNpg!uNmC3IjZIb+H*T#N~T<+brX)*;B-!m>-o^pSEVs>nMKaVH|#AD@c71 z5gw{*p`^q219#)J!fb<^h)s$(H3S(@*Y(d@O);!EbgN;EBeWsmgyy^7;~NJ2t#Y%+ zl~dL66u)uw+92EO2iq}qxPwD3?<$|-$0JY`XaivD&fBl?&W1e^2Hm?d7=nTLDSdWF zZ@>OKc<7rL$fgVQa{^mH$*OFhdotYn*Bw@&2~FS_$jBtF(*rtP?6{CWT-Qsff@f}P z@@`28?hN7)!>jBG2exFjUuI6+qh@@|BQ)*%)8T5kI0QUE(D)das60D3iBZtL1KXRx zb~+X8G3Lf9V!`Agf>u;C90udyjxWy|wsKQjgupn`@BNB6!Sm@MuFfFU<^;Az3+EYM z+@(O3;JYk74&dhG{v+@KND8ekO`1I%N9-n+R4FX(**49qVLbXBr73fj?8o8c*~hrB zZLcd!<QyTOg4~7B31^6<3(&;>f11E=2pvjiq+CrrBQwy~G*sq`A zGIM981ZFP;mCBOKD;vQ^!2EbMo+U3p0-Zi{aUGn;3D%kI;~S108gn?%i=^#!fb#tm z?Ae9d0Kra&zIpkWrre20AeRCzyR*WNQ_QRDT{($O#8E2Z;Pz~1CnW5#5d*lv2^dFN zWI1-yf&9>HTaU+WLZf65qpT!G;39PTE1*ZwdDm)P^4Bf*u7B3AX(f$Af~^7suSRWY z@_x@=$x6uT>XuWScar}MAyS@9N&-RoESH%wJGf6wA-^2=4lq#`axearl~bYEngA!6dnWBmuYt4hie~kGMyT?tL9q-n|tySRbE1B7C#1gYshX7KZz@uw!8lEJP=_5J_E#$Sn~ak#QNS;@2vPJ)Q$;juxex+8v|6 z8#JD;e6a>U9iNk7$L>4lyL$Czv8uTbQd&3%p!RCxMlY+~oqr*&vDpMF+l|cHH6sY5 zD!VklG)*Cc((awlRe>co9yU>Yrr8bTZ)0Nl!!8F+mL_j~UARWg@?ARo{5?)fDOx-4 z$+)M81+=(=Y*6m(TDZ_-sVjWqZE}|SHEkXG)aFgLu?&sw_C zFzUX(MfpcYvd}xUfI=vUt4iN5T&XviEy9R%x-C;g%TY{3FntDU7~4E7EWjmoZv~gcYJGX z7TE8~xQ0L#{fm9kSo7*uSMiXyqpv7BD?iR15OpQ#_tc`W=N!x5@*Mcueh0aj`q$~? zx8GtNE_+U$djt?G%{f#Lqk^T`bDN1j{|2&G6*|h|I!*LU&^=eov96!$!asO}gTA*K zOWKD%@p#m8%KsEw>q+~nw?cP@sjVV={3x-A7O?9QT=PEEZ=?sva@}C^+`8D~lQFaH zMkkbs#0*6S)dJ_7@qrn$(w|ixD}U{pf=<9k?*r*hvEesz_xGk2wPh5aLfR-by6Vip zGgJv9@v-HkN8r)L!KX)Oyx(rzK~p0xw+Pu%BjGNm=Pip1;T2aZcbisM;13`)pd&DP z^}v68Y6I$&i<8fgY*#|Y@`d&L!=(M_Vg*}4Am*(E4D9N7w|nRG;$Sju#&l^J5n;f0fF#?+g*<8GVnMBI@- zg6H=8dI>>dc%{cdgvG6>sgnj6r(LmuI<4Zp$wk*L4EN~Q;eGRWaMV93JG1rvp%8N5 z3BJ-=ou0r4$NAXDV7-AD)ap`FIno>De^TDo4Pm1zB$adZo;OC8)ABatDBs$K87_e> zwV6ouWuV3mjXa%}?Ejg3rBg$`jSLs$VV`{alOLBbWfARR^OGO+K z$B+uGf3zIZe=yLAs(}@qzSlQ$yCQ{D$$R&Y_Y@i>u`QN86^oE7tU88y*nR6Kbt>&f z>+T$1md)9pqT>jaPIHx6JLjSTt-yM&ff9VBdzBe_*&0|IpFT=9ggd%u87~U|?W#6G zPE(SS%nI-vj>8f!Xse{wr?()^J?On3$A&PFq|bP-2Lv0IdR0Qe(e24p)TL{GjDBTr z$wtg*{|Q~;xpSq_N++A$CYGV+@s6Y-@$6wgSCshC`&(W}wha22gn%Mc`~{Q1eA((G zb_7Qqy|2O^l;2Ra*FCwX!t5TMTA&ux_~H6*(b6qdqAQ1lpZW;z@*CM*Ug4K2O^PBb z9OdJ3o>6+_5K|-e=qOgY)1jow^h)I6y`GG)KhA8I@s?*5F%z`;=!I2Fvr=B65Ug%G zbDe^r{|FreXprLPZd{9J1uR{aPnH&z!{06kM6x|RHy~VROm6Qp_jWZ}MH)+hp>*mZ z5r15X$JQRVrtNke#;B<{gIM~wSlIN(0D!mLESi+X2Al_orTd89MeOE-Uh$JVE6E4T z8RhHAhN*oy1R6Ciif#cx=@&RITtCX^DC_?PXhHw2(A{PFUpf46FB}V>mwFS=bg0tj zHm2;?h7)*$VZ$EX)-bJH5ti8-OY(US-$CX|Uuy$Z-M5s%d3uwh)QPgy2;FiD?ZrZ2 zIZ`@@q54&D%u2hfN@`cri?q#+sq=LM<-em?sY1zm>{UV*b|iu4Q(EbVNsJ=%&}PHl zU{96vWM&!>Z@k}31J+X|9u9XnJ^LU=;DleJVU+d)=O1hC1Ol|kO`9|y8+A~i50()t z`}^ezR+S9AmAWMSxPwiAHA7+>uelWUNP`%mysfy-!Kcd5D~hSsnBKkavik~qvQ44| z6x|{gnsCANZq$c)fB5Q2KK34I8adPguhLG*oD4D@>`SNB8GF1BEet4P=7)ABIM9PK zD}n;9uCJ*vIk>f1K?9+o zpgvn|(llPMnRbroc`=UnhQemMNBQ24p|qDwp5^|G0^V zf%Z>zQpuo%b1}B9Y7V97{Vi;XZSe(cQ})cMx3jFd=B?Q=_OWPh{nJ8k-2DVbX7I7U zB_oge*d!TaEWZK|-5+UygcXaxl`6b6sYYPT=Ppf4cS&qiQ{vtgWWonEn60L%Q+3U% zh#J#@xdtL1ij?;1iRN5_TmQB-U=9Z|PEVG<$_1|EDSCf=qUb&TX(P!Trg!bsWni+2 z^?J@7tU@FL-l_vmc#@eRI%*ZuaO1aE3PZmsfTz~Up@*mNMA`;{Oc=aa9|smkK>-v2efq=&`I_nUaMJ__iolxYLS=|!@#60&s=^#OvH zM%K*w?VAQp6F}`r(#juNJK*?6AX4!AQR$Ci>hKW9g{y7E?>i8KR%Ch4e~c=G?>h0w z(c!Jg5RCd2k96VT#dSd5T!Y~j)eeRAqQzu(te(USQ{KRJVTN(P=hYpO|4i2)c-*$7 z#kO*P<9rqNUfB%rEEge?w5>5_qLOgpUsA1yBDpt!AI(a%k5Nw=wR9w+*G|I8&;B*f zsuepd_>m9s6dWnFeA%AUFRb^{^IKdJfQ;d)ob*EJZA?9} zt+eBMD{((7%&`XJlKlhV@6cgUYz=u5%qCbwR9JvFr{z@I<6{Fw&OWr64dI#@?~%mU z=!cCwO_kmNC(m8OnAwiaQsW2DkAkhKPvM+KSP+Lf!{#=w08xVZyjH8%4o@)STN`u* zm;}~Wih|Te0E($E>i2c;yt69-Rw|~zE6Hkj6KzwD$EeC`RLqqV9{eop){DHYPPInC zpi2P%gp8Q`=c81T#$0jZctPbz(wqU}rr%y~MM+L2_yWG>IDU8B{rE4R8Q!Q~{;!d@AW0q2>ZtjV2 z#B%~HC;;Vp`0l+ko$^2yeI;YkrsTKp9|rv2kPVf#Z@a)8FXu#>CB;gx6Kf*T_T*^TP)1j&s!4gJQ!6?Wy>n z&4UTX4mnk7(}bp+|TLyuJ4WV9et< z`n)h~AdtTP^_5PueT+uhT=$zy<&X+p;t?Np_2o`&x91XwXO2Z^ub76yXeFzb=qO-= zwKnkHKZuS+?Y55?s~C5@+S;d|6D=*3fZ6^0_Efy1awfmQ8%;8kZx-QSz|iNvD`ll7 z>{YvfS)`A)Ky}31K>}p>MfDNabJcIZc^OLqlC6`QO;&|f07PDsqdypHSmv!(<9H*d zdS@Nx`t1AUlk8*sb7QKc9)qRstd%9lRknMhKh&5#245Dsu)6pnxC@x$;|=N_j`NCr zL3+g-#cpv6-s*CA^;$&}tDJY^LlOyX%zR$)KrG4zCJdPsSf9>{IXSqpKrrvu$WBcR zBo$B-&p#rEPNyi;9SsVC`Ha$I8A1m4B(@9$-?9MEfce0BvZM$7m)e))Zf&d@{^0T2 zSRdomc;mEi&UUcGjNQyk>}wSR5N|uv?Ccu;%nf5%kl06il5dcy_@z@NL{Zm)&lJH( z-%r{I!bYG(!i6Ej@5|k{WAgG**I68Vai}6-o}uP=2bewbTIn^KB0N~$;xPNNG}~3_ z!4H^3%)qGc$Wq$c_tJX2>e9_R>W}0l!cTyb!h7WAGYYVITs4TyDe_R0vhmHj?`KPk zAmKqW{?H7lR}oY=6=ve{yr_=JxI#6;#J)b43-ruUVkglU_pVo{`f_hcivYVjxKOuE zzvN{Vq*7i{8Zt+Q3}d#+c_+fhhpzBF?-ug|4^$R80h@TKIU$>oii~`+-&~0fCB=7^ zB(%(*x^n_acr#MiGU(PKx@5q-5SfL<0<>&DS}uv=gx9)fx@qP)6J<=%_Rsl1ND&9& zt2QVbQv`>EXUml}_IKhn2TwQ`gOyeQH3#P5WiU>yug|#k-ZzTkRbMy1``#2_SNl@( zX^LWuos;w)_B2u-^aI%UB;YkYTBUi)jo%fp)LVhvKG=XC_No(?k^Cz};Ks4<)W$FR zob?gW4K)A$D)MUsy)Q8`{4jsF&83*2jfoAsOh7 z60iG>4nyB}3qCKzb%M0t9ncc8ui`hnC17i$X z$PP2_SriFHNZ@mH2rs9Pe1&b=;o^2i zfIK!XZq_1q=7h&>a@5OVR|*tu!wyq;`%0sg&z><_R$keFv~9R_b*kri@$Cq=aX7s> zm8o+f5nUpV8gdrQ77xHKeJQFm_9OdfU{Nc5KG+lsaWMuaVDdC+f>yinR5t+BvpQm< zKij3*E6y7*8`|G+8Fom_qUA(t=7;V~(xAA{_g5RHqbR^h0hjT0Fm;0F#V z&#zknTa2}w&Qk$Oo7X{NJ%3#eNRGbfD^5aAE+^h8x*gg_n`t@T{*a?Ss!K+{Mb(VA zu7N{%SwO%Q;G(3Gif%XCWq{isx|Vfa{W(Jv3?R1^bJoj`AwB0x+~zYQf8dd|>*H>P zz44oKS<-$}5aDS_UOxJZ+4bhwl~?WHIQkb9CbaQ+6Z0CAKzTbk32{8+f`t3|JQB`| z_)L3tw$sSLst44wj}{@+>BIO=6rPj7)C}0z%MYBe?~JWSKJJnf0`|9+er zTv{neG`!3{FpdB?z7N9iFQ6Y2RVykYN=rRa-S* zL;B%Nkh$!*clu4{rP_{9-Rd>f+@%dZOS_J9f*HE&HoD%Q zC5$`l-k`|`jCHdW{j49FwF4DGVQ56$ZFQwHgk!svFY0h!CojdT>s=DF_Z0>?_SwD1 zuoa|oTp+zRhzNMEN2dDI(-}>fRt}({^=rHA@4gQYiHVUTKYkxDbu&!_X5v_2cl^_a zt(oxL8d*B&x&6k=+=V%$<(Jz=)a#mEbPNF76oE^dyS~q{r<}i%vlL z2Jm;vMw;RGuw<9knD|GArZOERyoud(Xz|Vhc zP?ZeLlr~#Caz9_E`-YEe7Zz*Ff2pNZ2u9qR;VaH6^H>kS6WaMTI`K79x5QB%w;y0# zt*-j=Jhbuv7~QUYfWBbSo9Oze)kr}XqA;(V>cpxNX8;JmFFTduTr+Q36HuHl4cEv@ z2rYYnBqmQP;<^z=WkuuX1nY}WFbXT{9{+4|TNU&Bl<@v0P#PGJSO_&1BrXtO7Y=4& zi#Ctm7vHzCbdg{hH#hjk#*}@?#KSCkGf)FWa5JrpF&L0(5s6bR>P_@%7Ho1`DsGHL z`7RUZG#1Kkmev_ZR-l13#+V5>!9=grUx`m{Z&EzuxV!JNIji=&ZT#O^1gfxQ0}WE! z-heMfVe=c9D8ZSFK$N=q!6o4BB|BHH*uF==aS&J0NY+r3F*_L7Zif`=!1g#CLH8Lu zO2Anxd`pC)Qfq&xiX5Y`mQrd%GdXARp)0?5w9jZ+tV(^yYrU@zO%egJxbs-CD{CWa zOE&{##5q*a>-nV}x!&*9<=(jN@w|N_H8T^+mAW~BBXxO-0?d{sl+ipwZg&|&z5SXi zyLqZ|thD^TA9yYFPz$#{c~t#2xI?RoNO7O=0yQFlxu2g5p?o#M3L%Tkcvp!W!a%ms z?F(Ihhn=Qy4f`2oZ_e0C9>_AOq%!==kPmEwy3Y7)`mkRSSHFlW`4%p`+FxjJVCASG z)(Ez*!k0t0MQ3>MZp&3h{(Xw4ssGFp^7d*$_Ub}k&`GFXRFmdz2QGG_^5o+4(jwZ)eW(B=$im zACf}9_`_tBW{80WHV(&uli6%`X&(kCup5Vi&hJ}QI z$E%q(RV)@i|5}dEWM!52FLKIx(|drly>+@~HzO5TJy~Mx>K*b55OduT)0#yF1x65y zF#mE}*vK22!poK3gPxis$M{dLtci@ZsnO0`*rkrNL2*B@xQOHXiLk&;&Ub|ASy}!( zZ@wM+hPv1#A8I}O65GHzmH9V1L1~Pxn3Hs#lQkSh*`N`xL1$`$Ytrxv{enu;;je#7 z1=ey@)NtHhtIK!oChk_Yxt(<3jK*R8h1+@j^||5SRA-;@bkNVcKjoK~tey zuNW=-X0{NWVxyByx7(LTGGpr4ktg;D-c$luX# zLQ>TOcig#qri%t`xgsjgc$6$!@I=F#IvE*K9Pkh+Q$I858>Qw@(kw6at&%5!QCQ-( zsfAE6*;|rgN4*2cg((_YtE~zY<=yXtT@pShc)=nOQN#79I_B_)6kwv05n6U&98BFD zipKHJa%tPIu0<0CMFOrk@=o8t(o(fNB9Eerl;8C)V5&%NgYa4 z--|9)20IQFCfcJ7C1Z_84PpCXK~Dn(qrs>+rH&KW$%C z5N#8*vNj8%0E;5EGp=5k4Y=9(;UAK-Urox|Fovi6Cxl1e@V*wt{v33az6B}^dN&)A zN$4p(+adCRowT9V&d(v&}{h{YFd1GmR>%*?iHYU3`qLw;@@L~R|M&kWtFh*i#?a;vGcPN~#Wi@XThXLvB*rZg)gYg4^FoQ5}DQT_Sp zAa$xJ-spjn27<*~y_8mT$F>(<_gvL# zcj>F^HadGQU()3a+ni?%woPjf+ax>pxdL?+PZ?UNM*r*~k<^nP41hDzj8`;S;L~NJ zH$o9LgyY8NX1ojZ7}^dhM*GlHX@27_wPb*(;4Y9A<#nNhNue+`2b-&*?mw zvjRnT4?|^Rln^n!&uk6{M3!aAHNr(it!2sQ1ecX%O|V|$X2#@Sky3>lAFix1 zljnjK&cj0{^v!@+EnqhOj*#m|q@m7!|9V5Z-PW2#oL#{aoP7A!+x3g0Wp|g&`S#~o z_H_J$0SjILBBdFpa(v{hdvyhX6c1_|NH7ZHfo#x((!EoIF?0JA+J``pHT;~O?0-NlK6c}+ajR;&;vk{YEc7F}qj2Ra5(X?AX}KYDo!!iU72=k%|HWX1~PQ}LFpHQ$)itM&*MKpNgL@=3x{ zMmQg8(Z1Ia0V!=ty4W{YV)N=^ay}H^F6D@TmbNE(_RW>rWLy+h8!N~@D+SC0t#@=p z`)O9~bGJX>luA%vqO zUu(03{D>~k)|v;<(l0tq_oyWBqo=`7sY%Qs#)zH=ympuNF{#eLWNjJu4_nK*_y z7I+jKE-I`k$-?>F9Wa?Z+89?DL3+w@M*{K`RZZCeP^=rD6Q)1K5))v5LK-5G<} zfNRX0aS-I3v2q18|B-0NKdk-%R?*4Zrm|1K= zPj&@=M3dTh*4qcifgpqF&6Xhl!pXC^RD`W$sU$PH$-Z9$m13v#2~`R&g>aX(&_>m6 z0^o*Z_d@ll4rSW3;>6b*OGw+)$Cm1#G|$!%jRZ6v^W_ z*sS2B&1AK!y}SsL^s93K>!pqi%QjJN;Uo~FO_qSdI!}Qsp@9}kICpC>-v+T26%Eg1 z4x_IT$21dxdbV4~*}P9maVMK*@@aa?V3w&YTlnD4AykAb0^|LLz%pejGbW53Jf6U}#+$DslWOBZ2&%~#= z+s%qzPU>_Ozslk9vKH~%Ye(OHJX6z`&H%SDZ{}Jp9tT>7x#Z{nx?u8;EStb`;59;E zU*$llWKdEir@ph$@&oaR&4_a#E z6!}W?_eTy`x9<%aa8H^9=;dRGQqrPNyAR$nor2+y*i3hQx$@W;KW=g3H*%lat0hCqyP;jS5c2EC zihNcF)@OetPlb;7{?JPifK^d{c)+yIZUIaj#V)7Qvi!fqvq`@CjZ3ah%HR{>nNSjWM?67q4&Ri7{!m}X$qv>eQ0mQ!F>e?S}zJ2YD;bF zluMjVP+*xlDs*Px>C1>M2=LP5uaqRX1PoNB^~aE6E^3u9gS6eyJ9BRUu+}@V+SI}+ zRhdJCYC_RBe_Qg@1W6_851>B4RS=rakXqSUzwe($AV&YOHY|9OP{?hU_(wJT6Ivym zHv&36eDRj|^jWx9!k)x2u|v>3e=cja!%#ZLsV zbzc9YLTL}pfl*WpJoX%WC-(PqqymE9gFqY;x|c8ALxre2iCRPRtwGkm@y0S z{O^rRH$8Up1m0isz3lie-9)&j&P>itI%1i$~M53H*hy2%$o3ic!~EFtdu&UG38%PnWEnqTq8s(zHh}m^gkYg+YS5=VSB^;`D6E?+b7rt z@84>7y$bLX?%mp=QUluBd;skF{~yo6y#pB6R%>$SPROrwa1XOupM$&NcW!NK04OQy z6Hf!f0FM0qpH8r~dkwTtZf#ibU^Co4!PX;;)tYMV0p#<*egUBEmFwZj?XzdkZF48F zKKwr~Ar=wi3C_$qXaxV@!~-+bjh z>n6>u^$F;I#q$4+Si&}o(6}9wuMG-YMyx|9h6tAJ-12-p9kg=aIXlI@)lbLAk+q3D z_`NaF{qnRQ$4RcHrv_@PC%FJ1u3F45ZfC2`-7T|aDfoY{|91)0;Fz4QI-$EO(-3D^ Pcj;cyyIiPc{`Y?Y&)tnL literal 0 HcmV?d00001 diff --git a/openmdao/docs/openmdao_book/theory_manual/mpi.ipynb b/openmdao/docs/openmdao_book/theory_manual/mpi.ipynb index 612986a6b7..f1113c6b2e 100644 --- a/openmdao/docs/openmdao_book/theory_manual/mpi.ipynb +++ b/openmdao/docs/openmdao_book/theory_manual/mpi.ipynb @@ -34,33 +34,39 @@ "\n", "![Non-parallel example](images/Par1.png)\n", "\n", - "This model contains two components that don't depend on each other's outputs, and thus those calculations can be performed simultaneously by placing them in a `ParallelGroup`. When a model containing a `ParallelGroup` is run without using MPI, its components are just executed in succession. But when it is run using `mpirun` or `mpiexec`, its components are divided amongst the processors. To take fullest advantage of the available processors, the number of subsystems in the ParallelGroup should be equal to the number of available processors. However, it will still work if the number of processors is higher or lower than you need. If you don't give it enough processors, then some processors will sequentially execute some of the components. If the number of subsystems is evenly divisible by the number of processors, then you won't have idle time on any of the processors, so that is ideal. If you give your model too many processors, those processors will be idle during execution of the parallel subsystem.\n", + "This model contains two components that don't depend on each other's outputs, and thus those calculations can be performed simultaneously by placing them in a `ParallelGroup`. When a model containing a `ParallelGroup` is run without using MPI, its components are just executed in succession. But when it is run using `mpirun` or `mpiexec`, its components are divided amongst the processes. To ensure that all subsystems execute in parallel, you should run with at least as many processes as there are subsystems. If you don't provide enough processes, then some processes will sequentially execute some of the components. Some subsystems may require more processes than others. If you give your model more processes than are needed by the subsystems, those processes will be either be idle during execution of the parallel subsystem or will perform duplicate computations, depending upon how processes are assigned within a subsystem.\n", "\n", - "The following diagram shows the same example executing on 2 processors.\n", + "OpenMDAO can compute derivatives in forward or reverse mode, with the best choice of mode being determined by the ratio of the number of design variables vs. the number of responses. If the number of responses is greater than the number of design variables, then forward mode is best, and reverse is best if the number of design variables exceeds the number of responses. 'Best' in this case means requiring a smaller number of linear solves in order to compute the total jacobian matrix.\n", "\n", - "![Parallel subsystem example](images/Par2.png)\n", + "The following diagram shows forward mode derivative transfers for our example model running on 1 process.\n", "\n", - "We see here that every component that isn't under the ParallelGroup is executed on all processors. This is done to limit data transfer between the processors. Similarly, the input and output vectors on these components is the same on all processors. We sometimes call these duplicated variables, but it is clearer to call them non-parallel non-distributed variables.\n", + "![Non-parallel forward mode derivatives](images/nonpar_fwd.png)\n", "\n", - "In contrast, the inputs and outputs on the parallel components only exist on their execution processor. In this model, there are parallel outputs that need to be passed downstream to the final component. To make this happen, OpenMDAO broadcasts them from the rank that owns them to every rank. This can be seen in the diagram as the crossed arrows that connect to x1 and x2.\n", + "The next diagram shows the forward mode transfers for the same example executing on 2 processes. Note that the derivative values at the input and output of each component in the model are the same as they were in the 1 process case.\n", "\n", - "Since component execution is repeated on all processors, component authors need to be careful about file operations which can collide if they are called from all processors at once. The safest way to handle these is to restrict them to only write files on the root processor. In addition, the computation of derivatives is duplicated on all processes except for the components in the parallel group, which handle their own unique parts of the calculation. However this is only true in forward mode. \n", + "![Parallel forward model derivatives](images/par_fwd.png)\n", + "\n", + "We see here that every component that isn't under the ParallelGroup is executed on all processes. This is done to limit data transfer between the processes. Similarly, the input and output vectors on these components are the same on all processes. We sometimes call these duplicated variables, but it is clearer to call them non-parallel non-distributed variables.\n", + "\n", + "In contrast, the inputs and outputs on the parallel components only exist on their execution processor(s). In this model, there are parallel outputs that need to be passed downstream to the final component. To make this happen, OpenMDAO scatters them from the rank(s) that contain them to the rank(s) that don't. This can be seen in the diagram as the crossed arrows that connect to x1 and x2. Data transfers are done so as to minimize the amount of data passed between processes so, for example, the 'y' value from the duplicated 'y=2x' component, since it exists in both processes, is only passed to the connected 'x' input in the same process.\n", + "\n", + "Since component execution is repeated on all processes, component authors need to be careful about file operations which can collide if they are called from all processes at once. The safest way to handle these is to restrict them to only write files on the root processor. In addition, the computation of derivatives is duplicated on all processes except for the components in the parallel group, which handle their own unique parts of the calculation.\n", "\n", "## Reverse-mode Derivatives in Parallel Subsystems\n", "\n", - "Reverse-mode derivative calculation is the single exception where the computation on non-parallel, non-distributed portions is different on each processor. This can cause confusion if you are, for example, printing the values of the derivatives vectors when using the matrix-free API.\n", + "Reverse-mode derivative calculation uses different transfers than forward mode in order to ensure that the values of non-parallel non-distributed derivatives are consistent across processes and agree with derivatives from the same model if run in a single process.\n", "\n", - "To understand what is happening, let's examine how derivatives are computed in reverse mode for the example used above.\n", + "The following diagram shows reverse mode derivative transfers for our example model running on 1 process.\n", "\n", - "![Non-parallel reverse mode derivatives](images/Par3.png)\n", + "![Non-parallel reverse mode derivatives](images/nonpar_rev.png)\n", "\n", "In this diagram, our model has one derivative to compute. We start with a seed of 1.0 in the output, and propagate that through the model (as denoted by the red arrows), multiplying by the sub-jacobians in each component as we go. Whenever we have an internal output that is connected to multiple inputs, we need to sum the contributions that are propagated there in reverse mode. The end result is the derivative across these components.\n", "\n", "Now, let's examine this process under MPI with 2 processors:\n", "\n", - "![Parallel reverse mode derivatives](images/Par4.png)\n", + "![Parallel reverse mode derivatives](images/par_rev.png)\n", "\n", - "The biggest surprise here is that the parallel components receive a value that is double the corresponding value in the non-parallel example. This is because the output is computed as the sum of the inputs from each rank. This is a slightly unusual way to do it, but it is motivated by memory performance. The operation that transfers data from an output to an input, either of which may be local or remote, is done using a set of source indices and a set of target indices. These index sets may be large, and the scale with the full-model variable size. We found that we could save memory by using the same index sets, while swapping the source and target sets, for both forward and reverse modes. When this is done, different parts of the derivative calculation end up propagating on different ranks in the non-parallel part of the model, as the diagram shows. The correct derivative result can be extracted as a final step by summing up the values on all ranks, then dividing by the number of processors.\n", + "We see here, as in the forward case, the derivative values in the component inputs and outputs agree with those we saw in the non-parallel case. Note that, as mentioned above, we have to sum the values from multiple inputs if they are connected to the same output. However, when running on multiple processes, some of our inputs are duplicated, i.e. we have the *same* input existing in multiple processes. In that case, assuming the input is not distributed, we do not sum the multiple instances together but instead use only one of the values, either the value from the same process as the output if it exists, or the value from the lowest rank process where it does exist.\n", "\n", "\n", "## Distributed Components\n", diff --git a/openmdao/jacobians/dictionary_jacobian.py b/openmdao/jacobians/dictionary_jacobian.py index 1237b844ed..f038bb8acb 100644 --- a/openmdao/jacobians/dictionary_jacobian.py +++ b/openmdao/jacobians/dictionary_jacobian.py @@ -195,31 +195,22 @@ def _apply(self, system, d_inputs, d_outputs, d_residuals, mode): left_vec += subjac.dot(right_vec) else: # rev subjac = subjac.transpose() - # print("subjac (T): ", abs_key, self[abs_key].T) - # print("TIMES") - # print("dresids: ", right_vec) - # print("dinputs BEFORE:", left_vec) left_vec += subjac.dot(right_vec) - # print("dinputs AFTER:", left_vec) - - hasremote = abs_key in self._key_owner - if hasremote: - if True: # fwd: - owner = self._key_owner[abs_key] - if owner == system.comm.rank: - # print("SENDING", left_vec, "from", owner, abs_key) - system.comm.bcast(left_vec, root=owner) - elif owner is not None: - left_vec = system.comm.bcast(None, root=owner) - if fwd: - if res_name in d_res_names: - d_residuals._abs_set_val(res_name, left_vec) - else: # rev - if other_name in d_out_names: - d_outputs._abs_set_val(other_name, left_vec) - elif other_name in d_inp_names: - d_inputs._abs_set_val(other_name, left_vec) - # print("RECEIVED", left_vec, "from", owner, abs_key) + + if abs_key in self._key_owner: + owner = self._key_owner[abs_key] + if owner == system.comm.rank: + system.comm.bcast(left_vec, root=owner) + elif owner is not None: + left_vec = system.comm.bcast(None, root=owner) + if fwd: + if res_name in d_res_names: + d_residuals._abs_set_val(res_name, left_vec) + else: # rev + if other_name in d_out_names: + d_outputs._abs_set_val(other_name, left_vec) + elif other_name in d_inp_names: + d_inputs._abs_set_val(other_name, left_vec) class _CheckingJacobian(DictionaryJacobian): From 0f72aeb0b3d96d6caf11cbc841a09e8e3feb92cc Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 3 Nov 2023 10:20:48 -0400 Subject: [PATCH 50/70] one broken test --- openmdao/core/tests/test_distrib_derivs.py | 11 ++- openmdao/core/total_jac.py | 37 +++------- .../working_with_groups/src_indices.ipynb | 8 +-- openmdao/jacobians/dictionary_jacobian.py | 2 +- openmdao/jacobians/jacobian.py | 67 +------------------ .../linear/tests/test_linear_block_gs.py | 1 - openmdao/utils/array_utils.py | 22 ------ openmdao/utils/range_collection.py | 26 ------- openmdao/vectors/petsc_transfer.py | 20 +----- 9 files changed, 22 insertions(+), 172 deletions(-) diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index 21484c5cbc..61335d6d92 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -126,9 +126,6 @@ def compute(self, inputs, outputs): outputs['out_dist'] = inputs['in_dist'] * 5. def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): - Id = inputs['in_dist'] - Is = inputs['in_nd'] - if mode == 'fwd': if 'out_dist' in d_outputs: if 'in_dist' in d_inputs: @@ -844,7 +841,7 @@ def test_distrib_voi_group_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - assert_check_totals(prob.check_totals(method='fd', out_stream=None)) + assert_check_totals(prob.check_totals(method='fd', out_stream=None), rtol=1e-5) # rev mode @@ -860,7 +857,7 @@ def test_distrib_voi_group_fd(self): np.array([27.0, 24.96, 23.64, 23.04, 23.16, 24.0, 25.56]), 1e-6) - assert_check_totals(prob.check_totals(method='fd', out_stream=None)) + assert_check_totals(prob.check_totals(method='fd', out_stream=None), rtol=1e-5) def test_distrib_voi_group_fd2(self): prob = _setup_ivc_subivc_dist_parab_sum() @@ -1245,8 +1242,8 @@ def compute(self, inputs, outputs): prob.run_model() - assert_check_totals(prob.check_totals(method='fd', out_stream=None)) - assert_check_totals(prob.check_totals(method='cs', out_stream=None), rtol=1e-14) + assert_check_totals(prob.check_totals(method='fd', out_stream=None), atol=2e-5, rtol=2e-5) + assert_check_totals(prob.check_totals(method='cs', out_stream=None), rtol=1e-13) def run_mixed_distrib2_prob(self, mode, klass=MixedDistrib2): size = 5 diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index 9111c0d7b5..583b3a5b49 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -255,8 +255,6 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, self.has_lin_cons = has_lin_cons self.dist_input_range_map = {} - self.has_input_dist = {} - self.has_output_dist = {} self.total_relevant_systems = set() self.simul_coloring = None @@ -306,12 +304,10 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, self.dist_idx_map = {m: None for m in modes} self.modes = modes - self.of_meta, self.of_size, has_of_dist = \ + self.of_meta, self.of_size, _ = \ self._get_tuple_map(of, responses, all_abs2meta_out) - self.has_input_dist['rev'] = self.has_output_dist['fwd'] = has_of_dist - self.wrt_meta, self.wrt_size, has_wrt_dist = \ + self.wrt_meta, self.wrt_size, self.has_wrt_dist = \ self._get_tuple_map(wrt, design_vars, all_abs2meta_out) - self.has_input_dist['fwd'] = self.has_output_dist['rev'] = has_wrt_dist # always allocate a 2D dense array and we can assign views to dict keys later if # return format is 'dict' or 'flat_dict'. @@ -319,7 +315,7 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, # if we have distributed 'wrt' variables in fwd mode we have to broadcast the jac # columns from the owner of a given range of dist indices to everyone else. - if self.get_remote and has_wrt_dist and self.comm.size > 1: + if self.get_remote and self.has_wrt_dist and self.comm.size > 1: abs2idx = model._var_allprocs_abs2idx sizes = model._var_sizes['output'] meta = self.wrt_meta @@ -343,7 +339,6 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, # create scratch array for jac scatters self.jac_scratch = None - self.jac_dist_col_mask = None if self.comm.size > 1 and self.get_remote: # need 2 scratch vectors of the same size here @@ -365,20 +360,9 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, if 'rev' in modes: from openmdao.core.group import Group - # # find all groups doing FD - # fdgroups = [s.pathname for s in model.system_iter(recurse=True, typ=Group) - # if s._owns_approx_jac] - # allfdgroups = set() - # for grps in model.comm.allgather(fdgroups): - # allfdgroups.update(grps) - self.jac_scratch['rev'] = [scratch[0][:J.shape[1]]] if self.simul_coloring is not None: # when simul coloring, need two scratch arrays self.jac_scratch['rev'].append(scratch[1][:J.shape[1]]) - has_out_dist = self.has_output_dist['rev'] - - if has_out_dist: - self.jac_dist_col_mask = np.zeros(J.shape[1], dtype=bool) # create a column mask to zero out contributions to the Allreduce from # duplicated vars @@ -1347,19 +1331,16 @@ def _jac_setter_dist(self, i, mode): if self.jac_scatters[mode] is not None: self.src_petsc[mode].array = self.J[:, i] self.tgt_petsc[mode].array[:] = self.J[:, i] - # print(f"J[:, {i}] before:", self.J[:, i]) self.jac_scatters[mode].scatter(self.src_petsc[mode], self.tgt_petsc[mode], addv=False, mode=False) self.J[:, i] = self.tgt_petsc[mode].array - # print(f"J[:, {i}] after:", self.J[:, i]) else: # rev - if self.get_remote: - if self.rev_allreduce_mask is not None: - scratch = self.jac_scratch['rev'][0] - scratch[:] = 0.0 - scratch[self.rev_allreduce_mask] = self.J[i][self.rev_allreduce_mask] - self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) + if self.get_remote and self.rev_allreduce_mask is not None: + scratch = self.jac_scratch['rev'][0] + scratch[:] = 0.0 + scratch[self.rev_allreduce_mask] = self.J[i][self.rev_allreduce_mask] + self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) def single_jac_setter(self, i, mode, meta): """ @@ -1593,7 +1574,7 @@ def compute_totals(self): # if some of the wrt vars are distributed in fwd mode, we have to bcast from the rank # where each part of the distrib var exists - if self.get_remote and mode == 'fwd' and self.has_input_dist[mode]: + if self.get_remote and mode == 'fwd' and self.has_wrt_dist: for start, stop, rank in self.dist_input_range_map[mode]: contig = self.J[:, start:stop].copy() model.comm.Bcast(contig, root=rank) diff --git a/openmdao/docs/openmdao_book/features/core_features/working_with_groups/src_indices.ipynb b/openmdao/docs/openmdao_book/features/core_features/working_with_groups/src_indices.ipynb index b068a22c84..e75791eb35 100644 --- a/openmdao/docs/openmdao_book/features/core_features/working_with_groups/src_indices.ipynb +++ b/openmdao/docs/openmdao_book/features/core_features/working_with_groups/src_indices.ipynb @@ -264,8 +264,8 @@ " self.idxs = idxs\n", "\n", " def setup(self):\n", - " self.add_input('x', np.ones(len(self.idxs)))\n", - " self.add_output('y', 1.0)\n", + " self.add_input('x', np.ones(len(self.idxs)), distributed=True)\n", + " self.add_output('y', 1.0, distributed=True)\n", "\n", " def compute(self, inputs, outputs):\n", " outputs['y'] = np.sum(inputs['x'])*2.0\n", @@ -321,11 +321,11 @@ "%%px\n", "from openmdao.utils.assert_utils import assert_near_equal\n", "\n", - "assert_near_equal(p['C1.x'],\n", + "assert_near_equal(p.get_val('C1.x'),\n", " np.arange(3, dtype=float) if p.model.C1.comm.rank == 0 else np.arange(3, 5, dtype=float))\n", "\n", "# the output in each rank is based on the local inputs\n", - "assert_near_equal(p['C1.y'], 6. if p.model.C1.comm.rank == 0 else 14.)" + "assert_near_equal(p.get_val('C1.y'), 6. if p.model.C1.comm.rank == 0 else 14.)" ] } ], diff --git a/openmdao/jacobians/dictionary_jacobian.py b/openmdao/jacobians/dictionary_jacobian.py index f038bb8acb..aa20e2fd29 100644 --- a/openmdao/jacobians/dictionary_jacobian.py +++ b/openmdao/jacobians/dictionary_jacobian.py @@ -2,7 +2,7 @@ import numpy as np import scipy.sparse as sp -from openmdao.jacobians.jacobian import Jacobian, _get_remote_vars +from openmdao.jacobians.jacobian import Jacobian from openmdao.core.constants import INT_DTYPE diff --git a/openmdao/jacobians/jacobian.py b/openmdao/jacobians/jacobian.py index e5b58f0229..3410d54b67 100644 --- a/openmdao/jacobians/jacobian.py +++ b/openmdao/jacobians/jacobian.py @@ -418,10 +418,7 @@ def set_col(self, system, icol, column): if key in self._subjacs_info: subjac = self._subjacs_info[key] if subjac['cols'] is None: # dense - try: - subjac['val'][:, loc_idx] = column[start:end] - except Exception as ex: - print(ex) + subjac['val'][:, loc_idx] = column[start:end] else: # our COO format match_inds = np.nonzero(subjac['cols'] == loc_idx)[0] if match_inds.size > 0: @@ -462,65 +459,3 @@ def _restore_approx_sparsity(self): """ self._subjacs_info = self._system()._subjacs_info self._col_varnames = None # force recompute of internal index maps on next set_col - - def get_wrt_names(self): - """ - Get the list of all wrt names. - - Returns - ------- - list - List of all wrt names. - """ - return self._col_varnames - - def get_of_names(self): - """ - Get the list of all of names. - - Returns - ------- - list - List of all of names. - """ - seen = set() - ofs = [] - for of, _ in self._subjacs_info.keys(): - if of not in seen: - seen.add(of) - ofs.append(of) - - return ofs - - -def _get_remote_vars(system, varnames): - # vnames must be absolute names (no aliases) - - nprocs = system.comm.size - remote_vars = set() - - if nprocs > 1: - # If we have remote vars, pick an owning rank for each and use that - # to bcast to others later - all_abs2meta_out = system._var_allprocs_abs2meta['output'] - all_abs2meta_in = system._var_allprocs_abs2meta['input'] - abs2meta_out = system._var_abs2meta['output'] - abs2meta_in = system._var_abs2meta['input'] - - remote_lst = [n for n in varnames if n not in abs2meta_in and n not in abs2meta_out] - - seen = set() - - for vnames in system.comm.allgather(remote_lst): - for vname in vnames: - if vname not in seen: - seen.add(vname) - if vname in all_abs2meta_out: - dist = all_abs2meta_out[vname]['distributed'] - else: - dist = all_abs2meta_in[vname]['distributed'] - - if not dist: - remote_vars.add(vname) - - return remote_vars diff --git a/openmdao/solvers/linear/tests/test_linear_block_gs.py b/openmdao/solvers/linear/tests/test_linear_block_gs.py index 9c6b1e9ee9..dcdda4fa80 100644 --- a/openmdao/solvers/linear/tests/test_linear_block_gs.py +++ b/openmdao/solvers/linear/tests/test_linear_block_gs.py @@ -651,7 +651,6 @@ def compute(self, inputs, outputs): def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): self.count += 1 - # print(f'{self.name}: call to compute jacvecs have x? {"x" in d_inputs} have y? {"y" in d_inputs} have w? {"w" in d_inputs}') if mode == 'fwd': if 'z' in d_outputs: if 'x' in d_inputs: diff --git a/openmdao/utils/array_utils.py b/openmdao/utils/array_utils.py index 18640d81b7..89ab4eaa92 100644 --- a/openmdao/utils/array_utils.py +++ b/openmdao/utils/array_utils.py @@ -368,28 +368,6 @@ def _global2local_offsets(global_offsets): return offsets -def dist2local_src_inds(src_indices, sizes, rank): - """ - Given existing distributed src_indices, return a localized index array. - - Parameters - ---------- - src_indices : ndarray - Array of global src_indices. - sizes : ndarray - Array of sizes for a variable across procs. - rank : int - MPI rank of the current process. - - Returns - ------- - ndarray - Array of local src_indices. - """ - start = np.sum(sizes[:rank]) - return src_indices - start - - def get_input_idx_split(full_idxs, inputs, outputs, use_full_cols, is_total): """ Split an array of indices into vec outs + ins into two arrays of indices into outs and ins. diff --git a/openmdao/utils/range_collection.py b/openmdao/utils/range_collection.py index ee6bdd4b98..6e49ae625b 100644 --- a/openmdao/utils/range_collection.py +++ b/openmdao/utils/range_collection.py @@ -399,30 +399,6 @@ def metas2ranges(meta_iter, shape_name='shape'): start = stop -def metas2shapes(meta_iter, shape_name='shape'): - """ - Convert an iterator of metadata to an iterator of (name, shape) tuples. - - Parameters - ---------- - meta_iter : iterator of (name, meta) - Iterator of (name, meta) tuples, where name is the variable name and meta is the - corresponding metadata dictionary. - shape_name : str - Name of the metadata entry that contains the shape of the variable. Value can be either - 'shape' or 'global_shape'. Default is 'shape'. The value of the metadata entry must - be a tuple of integers. - - Yields - ------ - tuple - Tuple of the form (name, shape), where name is the variable name and shape is the shape - of the variable. - """ - for name, meta in meta_iter: - yield (name, meta[shape_name]) - - if __name__ == '__main__': meta = { 'x': {'shape': (2, 3)}, @@ -430,8 +406,6 @@ def metas2shapes(meta_iter, shape_name='shape'): 'z': {'shape': (6,)}, } - print(list(metas2shapes(meta.items()))) - ranges = list(metas2ranges(meta.items())) print(ranges) diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index fb9291b187..3c320bd89f 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -23,8 +23,7 @@ from petsc4py import PETSc from collections import defaultdict - from openmdao.vectors.default_transfer import DefaultTransfer, _fill, _setup_index_views - from openmdao.utils.array_utils import shape_to_len + from openmdao.vectors.default_transfer import DefaultTransfer, _setup_index_views class PETScTransfer(DefaultTransfer): """ @@ -160,13 +159,12 @@ def _setup_transfers_rev(group, desvars, responses): all_abs2meta_out = group._var_allprocs_abs2meta['output'] all_abs2meta_in = group._var_allprocs_abs2meta['input'] - # connections internal to this group + # connections internal to this group and all direct/indirect subsystems conns = group._conn_global_abs_in2out - relevant = group._relevant inp_boundary_set = set(all_abs2meta_in).difference(conns) - for resp, dvdct in relevant.items(): + for resp, dvdct in group._relevant.items(): if resp in all_abs2meta_out: # resp is continuous and inside this group is_dist_resp = all_abs2meta_out[resp]['distributed'] @@ -472,15 +470,3 @@ def _get_output_inds(group, abs_out, abs_in): start = end return output_inds, src_indices - - -def _get_output_dups(group, output): - return np.count_nonzero(group._var_sizes['output'][:, group._var_allprocs_abs2idx[output]]) - - -def _get_comp_inputs(graph, output): - compname = output.rpartition('.')[0] - if compname in graph: - return list(graph.predecessors(compname)) - else: # response will be connected directly to resp component inputs - return list(graph.predecessors(output)) From f838000d5ca78924d3aca66c98741f2d43113281 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 3 Nov 2023 11:08:39 -0400 Subject: [PATCH 51/70] split test into fwd/rev --- openmdao/core/tests/test_distrib_derivs.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index 61335d6d92..6265e81def 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -787,11 +787,9 @@ def test_distrib_voi_fd(self): assert_near_equal(J['sum.f_sum', 'p.x']['abs error'].reverse, 0.0, 1e-5) assert_near_equal(J['sum.f_sum', 'p.y']['abs error'].reverse, 0.0, 1e-5) - def test_distrib_voi_group_fd(self): + def _setup_distrib_voi_group_fd(self, mode, size=7): # Only supports groups where the inputs to the distributed component whose inputs are # distributed to procs via src_indices don't cross the boundary. - size = 7 - prob = om.Problem() model = prob.model @@ -829,10 +827,15 @@ def test_distrib_voi_group_fd(self): sub.approx_totals(method='fd') - prob.setup(mode='fwd', force_alloc_complex=True) + prob.setup(mode=mode, force_alloc_complex=True) prob.run_model() + return prob + + def test_distrib_voi_group_fd_fwd(self): + size = 7 + prob = self._setup_distrib_voi_group_fd('fwd', size) desvar = prob.driver.get_design_var_values() con = prob.driver.get_constraint_values() @@ -843,12 +846,9 @@ def test_distrib_voi_group_fd(self): assert_check_totals(prob.check_totals(method='fd', out_stream=None), rtol=1e-5) - # rev mode - - prob.setup(mode='rev', force_alloc_complex=True) - - prob.run_model() - + def test_distrib_voi_group_fd_rev(self): + size = 7 + prob = self._setup_distrib_voi_group_fd('rev', size) desvar = prob.driver.get_design_var_values() con = prob.driver.get_constraint_values() From f01970c3058adb99f540c250d03ae4796e270b09 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 3 Nov 2023 13:37:07 -0400 Subject: [PATCH 52/70] passing all tests --- openmdao/core/group.py | 54 +++++++++++----------- openmdao/core/tests/test_distrib_derivs.py | 3 +- openmdao/vectors/petsc_transfer.py | 16 +++---- 3 files changed, 37 insertions(+), 36 deletions(-) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index d5375510e3..8319e1170d 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -192,11 +192,11 @@ class Group(System): Dict of absolute response metadata. _relevance_graph : nx.DiGraph Graph of relevance connections. Always None except in the top level Group. - _fd_rev_xfer_correction_dist : set + _fd_rev_xfer_correction_dist : dict If one or more subgroups of this group is using finite difference to compute derivatives, this is the set of inputs to those subgroups that are upstream of a distributed response - within the same subgroup. These determine if an allreduce is necessary when transferring - data to a connected output in reverse mode. + within the same subgroup, keyed by active response. These determine if an allreduce is + necessary when transferring data to a connected output in reverse mode. """ def __init__(self, **kwargs): @@ -227,7 +227,7 @@ def __init__(self, **kwargs): self._abs_desvars = None self._abs_responses = None self._relevance_graph = None - self._fd_rev_xfer_correction_dist = set() + self._fd_rev_xfer_correction_dist = {} # TODO: we cannot set the solvers with property setters at the moment # because our lint check thinks that we are defining new attributes @@ -1322,7 +1322,7 @@ def _final_setup(self, comm, mode): # must call this before vector setup because it determines if we need to alloc commplex self._setup_partials() - self._fd_rev_xfer_correction_dist = set() + self._fd_rev_xfer_correction_dist = {} self._problem_meta['relevant'] = self._init_relevance(mode) @@ -3789,8 +3789,8 @@ def _apply_linear(self, jac, rel_systems, mode, scope_out=None, scope_in=None): # system doutput variables, taking into account distributed inputs. # Since the transfers are not correcting for those issues, we need to do it here. - # If we have a distributed constraint/obj within the FD group, - # we perform essentially an allreduce on the d_inputs vars that connect to + # If we have a distributed constraint/obj within the FD group and that con/obj is, + # active, we perform essentially an allreduce on the d_inputs vars that connect to # outside systems so they'll include the contribution from all procs. if self._fd_rev_xfer_correction_dist: seed_vars = self._problem_meta['seed_vars'] @@ -3804,25 +3804,27 @@ def _apply_linear(self, jac, rel_systems, mode, scope_out=None, scope_in=None): " supported under MPI in reverse mode if they " "depend on an group doing finite difference and " "containing distributed constraints/objectives.") - slices = self._dinputs.get_slice_dict() - inarr = self._dinputs.asarray() - data = {} - for inp in self._fd_rev_xfer_correction_dist: - if inp in slices: - arr = inarr[slices[inp]] - if np.any(arr): - data[inp] = arr - else: - data[inp] = None - - if data: - comm = self.comm - myrank = comm.rank - for rank, d in enumerate(comm.allgather(data)): - if rank != myrank: - for n, val in d.items(): - if val is not None and n in slices: - inarr[slices[n]] += val + seed_var = list(seed_vars)[0] + if seed_var in self._fd_rev_xfer_correction_dist: + slices = self._dinputs.get_slice_dict() + inarr = self._dinputs.asarray() + data = {} + for inp in self._fd_rev_xfer_correction_dist[seed_var]: + if inp in slices: + arr = inarr[slices[inp]] + if np.any(arr): + data[inp] = arr + else: + data[inp] = None + + if data: + comm = self.comm + myrank = comm.rank + for rank, d in enumerate(comm.allgather(data)): + if rank != myrank: + for n, val in d.items(): + if val is not None and n in slices: + inarr[slices[n]] += val # Apply recursion else: diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index 6265e81def..e7a401aaf9 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -793,11 +793,10 @@ def _setup_distrib_voi_group_fd(self, mode, size=7): prob = om.Problem() model = prob.model - ivc = om.IndepVarComp() + ivc = model.add_subsystem('p', om.IndepVarComp(), promotes=['*']) ivc.add_output('x', np.ones((size, ))) ivc.add_output('y', np.ones((size, ))) - model.add_subsystem('p', ivc, promotes=['*']) sub = model.add_subsystem('sub', om.Group(), promotes=['*']) ivc2 = om.IndepVarComp() diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 3c320bd89f..7a8c222c47 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -166,16 +166,16 @@ def _setup_transfers_rev(group, desvars, responses): for resp, dvdct in group._relevant.items(): if resp in all_abs2meta_out: # resp is continuous and inside this group - is_dist_resp = all_abs2meta_out[resp]['distributed'] - - for dv, tup in dvdct.items(): - # use only dvs outside of this group. - if dv not in allprocs_abs2prom: - rel = tup[0] - if is_dist_resp: + if all_abs2meta_out[resp]['distributed']: # distributed response + for dv, tup in dvdct.items(): + # use only dvs outside of this group. + if dv not in allprocs_abs2prom: + rel = tup[0] for inp in inp_boundary_set.intersection(rel['input']): if inp in abs2meta_in: - group._fd_rev_xfer_correction_dist.add(inp) + if resp not in group._fd_rev_xfer_correction_dist: + group._fd_rev_xfer_correction_dist[resp] = set() + group._fd_rev_xfer_correction_dist[resp].add(inp) # FD groups don't need reverse transfers return {} From 3721a440c0fa2f303b18caee18d6fd2cf4293721 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 6 Nov 2023 13:45:29 -0500 Subject: [PATCH 53/70] added test and made cosmetic improvement to check_totals output --- openmdao/core/problem.py | 9 ++++--- openmdao/core/tests/test_parallel_groups.py | 27 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index f8bf109736..c4e4c3257c 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -3262,7 +3262,8 @@ def _assemble_derivative_data(derivative_data, rel_error_tol, abs_error_tol, out out_buffer.write(' Directional Derivative (Jfor)') else: out_buffer.write(' Raw Forward Derivative (Jfor)') - out_buffer.write(f"\n {Jfor}\n\n") + Jstr = textwrap.indent(str(Jfor), ' ') + out_buffer.write(f"\n{Jstr}\n\n") fdtype = fd_opts['method'].upper() @@ -3274,7 +3275,8 @@ def _assemble_derivative_data(derivative_data, rel_error_tol, abs_error_tol, out out_buffer.write(' Directional Derivative (Jrev)') else: out_buffer.write(' Raw Reverse Derivative (Jrev)') - out_buffer.write(f"\n {Jrev}\n\n") + Jstr = textwrap.indent(str(Jrev), ' ') + out_buffer.write(f"\n{Jstr}\n\n") try: fds = derivative_info['J_fd'] @@ -3292,8 +3294,9 @@ def _assemble_derivative_data(derivative_data, rel_error_tol, abs_error_tol, out out_buffer.write(f" Directional {fdtype} Derivative (Jfd)" f"{stepstrs[i]}\n {fd}\n\n") else: + Jstr = textwrap.indent(str(fd), ' ') out_buffer.write(f" Raw {fdtype} Derivative (Jfd){stepstrs[i]}" - f"\n {fd}\n\n") + f"\n{Jstr}\n\n") out_buffer.write(' -' * 30 + '\n') diff --git a/openmdao/core/tests/test_parallel_groups.py b/openmdao/core/tests/test_parallel_groups.py index 90c1867b29..16672e73c0 100644 --- a/openmdao/core/tests/test_parallel_groups.py +++ b/openmdao/core/tests/test_parallel_groups.py @@ -214,6 +214,33 @@ def test_converge_diverge(self, solver, nlsolver, mode): J = prob.compute_totals(of=unknown_list, wrt=indep_list) assert_near_equal(J['c7.y1', 'iv.x'][0][0], -40.75, 1e-6) + @parameterized.expand(itertools.product([om.LinearRunOnce], + [om.NonlinearBlockGS, om.NonlinearRunOnce], + ['fwd', 'rev']), + name_func=_test_func_name) + def test_par_with_only_1_subsystem(self, solver, nlsolver, mode): + p = om.Problem() + model = p.model + p.model.linear_solver = solver() + p.model.nonlinear_solver = nlsolver() + + # model.add_subsystem('p1', om.IndepVarComp('x', 1.0)) + par = model.add_subsystem('par', om.ParallelGroup()) + par.add_subsystem('c1', om.ExecComp('y=2.0*x')) + model.add_subsystem('c2', om.ExecComp('y=3.0*x')) + + # model.connect('p1.x', 'par.c1.x') + model.connect('par.c1.y', 'c2.x') + + model.add_design_var('par.c1.x', lower=-50.0, upper=50.0) + model.add_constraint('par.c1.y', lower=-15.0, upper=15.0) + model.add_objective('c2.y') + + p.setup(check=False, mode=mode) + p.run_model() + + assert_check_totals(p.check_totals(out_stream=None), atol=1e-6) + def test_zero_shape(self): raise unittest.SkipTest("zero shapes not fully supported yet") class MultComp(ExplicitComponent): From 853f343b6c368618e0b8f5d45cbf917bde7c1e5e Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Tue, 7 Nov 2023 14:15:48 -0500 Subject: [PATCH 54/70] fix for missing subjac val --- openmdao/core/problem.py | 71 ++++++++++--------- openmdao/core/tests/test_parallel_groups.py | 33 ++++----- openmdao/jacobians/dictionary_jacobian.py | 4 ++ openmdao/test_suite/groups/parallel_groups.py | 14 ++-- 4 files changed, 66 insertions(+), 56 deletions(-) diff --git a/openmdao/core/problem.py b/openmdao/core/problem.py index c4e4c3257c..9904f49ebc 100644 --- a/openmdao/core/problem.py +++ b/openmdao/core/problem.py @@ -3256,47 +3256,48 @@ def _assemble_derivative_data(derivative_data, rel_error_tol, abs_error_tol, out out_buffer.write(f'\n MPI Rank {MPI.COMM_WORLD.rank}\n') out_buffer.write('\n') - # Raw Derivatives - if magnitudes[0].forward is not None: - if directional: - out_buffer.write(' Directional Derivative (Jfor)') - else: - out_buffer.write(' Raw Forward Derivative (Jfor)') - Jstr = textwrap.indent(str(Jfor), ' ') - out_buffer.write(f"\n{Jstr}\n\n") + with np.printoptions(linewidth=240): + # Raw Derivatives + if magnitudes[0].forward is not None: + if directional: + out_buffer.write(' Directional Derivative (Jfor)') + else: + out_buffer.write(' Raw Forward Derivative (Jfor)') + Jstr = textwrap.indent(str(Jfor), ' ') + out_buffer.write(f"\n{Jstr}\n\n") - fdtype = fd_opts['method'].upper() + fdtype = fd_opts['method'].upper() - if magnitudes[0].reverse is not None: - if directional: - if totals: - out_buffer.write(' Directional Derivative (Jrev) Dot Product') + if magnitudes[0].reverse is not None: + if directional: + if totals: + out_buffer.write(' Directional Derivative (Jrev) Dot Product') + else: + out_buffer.write(' Directional Derivative (Jrev)') else: - out_buffer.write(' Directional Derivative (Jrev)') - else: - out_buffer.write(' Raw Reverse Derivative (Jrev)') - Jstr = textwrap.indent(str(Jrev), ' ') - out_buffer.write(f"\n{Jstr}\n\n") + out_buffer.write(' Raw Reverse Derivative (Jrev)') + Jstr = textwrap.indent(str(Jrev), ' ') + out_buffer.write(f"\n{Jstr}\n\n") - try: - fds = derivative_info['J_fd'] - except KeyError: - fds = [0.] + try: + fds = derivative_info['J_fd'] + except KeyError: + fds = [0.] - for i in range(len(magnitudes)): - fd = fds[i] + for i in range(len(magnitudes)): + fd = fds[i] - if directional: - if totals and magnitudes[i].reverse is not None: - out_buffer.write(f' Directional {fdtype} Derivative (Jfd) ' - f'Dot Product{stepstrs[i]}\n {fd}\n\n') + if directional: + if totals and magnitudes[i].reverse is not None: + out_buffer.write(f' Directional {fdtype} Derivative (Jfd) ' + f'Dot Product{stepstrs[i]}\n {fd}\n\n') + else: + out_buffer.write(f" Directional {fdtype} Derivative (Jfd)" + f"{stepstrs[i]}\n {fd}\n\n") else: - out_buffer.write(f" Directional {fdtype} Derivative (Jfd)" - f"{stepstrs[i]}\n {fd}\n\n") - else: - Jstr = textwrap.indent(str(fd), ' ') - out_buffer.write(f" Raw {fdtype} Derivative (Jfd){stepstrs[i]}" - f"\n{Jstr}\n\n") + Jstr = textwrap.indent(str(fd), ' ') + out_buffer.write(f" Raw {fdtype} Derivative (Jfd){stepstrs[i]}" + f"\n{Jstr}\n\n") out_buffer.write(' -' * 30 + '\n') @@ -3354,7 +3355,7 @@ def _assemble_derivative_data(derivative_data, rel_error_tol, abs_error_tol, out f"{ders}.\nThis can happen if a component 'compute_jacvec_product' " "or 'apply_linear'\nmethod does not properly reduce the value of a distributed " "output when computing the\nderivative of that output with respect to a serial " - "input.\nOpenMDAO 3.25 changed the convention used" + "input.\nOpenMDAO 3.25 changed the convention used " "when transferring data between distributed and non-distributed \nvariables " "within a matrix free component. See POEM 75 for details.") diff --git a/openmdao/core/tests/test_parallel_groups.py b/openmdao/core/tests/test_parallel_groups.py index 16672e73c0..0fbe125a5b 100644 --- a/openmdao/core/tests/test_parallel_groups.py +++ b/openmdao/core/tests/test_parallel_groups.py @@ -24,6 +24,8 @@ from openmdao.test_suite.groups.parallel_groups import \ FanOutGrouped, FanInGrouped2, Diamond, ConvergeDiverge +from openmdao.core.tests.test_distrib_derivs import DistribExecComp + from openmdao.utils.assert_utils import assert_near_equal, assert_check_totals from openmdao.utils.logger_utils import TestLogger from openmdao.utils.array_utils import evenly_distrib_idxs @@ -214,32 +216,31 @@ def test_converge_diverge(self, solver, nlsolver, mode): J = prob.compute_totals(of=unknown_list, wrt=indep_list) assert_near_equal(J['c7.y1', 'iv.x'][0][0], -40.75, 1e-6) - @parameterized.expand(itertools.product([om.LinearRunOnce], - [om.NonlinearBlockGS, om.NonlinearRunOnce], - ['fwd', 'rev']), - name_func=_test_func_name) - def test_par_with_only_1_subsystem(self, solver, nlsolver, mode): + @parameterized.expand(['fwd', 'rev'], name_func=_test_func_name) + def test_par_with_only_1_subsystem(self, mode): p = om.Problem() model = p.model - p.model.linear_solver = solver() - p.model.nonlinear_solver = nlsolver() - # model.add_subsystem('p1', om.IndepVarComp('x', 1.0)) + model.add_subsystem('p1', om.IndepVarComp('x', np.ones(3))) par = model.add_subsystem('par', om.ParallelGroup()) - par.add_subsystem('c1', om.ExecComp('y=2.0*x')) - model.add_subsystem('c2', om.ExecComp('y=3.0*x')) + G = par.add_subsystem('G', om.Group()) + G.add_subsystem('c1', om.ExecComp('y=2.0*x', shape=3)) + G.add_subsystem('c2', DistribExecComp(['y=5.0*x', 'y=7.0*x'], arr_size=3)) + model.add_subsystem('c2', om.ExecComp('y=3.0*x', shape=3)) - # model.connect('p1.x', 'par.c1.x') - model.connect('par.c1.y', 'c2.x') + model.connect('p1.x', 'par.G.c1.x') + model.connect('par.G.c1.y', 'par.G.c2.x') + model.connect('par.G.c2.y', 'c2.x', src_indices=om.slicer[:]) - model.add_design_var('par.c1.x', lower=-50.0, upper=50.0) - model.add_constraint('par.c1.y', lower=-15.0, upper=15.0) - model.add_objective('c2.y') + model.add_design_var('p1.x', lower=-50.0, upper=50.0) + model.add_constraint('par.G.c1.y', lower=-15.0, upper=15.0) + model.add_constraint('par.G.c2.y', lower=-15.0, upper=15.0) + model.add_objective('c2.y', index=0) p.setup(check=False, mode=mode) p.run_model() - assert_check_totals(p.check_totals(out_stream=None), atol=1e-6) + assert_check_totals(p.check_totals(out_stream=None), rtol=2e-5, atol=2e-5) def test_zero_shape(self): raise unittest.SkipTest("zero shapes not fully supported yet") diff --git a/openmdao/jacobians/dictionary_jacobian.py b/openmdao/jacobians/dictionary_jacobian.py index aa20e2fd29..3ac1373f82 100644 --- a/openmdao/jacobians/dictionary_jacobian.py +++ b/openmdao/jacobians/dictionary_jacobian.py @@ -330,11 +330,15 @@ def set_col(self, system, icol, column): if key in self._subjacs_info: subjac = self._subjacs_info[key] if subjac['cols'] is None: + if subjac['val'] is None: # can happen for matrix free comp + subjac['val'] = np.zeros(subjac['shape']) subjac['val'][:, loc_idx] = column[start:end] else: match_inds = np.nonzero(subjac['cols'] == loc_idx)[0] if match_inds.size > 0: row_inds = subjac['rows'][match_inds] + if subjac['val'] is None: + subjac['val'] = np.zeros(len(subjac['rows'])) subjac['val'][match_inds] = column[start:end][row_inds] else: row_inds = np.zeros(0, dtype=INT_DTYPE) diff --git a/openmdao/test_suite/groups/parallel_groups.py b/openmdao/test_suite/groups/parallel_groups.py index c1c2afa560..76bab2d986 100644 --- a/openmdao/test_suite/groups/parallel_groups.py +++ b/openmdao/test_suite/groups/parallel_groups.py @@ -233,16 +233,18 @@ class ConvergeDiverge(om.Group): Used for testing parallel reverse scatters. """ - def __init__(self): + def __init__(self, parallel=True, inner_ivc=True): super().__init__() - self.add_subsystem('iv', om.IndepVarComp('x', 2.0)) + if inner_ivc: + self.add_subsystem('iv', om.IndepVarComp('x', 2.0)) self.add_subsystem('c1', om.ExecComp(['y1 = 2.0*x1**2', 'y2 = 3.0*x1' ])) - g1 = self.add_subsystem('g1', om.ParallelGroup()) + grp = om.ParallelGroup() if parallel else om.Group() + g1 = self.add_subsystem('g1', grp) g1.add_subsystem('c2', om.ExecComp('y1 = 0.5*x1')) g1.add_subsystem('c3', om.ExecComp('y1 = 3.5*x1')) @@ -250,14 +252,16 @@ def __init__(self): 'y2 = 3.0*x1 - 5.0*x2' ])) - g2 = self.add_subsystem('g2', om.ParallelGroup()) + grp = om.ParallelGroup() if parallel else om.Group() + g2 = self.add_subsystem('g2', grp) g2.add_subsystem('c5', om.ExecComp('y1 = 0.8*x1')) g2.add_subsystem('c6', om.ExecComp('y1 = 0.5*x1')) self.add_subsystem('c7', om.ExecComp('y1 = x1 + 3.0*x2')) # make connections - self.connect('iv.x', 'c1.x1') + if inner_ivc: + self.connect('iv.x', 'c1.x1') self.connect('c1.y1', 'g1.c2.x1') self.connect('c1.y2', 'g1.c3.x1') From af8766d7f93d88161b774623c0e8c611fd1a8126 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Tue, 7 Nov 2023 15:28:39 -0500 Subject: [PATCH 55/70] fix list_pre_post command --- openmdao/utils/om.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmdao/utils/om.py b/openmdao/utils/om.py index 35da475721..3d8b0f029b 100644 --- a/openmdao/utils/om.py +++ b/openmdao/utils/om.py @@ -433,7 +433,7 @@ def _list_pre_post(prob): prob.list_pre_post(outfile=options.outfile) # register the hook - hooks._register_hook('setup', class_name='Problem', inst_id=options.problem, + hooks._register_hook('final_setup', class_name='Problem', inst_id=options.problem, post=_list_pre_post, exit=True) _load_and_exec(options.file[0], user_args) From 11c9f28102d1181d0c8bf242a383fab9546b1c46 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Wed, 8 Nov 2023 22:59:47 -0500 Subject: [PATCH 56/70] updated RangeMappers and added tests --- .../approximation_scheme.py | 3 +- openmdao/core/group.py | 59 +-- openmdao/core/system.py | 6 +- openmdao/core/tests/test_mpi_coloring_bug.py | 24 +- openmdao/core/total_jac.py | 5 +- openmdao/devtools/debug.py | 53 ++- openmdao/jacobians/dictionary_jacobian.py | 3 + openmdao/utils/range_collection.py | 418 ------------------ openmdao/utils/rangemapper.py | 405 +++++++++++++++++ openmdao/utils/tests/test_rangemapper.py | 86 ++++ openmdao/vectors/petsc_transfer.py | 6 +- 11 files changed, 579 insertions(+), 489 deletions(-) delete mode 100644 openmdao/utils/range_collection.py create mode 100644 openmdao/utils/rangemapper.py create mode 100644 openmdao/utils/tests/test_rangemapper.py diff --git a/openmdao/approximation_schemes/approximation_scheme.py b/openmdao/approximation_schemes/approximation_scheme.py index 65ea042d49..de5602a510 100644 --- a/openmdao/approximation_schemes/approximation_scheme.py +++ b/openmdao/approximation_schemes/approximation_scheme.py @@ -272,9 +272,10 @@ def _init_approximations(self, system): in_idx = [list(in_idx)] vec_idx = [vec_idx] else: - vec_idx = LocalRangeIterable(system, wrt) if directional and vec is not None: vec_idx = [v for v in vec_idx if v is not None] + else: + vec_idx = LocalRangeIterable(system, wrt) # Directional derivatives for quick deriv checking. # Place the indices in a list so that they are all stepped at the same time. diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 8319e1170d..327f313551 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -27,7 +27,7 @@ shape_to_len from openmdao.utils.general_utils import common_subpath, all_ancestors, \ convert_src_inds, _contains_all, shape2tuple, get_connection_owner, ensure_compatible, \ - _src_name_iter, meta2src_iter, get_rev_conns + meta2src_iter, get_rev_conns from openmdao.utils.units import is_compatible, unit_conversion, _has_val_mismatch, _find_unit, \ _is_unitless, simplify_unit from openmdao.utils.graph_utils import get_sccs_topo, get_out_of_order_nodes @@ -193,10 +193,11 @@ class Group(System): _relevance_graph : nx.DiGraph Graph of relevance connections. Always None except in the top level Group. _fd_rev_xfer_correction_dist : dict - If one or more subgroups of this group is using finite difference to compute derivatives, - this is the set of inputs to those subgroups that are upstream of a distributed response - within the same subgroup, keyed by active response. These determine if an allreduce is - necessary when transferring data to a connected output in reverse mode. + If this group is using finite difference to compute derivatives, + this is the set of inputs that are upstream of a distributed response + within this group, keyed by active response. These determine if contributions + from all ranks will be added together to get the correct input values when derivatives + in the larger model are being solved using reverse mode. """ def __init__(self, **kwargs): @@ -3799,32 +3800,34 @@ def _apply_linear(self, jac, rel_systems, mode, scope_out=None, scope_in=None): prefix = self.pathname + '.' seed_vars = [n for n in seed_vars if n.startswith(prefix)] if len(seed_vars) > 1: + # TODO: get this working with coloring raise RuntimeError("Multiple simultaneous seed variables " f"{sorted(seed_vars)} within an FD group are not" " supported under MPI in reverse mode if they " - "depend on an group doing finite difference and " + "depend on a group doing finite difference and " "containing distributed constraints/objectives.") - seed_var = list(seed_vars)[0] - if seed_var in self._fd_rev_xfer_correction_dist: - slices = self._dinputs.get_slice_dict() - inarr = self._dinputs.asarray() - data = {} - for inp in self._fd_rev_xfer_correction_dist[seed_var]: - if inp in slices: - arr = inarr[slices[inp]] - if np.any(arr): - data[inp] = arr - else: - data[inp] = None - - if data: - comm = self.comm - myrank = comm.rank - for rank, d in enumerate(comm.allgather(data)): - if rank != myrank: - for n, val in d.items(): - if val is not None and n in slices: - inarr[slices[n]] += val + for seed_var in seed_vars: + if seed_var in self._fd_rev_xfer_correction_dist: + slices = self._dinputs.get_slice_dict() + inarr = self._dinputs.asarray() + data = {} + for inp in self._fd_rev_xfer_correction_dist[seed_var]: + if inp in slices: + arr = inarr[slices[inp]] + if np.any(arr): + data[inp] = arr + else: + data[inp] = None + + if data: + comm = self.comm + myrank = comm.rank + for rank, d in enumerate(comm.allgather(data)): + if rank != myrank: + for n, val in d.items(): + if val is not None and n in slices: + inarr[slices[n]] += val + break # there's only 1 in the seed_vars set # Apply recursion else: @@ -5053,7 +5056,7 @@ def _setup_iteration_lists(self): graph = self.get_relevance_graph(dvs, responses) dvs = set([meta['source'] for meta in dvs.values()]) - responses = [meta['source'] for meta in responses.values()] + responses = set([meta['source'] for meta in responses.values()]) # we don't want _auto_ivc dependency to force all subsystems to be iterated, so split # the _auto_ivc node into two nodes, one for design vars and one for everything else. diff --git a/openmdao/core/system.py b/openmdao/core/system.py index 1d0254ff21..31deda8486 100644 --- a/openmdao/core/system.py +++ b/openmdao/core/system.py @@ -6298,7 +6298,7 @@ def comm_info_iter(self): for s in self._subsystems_myproc: yield from s.comm_info_iter() - def dist_range_iter(self, io, top_comm): + def dist_size_iter(self, io, top_comm): """ Yield names and distributed ranges of all local and remote variables in this system. @@ -6324,13 +6324,11 @@ def dist_range_iter(self, io, top_comm): mytopranks = topranks[toprank - myrank: toprank - myrank + self.comm.size] - total = 0 for rank in range(self.comm.size): for ivar, vname in enumerate(vmeta[io]): sz = sizes[io][rank, ivar] if sz > 0: - yield (vname, mytopranks[rank]), total, total + sz - total += sz + yield (vname, mytopranks[rank]), sz def local_range_iter(self, io): """ diff --git a/openmdao/core/tests/test_mpi_coloring_bug.py b/openmdao/core/tests/test_mpi_coloring_bug.py index e819267126..b147375419 100644 --- a/openmdao/core/tests/test_mpi_coloring_bug.py +++ b/openmdao/core/tests/test_mpi_coloring_bug.py @@ -516,23 +516,23 @@ def test_bug2(self): p = om.Problem() phases = p.model.add_subsystem('phases', om.ParallelGroup()) phase1 = phases.add_subsystem('phase1', om.Group()) - phase2 = phases.add_subsystem('phase2', om.Group()) phase1.add_subsystem('indep', om.IndepVarComp('x', val=np.ones(size))) - phase2.add_subsystem('indep', om.IndepVarComp('x', val=np.ones(size))) phase1.add_subsystem('comp1', om.ExecComp('y=3.0*x', x=np.ones(size), y=np.ones(size))) - # phase1.add_subsystem('comp2', om.ExecComp('y=5.0*x', x=np.ones(size), y=np.ones(size))) - phase2.add_subsystem('comp1', om.ExecComp('y=7.0*x', x=np.ones(size), y=np.ones(size))) - # phase2.add_subsystem('comp2', om.ExecComp('y=9.0*x', x=np.ones(size), y=np.ones(size))) + phase1.add_subsystem('comp2', om.ExecComp('y=5.0*x', x=np.ones(size), y=np.ones(size))) phase1.connect('indep.x', 'comp1.x') - # phase1.connect('comp1.y', 'comp2.x') - phase2.connect('indep.x', 'comp1.x') - # phase2.connect('comp1.y', 'comp2.x') - + phase1.connect('comp1.y', 'comp2.x') phase1.add_design_var('indep.x') - phase2.add_design_var('indep.x') - # phase1.add_constraint('comp2.y', lower=0.0) - # phase2.add_constraint('comp2.y', lower=0.0) + phase1.add_constraint('comp2.y', lower=0.0) phase1.add_constraint('comp1.y', lower=0.0) + + phase2 = phases.add_subsystem('phase2', om.Group()) + phase2.add_subsystem('indep', om.IndepVarComp('x', val=np.ones(size))) + phase2.add_subsystem('comp1', om.ExecComp('y=7.0*x', x=np.ones(size), y=np.ones(size))) + phase2.add_subsystem('comp2', om.ExecComp('y=9.0*x', x=np.ones(size), y=np.ones(size))) + phase2.connect('indep.x', 'comp1.x') + phase2.connect('comp1.y', 'comp2.x') + phase2.add_design_var('indep.x') + phase2.add_constraint('comp2.y', lower=0.0) phase2.add_constraint('comp1.y', lower=0.0) p.setup(mode='rev') diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index 583b3a5b49..0b1aeef9c1 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -358,8 +358,6 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, if self.simul_coloring is not None: # when simul coloring, need two scratch arrays self.jac_scratch['fwd'].append(scratch[1][:J.shape[0]]) if 'rev' in modes: - from openmdao.core.group import Group - self.jac_scratch['rev'] = [scratch[0][:J.shape[1]]] if self.simul_coloring is not None: # when simul coloring, need two scratch arrays self.jac_scratch['rev'].append(scratch[1][:J.shape[1]]) @@ -374,8 +372,7 @@ def __init__(self, problem, of, wrt, use_abs_names, return_format, approx=False, for name, pname in zip(wrt, prom_wrt): vmeta = all_abs2meta_out[name] if pname in voimeta: - meta = voimeta[pname] - end += meta['size'] + end += voimeta[pname]['size'] else: end += vmeta['global_size'] diff --git a/openmdao/devtools/debug.py b/openmdao/devtools/debug.py index 7fbcbd2412..bd269bc504 100644 --- a/openmdao/devtools/debug.py +++ b/openmdao/devtools/debug.py @@ -14,7 +14,7 @@ from openmdao.utils.om_warnings import issue_warning, MPIWarning from openmdao.utils.reports_system import register_report from openmdao.utils.file_utils import text2html, _load_and_exec -from openmdao.utils.range_collection import DataRangeMapper +from openmdao.utils.rangemapper import RangeMapper from openmdao.visualization.tables.table_builder import generate_table @@ -689,11 +689,11 @@ def show_dist_var_conns(group, rev=False, out_stream=_DEFAULT_OUT_STREAM): for g in group.system_iter(typ=Group, include_self=True): if g._transfers[direction]: - in_ranges = list(g.dist_range_iter('input', group.comm)) - out_ranges = list(g.dist_range_iter('output', group.comm)) + in_ranges = list(g.dist_size_iter('input', group.comm)) + out_ranges = list(g.dist_size_iter('output', group.comm)) - inmapper = DataRangeMapper.create(in_ranges) - outmapper = DataRangeMapper.create(out_ranges) + inmapper = RangeMapper.create(in_ranges) + outmapper = RangeMapper.create(out_ranges) gprint = False @@ -708,9 +708,9 @@ def show_dist_var_conns(group, rev=False, out_stream=_DEFAULT_OUT_STREAM): conns = {} for iidx, oidx in zip(transfer._in_inds, transfer._out_inds): - idata, irind = inmapper.index2rel_data(iidx) + idata, irind = inmapper.index2key_rel(iidx) ivar, irank = idata - odata, orind = outmapper.index2rel_data(oidx) + odata, orind = outmapper.index2key_rel(oidx) ovar, orank = odata if odata not in conns: @@ -733,18 +733,15 @@ def show_dist_var_conns(group, rev=False, out_stream=_DEFAULT_OUT_STREAM): oinds = [d[0] for d in dlist] iinds = [d[1] for d in dlist] - orange = outmapper.data2range(odata) - irange = inmapper.data2range(idata) + orange = outmapper.key2range(odata) + irange = inmapper.key2range(idata) - oslc = is_full_slice(orange, oinds) - if oslc: - islc = is_full_slice(irange, iinds) - if islc: - s = f"{ovar}[:] {arrow} {ivar}[:]" - if s not in strs: - strs[s] = set() - strs[s].add(ranktup) - continue + if is_full_slice(orange, oinds) and is_full_slice(irange, iinds): + s = f"{ovar} {arrow} {ivar}" + if s not in strs: + strs[s] = set() + strs[s].add(ranktup) + continue for oidx, iidx in zip(oinds, iinds): s = f"{ovar}[{oidx}] {arrow} {ivar}[{iidx}]" @@ -802,12 +799,30 @@ def show_dist_var_conns(group, rev=False, out_stream=_DEFAULT_OUT_STREAM): if np.all(oranks == oranks[0]): orstr = str(oranks[0]) else: - orstr = str(sorted(oranks)) + sorted_ranks = sorted(oranks) + orstr = str(sorted_ranks) + if len(sorted_ranks) > 3: + for j, r in enumerate(sorted_ranks): + if j == 0 or r - val == 1: + val = r + else: + break + else: + orstr = f"[{sorted_ranks[0]} to {sorted_ranks[-1]}]" if np.all(iranks == iranks[0]): irstr = str(iranks[0]) else: + sorted_ranks = sorted(iranks) irstr = str(sorted(iranks)) + if len(sorted_ranks) > 3: + for j, r in enumerate(sorted_ranks): + if j == 0 or r - val == 1: + val = r + else: + break + else: + irstr = f"[{sorted_ranks[0]} to {sorted_ranks[-1]}]" if orstr == irstr and '[' not in orstr: printer(f"{pad} {s} rank {orstr}", file=out_stream) diff --git a/openmdao/jacobians/dictionary_jacobian.py b/openmdao/jacobians/dictionary_jacobian.py index 3ac1373f82..b4b807d509 100644 --- a/openmdao/jacobians/dictionary_jacobian.py +++ b/openmdao/jacobians/dictionary_jacobian.py @@ -50,6 +50,9 @@ def _iter_abs_keys(self, system): List of keys matching this jacobian for the current system. """ if self._iter_keys is None: + # determine the set of remote keys (keys where either of or wrt is remote somewhere) + # only if we're under MPI with comm size > 1 and the given system is a Group that + # computes its derivatives using finite difference or complex step. include_remotes = system.pathname and \ system.comm.size > 1 and system._owns_approx_jac and system._subsystems_allprocs subjacs = self._subjacs_info diff --git a/openmdao/utils/range_collection.py b/openmdao/utils/range_collection.py deleted file mode 100644 index 6e49ae625b..0000000000 --- a/openmdao/utils/range_collection.py +++ /dev/null @@ -1,418 +0,0 @@ -""" -A collection of classes for mapping indices to variable names and vice versa. -""" - -from openmdao.utils.array_utils import shape_to_len - - -# default size of array for which we use a FlatRangeMapper instead of a RangeTree -_MAX_FLAT_RANGE_SIZE = 10000 - - -class DataRangeMapper(object): - """ - A mapper of indices to variable names and vice versa. - - Parameters - ---------- - ranges : list of (data, start, stop) - Ordered list of (data, start, stop) tuples, where start and stop define the range of - indices for the data. Ranges must be contiguous. data must be hashable. - - Attributes - ---------- - size : int - Total size of all of the ranges combined. - _data2range : dict - Dictionary mapping data to an index range. - _range2data : dict - Dictionary mapping an index range to data. - """ - - def __init__(self, ranges): - """ - Initialize a DataRangeMapper. - """ - self._data2range = {} - self._range2data = {} - self.size = ranges[-1][2] - ranges[0][1] - - @staticmethod - def create(ranges, max_flat_range_size=_MAX_FLAT_RANGE_SIZE): - """ - Return a mapper that maps indices to variable names and relative indices. - - Parameters - ---------- - ranges : list of (data, start, stop) - Ordered list of (data, start, stop) tuples, where start and stop define the range - of indices for the data. Ranges must be contiguous. - max_flat_range_size : int - If the total array size is less than this, a FlatRangeMapper will be returned instead - of a RangeTree. - - Returns - ------- - FlatRangeMapper or RangeTree - A mapper that maps indices to variable data and relative indices. - """ - size = ranges[-1][2] - ranges[0][1] - return FlatRangeMapper(ranges) if size < max_flat_range_size else RangeTree(ranges) - - def add_range(self, data, start, stop): - """ - Add a range to the mapper. - - Parameters - ---------- - data : object (must be hashable) - Data corresponding to an index range. - start : int - Starting index of the variable. - stop : int - Ending index of the variable. - """ - self._data2range[data] = (start, stop) - self._range2data[start, stop] = data - - def data2range(self, data): - """ - Get the range corresponding to the given name and rank. - - Parameters - ---------- - data : object (must be hashable) - Data corresponding to an index range. - - Returns - ------- - tuple of (int, int) - The range of indices corresponding to the given data. - """ - return self._data2range[data] - - def _index2data(self, idx): - """ - Find the data corresponding to the given index. - - Parameters - ---------- - idx : int - The index into the full array. - - Returns - ------- - object or None - The data corresponding to the given index, or None if not found. - """ - raise NotImplementedError("_index2data method must be implemented by subclass.") - - def __getitem__(self, idx): - """ - Find the data corresponding to the given index. - - Parameters - ---------- - idx : int - The index into the full array. - """ - return self._index2data(idx) - - def indices2data(self, idxs): - """ - Find the data objects corresponding to the given indices. - - Parameters - ---------- - idxs : list of int - The indices into the full array. - - Returns - ------- - list of (object, int) - The data corresponding to each of the given indices. - """ - data = [self._index2data(idx) for idx in idxs] - if None in data: - missing = [] - for idx in idxs: - d = self._index2data(idx) - if d is None: - missing.append(idx) - raise RuntimeError(f"Indices {sorted(missing)} are not in any range.") - - return data - - -class RangeTreeNode(DataRangeMapper): - """ - A node in a binary search tree of ranges, mapping data to an index range. - - Parameters - ---------- - data : object - Data corresponding to an index range. - start : int - Starting index of the variable. - stop : int - Ending index of the variable. - - Attributes - ---------- - data : object - Data corresponding to an index range. - start : int - Starting index of the variable. - stop : int - Ending index of the variable. - left : RangeTreeNode or None - Left child node. - right : RangeTreeNode or None - Right child node. - """ - - __slots__ = ['data', 'start', 'stop', 'left', 'right'] - - def __init__(self, data, start, stop): - """ - Initialize a RangeTreeNode. - """ - self.data = data - self.start = start - self.stop = stop - self.left = None - self.right = None - - -class RangeTree(DataRangeMapper): - """ - A binary search tree of ranges, mapping data to an index range. - - Allows for fast lookup of the data corresponding to a given index. The ranges must be - contiguous, but they can be of different sizes. - - Search complexity is O(log2 n). Uses less memory than FlatRangeMapper when total array size is - large. - - Parameters - ---------- - ranges : list of (data, start, stop) - Ordered list of (data, start, stop) tuples, where start and stop define the range of - indices for the data. Ranges must be contiguous. data must be hashable. - - Attributes - ---------- - size : int - Total size of all of the ranges combined. - root : RangeTreeNode - Root node of the binary search tree. - """ - - def __init__(self, ranges): - """ - Initialize a RangeTree. - """ - super().__init__(ranges) - self.size = ranges[-1][2] - ranges[0][1] - self.root = self.build(ranges) - - def _index2data(self, idx): - """ - Find the data corresponding to the given index. - - Parameters - ---------- - idx : int - The index into the full array. - - Returns - ------- - object or None - The data corresponding to the given index, or None if not found. - int or None - The rank corresponding to the given index, or None if not found. - """ - node = self.root - while node is not None: - if idx < node.start: - node = node.left - elif idx >= node.stop: - node = node.right - else: - return node.data - - def index2rel_data(self, idx): - """ - Find the data and relative index corresponding to the matched range. - - Parameters - ---------- - idx : int - The index into the full array. - - Returns - ------- - obj or None - The data corresponding to the matched range, or None if not found. - int or None - The relative index into the matched range, or None if not found. - """ - node = self.root - while node is not None: - if idx < node.start: - node = node.left - elif idx >= node.stop: - node = node.right - else: - return node.data, idx - node.start - - return None, None - - def build(self, ranges): - """ - Build a binary search tree to map indices to variable data. - - Parameters - ---------- - ranges : list of (data, start, stop) - List of (data, start, stop) tuples, where start and stop - define the range of indices for the data. Ranges must be contiguous. - data must be hashable. - - Returns - ------- - RangeTreeNode - Root node of the binary search tree. - """ - half = len(ranges) // 2 - data, start, stop = ranges[half] - - node = RangeTreeNode(data, start, stop) - self.add_range(data, start, stop) - - left_slices = ranges[:half] - if left_slices: - node.left = self.build(left_slices) - - right_slices = ranges[half + 1:] - if right_slices: - node.right = self.build(right_slices) - - return node - - -class FlatRangeMapper(DataRangeMapper): - """ - A flat list mapping indices to variable data and relative indices. - - Parameters - ---------- - ranges : list of (data, start, stop) - Ordered list of (data, start, stop) tuples, where start and stop define the range of - indices for that data. Ranges must be contiguous. data must be hashable. - - Attributes - ---------- - ranges : list of (data, start, stop) - List of (data, start, stop) tuples, where start and stop define the range of - indices for that data. Ranges must be contiguous. data must be hashable. - """ - - def __init__(self, ranges): - """ - Initialize a FlatRangeMapper. - """ - super().__init__(ranges) - self.ranges = [None] * self.size - for rng in ranges: - data, start, stop = rng - self.ranges[start:stop] = [rng] * (stop - start) - self.add_range(data, start, stop) - - def _index2data(self, idx): - """ - Find the data corresponding to the given index. - - Parameters - ---------- - idx : int - The index into the full array. - - Returns - ------- - object or None - The data corresponding to the given index, or None if not found. - """ - try: - return self.ranges[idx][0] - except IndexError: - return None - - def index2rel_data(self, idx): - """ - Find the data and relative index corresponding to the matched range. - - Parameters - ---------- - idx : int - The index into the full array. - - Returns - ------- - object or None - The data corresponding to the matched range, or None if not found. - int or None - The relative index into the matched range, or None if not found. - """ - try: - data, start, _ = self.ranges[idx] - except IndexError: - return (None, None) - - return (data, idx - start) - - -def metas2ranges(meta_iter, shape_name='shape'): - """ - Convert an iterator of metadata to an iterator of (name, start, stop) tuples. - - Parameters - ---------- - meta_iter : iterator of (name, meta) - Iterator of (name, meta) tuples, where name is the variable name and meta is the - corresponding metadata dictionary. - shape_name : str - Name of the metadata entry that contains the shape of the variable. Value can be either - 'shape' or 'global_shape'. Default is 'shape'. The value of the metadata entry must - be a tuple of integers. - - Yields - ------ - tuple - Tuple of the form (name, start, stop), where name is the variable name, start is the start - of the variable range, and stop is the end of the variable range. - """ - start = stop = 0 - for name, meta in meta_iter: - stop += shape_to_len(meta[shape_name]) - yield (name, start, stop) - start = stop - - -if __name__ == '__main__': - meta = { - 'x': {'shape': (2, 3)}, - 'y': {'shape': (4, 5)}, - 'z': {'shape': (6,)}, - } - - ranges = list(metas2ranges(meta.items())) - print(ranges) - - rtree = RangeTree(ranges) - flat = FlatRangeMapper(ranges) - - for i in range(34): - rname, rind = rtree.index2rel_data(i) - fname, find = flat.index2rel_data(i) - print(i, rname, rind, fname, find) diff --git a/openmdao/utils/rangemapper.py b/openmdao/utils/rangemapper.py new file mode 100644 index 0000000000..62ec4db92e --- /dev/null +++ b/openmdao/utils/rangemapper.py @@ -0,0 +1,405 @@ +""" +A collection of classes for mapping indices to variable names and vice versa. +""" + +from openmdao.utils.array_utils import shape_to_len + + +# default size of array for which we use a FlatRangeMapper instead of a RangeTree +MAX_FLAT_RANGE_SIZE = 10000 + + +class RangeMapper(object): + """ + A mapper of indices to variable names and vice versa. + + Parameters + ---------- + sizes : iterable of (key, size) tuples + Iterable of (key, size) tuples. key must be hashable. + + Attributes + ---------- + size : int + Total size of all of the sizes combined. + _key2range : dict + Dictionary mapping key to an index range. + """ + + def __init__(self, sizes): + """ + Initialize a RangeMapper. + """ + self._key2range = {} + self.size = sum(size for _, size in sizes) + + @staticmethod + def create(sizes, max_flat_range_size=MAX_FLAT_RANGE_SIZE): + """ + Return a mapper that maps indices to variable names and relative indices. + + Parameters + ---------- + sizes : list of (key, size) + Iterable of (key, size) tuples. + max_flat_range_size : int + If the total array size is less than this, a FlatRangeMapper will be returned instead + of a RangeTree. + + Returns + ------- + FlatRangeMapper or RangeTree + A mapper that maps indices to variable key and relative indices. + """ + size = sum(size for _, size in sizes) + return FlatRangeMapper(sizes) if size < max_flat_range_size else RangeTree(sizes) + + def key2range(self, key): + """ + Get the range corresponding to the given key. + + Parameters + ---------- + key : object (must be hashable) + Data corresponding to an index range. + + Returns + ------- + tuple of (int, int) + The range of indices corresponding to the given key. + """ + return self._key2range[key] + + def key2size(self, key): + """ + Get the size corresponding to the given key. + + Parameters + ---------- + key : object (must be hashable) + Key corresponding to an index range. + + Returns + ------- + int + The size corresponding to the given key. + """ + start, stop = self._key2range[key] + return stop - start + + def __getitem__(self, idx): + """ + Find the key corresponding to the given index. + + Parameters + ---------- + idx : int + The index into the full array. + """ + raise NotImplementedError("__getitem__ method must be implemented by subclass.") + + def __iter__(self): + """ + Iterate over (key, start, stop) tuples. + + Yields + ------ + (obj, int, int) + (key, start index, stop index), where key is a hashable object. + """ + raise NotImplementedError("__getitem__ method must be implemented by subclass.") + + def dump(self): + """ + Dump the contents of the mapper to stdout. + """ + for key, (start, stop) in self._key2range.items(): + print(f'{key}: {start} - {stop}') + + +class RangeTreeNode(RangeMapper): + """ + A node in a binary search tree of sizes, mapping key to an index range. + + Parameters + ---------- + key : object + Data corresponding to an index range. + start : int + Starting index of the variable. + stop : int + Ending index of the variable. + + Attributes + ---------- + key : object + Data corresponding to an index range. + start : int + Starting index of the variable. + stop : int + Ending index of the variable. + left : RangeTreeNode or None + Left child node. + right : RangeTreeNode or None + Right child node. + """ + + __slots__ = ['key', 'start', 'stop', 'left', 'right'] + + def __init__(self, key, start, stop): + """ + Initialize a RangeTreeNode. + """ + self.key = key + self.start = start + self.stop = stop + self.left = None + self.right = None + + def __repr__(self): + """ + Return a string representation of the RangeTreeNode. + """ + return f"RangeTreeNode({self.key}, ({self.start}:{self.stop}))" + + +def _size_of_ranges(ranges): + size = 0 + for _, start, stop in ranges: + size += stop - start + + return size + + +class RangeTree(RangeMapper): + """ + A binary search tree of sizes, mapping key to an index range. + + Allows for fast lookup of the key corresponding to a given index. The sizes must be + contiguous, but they can be of different sizes. + + Search complexity is O(log2 n). Uses less memory than FlatRangeMapper when total array size is + large. + + Parameters + ---------- + sizes : list of (key, start, stop) + Ordered list of (key, start, stop) tuples, where start and stop define the range of + indices for the key. Ranges must be contiguous. key must be hashable. + + Attributes + ---------- + size : int + Total size of all of the sizes combined. + root : RangeTreeNode + Root node of the binary search tree. + """ + + def __init__(self, sizes): + """ + Initialize a RangeTree. + """ + super().__init__(sizes) + ranges = [] + start = stop = 0 + for key, size in sizes: + stop += size + ranges.append((key, start, stop)) + start = stop + + self.root = self.build(ranges) + + def __getitem__(self, idx): + """ + Find the key corresponding to the given index. + + Parameters + ---------- + idx : int + The index into the full array. + + Returns + ------- + object or None + The key corresponding to the given index, or None if not found. + int or None + The rank corresponding to the given index, or None if not found. + """ + node = self.root + while node is not None: + if idx < node.start: + node = node.left + elif idx >= node.stop: + node = node.right + else: + return node.key + + def __iter__(self): + """ + Iterate over (key, start, stop) tuples. + + Yields + ------ + (obj, int, int) + (key, start index, stop index), where key is a hashable object. + """ + node = self.root + stack = [[node, node.left, node.right]] + while stack: + sub = stack[-1] + node, left, right = sub + if left: + stack.append([left, left.left, left.right]) + sub[1] = None # zero left + else: + if right: + stack.append([right, right.left, right.right]) + sub[2] = None # zero right + else: + stack.pop() + yield (node.key, node.start, node.stop) + + def index2key_rel(self, idx): + """ + Find the key and relative index corresponding to the matched range. + + Parameters + ---------- + idx : int + The index into the full array. + + Returns + ------- + obj or None + The key corresponding to the matched range, or None if not found. + int or None + The relative index into the matched range, or None if not found. + """ + node = self.root + while node is not None: + if idx < node.start: + node = node.left + elif idx >= node.stop: + node = node.right + else: + return node.key, idx - node.start + + return None, None + + def build(self, ranges): + """ + Build a binary search tree to map indices to variable key. + + Parameters + ---------- + ranges : list of (key, start, stop) + List of (key, start, stop) tuples, where start and stop + define the range of indices for the key. Ranges must be ordered and contiguous. + key must be hashable. + + Returns + ------- + RangeTreeNode + Root node of the binary search tree. + """ + mid = len(ranges) // 2 + + key, start, stop = ranges[mid] + + node = RangeTreeNode(key, start, stop) + self._key2range[key] = (start, stop) + + left_slices = ranges[:mid] + right_slices = ranges[mid + 1:] + + if left_slices: + node.left = self.build(left_slices) + + if right_slices: + node.right = self.build(right_slices) + + return node + + +class FlatRangeMapper(RangeMapper): + """ + A flat list mapping indices to variable key and relative indices. + + Parameters + ---------- + sizes : list of (key, size) + Ordered list of (key, size) tuples. key must be hashable. + + Attributes + ---------- + ranges : list of (key, start, stop) + List of (key, start, stop) tuples, where start and stop define the range of + indices for that key. Ranges must be contiguous. key must be hashable. + """ + + def __init__(self, sizes): + """ + Initialize a FlatRangeMapper. + """ + super().__init__(sizes) + self.ranges = [None] * self.size + start = stop = 0 + for key, size in sizes: + stop += size + self.ranges[start:stop] = [(key, start, stop)] * size + self._key2range[key] = (start, stop) + start = stop + + def __getitem__(self, idx): + """ + Find the key corresponding to the given index. + + Parameters + ---------- + idx : int + The index into the full array. + + Returns + ------- + object or None + The key corresponding to the given index, or None if not found. + """ + try: + return self.ranges[idx][0] + except IndexError: + return None + + def __iter__(self): + """ + Iterate over (key, start, stop) tuples. + + Yields + ------ + (obj, int, int) + (key, start index, stop index), where key is a hashable object. + """ + for key, (start, stop) in self._key2range.items(): + yield (key, start, stop) + + def index2key_rel(self, idx): + """ + Find the key and relative index corresponding to the matched range. + + Parameters + ---------- + idx : int + The index into the full array. + + Returns + ------- + object or None + The key corresponding to the matched range, or None if not found. + int or None + The relative index into the matched range, or None if not found. + """ + try: + key, start, _ = self.ranges[idx] + except IndexError: + return (None, None) + + return (key, idx - start) diff --git a/openmdao/utils/tests/test_rangemapper.py b/openmdao/utils/tests/test_rangemapper.py new file mode 100644 index 0000000000..d29411f7e2 --- /dev/null +++ b/openmdao/utils/tests/test_rangemapper.py @@ -0,0 +1,86 @@ +import unittest + +from openmdao.utils.assert_utils import assert_near_equal +from openmdao.utils.rangemapper import RangeMapper, RangeTree, FlatRangeMapper, MAX_FLAT_RANGE_SIZE + + +_data = { + 'a': 1, # 0:1 + 'b': 8, # 1:9 + 'x': 6, # 9:15 + 'y': 21, # 15:36 + 'z': 6, # 36:42 +} + + +class TestRangeMapper(unittest.TestCase): + def test_create(self): + mapper = RangeMapper.create(_data.items()) + self.assertEqual(type(mapper), FlatRangeMapper) + mapper = RangeMapper.create(_data.items(), max_flat_range_size=40) + self.assertEqual(type(mapper), RangeTree) + + def test_get_item(self): + for mclass in (RangeTree, FlatRangeMapper): + with self.subTest(msg=f'{mclass.__name__} test'): + mapper = mclass(_data.items()) + inds = [0, 1, 7, 9, 14, 15, 22, 41, 42, 43] + expected = ['a', 'b', 'b', 'x', 'x', 'y', 'y', 'z', None, None] + for i, ex_i in zip(inds, expected): + got = mapper[i] + self.assertEqual(got, ex_i) + + def test_key2range(self): + for mclass in (RangeTree, FlatRangeMapper): + with self.subTest(msg=f'{mclass.__name__} test'): + mapper = mclass(_data.items()) + keys = ['a', 'b', 'x', 'y', 'z'] + expected = [(0, 1), (1, 9), (9, 15), (15, 36), (36, 42)] + for key, ex in zip(keys, expected): + got = mapper.key2range(key) + self.assertEqual(got, ex) + + try: + mapper.key2range('bad') + except KeyError: + pass + else: + self.fail("Expected KeyError") + + def test_key2size(self): + for mclass in (RangeTree, FlatRangeMapper): + with self.subTest(msg=f'{mclass.__name__} test'): + mapper = mclass(_data.items()) + keys = ['a', 'b', 'x', 'y', 'z'] + expected = [1, 8, 6, 21, 6] + for key, ex in zip(keys, expected): + got = mapper.key2size(key) + self.assertEqual(got, ex) + + try: + mapper.key2size('bad') + except KeyError: + pass + else: + self.fail("Expected KeyError") + + def test_index2key_rel(self): + for mclass in (RangeTree, FlatRangeMapper): + with self.subTest(msg=f'{mclass.__name__} test'): + mapper = mclass(_data.items()) + inds = [0, 1, 7, 9, 14, 15, 22, 41, 42, 43] + expected = [('a',0), ('b',0), ('b',6), ('x',0), ('x',5), ('y',0), ('y',7), ('z',5), (None, None), (None, None)] + for i, ex in zip(inds, expected): + got = mapper.index2key_rel(i) + self.assertEqual(got, ex) + + def test_iter(self): + for mclass in (RangeTree, FlatRangeMapper): + with self.subTest(msg=f'{mclass.__name__} test'): + mapper = mclass(_data.items()) + expected = [('a', 0, 1), ('b', 1, 9), ('x', 9, 15), ('y', 15, 36), ('z', 36, 42)] + for got, ex in zip(mapper, expected): + self.assertEqual(got, ex) + +if __name__ == "__main__": + unittest.main() diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index 7a8c222c47..f4d8dceb78 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -229,7 +229,7 @@ def _setup_transfers_rev(group, desvars, responses): xfer_in[sub_out] xfer_out[sub_out] elif out_is_dup and inp_missing > 0 and (iowninput or distrib_in): - # if this proc owns the input or the input is distributyed, + # if this proc owns the input or the input is distributed, # and the output is duplicated, then we send the owning/distrib input # to each duplicated output that doesn't have a corresponding connected # input on the same proc. @@ -278,8 +278,8 @@ def _setup_transfers_rev(group, desvars, responses): xfer_out[sub_out].append(output_inds) if has_rev_par_coloring and iidxlist_nc: - # keep transfers separate that shouldn't happen when partial - # coloring is active + # keep transfers separate that shouldn't happen when parallel + # deriv coloring is active if len(iidxlist_nc) > 1: input_inds = _merge(iidxlist_nc, size_nc) output_inds = _merge(oidxlist_nc, size_nc) From 690f3166f8bf1a468b8c1013f4a4b5f279a96ad5 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Thu, 9 Nov 2023 08:44:38 -0500 Subject: [PATCH 57/70] test fixes --- openmdao/approximation_schemes/approximation_scheme.py | 3 +-- openmdao/core/group.py | 2 +- openmdao/utils/rangemapper.py | 2 -- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/openmdao/approximation_schemes/approximation_scheme.py b/openmdao/approximation_schemes/approximation_scheme.py index de5602a510..65ea042d49 100644 --- a/openmdao/approximation_schemes/approximation_scheme.py +++ b/openmdao/approximation_schemes/approximation_scheme.py @@ -272,10 +272,9 @@ def _init_approximations(self, system): in_idx = [list(in_idx)] vec_idx = [vec_idx] else: + vec_idx = LocalRangeIterable(system, wrt) if directional and vec is not None: vec_idx = [v for v in vec_idx if v is not None] - else: - vec_idx = LocalRangeIterable(system, wrt) # Directional derivatives for quick deriv checking. # Place the indices in a list so that they are all stepped at the same time. diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 327f313551..fc46e08060 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -5056,7 +5056,7 @@ def _setup_iteration_lists(self): graph = self.get_relevance_graph(dvs, responses) dvs = set([meta['source'] for meta in dvs.values()]) - responses = set([meta['source'] for meta in responses.values()]) + responses = list(set([meta['source'] for meta in responses.values()])) # we don't want _auto_ivc dependency to force all subsystems to be iterated, so split # the _auto_ivc node into two nodes, one for design vars and one for everything else. diff --git a/openmdao/utils/rangemapper.py b/openmdao/utils/rangemapper.py index 62ec4db92e..b0a7b6fd56 100644 --- a/openmdao/utils/rangemapper.py +++ b/openmdao/utils/rangemapper.py @@ -189,8 +189,6 @@ class RangeTree(RangeMapper): Attributes ---------- - size : int - Total size of all of the sizes combined. root : RangeTreeNode Root node of the binary search tree. """ From 1e3be430f1b6ed814c09415f761070bc9a5c583a Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Thu, 9 Nov 2023 09:39:09 -0500 Subject: [PATCH 58/70] docstring fix --- openmdao/utils/rangemapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmdao/utils/rangemapper.py b/openmdao/utils/rangemapper.py index b0a7b6fd56..382ec7bb69 100644 --- a/openmdao/utils/rangemapper.py +++ b/openmdao/utils/rangemapper.py @@ -40,7 +40,7 @@ def create(sizes, max_flat_range_size=MAX_FLAT_RANGE_SIZE): Parameters ---------- - sizes : list of (key, size) + sizes : iterable of (key, size) Iterable of (key, size) tuples. max_flat_range_size : int If the total array size is less than this, a FlatRangeMapper will be returned instead From 92f0b2c964b1f68f41b4915b852ba20cb130d611 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 13 Nov 2023 12:48:11 -0500 Subject: [PATCH 59/70] fix to seed_vars --- openmdao/core/total_jac.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index 0b1aeef9c1..a2f894b595 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -818,11 +818,12 @@ def _create_in_idx_map(self, mode): imeta = defaultdict(bool) imeta['coloring'] = simul_coloring all_rel_systems = set() - all_vois = set() cache = False imeta['itermeta'] = itermeta = [] locs = None for ilist in simul_coloring.color_iter(mode): + all_vois = set() + for i in ilist: rel_systems, cache_lin_sol, voiname = idx_map[i] all_rel_systems = _update_rel_systems(all_rel_systems, rel_systems) From d851549c53f1b6095d938a5ebd4469ba852922aa Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 13 Nov 2023 15:25:25 -0500 Subject: [PATCH 60/70] cleanup of multi seed_var case --- openmdao/core/group.py | 47 +++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index fc46e08060..9eb289a1d9 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -3793,41 +3793,32 @@ def _apply_linear(self, jac, rel_systems, mode, scope_out=None, scope_in=None): # If we have a distributed constraint/obj within the FD group and that con/obj is, # active, we perform essentially an allreduce on the d_inputs vars that connect to # outside systems so they'll include the contribution from all procs. - if self._fd_rev_xfer_correction_dist: + if self._fd_rev_xfer_correction_dist and mode == 'rev': seed_vars = self._problem_meta['seed_vars'] if seed_vars is not None: - if len(seed_vars) > 1: - prefix = self.pathname + '.' - seed_vars = [n for n in seed_vars if n.startswith(prefix)] - if len(seed_vars) > 1: - # TODO: get this working with coloring - raise RuntimeError("Multiple simultaneous seed variables " - f"{sorted(seed_vars)} within an FD group are not" - " supported under MPI in reverse mode if they " - "depend on a group doing finite difference and " - "containing distributed constraints/objectives.") + seed_vars = [n for n in seed_vars if n in self._fd_rev_xfer_correction_dist] + slices = self._dinputs.get_slice_dict() + inarr = self._dinputs.asarray() + data = {} for seed_var in seed_vars: - if seed_var in self._fd_rev_xfer_correction_dist: - slices = self._dinputs.get_slice_dict() - inarr = self._dinputs.asarray() - data = {} - for inp in self._fd_rev_xfer_correction_dist[seed_var]: - if inp in slices: + for inp in self._fd_rev_xfer_correction_dist[seed_var]: + if inp not in data: + if inp in slices: # inp is a local input arr = inarr[slices[inp]] if np.any(arr): data[inp] = arr else: - data[inp] = None - - if data: - comm = self.comm - myrank = comm.rank - for rank, d in enumerate(comm.allgather(data)): - if rank != myrank: - for n, val in d.items(): - if val is not None and n in slices: - inarr[slices[n]] += val - break # there's only 1 in the seed_vars set + data[inp] = None # don't send an array of zeros + else: + data[inp] = None # prevent possible MPI hangs + + if data: + myrank = self.comm.rank + for rank, d in enumerate(self.comm.allgather(data)): + if rank != myrank: + for n, val in d.items(): + if val is not None and n in slices: + inarr[slices[n]] += val # Apply recursion else: From f9c87d89c56bbca778d96366b134335cd1f72a6e Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Tue, 14 Nov 2023 13:21:44 -0500 Subject: [PATCH 61/70] fixed and added test for one of andrew's cases --- openmdao/core/tests/test_distrib_derivs.py | 70 +++++++++++++++++++ openmdao/devtools/debug.py | 4 +- .../drivers/tests/test_scipy_optimizer.py | 4 +- openmdao/vectors/petsc_transfer.py | 34 ++++++--- 4 files changed, 99 insertions(+), 13 deletions(-) diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index e7a401aaf9..d42fc54b49 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -2656,6 +2656,76 @@ def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): prob.run_model() assert_check_totals(prob.check_totals("ParallelSum.sum", "ivc.x")) + +class DummyComp(om.ExplicitComponent): + def initialize(self): + self.options.declare('a',default=0.) + self.options.declare('b',default=0.) + + def setup(self): + self.add_input('x') + self.add_output('y', 0.) + + def compute(self, inputs, outputs): + outputs['y'] = self.options['a']*inputs['x'] + self.options['b'] + + def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): + if mode=='rev': + if 'y' in d_outputs: + if 'x' in d_inputs: + d_inputs['x'] += self.options['a'] * d_outputs['y'] + print(self.pathname, 'compute_jvp: dinputs[x]', d_inputs['x']) + else: + raise RuntimeError("fwd mode not supported") + +class DummyGroup(om.ParallelGroup): + def setup(self): + self.add_subsystem('C1',DummyComp(a=1,b=2.)) + self.add_subsystem('C2',DummyComp(a=3.,b=4.)) + + +class TestLocalSrcIndsParColoring2(unittest.TestCase): + N_PROCS = 2 + + def test_local_src_inds(self): + # this uses parallel coloring with src_indices indexing into a local array + prob = om.Problem() + model = prob.model + model.add_subsystem('dvs',om.IndepVarComp('x',[1.,2.]), promotes=['*']) + model.add_subsystem('par',DummyGroup()) + model.connect('x','par.C1.x',src_indices=[0]) + model.connect('x','par.C2.x',src_indices=[1]) + + prob.model.add_design_var('x',lower=0.,upper=1.) + + # None or string + deriv_color = 'deriv_color' + + # compute derivatives for made-up y constraints in parallel + prob.model.add_constraint('par.C1.y', + lower=1.0, + parallel_deriv_color=deriv_color) + prob.model.add_constraint('par.C2.y', + lower=1.0, + parallel_deriv_color=deriv_color) + + prob.setup(mode='rev') + prob.run_model() + prob.check_totals(compact_print=False, + show_progress=False, + directional=False, + show_only_incorrect=True) + + +class TestLocalSrcIndsParColoring3(TestLocalSrcIndsParColoring2): + N_PROCS = 3 + + +class TestLocalSrcIndsParColoring4(TestLocalSrcIndsParColoring2): + N_PROCS = 4 + + + if __name__ == "__main__": from openmdao.utils.mpi import mpirun_tests mpirun_tests() diff --git a/openmdao/devtools/debug.py b/openmdao/devtools/debug.py index bd269bc504..c32c15ee2e 100644 --- a/openmdao/devtools/debug.py +++ b/openmdao/devtools/debug.py @@ -701,7 +701,7 @@ def show_dist_var_conns(group, rev=False, out_stream=_DEFAULT_OUT_STREAM): skip = len(gprefix) for sub, transfer in g._transfers[direction].items(): - if sub is not None: + if sub is not None and (not isinstance(sub, tuple) or sub[0] is not None): if not gprint: gdict[g.pathname] = {} gprint = True @@ -749,7 +749,7 @@ def show_dist_var_conns(group, rev=False, out_stream=_DEFAULT_OUT_STREAM): strs[s] = set() strs[s].add(ranktup) - gdict[g.pathname][sub] = strs + gdict[g.pathname][str(sub)] = strs do_ranks = False diff --git a/openmdao/drivers/tests/test_scipy_optimizer.py b/openmdao/drivers/tests/test_scipy_optimizer.py index 9022294e0a..5330e3abab 100644 --- a/openmdao/drivers/tests/test_scipy_optimizer.py +++ b/openmdao/drivers/tests/test_scipy_optimizer.py @@ -17,7 +17,7 @@ from openmdao.test_suite.components.sellar_feature import SellarMDA from openmdao.test_suite.components.simple_comps import NonSquareArrayComp from openmdao.test_suite.groups.sin_fitter import SineFitter -from openmdao.utils.assert_utils import assert_near_equal, assert_warning +from openmdao.utils.assert_utils import assert_near_equal, assert_warning, assert_check_totals from openmdao.utils.general_utils import run_driver from openmdao.utils.testing_utils import set_env_vars_context from openmdao.utils.mpi import MPI @@ -155,6 +155,8 @@ def test_opt_distcomp(self): con = prob.driver.get_constraint_values() obj = prob.driver.get_objective_values() + assert_check_totals(prob.check_totals(method='cs', out_stream=None)) + assert_near_equal(obj['sum.f_sum'], 0.0, 2e-6) assert_near_equal(con['parab.f_xy'], np.zeros(7), diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index f4d8dceb78..d00947a136 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -229,10 +229,10 @@ def _setup_transfers_rev(group, desvars, responses): xfer_in[sub_out] xfer_out[sub_out] elif out_is_dup and inp_missing > 0 and (iowninput or distrib_in): - # if this proc owns the input or the input is distributed, + # if this rank owns the input or the input is distributed, # and the output is duplicated, then we send the owning/distrib input # to each duplicated output that doesn't have a corresponding connected - # input on the same proc. + # input on the same rank. oidxlist = [] iidxlist = [] oidxlist_nc = [] @@ -241,7 +241,7 @@ def _setup_transfers_rev(group, desvars, responses): for rnk, osize, isize in zip(range(group.comm.size), group.get_var_sizes(abs_out, 'output'), group.get_var_sizes(abs_in, 'input')): - if rnk == myrank: + if rnk == myrank: # transfer to output on same rank oidxlist.append(output_inds) iidxlist.append(input_inds) size += len(input_inds) @@ -257,6 +257,8 @@ def _setup_transfers_rev(group, desvars, responses): continue if has_rev_par_coloring: + # these transfers will only happen if parallel coloring is + # not active for the current seed response oidxlist_nc.append(oarr) iidxlist_nc.append(input_inds) size_nc += len(input_inds) @@ -302,8 +304,7 @@ def _setup_transfers_rev(group, desvars, responses): xfer_in[sub_out].append(input_inds) xfer_out[sub_out].append(output_inds) else: - # not a local input but still need entries in the transfer dicts to - # avoid hangs + # remote input but still need entries in the transfer dicts to avoid hangs xfer_in[sub_out] xfer_out[sub_out] if has_rev_par_coloring: @@ -429,20 +430,33 @@ def _merge(inds_list, tot_size): def _get_output_inds(group, abs_out, abs_in): owner = group._owning_rank[abs_out] - src_indices = group._var_abs2meta['input'][abs_in]['src_indices'] - if src_indices is not None: - src_indices = src_indices.shaped_array() + meta_in = group._var_abs2meta['input'][abs_in] + out_dist = group._var_allprocs_abs2meta['output'][abs_out]['distributed'] + in_dist = meta_in['distributed'] + src_indices = meta_in['src_indices'] rank = group.comm.rank if abs_out in group._var_abs2meta['output'] else owner out_idx = group._var_allprocs_abs2idx[abs_out] offsets = group._get_var_offsets()['output'][:, out_idx] sizes = group._var_sizes['output'][:, out_idx] + + if src_indices is None: + orig_src_inds = src_indices + else: + src_indices = src_indices.shaped_array() + orig_src_inds = src_indices + if not out_dist and not in_dist: # convert from local to distributed src_indices + off = np.sum(sizes[:rank]) + if off > 0.: # adjust for local offsets + # don't do += to avoid modifying stored value + src_indices = src_indices + off + # NOTE: src_indices are relative to a single, possibly distributed variable, # while the output_inds that we compute are relative to the full distributed # array that contains all local variables from each rank stacked in rank order. if src_indices is None: - if group._var_allprocs_abs2meta['output'][abs_out]['distributed']: + if out_dist: # input in this case is non-distributed (else src_indices would be # defined by now). dist output to non-distributed input conns w/o # src_indices are not allowed. @@ -469,4 +483,4 @@ def _get_output_inds(group, abs_out, abs_in): start = end - return output_inds, src_indices + return output_inds, orig_src_inds From d087796471a6d69c03605165d5385e99d0b10e92 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Tue, 14 Nov 2023 13:34:49 -0500 Subject: [PATCH 62/70] cleanup --- openmdao/core/tests/test_distrib_derivs.py | 33 ++++++++++------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index d42fc54b49..1ad068fc77 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -2674,9 +2674,10 @@ def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): if 'y' in d_outputs: if 'x' in d_inputs: d_inputs['x'] += self.options['a'] * d_outputs['y'] - print(self.pathname, 'compute_jvp: dinputs[x]', d_inputs['x']) else: - raise RuntimeError("fwd mode not supported") + if 'y' in d_outputs: + if 'x' in d_inputs: + d_outputs['y'] += self.options['a'] * d_inputs['x'] class DummyGroup(om.ParallelGroup): def setup(self): @@ -2687,7 +2688,7 @@ def setup(self): class TestLocalSrcIndsParColoring2(unittest.TestCase): N_PROCS = 2 - def test_local_src_inds(self): + def check_model(self, mode): # this uses parallel coloring with src_indices indexing into a local array prob = om.Problem() model = prob.model @@ -2696,25 +2697,21 @@ def test_local_src_inds(self): model.connect('x','par.C1.x',src_indices=[0]) model.connect('x','par.C2.x',src_indices=[1]) - prob.model.add_design_var('x',lower=0.,upper=1.) - - # None or string - deriv_color = 'deriv_color' + model.add_design_var('x',lower=0.,upper=1.) # compute derivatives for made-up y constraints in parallel - prob.model.add_constraint('par.C1.y', - lower=1.0, - parallel_deriv_color=deriv_color) - prob.model.add_constraint('par.C2.y', - lower=1.0, - parallel_deriv_color=deriv_color) + model.add_constraint('par.C1.y', lower=1.0, parallel_deriv_color='deriv_color') + model.add_constraint('par.C2.y', lower=1.0, parallel_deriv_color='deriv_color') - prob.setup(mode='rev') + prob.setup(mode=mode) prob.run_model() - prob.check_totals(compact_print=False, - show_progress=False, - directional=False, - show_only_incorrect=True) + assert_check_totals(prob.check_totals(out_stream=None)) + + def test_local_src_inds_fwd(self): + self.check_model(mode='fwd') + + def test_local_src_inds_rev(self): + self.check_model(mode='rev') class TestLocalSrcIndsParColoring3(TestLocalSrcIndsParColoring2): From a5491426bd54350683aed7ade8bd59c90a8b1c18 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Tue, 14 Nov 2023 14:28:07 -0500 Subject: [PATCH 63/70] added test for andrew's bcast case --- openmdao/core/tests/test_distrib_derivs.py | 3 + openmdao/core/tests/test_parallel_groups.py | 104 ++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/openmdao/core/tests/test_distrib_derivs.py b/openmdao/core/tests/test_distrib_derivs.py index 1ad068fc77..0cd685a4be 100644 --- a/openmdao/core/tests/test_distrib_derivs.py +++ b/openmdao/core/tests/test_distrib_derivs.py @@ -2685,6 +2685,7 @@ def setup(self): self.add_subsystem('C2',DummyComp(a=3.,b=4.)) +@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") class TestLocalSrcIndsParColoring2(unittest.TestCase): N_PROCS = 2 @@ -2714,10 +2715,12 @@ def test_local_src_inds_rev(self): self.check_model(mode='rev') +@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") class TestLocalSrcIndsParColoring3(TestLocalSrcIndsParColoring2): N_PROCS = 3 +@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") class TestLocalSrcIndsParColoring4(TestLocalSrcIndsParColoring2): N_PROCS = 4 diff --git a/openmdao/core/tests/test_parallel_groups.py b/openmdao/core/tests/test_parallel_groups.py index 0fbe125a5b..2f6784462f 100644 --- a/openmdao/core/tests/test_parallel_groups.py +++ b/openmdao/core/tests/test_parallel_groups.py @@ -745,6 +745,110 @@ def test_fd_rev_mode(self): assert_check_totals(data, atol=1e-5) + +class DoubleComp(om.ExplicitComponent): + # dummy component to use outputs of parallel group + def setup(self): + self.add_input('x') + self.add_output('y', 0.) + + def compute(self, inputs, outputs): + outputs['y'] = 2. * inputs['x'] + + def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): + if mode=='rev': + if 'y' in d_outputs: + if 'x' in d_inputs: + d_inputs['x'] += 2. * d_outputs['y'] + else: + if 'y' in d_outputs: + if 'x' in d_inputs: + d_outputs['y'] += 2. * d_inputs['x'] + + +class BcastComp(om.ExplicitComponent): + # dummy component to be evaluated in pararallel by om.ParallelGroup + def setup(self): + self.add_input('x', shape_by_conn=True) + self.add_output('y', 0.0) + + def compute(self,inputs,outputs): + y = None + if self.comm.rank==0: # pretend that this must be computed on one or a subset of procs... + # which may be necessary for various solvers/analyses + y = np.sum(inputs['x']) + + # bcast to all procs + outputs['y'] = self.comm.bcast(y, root=0) + + def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): + if mode == 'rev': + if 'y' in d_outputs: + # this used to give bad derivatives in parallel group due to non-uniform d_outputs['y'] across procs... + if self.comm.rank==0: # compute on first proc + d_inputs['x'] += d_outputs['y'] + + # bcast to all procs + d_inputs['x'] = self.comm.bcast(d_inputs['x'], root=0) + else: + if 'y' in d_outputs: + if self.comm.rank==0: # compute on first proc + d_outputs['y'] += np.sum(d_inputs['x']) + + # bcast to all procs + d_outputs['y'] = self.comm.bcast(d_outputs['y'], root=0) + + +@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") +class TestSingleRankRunWithBcast(unittest.TestCase): + + N_PROCS = 2 + + def setup_model(self, mode): + prob = om.Problem() + model = prob.model + + # dummy structural design variables + model.add_subsystem('indep', om.IndepVarComp('x', 0.01*np.ones(20))) + par = model.add_subsystem('par', om.ParallelGroup()) + + par.add_subsystem('C1', BcastComp()) + par.add_subsystem('C2', BcastComp()) + + model.connect('indep.x', f'par.C1.x') + model.connect('indep.x', f'par.C2.x') + + # add component that uses outputs from parallel components + model.add_subsystem('dummy_comp', DoubleComp()) + model.connect('par.C1.y','dummy_comp.x') + + + prob.setup(mode=mode) + prob.run_model() + + return prob + + def test_bcast_fwd(self): + prob = self.setup_model(mode='fwd') + + for i in range(1,3): + y = prob.get_val(f'par.C{i}.y',get_remote=True) + assert_near_equal(y, 0.2, 1e-6) + + assert_check_totals(prob.check_totals(of=['dummy_comp.y','par.C1.y'], + wrt=['indep.x'], out_stream=None)) + + def test_bcast_rev(self): + prob = self.setup_model(mode='rev') + + for i in range(1,3): + y = prob.get_val(f'par.C{i}.y',get_remote=True) + assert_near_equal(y, 0.2, 1e-6) + + assert_check_totals(prob.check_totals(of=['dummy_comp.y','par.C1.y'], + wrt=['indep.x'], out_stream=None)) + + if __name__ == "__main__": from openmdao.utils.mpi import mpirun_tests mpirun_tests() From dd15037ca428973fcb61c4473f34d1f21e3c64ac Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Tue, 14 Nov 2023 14:37:14 -0500 Subject: [PATCH 64/70] added serial tests for several group layouts (fan in, fan out, etc.) --- openmdao/core/tests/test_parallel_groups.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/openmdao/core/tests/test_parallel_groups.py b/openmdao/core/tests/test_parallel_groups.py index 2f6784462f..1e27a06c8b 100644 --- a/openmdao/core/tests/test_parallel_groups.py +++ b/openmdao/core/tests/test_parallel_groups.py @@ -54,11 +54,8 @@ def _test_func_name(func, num, param): return func.__name__ + '_' + '_'.join(args) -@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") class TestParallelGroups(unittest.TestCase): - N_PROCS = 2 - @parameterized.expand(itertools.product([(om.LinearRunOnce, None)], [om.NonlinearBlockGS, om.NonlinearRunOnce]), name_func=_test_func_name) @@ -216,6 +213,12 @@ def test_converge_diverge(self, solver, nlsolver, mode): J = prob.compute_totals(of=unknown_list, wrt=indep_list) assert_near_equal(J['c7.y1', 'iv.x'][0][0], -40.75, 1e-6) + +@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") +class TestParallelGroupsMPI2(TestParallelGroups): + + N_PROCS = 2 + @parameterized.expand(['fwd', 'rev'], name_func=_test_func_name) def test_par_with_only_1_subsystem(self, mode): p = om.Problem() @@ -349,6 +352,7 @@ def test_setup_messages_only_on_proc0(self): else: self.fail("Didn't find '%s' in info messages." % msg) + @unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") class TestDistDriverVars(unittest.TestCase): From b2cbf7683f475b49389d5a8d8ada823b61b1f4a4 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Tue, 14 Nov 2023 16:13:34 -0500 Subject: [PATCH 65/70] pep8 fix --- openmdao/vectors/default_transfer.py | 2 +- openmdao/vectors/petsc_transfer.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/openmdao/vectors/default_transfer.py b/openmdao/vectors/default_transfer.py index 1f4db1b2a2..b0551619ef 100644 --- a/openmdao/vectors/default_transfer.py +++ b/openmdao/vectors/default_transfer.py @@ -18,7 +18,7 @@ def _fill(arr, indices_list): ---------- arr : ndarray Array to be filled. - indices_list : list of int ndarrays + indices_list : list of int ndarrays or ranges List of ranges/indices to be placed into arr. """ start = end = 0 diff --git a/openmdao/vectors/petsc_transfer.py b/openmdao/vectors/petsc_transfer.py index d00947a136..4d1cef2cc5 100644 --- a/openmdao/vectors/petsc_transfer.py +++ b/openmdao/vectors/petsc_transfer.py @@ -440,7 +440,6 @@ def _get_output_inds(group, abs_out, abs_in): offsets = group._get_var_offsets()['output'][:, out_idx] sizes = group._var_sizes['output'][:, out_idx] - if src_indices is None: orig_src_inds = src_indices else: From c1d5def5eb7c366b268a092526602c36c43466eb Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 17 Nov 2023 13:02:42 -0500 Subject: [PATCH 66/70] trying a change to par_deriv_jac_setter --- openmdao/core/group.py | 1 + openmdao/core/total_jac.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 9eb289a1d9..88969ac17b 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -3857,6 +3857,7 @@ def _solve_linear(self, mode, rel_systems, scope_out=_UNDEFINED, scope_in=_UNDEF scope_in : set, None, or _UNDEFINED Inputs relevant to possible lower level calls to _apply_linear on Components. """ + # print(f"'{self.pathname}'", 'solve_linear', mode) if self._owns_approx_jac: # No subsolves if we are approximating our jacobian. Instead, we behave like an # ExplicitComponent and pass on the values in the derivatives vectors. diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index a2f894b595..d261fba8ec 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -1233,17 +1233,19 @@ def par_deriv_input_setter(self, inds, imeta, mode): all_rel_systems = set() vec_names = set() - dist = self.comm.size > 1 - for i in inds: - if not dist or self.in_loc_idxs[mode][i] >= 0: + if self.in_loc_idxs[mode][i] >= 0: rel_systems, vnames, _ = self.single_input_setter(i, imeta, mode) - all_rel_systems = _update_rel_systems(all_rel_systems, rel_systems) if vnames is not None: vec_names.add(vnames[0]) + else: + rel_systems, _, _ = self.in_idx_map[mode][i] + all_rel_systems = _update_rel_systems(all_rel_systems, rel_systems) self.model._problem_meta['parallel_deriv_color'] = imeta['par_deriv_color'] + # print("par_deriv_input_setter: all_rel_systems =", sorted(all_rel_systems), "par_deriv_color =", self.model._problem_meta['parallel_deriv_color']) + if vec_names: return all_rel_systems, sorted(vec_names), (inds[0], mode) else: @@ -1338,6 +1340,7 @@ def _jac_setter_dist(self, i, mode): scratch = self.jac_scratch['rev'][0] scratch[:] = 0.0 scratch[self.rev_allreduce_mask] = self.J[i][self.rev_allreduce_mask] + # print("_jac_setter_dist: Allreduce in jac_setter_dist on rank", self.comm.rank, "par_deriv_color =", self.model._problem_meta['parallel_deriv_color']) self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) def single_jac_setter(self, i, mode, meta): @@ -1370,8 +1373,7 @@ def par_deriv_jac_setter(self, inds, mode, meta): meta : dict Metadata dict. """ - dist = self.comm.size > 1 - if dist: + if self.comm.size > 1: for i in inds: if self.in_loc_idxs[mode][i] >= 0: self.simple_single_jac_scatter(i, mode) @@ -1389,8 +1391,10 @@ def par_deriv_jac_setter(self, inds, mode, meta): else: # rev if i < 0: byrank = self.comm.allgather((i, None)) + # print("rank=", self.comm.rank, "par_deriv_jac_setter: allgather None inds=", inds, "par_deriv_color =", self.model._problem_meta['parallel_deriv_color']) else: byrank = self.comm.allgather((i, self.J[i])) + # print("rank=", self.comm.rank, f"par_deriv_jac_setter: allgather {self.J[i]} inds=", inds, "par_deriv_color =", self.model._problem_meta['parallel_deriv_color']) for ind, row in byrank: if row is not None: self.J[ind, :] = row From 59dab6df3943a0d4f54b38519aa54d16d398d22a Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Fri, 17 Nov 2023 14:53:10 -0500 Subject: [PATCH 67/70] cleanup --- openmdao/core/group.py | 1 - openmdao/core/total_jac.py | 5 ----- 2 files changed, 6 deletions(-) diff --git a/openmdao/core/group.py b/openmdao/core/group.py index 88969ac17b..9eb289a1d9 100644 --- a/openmdao/core/group.py +++ b/openmdao/core/group.py @@ -3857,7 +3857,6 @@ def _solve_linear(self, mode, rel_systems, scope_out=_UNDEFINED, scope_in=_UNDEF scope_in : set, None, or _UNDEFINED Inputs relevant to possible lower level calls to _apply_linear on Components. """ - # print(f"'{self.pathname}'", 'solve_linear', mode) if self._owns_approx_jac: # No subsolves if we are approximating our jacobian. Instead, we behave like an # ExplicitComponent and pass on the values in the derivatives vectors. diff --git a/openmdao/core/total_jac.py b/openmdao/core/total_jac.py index d261fba8ec..df93ab1844 100644 --- a/openmdao/core/total_jac.py +++ b/openmdao/core/total_jac.py @@ -1244,8 +1244,6 @@ def par_deriv_input_setter(self, inds, imeta, mode): self.model._problem_meta['parallel_deriv_color'] = imeta['par_deriv_color'] - # print("par_deriv_input_setter: all_rel_systems =", sorted(all_rel_systems), "par_deriv_color =", self.model._problem_meta['parallel_deriv_color']) - if vec_names: return all_rel_systems, sorted(vec_names), (inds[0], mode) else: @@ -1340,7 +1338,6 @@ def _jac_setter_dist(self, i, mode): scratch = self.jac_scratch['rev'][0] scratch[:] = 0.0 scratch[self.rev_allreduce_mask] = self.J[i][self.rev_allreduce_mask] - # print("_jac_setter_dist: Allreduce in jac_setter_dist on rank", self.comm.rank, "par_deriv_color =", self.model._problem_meta['parallel_deriv_color']) self.comm.Allreduce(scratch, self.J[i], op=MPI.SUM) def single_jac_setter(self, i, mode, meta): @@ -1391,10 +1388,8 @@ def par_deriv_jac_setter(self, inds, mode, meta): else: # rev if i < 0: byrank = self.comm.allgather((i, None)) - # print("rank=", self.comm.rank, "par_deriv_jac_setter: allgather None inds=", inds, "par_deriv_color =", self.model._problem_meta['parallel_deriv_color']) else: byrank = self.comm.allgather((i, self.J[i])) - # print("rank=", self.comm.rank, f"par_deriv_jac_setter: allgather {self.J[i]} inds=", inds, "par_deriv_color =", self.model._problem_meta['parallel_deriv_color']) for ind, row in byrank: if row is not None: self.J[ind, :] = row From e6b393953f2b89f1313dc2b7f0e20d2e654a133a Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 27 Nov 2023 15:10:44 -0500 Subject: [PATCH 68/70] fix for distrib response specified by input name --- openmdao/utils/coloring.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/openmdao/utils/coloring.py b/openmdao/utils/coloring.py index 2cfffc729c..7ae112e36a 100644 --- a/openmdao/utils/coloring.py +++ b/openmdao/utils/coloring.py @@ -2697,12 +2697,10 @@ def _initialize_model_approx(model, driver, of=None, wrt=None): """ Set up internal data structures needed for computing approx totals. """ - design_vars = driver._designvars - if of is None: of = driver._get_ordered_nl_responses() if wrt is None: - wrt = list(design_vars) + wrt = list(driver._designvars) # Initialization based on driver (or user) -requested "of" and "wrt". if (not model._owns_approx_jac or model._owns_approx_of is None or @@ -2712,19 +2710,12 @@ def _initialize_model_approx(model, driver, of=None, wrt=None): model._owns_approx_wrt = wrt # Support for indices defined on driver vars. - if MPI and model.comm.size > 1: - of_idx = model._owns_approx_of_idx - for key, meta in driver._responses.items(): - if meta['indices'] is not None: - of_idx[key] = meta['indices'] - else: - model._owns_approx_of_idx = { - key: meta['indices'] - for key, meta in _src_or_alias_item_iter(driver._responses) - if meta['indices'] is not None - } + model._owns_approx_of_idx = { + key: meta['indices'] for key, meta in _src_or_alias_item_iter(driver._responses) + if meta['indices'] is not None + } model._owns_approx_wrt_idx = { - key: meta['indices'] for key, meta in _src_or_alias_item_iter(design_vars) + key: meta['indices'] for key, meta in _src_or_alias_item_iter(driver._designvars) if meta['indices'] is not None } From af8f2ec01518afc8753a6cbfeff059376b42ea09 Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Mon, 27 Nov 2023 15:41:55 -0500 Subject: [PATCH 69/70] added some tests --- openmdao/core/tests/test_mixed_dist.py | 141 +++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 openmdao/core/tests/test_mixed_dist.py diff --git a/openmdao/core/tests/test_mixed_dist.py b/openmdao/core/tests/test_mixed_dist.py new file mode 100644 index 0000000000..2d749e1796 --- /dev/null +++ b/openmdao/core/tests/test_mixed_dist.py @@ -0,0 +1,141 @@ +import unittest + +import numpy as np +import openmdao.api as om +from openmdao.utils.mpi import MPI +from openmdao.utils.assert_utils import assert_check_totals + + +if MPI: + try: + from openmdao.vectors.petsc_vector import PETScVector + except ImportError: + PETScVector = None + + dist_shape = 1 if MPI.COMM_WORLD.rank > 0 else 2 +else: + dist_shape = 2 + + +class SerialComp(om.ExplicitComponent): + def setup(self): + self.add_input('x') + self.add_output('y') + + def compute(self, inputs, outputs): + outputs['y'] = 2.0* inputs['x'] + + def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): + if mode == 'fwd': + if 'y' in d_outputs: + if 'x' in d_inputs: + d_outputs['y'] += 2.0 * d_inputs['x'] + if mode == 'rev': + if 'y' in d_outputs: + if 'x' in d_inputs: + d_inputs['x'] += 2.0 * d_outputs['y'] + + +class MixedSerialInComp(om.ExplicitComponent): + def setup(self): + self.add_input('x') + self.add_output('yd', shape = dist_shape, distributed=True) + + def compute(self, inputs, outputs): + outputs['yd'][:] = 0.5 * inputs['x'] + + def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): + if mode == 'fwd': + if 'yd' in d_outputs: + if 'x' in d_inputs: + d_outputs['yd'] += 0.5 * d_inputs['x'] + if mode == 'rev': + if 'yd' in d_outputs: + if 'x' in d_inputs: + d_inputs['x'] += 0.5 * self.comm.allreduce(np.sum(d_outputs['yd'])) + + +class MixedSerialOutComp(om.ExplicitComponent): + def setup(self): + self.add_input('x') + self.add_input('xd', shape = dist_shape, distributed=True) + self.add_output('y') + + def compute(self, inputs, outputs): + outputs['y'] = 2.0 * inputs['x'] + self.comm.allreduce(3.0 * np.sum(inputs['xd'])) + + def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): + if mode == 'fwd': + if 'y' in d_outputs: + if 'x' in d_inputs: + d_outputs['y'] += 2.0 * d_inputs['x'] + if 'xd' in d_inputs: + d_outputs['y'] += 3.0 * self.comm.allreduce(np.sum(d_inputs['xd'])) + if mode == 'rev': + if 'y' in d_outputs: + if 'x' in d_inputs: + d_inputs['x'] += 2.0 * d_outputs['y'] + if 'xd' in d_inputs: + d_inputs['xd'] += 3.0 * d_outputs['y'] + + +class DistComp(om.ExplicitComponent): + def setup(self): + self.add_input('xd', shape = dist_shape, distributed=True) + self.add_output('yd', shape = dist_shape, distributed=True) + + def compute(self, inputs, outputs): + outputs['yd'] = 3.0 * inputs['xd'] + + def compute_jacvec_product(self, inputs, d_inputs, d_outputs, mode): + if mode == 'fwd': + if 'yd' in d_outputs: + if 'xd' in d_inputs: + d_outputs['yd'] += 3.0 * d_inputs['xd'] + if mode == 'rev': + if 'yd' in d_outputs: + if 'xd' in d_inputs: + d_inputs['xd'] += 3.0 * d_outputs['yd'] + + +def create_problem(): + prob = om.Problem() + model = prob.model + model.add_subsystem('ivc', om.IndepVarComp('x', val = 1.0)) + + model.add_subsystem('S', SerialComp()) # x -> y + model.add_subsystem('MI', MixedSerialInComp()) # x -> yd + model.add_subsystem('D', DistComp()) # xd -> yd + model.add_subsystem('MO', MixedSerialOutComp()) # x, xd -> y + model.connect('ivc.x', 'S.x') + model.connect('S.y','MI.x') + model.connect('MI.yd', 'D.xd') + model.connect('D.yd', 'MO.xd') + model.connect('S.y', 'MO.x') + model.add_design_var("ivc.x") + model.add_objective("MO.xd", index=0) # adding objective using input name + return prob + + +class TestMixedDist(unittest.TestCase): + def test_mixed_dist(self): + prob = create_problem() + prob.setup(mode='rev') + prob.run_model() + assert_check_totals(prob.check_totals()) + + + +@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") +class TestMixedDist2(TestMixedDist): + N_PROCS = 2 + + +@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") +class TestMixedDist3(TestMixedDist): + N_PROCS = 3 + + +if __name__ == '__main__': + from openmdao.utils.mpi import mpirun_tests + mpirun_tests() From 6e16d7c4d8ee2e3d740586f2052202445990c0bc Mon Sep 17 00:00:00 2001 From: Bret Naylor Date: Tue, 28 Nov 2023 13:53:38 -0500 Subject: [PATCH 70/70] fixed docstring and test --- openmdao/core/tests/test_mixed_dist.py | 13 +++++-------- openmdao/vectors/vector.py | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/openmdao/core/tests/test_mixed_dist.py b/openmdao/core/tests/test_mixed_dist.py index 2d749e1796..f5e8324c52 100644 --- a/openmdao/core/tests/test_mixed_dist.py +++ b/openmdao/core/tests/test_mixed_dist.py @@ -117,7 +117,10 @@ def create_problem(): return prob -class TestMixedDist(unittest.TestCase): +@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") +class TestMixedDist2(unittest.TestCase): + N_PROCS = 2 + def test_mixed_dist(self): prob = create_problem() prob.setup(mode='rev') @@ -125,14 +128,8 @@ def test_mixed_dist(self): assert_check_totals(prob.check_totals()) - -@unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") -class TestMixedDist2(TestMixedDist): - N_PROCS = 2 - - @unittest.skipUnless(MPI and PETScVector, "MPI and PETSc are required.") -class TestMixedDist3(TestMixedDist): +class TestMixedDist3(TestMixedDist2): N_PROCS = 3 diff --git a/openmdao/vectors/vector.py b/openmdao/vectors/vector.py index d8df6dd2b2..ab4452906d 100644 --- a/openmdao/vectors/vector.py +++ b/openmdao/vectors/vector.py @@ -404,7 +404,7 @@ def _abs_get_val(self, name, flat=True): def _abs_set_val(self, name, val): """ - Get the variable value using the absolute name. + Set the variable value using the absolute name. No error checking is performed on the name.