Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fixed bug that made Gatherv call fail #1847

Merged
merged 1 commit into from Jan 21, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 1 addition & 2 deletions openmdao/core/driver.py
Expand Up @@ -570,8 +570,7 @@ def _get_voi_val(self, name, meta, remote_vois, driver_scaling=True,
local_val = local_val[local_indices]

if get_remote:
if not local_val.flags['C_CONTIGUOUS']:
local_val = np.ascontiguousarray(local_val)
local_val = np.ascontiguousarray(local_val)
offsets = np.zeros(sizes.size, dtype=INT_DTYPE)
offsets[1:] = np.cumsum(sizes[:-1])
val = np.zeros(np.sum(sizes))
Expand Down
14 changes: 9 additions & 5 deletions openmdao/core/system.py
Expand Up @@ -1674,16 +1674,14 @@ def _setup_connections(self):
"""
pass

def _setup_vectors(self, root_vectors, alloc_complex=False):
def _setup_vectors(self, root_vectors):
"""
Compute all vectors for all vec names and assign excluded variables lists.

Parameters
----------
root_vectors : dict of dict of Vector
Root vectors: first key is 'input', 'output', or 'residual'; second key is vec_name.
alloc_complex : bool
Whether to allocate any imaginary storage to perform complex step. Default is False.
"""
self._vectors = vectors = {'input': OrderedDict(),
'output': OrderedDict(),
Expand Down Expand Up @@ -4273,7 +4271,10 @@ def _abs_get_val(self, abs_name, get_remote=False, rank=None, vec_name=None, kin
# TODO: could cache these offsets
offsets = np.zeros(sizes.size, dtype=INT_DTYPE)
offsets[1:] = np.cumsum(sizes[:-1])
loc_val = val if val is not _UNDEFINED else np.zeros(sizes[myrank])
if val is _UNDEFINED:
loc_val = np.zeros(sizes[myrank])
else:
loc_val = np.ascontiguousarray(val)
val = np.zeros(np.sum(sizes))
self.comm.Allgatherv(loc_val, [val, sizes, offsets, MPI.DOUBLE])
if not flat:
Expand All @@ -4291,7 +4292,10 @@ def _abs_get_val(self, abs_name, get_remote=False, rank=None, vec_name=None, kin
# TODO: could cache these offsets
offsets = np.zeros(sizes.size, dtype=INT_DTYPE)
offsets[1:] = np.cumsum(sizes[:-1])
loc_val = val if val is not _UNDEFINED else np.zeros(sizes[idx])
if val is _UNDEFINED:
loc_val = np.zeros(sizes[idx])
else:
loc_val = np.ascontiguousarray(val)
val = np.zeros(np.sum(sizes))
self.comm.Gatherv(loc_val, [val, sizes, offsets, MPI.DOUBLE], root=rank)
if not flat:
Expand Down
2 changes: 1 addition & 1 deletion openmdao/core/tests/test_distrib_list_vars.py
Expand Up @@ -77,7 +77,7 @@ def test_distributed_array_list_vars(self):
prob.model.add_subsystem('plus', DistributedAdder(size=size), promotes=['x', 'y'])
prob.model.add_subsystem('summer', Summer(size=size), promotes=[('invec', 'y'), 'sum'])

prob.setup()
prob.setup(force_alloc_complex=True) # force complex array storage to detect mpi bug

prob['x'] = np.arange(size)

Expand Down
18 changes: 8 additions & 10 deletions openmdao/error_checking/check_config.py
Expand Up @@ -651,16 +651,16 @@ def _check_config(prob):
_load_and_exec(options.file[0], user_args)


def check_allocate_complex_ln(model, under_cs):
def check_allocate_complex_ln(group, under_cs):
"""
Return True if linear vector should be complex.

This happens when a solver needs derivatives under complex step.

Parameters
----------
model : <Group>
Model to be checked, usually the root model.
group : <Group>
Group to be checked.
under_cs : bool
Flag indicates if complex vectors were allocated in a containing Group or were force
allocated in setup.
Expand All @@ -670,16 +670,14 @@ def check_allocate_complex_ln(model, under_cs):
bool
True if linear vector should be complex.
"""
under_cs |= 'cs' in model._approx_schemes
under_cs |= 'cs' in group._approx_schemes

if under_cs and model.nonlinear_solver is not None and \
model.nonlinear_solver.supports['gradients']:
if under_cs and group.nonlinear_solver is not None and \
group.nonlinear_solver.supports['gradients']:
return True

for sub, _ in model._subsystems_allprocs.values():
chk = check_allocate_complex_ln(sub, under_cs)

if chk:
for sub, _ in group._subsystems_allprocs.values():
if isinstance(sub, Group) and check_allocate_complex_ln(sub, under_cs):
return True

return False